From 923fecd7eec57802b8037ee014026430e6b2e03a Mon Sep 17 00:00:00 2001 From: Victor Elias Date: Mon, 22 Aug 2022 17:30:45 -0300 Subject: [PATCH 01/12] drivers: Implement delete for S3 and GS Right now it's not in the public Session interface and clients have to hackily cast the sessions. Sue me. --- drivers/gs.go | 14 ++++++++++++++ drivers/s3.go | 17 +++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/drivers/gs.go b/drivers/gs.go index ebd48da..6d60a15 100644 --- a/drivers/gs.go +++ b/drivers/gs.go @@ -156,6 +156,20 @@ func (os *gsSession) createClient() error { return nil } +func (os *gsSession) Delete(ctx context.Context, name string) error { + if !os.useFullAPI { + return errors.New("delete not supported for non full api") + } + if os.client == nil { + if err := os.createClient(); err != nil { + return err + } + } + return os.client.Bucket(os.bucket). + Object(name). + Delete(ctx) +} + func (os *gsSession) SaveData(ctx context.Context, name string, data io.Reader, meta map[string]string, timeout time.Duration) (string, error) { if os.useFullAPI { if os.client == nil { diff --git a/drivers/s3.go b/drivers/s3.go index 02a2799..d658eb0 100644 --- a/drivers/s3.go +++ b/drivers/s3.go @@ -8,6 +8,7 @@ import ( "crypto/sha256" "encoding/base64" "encoding/hex" + "errors" "fmt" "io" "mime/multipart" @@ -372,6 +373,22 @@ func (os *s3Session) saveDataPut(ctx context.Context, name string, data io.Reade return url, nil } +func (os *s3Session) Delete(ctx context.Context, name string) error { + if os.s3svc == nil { + return errors.New("delete not supported for non full api") + } + params := &s3.DeleteObjectInput{ + Bucket: aws.String(os.bucket), + Key: aws.String(name), + } + if *params.Key == "" { + // if name is not specified, assume that this session already created with specific key + params.Key = aws.String(os.key) + } + _, err := os.s3svc.DeleteObjectWithContext(ctx, params) + return err +} + func (os *s3Session) SaveData(ctx context.Context, name string, data io.Reader, meta map[string]string, timeout time.Duration) (string, error) { if os.s3svc != nil { return os.saveDataPut(ctx, name, data, meta, timeout) From e56c098608a1a892b0ad0bced49fd59790d08f7a Mon Sep 17 00:00:00 2001 From: Thom Shutt Date: Tue, 31 Jan 2023 16:19:03 +0000 Subject: [PATCH 02/12] Add deletion method to interface --- Makefile | 10 ++++++++++ drivers/drivers.go | 3 +++ drivers/fs.go | 4 ++++ drivers/fs_test.go | 27 ++++++++++++++++++++++++++- drivers/gs.go | 2 +- drivers/ipfs.go | 8 +++++++- drivers/local.go | 4 ++++ drivers/s3.go | 2 +- drivers/session_mock.go | 4 ++++ 9 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1e5f5e1 --- /dev/null +++ b/Makefile @@ -0,0 +1,10 @@ +.PHONY: all +all: build test + +.PHONY: build +build: + go build ./... + +.PHONY: tets +test: + go test -race ./... diff --git a/drivers/drivers.go b/drivers/drivers.go index 3e020e9..596464c 100644 --- a/drivers/drivers.go +++ b/drivers/drivers.go @@ -131,6 +131,9 @@ type OSSession interface { // ListFiles return list of files ListFiles(ctx context.Context, prefix, delim string) (PageInfo, error) + // DeleteFile deletes a single file. 'name' should be the full path + filename + DeleteFile(ctx context.Context, name string) error + ReadData(ctx context.Context, name string) (*FileInfoReader, error) } diff --git a/drivers/fs.go b/drivers/fs.go index e8fff77..0f15028 100644 --- a/drivers/fs.go +++ b/drivers/fs.go @@ -121,6 +121,10 @@ func (ostore *FSSession) ListFiles(ctx context.Context, dir, delim string) (Page return pi, nil } +func (ostore *FSSession) DeleteFile(ctx context.Context, name string) error { + return os.Remove(name) +} + func (ostore *FSSession) ReadData(ctx context.Context, name string) (*FileInfoReader, error) { prefix := "" if ostore.os.baseURI != nil { diff --git a/drivers/fs_test.go b/drivers/fs_test.go index 70a850f..32f0985 100644 --- a/drivers/fs_test.go +++ b/drivers/fs_test.go @@ -4,11 +4,14 @@ import ( "bytes" "context" "crypto/rand" - "github.com/stretchr/testify/assert" "io" + "io/ioutil" "net/url" "os" "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func readFile(session *FSSession, name string) []byte { @@ -66,3 +69,25 @@ func TestFsOS(t *testing.T) { assert.Equal(1, len(files.Directories())) assert.Equal("name1", files.Directories()[0]) } + +func TestDeleteFile(t *testing.T) { + file, err := ioutil.TempFile(os.TempDir(), "TestDeleteFileefix") + require.NoError(t, err) + + // Defer a removal of the file so that we don't litter the filesystem when this test fails + defer os.Remove(file.Name()) + + // Confirm that the file exists + _, err = os.Stat(file.Name()) + require.NoError(t, err) + + // Try to delete the file + u, err := url.Parse(os.TempDir()) + require.NoError(t, err) + sess := NewFSDriver(u).NewSession("driver-TestDeleteFile") + require.NoError(t, sess.DeleteFile(context.Background(), file.Name())) + + // Check the file no longer exists + _, err = os.Stat(file.Name()) + require.ErrorContains(t, err, "no such file or directory") +} diff --git a/drivers/gs.go b/drivers/gs.go index 6d60a15..bd32110 100644 --- a/drivers/gs.go +++ b/drivers/gs.go @@ -156,7 +156,7 @@ func (os *gsSession) createClient() error { return nil } -func (os *gsSession) Delete(ctx context.Context, name string) error { +func (os *gsSession) DeleteFile(ctx context.Context, name string) error { if !os.useFullAPI { return errors.New("delete not supported for non full api") } diff --git a/drivers/ipfs.go b/drivers/ipfs.go index 78f6da0..4382594 100644 --- a/drivers/ipfs.go +++ b/drivers/ipfs.go @@ -2,12 +2,14 @@ package drivers import ( "context" - "github.com/livepeer/go-tools/clients" + "errors" "io" "net/http" "path" "sync" "time" + + "github.com/livepeer/go-tools/clients" ) type IpfsOS struct { @@ -107,6 +109,10 @@ func (session *IpfsSession) GetInfo() *OSInfo { return nil } +func (ostore *IpfsSession) DeleteFile(ctx context.Context, name string) error { + return errors.New("unsupported method: cannot delete files from IPFS") +} + func (session *IpfsSession) SaveData(ctx context.Context, name string, data io.Reader, meta map[string]string, timeout time.Duration) (string, error) { // concatenate filename with name argument to get full filename, both may be empty fullPath := session.getAbsolutePath(name) diff --git a/drivers/local.go b/drivers/local.go index e66af92..5081500 100644 --- a/drivers/local.go +++ b/drivers/local.go @@ -81,6 +81,10 @@ func (ostore *MemorySession) EndSession() { ostore.os.lock.Unlock() } +func (ostore *MemorySession) DeleteFile(ctx context.Context, name string) error { + return errors.New("unsupported method: cannot delete files from in-memory session") +} + func (ostore *MemorySession) ListFiles(ctx context.Context, prefix, delim string) (PageInfo, error) { pi := &singlePageInfo{} if prefix == "" { diff --git a/drivers/s3.go b/drivers/s3.go index d658eb0..67cc382 100644 --- a/drivers/s3.go +++ b/drivers/s3.go @@ -373,7 +373,7 @@ func (os *s3Session) saveDataPut(ctx context.Context, name string, data io.Reade return url, nil } -func (os *s3Session) Delete(ctx context.Context, name string) error { +func (os *s3Session) DeleteFile(ctx context.Context, name string) error { if os.s3svc == nil { return errors.New("delete not supported for non full api") } diff --git a/drivers/session_mock.go b/drivers/session_mock.go index 84f6c93..6c25944 100644 --- a/drivers/session_mock.go +++ b/drivers/session_mock.go @@ -58,6 +58,10 @@ func (s *MockOSSession) ListFiles(ctx context.Context, prefix, delim string) (Pa return nil, nil } +func (s *MockOSSession) DeleteFile(ctx context.Context, name string) error { + return nil +} + func (s *MockOSSession) ReadData(ctx context.Context, name string) (*FileInfoReader, error) { args := s.Called(ctx, name) var fi *FileInfoReader From be6a3253d56fc2722436d33b3c53b5e594c35382 Mon Sep 17 00:00:00 2001 From: Thom Shutt Date: Tue, 31 Jan 2023 17:45:45 +0000 Subject: [PATCH 03/12] Use base when deleting with filesystem driver --- drivers/fs.go | 3 ++- drivers/fs_test.go | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/drivers/fs.go b/drivers/fs.go index 0f15028..e303ed4 100644 --- a/drivers/fs.go +++ b/drivers/fs.go @@ -7,6 +7,7 @@ import ( "net/url" "os" "path" + "path/filepath" "strings" "sync" "time" @@ -122,7 +123,7 @@ func (ostore *FSSession) ListFiles(ctx context.Context, dir, delim string) (Page } func (ostore *FSSession) DeleteFile(ctx context.Context, name string) error { - return os.Remove(name) + return os.Remove(filepath.Join(ostore.path, name)) } func (ostore *FSSession) ReadData(ctx context.Context, name string) (*FileInfoReader, error) { diff --git a/drivers/fs_test.go b/drivers/fs_test.go index 32f0985..942f356 100644 --- a/drivers/fs_test.go +++ b/drivers/fs_test.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "net/url" "os" + "path/filepath" "testing" "github.com/stretchr/testify/assert" @@ -84,8 +85,8 @@ func TestDeleteFile(t *testing.T) { // Try to delete the file u, err := url.Parse(os.TempDir()) require.NoError(t, err) - sess := NewFSDriver(u).NewSession("driver-TestDeleteFile") - require.NoError(t, sess.DeleteFile(context.Background(), file.Name())) + sess := NewFSDriver(u).NewSession(os.TempDir()) + require.NoError(t, sess.DeleteFile(context.Background(), filepath.Base(file.Name()))) // Check the file no longer exists _, err = os.Stat(file.Name()) From 5995dcae3d2c50155326b9828725c7c8db9c19f9 Mon Sep 17 00:00:00 2001 From: Thom Shutt Date: Tue, 31 Jan 2023 18:04:58 +0000 Subject: [PATCH 04/12] Use os.key for GS --- drivers/gs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/gs.go b/drivers/gs.go index bd32110..1ca6401 100644 --- a/drivers/gs.go +++ b/drivers/gs.go @@ -166,7 +166,7 @@ func (os *gsSession) DeleteFile(ctx context.Context, name string) error { } } return os.client.Bucket(os.bucket). - Object(name). + Object(os.key + "/" + name). Delete(ctx) } From bbc8b02cdf1deaa4c8ec9531e6da5444ddff1dea Mon Sep 17 00:00:00 2001 From: Thom Shutt Date: Wed, 1 Feb 2023 12:18:55 +0000 Subject: [PATCH 05/12] Update drivers/fs_test.go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Rafał Leszko --- drivers/fs_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/fs_test.go b/drivers/fs_test.go index 942f356..b998880 100644 --- a/drivers/fs_test.go +++ b/drivers/fs_test.go @@ -72,7 +72,7 @@ func TestFsOS(t *testing.T) { } func TestDeleteFile(t *testing.T) { - file, err := ioutil.TempFile(os.TempDir(), "TestDeleteFileefix") + file, err := os.CreateTemp("", "TestDeleteFileefix") require.NoError(t, err) // Defer a removal of the file so that we don't litter the filesystem when this test fails From 5a87ff3d20b4910f9273c5ab70e92d05db72781a Mon Sep 17 00:00:00 2001 From: Thom Shutt Date: Wed, 1 Feb 2023 12:31:24 +0000 Subject: [PATCH 06/12] Used shared ErrNotSupported --- drivers/drivers.go | 1 + drivers/ipfs.go | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/drivers.go b/drivers/drivers.go index 596464c..e6fbee6 100644 --- a/drivers/drivers.go +++ b/drivers/drivers.go @@ -24,6 +24,7 @@ var ext2mime = map[string]string{ } var ErrFormatMime = fmt.Errorf("unknown file extension") +var ErrNotSupported = fmt.Errorf("not supported") // NodeStorage is current node's primary driver var NodeStorage OSDriver diff --git a/drivers/ipfs.go b/drivers/ipfs.go index 4382594..8047be5 100644 --- a/drivers/ipfs.go +++ b/drivers/ipfs.go @@ -2,7 +2,6 @@ package drivers import ( "context" - "errors" "io" "net/http" "path" @@ -110,7 +109,7 @@ func (session *IpfsSession) GetInfo() *OSInfo { } func (ostore *IpfsSession) DeleteFile(ctx context.Context, name string) error { - return errors.New("unsupported method: cannot delete files from IPFS") + return ErrNotSupported } func (session *IpfsSession) SaveData(ctx context.Context, name string, data io.Reader, meta map[string]string, timeout time.Duration) (string, error) { From bedaf7396446901c8abff4f4ac8e1fde356d7629 Mon Sep 17 00:00:00 2001 From: Thom Shutt Date: Wed, 1 Feb 2023 12:32:26 +0000 Subject: [PATCH 07/12] ErrNotSupported in memory implementation --- drivers/local.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/local.go b/drivers/local.go index 5081500..786cf7a 100644 --- a/drivers/local.go +++ b/drivers/local.go @@ -82,7 +82,7 @@ func (ostore *MemorySession) EndSession() { } func (ostore *MemorySession) DeleteFile(ctx context.Context, name string) error { - return errors.New("unsupported method: cannot delete files from in-memory session") + return ErrNotSupported } func (ostore *MemorySession) ListFiles(ctx context.Context, prefix, delim string) (PageInfo, error) { From 24ebc23571967027584748531454473139e14c79 Mon Sep 17 00:00:00 2001 From: Thom Shutt Date: Wed, 1 Feb 2023 14:51:35 +0000 Subject: [PATCH 08/12] Address code review comments --- drivers/fs_test.go | 1 - drivers/s3.go | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/drivers/fs_test.go b/drivers/fs_test.go index b998880..bdb7677 100644 --- a/drivers/fs_test.go +++ b/drivers/fs_test.go @@ -5,7 +5,6 @@ import ( "context" "crypto/rand" "io" - "io/ioutil" "net/url" "os" "path/filepath" diff --git a/drivers/s3.go b/drivers/s3.go index 67cc382..95f6ce6 100644 --- a/drivers/s3.go +++ b/drivers/s3.go @@ -381,9 +381,8 @@ func (os *s3Session) DeleteFile(ctx context.Context, name string) error { Bucket: aws.String(os.bucket), Key: aws.String(name), } - if *params.Key == "" { - // if name is not specified, assume that this session already created with specific key - params.Key = aws.String(os.key) + if os.key != "" && !strings.HasPrefix(name, os.key+"/") { + params.Key = aws.String(path.Join(os.key, name)) } _, err := os.s3svc.DeleteObjectWithContext(ctx, params) return err From 6b33179cb3d31202b10c6a1b1b133fa8afc590b2 Mon Sep 17 00:00:00 2001 From: Thom Shutt Date: Wed, 1 Feb 2023 15:02:26 +0000 Subject: [PATCH 09/12] Update comment to reflect reality --- drivers/drivers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/drivers.go b/drivers/drivers.go index e6fbee6..56e4e37 100644 --- a/drivers/drivers.go +++ b/drivers/drivers.go @@ -132,7 +132,7 @@ type OSSession interface { // ListFiles return list of files ListFiles(ctx context.Context, prefix, delim string) (PageInfo, error) - // DeleteFile deletes a single file. 'name' should be the full path + filename + // DeleteFile deletes a single file. 'name' should be the relative filename DeleteFile(ctx context.Context, name string) error ReadData(ctx context.Context, name string) (*FileInfoReader, error) From b843f9e8b0b8b7ead855b09e62ca2a5c0e35d7f4 Mon Sep 17 00:00:00 2001 From: Thom Shutt Date: Wed, 1 Feb 2023 15:22:02 +0000 Subject: [PATCH 10/12] Start fixing linter warnings --- Makefile | 6 +++++- drivers/gs.go | 1 - drivers/local_test.go | 43 ++++++++++++++++++++++++++----------------- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 1e5f5e1..6f65afb 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,14 @@ .PHONY: all -all: build test +all: fmt build test .PHONY: build build: go build ./... +.PHONY: fmt +fmt: + go fmt ./... + .PHONY: tets test: go test -race ./... diff --git a/drivers/gs.go b/drivers/gs.go index 1ca6401..b6f7cdf 100644 --- a/drivers/gs.go +++ b/drivers/gs.go @@ -349,7 +349,6 @@ func gsGetFields(sess *s3Session) map[string]string { // gsCreatePolicy returns policy, signature func gsCreatePolicy(signer *gsSigner, bucket, region, path string) (string, string) { const timeFormat = "2006-01-02T15:04:05.999Z" - const shortTimeFormat = "20060102" expireAt := time.Now().Add(S3_POLICY_EXPIRE_IN_HOURS * time.Hour) expireFmt := expireAt.UTC().Format(timeFormat) diff --git a/drivers/local_test.go b/drivers/local_test.go index d006935..4571a52 100644 --- a/drivers/local_test.go +++ b/drivers/local_test.go @@ -2,11 +2,11 @@ package drivers import ( "context" - "fmt" - "github.com/stretchr/testify/assert" "net/url" "strings" "testing" + + "github.com/stretchr/testify/require" ) func TestLocalOS(t *testing.T) { @@ -18,39 +18,48 @@ func TestLocalOS(t *testing.T) { defer func() { dataCacheLen = oldDataCacheLen }() - assert := assert.New(t) + u, err := url.Parse("fake.com/url") - assert.NoError((err)) + require.NoError(t, err) + os := NewMemoryDriver(u) sess := os.NewSession(("sesspath")).(*MemorySession) path, err := sess.SaveData(context.TODO(), "name1/1.ts", strings.NewReader(tempData1), nil, 0) - fmt.Println(path) - assert.Equal("fake.com/url/stream/sesspath/name1/1.ts", path) + require.NoError(t, err) + require.Equal(t, "fake.com/url/stream/sesspath/name1/1.ts", path) + data := sess.GetData("sesspath/name1/1.ts") - fmt.Printf("got Data: '%s'\n", data) - assert.Equal(tempData1, string(data)) - path, err = sess.SaveData(context.TODO(), "name1/1.ts", strings.NewReader(tempData2), nil, 0) + require.Equal(t, tempData1, string(data)) + + _, err = sess.SaveData(context.TODO(), "name1/1.ts", strings.NewReader(tempData2), nil, 0) + require.NoError(t, err) + data = sess.GetData("sesspath/name1/1.ts") - assert.Equal(tempData2, string(data)) + require.Equal(t, tempData2, string(data)) + path, err = sess.SaveData(context.TODO(), "name1/2.ts", strings.NewReader(tempData3), nil, 0) + require.NoError(t, err) + data = sess.GetData("sesspath/name1/2.ts") - assert.Equal(tempData3, string(data)) + require.Equal(t, tempData3, string(data)) + // Test trim prefix when baseURI != nil data = sess.GetData(path) - assert.Equal(tempData3, string(data)) + require.Equal(t, tempData3, string(data)) data = sess.GetData("sesspath/name1/1.ts") - assert.Nil(data) + require.Nil(t, data) sess.EndSession() + data = sess.GetData("sesspath/name1/2.ts") - assert.Nil(data) + require.Nil(t, data) // Test trim prefix when baseURI = nil os = NewMemoryDriver(nil) sess = os.NewSession("sesspath").(*MemorySession) path, err = sess.SaveData(context.TODO(), "name1/1.ts", strings.NewReader(tempData1), nil, 0) - assert.Nil(err) - assert.Equal("/stream/sesspath/name1/1.ts", path) + require.NoError(t, err) + require.Equal(t, "/stream/sesspath/name1/1.ts", path) data = sess.GetData(path) - assert.Equal(tempData1, string(data)) + require.Equal(t, tempData1, string(data)) } From 39916cf064dbc56122b62dc4081c5eee3ee221c5 Mon Sep 17 00:00:00 2001 From: Thom Shutt Date: Wed, 1 Feb 2023 15:27:28 +0000 Subject: [PATCH 11/12] Remove unused var --- drivers/ipfs.go | 1 - 1 file changed, 1 deletion(-) diff --git a/drivers/ipfs.go b/drivers/ipfs.go index 8047be5..56703a7 100644 --- a/drivers/ipfs.go +++ b/drivers/ipfs.go @@ -19,7 +19,6 @@ type IpfsOS struct { type IpfsSession struct { os *IpfsOS filename string - ended bool client clients.IPFS dCache map[string]*dataCache dLock sync.RWMutex From 46c12859f5ba3e8ae85cd8c88ec0f3c7012240fc Mon Sep 17 00:00:00 2001 From: Thom Shutt Date: Wed, 1 Feb 2023 18:23:04 +0000 Subject: [PATCH 12/12] Add deletion integration test --- drivers/s3_test.go | 91 +++++++++++++++++++++++++++++++++------------- 1 file changed, 65 insertions(+), 26 deletions(-) diff --git a/drivers/s3_test.go b/drivers/s3_test.go index afabe33..6d9b23f 100644 --- a/drivers/s3_test.go +++ b/drivers/s3_test.go @@ -5,34 +5,41 @@ import ( "context" "crypto/rand" "fmt" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" "net/url" "os" "path" "strings" "testing" "time" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" ) -func S3UploadTest(assert *assert.Assertions, fullUriStr, saveName string) { +func S3UploadTest(require *require.Assertions, fullUriStr, saveName string) { testData := make([]byte, 1024*10) - rand.Read(testData) + _, err := rand.Read(testData) + require.NoError(err) + fullUri, _ := url.Parse(fullUriStr) os, err := ParseOSURL(fullUriStr, true) - assert.NoError(err) + require.NoError(err) + session := os.NewSession("") outUriStr, err := session.SaveData(context.Background(), saveName, bytes.NewReader(testData), nil, 10*time.Second) - assert.NoError(err) + require.NoError(err) + var data *FileInfoReader // for specific key session, saveName is empty, otherwise, it's the key data, err = session.ReadData(context.Background(), saveName) - assert.NoError(err) - assert.Equal(*data.Size, int64(len(testData))) + require.NoError(err) + require.Equal(*data.Size, int64(len(testData))) + osBuf := new(bytes.Buffer) osBuf.ReadFrom(data.Body) osData := osBuf.Bytes() - assert.Equal(testData, osData) + require.Equal(testData, osData) + // also test that the object is accessible through full output path with same URL structure if saveName != "" { outUri, _ := url.Parse(outUriStr) @@ -45,15 +52,15 @@ func S3UploadTest(assert *assert.Assertions, fullUriStr, saveName string) { unifiedUrl := fullUri.Scheme + "://" + path.Clean(fmt.Sprintf("%s:%s@%s/%s/%s", fullUri.User.Username(), password, fullUri.Host, bucket, outUri.Path)) os, err := ParseOSURL(unifiedUrl, true) - assert.NoError(err) + require.NoError(err) session := os.NewSession("") data, err = session.ReadData(context.Background(), "") - assert.NoError(err) - assert.Equal(*data.Size, int64(len(testData))) + require.NoError(err) + require.Equal(*data.Size, int64(len(testData))) osBuf := new(bytes.Buffer) osBuf.ReadFrom(data.Body) osData := osBuf.Bytes() - assert.Equal(testData, osData) + require.Equal(testData, osData) } } @@ -62,17 +69,49 @@ func TestAwsS3Upload(t *testing.T) { s3secret := os.Getenv("AWS_S3_SECRET") s3region := os.Getenv("AWS_S3_REGION") s3bucket := os.Getenv("AWS_S3_BUCKET") - assert := assert.New(t) + require := require.New(t) if s3key != "" && s3secret != "" && s3region != "" && s3bucket != "" { // test full path in URI testUriKey := "test/" + uuid.New().String() + ".ts" fullUrl := fmt.Sprintf("s3://%s:%s@%s/%s/%s", s3key, s3secret, s3region, s3bucket, testUriKey) - S3UploadTest(assert, fullUrl, "") + S3UploadTest(require, fullUrl, "") // test key in SaveData arg fullUrl = fmt.Sprintf("s3://%s:%s@%s/%s", s3key, s3secret, s3region, s3bucket) - S3UploadTest(assert, fullUrl, testUriKey) + S3UploadTest(require, fullUrl, testUriKey) + } else { + t.Skip("No S3 credentials, test skipped") + } +} + +func TestAwsS3Deletion(t *testing.T) { + s3key := os.Getenv("AWS_S3_KEY") + s3secret := os.Getenv("AWS_S3_SECRET") + s3region := os.Getenv("AWS_S3_REGION") + s3bucket := os.Getenv("AWS_S3_BUCKET") + require := require.New(t) + if s3key != "" && s3secret != "" && s3region != "" && s3bucket != "" { + // test full path in URI + testUriKey := "test/" + uuid.New().String() + ".ts" + fullUrl := fmt.Sprintf("s3://%s:%s@%s/%s/%s", s3key, s3secret, s3region, s3bucket, testUriKey) + S3UploadTest(require, fullUrl, "") + + os, err := ParseOSURL(fullUrl, true) + require.NoError(err) + session := os.NewSession("") + + // Confirm we can read the file that we wrote to S3 + _, err = session.ReadData(context.Background(), "") + require.NoError(err) + + // Delete the file + err = session.DeleteFile(context.Background(), "") + require.NoError(err) + + // Confirm we can no longer read it + _, err = session.ReadData(context.Background(), "") + require.ErrorContains(err, "The specified key does not exist") } else { - fmt.Println("No S3 credentials, test skipped") + t.Skip("No S3 credentials, test skipped") } } @@ -80,17 +119,17 @@ func TestMinioS3Upload(t *testing.T) { s3key := os.Getenv("MINIO_S3_KEY") s3secret := os.Getenv("MINIO_S3_SECRET") s3bucket := os.Getenv("MINIO_S3_BUCKET") - assert := assert.New(t) + require := require.New(t) if s3key != "" && s3secret != "" { // test full path in URI testUriKey := "test/" + uuid.New().String() + ".ts" fullUrl := fmt.Sprintf("s3+http://%s:%s@localhost:9000/%s/%s", s3key, s3secret, s3bucket, testUriKey) - S3UploadTest(assert, fullUrl, "") + S3UploadTest(require, fullUrl, "") // test key in SaveData arg fullUrl = fmt.Sprintf("s3+http://%s:%s@localhost:9000/%s", s3key, s3secret, s3bucket) - S3UploadTest(assert, fullUrl, testUriKey) + S3UploadTest(require, fullUrl, testUriKey) } else { - fmt.Println("No S3 credentials, test skipped") + t.Skip("No S3 credentials, test skipped") } } @@ -99,19 +138,19 @@ func TestStorjS3Read(t *testing.T) { s3secret := os.Getenv("STORJ_S3_SECRET") s3bucket := os.Getenv("STORJ_S3_BUCKET") s3Path := os.Getenv("STORJ_S3_PATH") - assert := assert.New(t) + require := require.New(t) if s3key != "" && s3secret != "" && s3bucket != "" && s3Path != "" { fullUrl := fmt.Sprintf("s3+https://%s:%s@gateway.storjshare.io/%s", s3key, s3secret, s3bucket) os, err := ParseOSURL(fullUrl, true) - assert.NoError(err) + require.NoError(err) session := os.NewSession("") data, err := session.ReadData(context.Background(), s3Path) - assert.NoError(err) + require.NoError(err) osBuf := new(bytes.Buffer) osBuf.ReadFrom(data.Body) osData := osBuf.Bytes() - assert.True(len(osData) > 0) + require.True(len(osData) > 0) } else { - fmt.Println("No S3 credentials, test skipped") + t.Skip("No S3 credentials, test skipped") } }