From 56e074a3ad0842c34e83c670fbb5dfc988460ee7 Mon Sep 17 00:00:00 2001 From: Shion Ichikawa Date: Sat, 11 May 2024 08:49:12 +0900 Subject: [PATCH 1/5] =?UTF-8?q?=E2=9C=A8=20(pkg/gcs)NewFolderRef?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/gcs/folder.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/gcs/folder.go b/pkg/gcs/folder.go index a36a921..2fd0182 100644 --- a/pkg/gcs/folder.go +++ b/pkg/gcs/folder.go @@ -14,6 +14,16 @@ type FolderRef struct { bucket *storage.BucketHandle } +func NewFolderRef(bucket *storage.BucketHandle, basePath string) FolderRef { + if basePath != "" && basePath[len(basePath)-1] != '/' { + basePath += "/" + } + return FolderRef{ + basePath: basePath, + bucket: bucket, + } +} + func (f FolderRef) Object(name string) ObjectRef { return ObjectRef{ objName: f.basePath + name, From d55163483566002381d5bf664f0c2afd41bc8c49 Mon Sep 17 00:00:00 2001 From: Shion Ichikawa Date: Sat, 11 May 2024 08:49:43 +0900 Subject: [PATCH 2/5] =?UTF-8?q?=E2=9C=A8=20(pkg/gcs)=20add=20security=20op?= =?UTF-8?q?tions=20for=20IssueLink()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/gcs/object.go | 55 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/pkg/gcs/object.go b/pkg/gcs/object.go index 56b826a..6074842 100644 --- a/pkg/gcs/object.go +++ b/pkg/gcs/object.go @@ -4,6 +4,7 @@ import ( "cloud.google.com/go/storage" "context" "io" + "net/url" "time" ) @@ -33,14 +34,56 @@ func (r ObjectRef) Download(ctx context.Context) ([]byte, error) { return data, nil } -func (r ObjectRef) IssueLink() (SignedObjectLink, error) { - url, err := r.bucket.SignedURL(r.objName, &storage.SignedURLOptions{ - Method: "GET", - Expires: time.Now().Add(time.Minute * 1), - Scheme: storage.SigningSchemeV4, +// IssueLinkOptions +// ExpiresIn is the duration the link is valid for. +// AuthHeaders are the headers to sign. +// AuthMetaHeaders are the metadata headers to sign. The key will be prefixed with "x-goog-meta-". +// headers starting with "x-goog-meta-" are considered metadata headers, and it will be set as metadata in the object. +// Both AuthHeaders and AuthMetaHeaders are required when using the URL. +// AuthQueries are the query parameters to sign. +type IssueLinkOptions struct { + ExpiresIn time.Duration + AuthHeaders map[string]string + AuthMetaHeaders map[string]string + AuthQueries map[string]string +} + +// IssueLink +// removeAuthQueries will remove the query parameters from the signed URL. +// It will be useful for preventing CSRF attacks. +func (r ObjectRef) IssueLink(ops IssueLinkOptions, removeAuthQueries bool) (SignedObjectLink, error) { + headers := make([]string, 0, len(ops.AuthHeaders)) + for k, v := range ops.AuthHeaders { + headers = append(headers, k+":"+v) + } + for k, v := range ops.AuthMetaHeaders { + headers = append(headers, "x-goog-meta-"+k+":"+v) + } + queries := url.Values{} + for k, v := range ops.AuthQueries { + queries.Set(k, v) + } + signedUrl, err := r.bucket.SignedURL(r.objName, &storage.SignedURLOptions{ + Method: "GET", + Expires: time.Now().Add(ops.ExpiresIn), + Headers: headers, + QueryParameters: queries, + Scheme: storage.SigningSchemeV4, }) if err != nil { return "", err } - return SignedObjectLink(url), nil + if removeAuthQueries { + u, err := url.Parse(signedUrl) + if err != nil { + return "", err + } + q := u.Query() + for k := range ops.AuthQueries { + q.Del(k) + } + u.RawQuery = q.Encode() + signedUrl = u.String() + } + return SignedObjectLink(signedUrl), nil } From 7f5819f9f6d932c5caf1fdd78944f0a6367eddf5 Mon Sep 17 00:00:00 2001 From: Shion Ichikawa Date: Sat, 11 May 2024 08:52:26 +0900 Subject: [PATCH 3/5] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20(pkg/gcs)=20IssueLink?= =?UTF-8?q?=20->=20IssueDownloadSignedURL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/gcs/object.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/gcs/object.go b/pkg/gcs/object.go index 6074842..e7ee82f 100644 --- a/pkg/gcs/object.go +++ b/pkg/gcs/object.go @@ -48,10 +48,10 @@ type IssueLinkOptions struct { AuthQueries map[string]string } -// IssueLink +// IssueDownloadSignedURL // removeAuthQueries will remove the query parameters from the signed URL. // It will be useful for preventing CSRF attacks. -func (r ObjectRef) IssueLink(ops IssueLinkOptions, removeAuthQueries bool) (SignedObjectLink, error) { +func (r ObjectRef) IssueDownloadSignedURL(ops IssueLinkOptions, removeAuthQueries bool) (SignedObjectLink, error) { headers := make([]string, 0, len(ops.AuthHeaders)) for k, v := range ops.AuthHeaders { headers = append(headers, k+":"+v) From cd68a99aaf9e4ab8f3ae6b8fb3fb5cb01d20af56 Mon Sep 17 00:00:00 2001 From: Shion Ichikawa Date: Sat, 11 May 2024 10:20:42 +0900 Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=93=8C=20go=20mod=20tidy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 239b157..1e028f4 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ toolchain go1.22.0 require ( cloud.google.com/go/firestore v1.14.0 + cloud.google.com/go/storage v1.38.0 firebase.google.com/go/v4 v4.13.0 github.com/gin-gonic/gin v1.9.1 github.com/go-resty/resty/v2 v2.11.0 @@ -26,7 +27,6 @@ require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v1.1.6 // indirect cloud.google.com/go/longrunning v0.5.5 // indirect - cloud.google.com/go/storage v1.38.0 // indirect github.com/MicahParks/keyfunc v1.9.0 // indirect github.com/bytedance/sonic v1.11.0 // indirect github.com/chenzhuoyu/base64x v0.0.0-20230717121745-296ad89f973d // indirect From 3bdc8b6eb6d1dd015bf62339f4476cd9dd8131e1 Mon Sep 17 00:00:00 2001 From: Shion Ichikawa Date: Sat, 11 May 2024 10:21:28 +0900 Subject: [PATCH 5/5] =?UTF-8?q?=E2=9C=85=20(pkg/gcs)=20test=20for=20IssueD?= =?UTF-8?q?ownloadSignedURL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkg/gcs/object_test.go | 102 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 pkg/gcs/object_test.go diff --git a/pkg/gcs/object_test.go b/pkg/gcs/object_test.go new file mode 100644 index 0000000..c6cec2d --- /dev/null +++ b/pkg/gcs/object_test.go @@ -0,0 +1,102 @@ +package gcs + +import ( + "cloud.google.com/go/storage" + "context" + "fmt" + "google.golang.org/api/option" + "io" + "net/http" + "net/url" + "testing" + "time" + "ynufes-mypage-backend/pkg/setting" +) + +func TestObjectRef_IssueDownloadSignedURL(t *testing.T) { + ctx := context.Background() + conf := setting.Get() + certPath := conf.Infrastructure.Firebase.JsonCredentialFile + c, err := storage.NewClient(ctx, option.WithCredentialsFile(certPath)) + if err != nil { + t.Fatal(err) + } + b := c.Bucket("ynufes-mypage-staging-bucket") + ref := NewFolderRef(b, "some-folder") + uploadRef, err := ref.Upload(ctx, "some-file", []byte("some-content")) + if err != nil { + t.Fatal(err) + return + } + issueOpt := IssueLinkOptions{ + ExpiresIn: 5 * time.Second, + AuthHeaders: map[string]string{ + "user": "shion-test", + }, + AuthMetaHeaders: map[string]string{ + "user": "shion-meta", + }, + AuthQueries: map[string]string{ + "user-query": "shion-query", + }, + } + targetUrl, err := uploadRef.IssueDownloadSignedURL(issueOpt, true) + if err != nil { + t.Fatal(err) + } + if err := verifySignedURL(targetUrl, issueOpt, true); err != nil { + t.Fatal(err) + } +} + +//func uploadTestFile( +// ctx context.Context, +// bucket *storage.BucketHandle, +// fileName string, +// content []byte, +//) (ObjectRef, error) { +// ref := NewFolderRef(bucket, "some-folder") +// return ref.Upload(ctx, fileName, content) +//} + +func verifySignedURL( + target SignedObjectLink, + opt IssueLinkOptions, + addQuery bool, +) error { + targetURL, err := url.Parse(string(target)) + if err != nil { + return fmt.Errorf("failed to parse signed URL: %w", err) + } + if addQuery { + q := targetURL.Query() + for k, v := range opt.AuthQueries { + q.Set(k, v) + } + targetURL.RawQuery = q.Encode() + } + fmt.Println("targetURL: ", targetURL.String()) + req, err := http.NewRequest("GET", targetURL.String(), nil) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + for k, v := range opt.AuthHeaders { + req.Header.Set(k, v) + } + for k, v := range opt.AuthMetaHeaders { + req.Header.Set("x-goog-meta-"+k, v) + } + client := http.DefaultClient + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("failed to send request: %w", err) + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + payload, _ := io.ReadAll(resp.Body) + if string(payload) != "some-content" { + return fmt.Errorf("unexpected payload: %s", string(payload)) + } + return nil +}