From f1f907a2ae34a6457609051227f2c5f7404a792d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Wed, 24 Jan 2024 13:43:12 +0100 Subject: [PATCH 1/5] Add a basic support for parsing urls from virtual private cloud --- src/parse.jl | 6 ++++++ test/runtests.jl | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/parse.jl b/src/parse.jl index b98abf3..3a35716 100644 --- a/src/parse.jl +++ b/src/parse.jl @@ -18,6 +18,7 @@ const AWS_REGIONS = Set{String}([ "ap-southeast-2", "ap-northeast-1", "ca-central-1", + "ca-west-1", "eu-central-1", "eu-west-1", "eu-west-2", @@ -26,6 +27,7 @@ const AWS_REGIONS = Set{String}([ "eu-south-2", "eu-north-1", "eu-central-2", + "il-central-1", "me-south-1", "me-central-1", "sa-east-1", @@ -136,6 +138,10 @@ function parseAWSBucketRegionKey(url; parseLocal::Bool=false) # https://bucket-name.s3.amazonaws.com m = match(r"^https://(?[^\.]+)\.s3(?-accelerate)?(?:\.(?[^\.]+))?\.amazonaws\.com(?:/(?.+))?$"i, url) m !== nothing && return _validate_aws(true, !isnothing(m[:accelerate]), nothing, m[:bucket], m[:region], m[:key]) + + # https://bucket.vpce-1a2b3c4d-5e6f.s3.region-code.vpce.amazonaws.com + m = match(r"^https://bucket\.vpce[^\.]+\.s3\.(?[^\.]+)\.vpce\.amazonaws\.com/(?[^/]+)(?:/(?.+))?$"i, url) + m !== nothing && return _validate_aws(true, false, nothing, m[:bucket], m[:region], m[:key]) # https://s3.region-code.amazonaws.com/bucket-name/key-name # https://s3.region-code.amazonaws.com/bucket-name m = match(r"^https://s3(?:\.(?[^\.]+))?\.amazonaws\.com/(?[^/]+)(?:/(?.+))?$"i, url) diff --git a/test/runtests.jl b/test/runtests.jl index af43f21..0b78ee5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -531,6 +531,9 @@ end ("S3://bucket-name", (true, false, nothing, "bucket-name", "", "")), ("HTtp://127.0.0.1:27181/bucket-name/key-name", (true, false, "HTtp://127.0.0.1:27181", "bucket-name", "", "key-name")), ("htTP://127.0.0.1:27181/bucket-name", (true, false, "htTP://127.0.0.1:27181", "bucket-name", "", "")), + + ("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name", (true, false, nothing, "bucket-name", "us-west-2", "")), + ("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name/key-name", (true, false, nothing, "bucket-name", "us-west-2", "key-name")), ] for (url, parts) in s3 ok, accelerate, host, bucket, reg, key = CloudStore.parseAWSBucketRegionKey(url; parseLocal=true) From b2d120f4104187740f1fdafc3e8f4e8ccaecd0a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Wed, 24 Jan 2024 13:46:27 +0100 Subject: [PATCH 2/5] Improve comment --- src/parse.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/parse.jl b/src/parse.jl index 3a35716..7255d3f 100644 --- a/src/parse.jl +++ b/src/parse.jl @@ -139,7 +139,8 @@ function parseAWSBucketRegionKey(url; parseLocal::Bool=false) m = match(r"^https://(?[^\.]+)\.s3(?-accelerate)?(?:\.(?[^\.]+))?\.amazonaws\.com(?:/(?.+))?$"i, url) m !== nothing && return _validate_aws(true, !isnothing(m[:accelerate]), nothing, m[:bucket], m[:region], m[:key]) - # https://bucket.vpce-1a2b3c4d-5e6f.s3.region-code.vpce.amazonaws.com + # https://bucket.vpce-1a2b3c4d-5e6f.s3.region-code.vpce.amazonaws.com/bucket-name/key-name + # https://bucket.vpce-1a2b3c4d-5e6f.s3.region-code.vpce.amazonaws.com/bucket-name m = match(r"^https://bucket\.vpce[^\.]+\.s3\.(?[^\.]+)\.vpce\.amazonaws\.com/(?[^/]+)(?:/(?.+))?$"i, url) m !== nothing && return _validate_aws(true, false, nothing, m[:bucket], m[:region], m[:key]) # https://s3.region-code.amazonaws.com/bucket-name/key-name From b938a60a084162f837efb3ada8ed0e2679c59553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Wed, 24 Jan 2024 13:59:41 +0100 Subject: [PATCH 3/5] v1.6.2 --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 450e966..912563d 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "CloudStore" uuid = "3365d9ee-d53b-4a56-812d-5344d5b716d7" authors = ["quinnj "] -version = "1.6.1" +version = "1.6.2" [deps] Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" From 04b88abc481f90314395966f23733e2a4d30d2de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Wed, 24 Jan 2024 14:22:41 +0100 Subject: [PATCH 4/5] PR feedback --- src/parse.jl | 1 + test/runtests.jl | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/src/parse.jl b/src/parse.jl index 7255d3f..840356c 100644 --- a/src/parse.jl +++ b/src/parse.jl @@ -139,6 +139,7 @@ function parseAWSBucketRegionKey(url; parseLocal::Bool=false) m = match(r"^https://(?[^\.]+)\.s3(?-accelerate)?(?:\.(?[^\.]+))?\.amazonaws\.com(?:/(?.+))?$"i, url) m !== nothing && return _validate_aws(true, !isnothing(m[:accelerate]), nothing, m[:bucket], m[:region], m[:key]) + ### See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/privatelink-interface-endpoints.html # https://bucket.vpce-1a2b3c4d-5e6f.s3.region-code.vpce.amazonaws.com/bucket-name/key-name # https://bucket.vpce-1a2b3c4d-5e6f.s3.region-code.vpce.amazonaws.com/bucket-name m = match(r"^https://bucket\.vpce[^\.]+\.s3\.(?[^\.]+)\.vpce\.amazonaws\.com/(?[^/]+)(?:/(?.+))?$"i, url) diff --git a/test/runtests.jl b/test/runtests.jl index 0b78ee5..bddd74f 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -590,6 +590,12 @@ end "httP://S3.AmAzonAws.com/bucket-name", "httP://bucket-name/key-name", "httP://bucket-name", + + "http://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name", + "http://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name/key-name", + "https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.XvpceX.amazonaws.com/bucket-name", + "https://bucket.Xvpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name", + "https://XbucketX.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name/key-name", ] for url in invalid_s3 ok, accelerate, host, bucket, reg, key = CloudStore.parseAWSBucketRegionKey(url; parseLocal=true) @@ -921,6 +927,10 @@ end @test CloudStore.validate_bucket_name("a.b-c1", false) == "a.b-c1" @test CloudStore.validate_bucket_name("a"^63, false) == "a"^63 @test CloudStore.validate_bucket_name("a"^3, false) == "a"^3 + + @test_throws "ArgumentError: Validation failed for `region`" CloudStore.parseAWSBucketRegionKey("https://bucket.vpce-1a2b3c4d-5e6f.s3.xx-xxxx-x.vpce.amazonaws.com/bucket-name") + @test_throws "ArgumentError: Validation failed for `bucket`" CloudStore.parseAWSBucketRegionKey("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bn") + @test_throws "ArgumentError: Validation failed for `key` " CloudStore.parseAWSBucketRegionKey("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name/key-n$("a" ^ 1024)me") end @testset "validate_container_name" begin From 5fd14ebce2f6ce195f9032ea73bff3501d760d66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?= Date: Wed, 24 Jan 2024 14:35:48 +0100 Subject: [PATCH 5/5] 1.6 compat --- test/runtests.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/runtests.jl b/test/runtests.jl index bddd74f..fa535c5 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -928,9 +928,9 @@ end @test CloudStore.validate_bucket_name("a"^63, false) == "a"^63 @test CloudStore.validate_bucket_name("a"^3, false) == "a"^3 - @test_throws "ArgumentError: Validation failed for `region`" CloudStore.parseAWSBucketRegionKey("https://bucket.vpce-1a2b3c4d-5e6f.s3.xx-xxxx-x.vpce.amazonaws.com/bucket-name") - @test_throws "ArgumentError: Validation failed for `bucket`" CloudStore.parseAWSBucketRegionKey("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bn") - @test_throws "ArgumentError: Validation failed for `key` " CloudStore.parseAWSBucketRegionKey("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name/key-n$("a" ^ 1024)me") + @test_throws ArgumentError("Validation failed for `region` \"xx-xxxx-x\"") CloudStore.parseAWSBucketRegionKey("https://bucket.vpce-1a2b3c4d-5e6f.s3.xx-xxxx-x.vpce.amazonaws.com/bucket-name") + @test_throws ArgumentError("Validation failed for `bucket` name \"bn\": Bucket names must be between 3 (min) and 63 (max) characters long.") CloudStore.parseAWSBucketRegionKey("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bn") + @test_throws ArgumentError("Validation failed for `key` \"key-n$("a" ^ 1024)me\": The key name must be shorter than 1025 bytes.") CloudStore.parseAWSBucketRegionKey("https://bucket.vpce-1a2b3c4d-5e6f.s3.us-west-2.vpce.amazonaws.com/bucket-name/key-n$("a" ^ 1024)me") end @testset "validate_container_name" begin