From b150aca1a93d94f4cd470448613e25c33d7a6702 Mon Sep 17 00:00:00 2001 From: Ivan Chesnov Date: Tue, 9 Jul 2024 17:30:20 +0300 Subject: [PATCH] GH-40968: [C++][Gandiva] add RE2::Options set_dot_nl(true) for Like function --- cpp/src/gandiva/regex_functions_holder.cc | 13 ++++--- cpp/src/gandiva/regex_functions_holder.h | 2 +- .../gandiva/regex_functions_holder_test.cc | 34 ++++++++++++++----- 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/cpp/src/gandiva/regex_functions_holder.cc b/cpp/src/gandiva/regex_functions_holder.cc index 30d68cbc87ab3..6cac3fc701be5 100644 --- a/cpp/src/gandiva/regex_functions_holder.cc +++ b/cpp/src/gandiva/regex_functions_holder.cc @@ -99,13 +99,14 @@ Status LikeHolder::Make(const FunctionNode& node, std::shared_ptr* h "'like' function requires a string literal as the second parameter")); RE2::Options regex_op; + regex_op.set_dot_nl(true); if (node.descriptor()->name() == "ilike") { regex_op.set_case_sensitive(false); // set case-insensitive for ilike function. return Make(std::get(literal->holder()), holder, regex_op); } if (node.children().size() == 2) { - return Make(std::get(literal->holder()), holder); + return Make(std::get(literal->holder()), holder, regex_op); } else { auto escape_char = dynamic_cast(node.children().at(2).get()); ARROW_RETURN_IF( @@ -118,7 +119,7 @@ Status LikeHolder::Make(const FunctionNode& node, std::shared_ptr* h Status::Invalid( "'like' function requires a string literal as the third parameter")); return Make(std::get(literal->holder()), - std::get(escape_char->holder()), holder); + std::get(escape_char->holder()), holder, regex_op); } } @@ -127,7 +128,9 @@ Status LikeHolder::Make(const std::string& sql_pattern, std::string pcre_pattern; ARROW_RETURN_NOT_OK(RegexUtil::SqlLikePatternToPcre(sql_pattern, pcre_pattern)); - auto lholder = std::shared_ptr(new LikeHolder(pcre_pattern)); + RE2::Options regex_op; + regex_op.set_dot_nl(true); + auto lholder = std::shared_ptr(new LikeHolder(pcre_pattern, regex_op)); ARROW_RETURN_IF(!lholder->regex_.ok(), Status::Invalid("Building RE2 pattern '", pcre_pattern, "' failed with: ", lholder->regex_.error())); @@ -137,7 +140,7 @@ Status LikeHolder::Make(const std::string& sql_pattern, } Status LikeHolder::Make(const std::string& sql_pattern, const std::string& escape_char, - std::shared_ptr* holder) { + std::shared_ptr* holder, RE2::Options regex_op) { ARROW_RETURN_IF(escape_char.length() > 1, Status::Invalid("The length of escape char ", escape_char, " in 'like' function is greater than 1")); @@ -149,7 +152,7 @@ Status LikeHolder::Make(const std::string& sql_pattern, const std::string& escap ARROW_RETURN_NOT_OK(RegexUtil::SqlLikePatternToPcre(sql_pattern, pcre_pattern)); } - auto lholder = std::shared_ptr(new LikeHolder(pcre_pattern)); + auto lholder = std::shared_ptr(new LikeHolder(pcre_pattern, regex_op)); ARROW_RETURN_IF(!lholder->regex_.ok(), Status::Invalid("Building RE2 pattern '", pcre_pattern, "' failed with: ", lholder->regex_.error())); diff --git a/cpp/src/gandiva/regex_functions_holder.h b/cpp/src/gandiva/regex_functions_holder.h index ecf4095f3d473..08ffa4a8234f3 100644 --- a/cpp/src/gandiva/regex_functions_holder.h +++ b/cpp/src/gandiva/regex_functions_holder.h @@ -40,7 +40,7 @@ class GANDIVA_EXPORT LikeHolder : public FunctionHolder { static Status Make(const std::string& sql_pattern, std::shared_ptr* holder); static Status Make(const std::string& sql_pattern, const std::string& escape_char, - std::shared_ptr* holder); + std::shared_ptr* holder, RE2::Options regex_op); static Status Make(const std::string& sql_pattern, std::shared_ptr* holder, RE2::Options regex_op); diff --git a/cpp/src/gandiva/regex_functions_holder_test.cc b/cpp/src/gandiva/regex_functions_holder_test.cc index 930f3a7ade718..7efc75e8d508c 100644 --- a/cpp/src/gandiva/regex_functions_holder_test.cc +++ b/cpp/src/gandiva/regex_functions_holder_test.cc @@ -27,6 +27,8 @@ namespace gandiva { class TestLikeHolder : public ::testing::Test { public: RE2::Options regex_op; + void SetUp() { regex_op.set_dot_nl(true); } + FunctionNode BuildLike(std::string pattern) { auto field = std::make_shared(arrow::field("in", arrow::utf8())); auto pattern_node = @@ -75,6 +77,15 @@ TEST_F(TestLikeHolder, TestMatchOne) { EXPECT_FALSE(like("dabc")); } +TEST_F(TestLikeHolder, TestPcreSpecialWithNewLine) { + std::shared_ptr like_holder; + auto status = LikeHolder::Make("%Space1.%", &like_holder, regex_op); + + auto& like = *like_holder; + EXPECT_TRUE( + like("[name: \"Space1.protect\"\nargs: \"count\"\ncolumn_name: \"pass_count\"]")); +} + TEST_F(TestLikeHolder, TestPcreSpecial) { std::shared_ptr like_holder; @@ -104,17 +115,25 @@ TEST_F(TestLikeHolder, TestDot) { EXPECT_FALSE(like("abcd")); } +TEST_F(TestLikeHolder, TestMatchWithNewLine) { + std::shared_ptr like_holder; + auto status = LikeHolder::Make("%abc%", "\\", &like_holder, regex_op); + + auto& like = *like_holder; + EXPECT_TRUE(like("abc\nd")); +} + TEST_F(TestLikeHolder, TestMatchSubString) { std::shared_ptr like_holder; - auto status = LikeHolder::Make("%abc%", "\\", &like_holder); + auto status = LikeHolder::Make("%abc%", "\\", &like_holder, regex_op); EXPECT_EQ(status.ok(), true) << status.message(); auto& like = *like_holder; EXPECT_TRUE(like("abc")); EXPECT_FALSE(like("xxabdc")); - status = LikeHolder::Make("%ab-.^$*+?()[]{}|—/c\\%%", "\\", &like_holder); + status = LikeHolder::Make("%ab-.^$*+?()[]{}|—/c\\%%", "\\", &like_holder, regex_op); EXPECT_EQ(status.ok(), true) << status.message(); auto& like_reserved_char = *like_holder; @@ -192,7 +211,7 @@ TEST_F(TestLikeHolder, TestOptimise) { TEST_F(TestLikeHolder, TestMatchOneEscape) { std::shared_ptr like_holder; - auto status = LikeHolder::Make("ab\\_", "\\", &like_holder); + auto status = LikeHolder::Make("ab\\_", "\\", &like_holder, regex_op); EXPECT_EQ(status.ok(), true) << status.message(); auto& like = *like_holder; @@ -209,8 +228,7 @@ TEST_F(TestLikeHolder, TestMatchOneEscape) { TEST_F(TestLikeHolder, TestMatchManyEscape) { std::shared_ptr like_holder; - auto status = LikeHolder::Make("ab\\%", "\\", &like_holder); - EXPECT_EQ(status.ok(), true) << status.message(); + auto status = LikeHolder::Make("ab\\%", "\\", &like_holder, regex_op); auto& like = *like_holder; @@ -226,7 +244,7 @@ TEST_F(TestLikeHolder, TestMatchManyEscape) { TEST_F(TestLikeHolder, TestMatchEscape) { std::shared_ptr like_holder; - auto status = LikeHolder::Make("ab\\\\", "\\", &like_holder); + auto status = LikeHolder::Make("ab\\\\", "\\", &like_holder, regex_op); EXPECT_EQ(status.ok(), true) << status.message(); auto& like = *like_holder; @@ -239,7 +257,7 @@ TEST_F(TestLikeHolder, TestMatchEscape) { TEST_F(TestLikeHolder, TestEmptyEscapeChar) { std::shared_ptr like_holder; - auto status = LikeHolder::Make("ab\\_", "", &like_holder); + auto status = LikeHolder::Make("ab\\_", "", &like_holder, regex_op); EXPECT_EQ(status.ok(), true) << status.message(); auto& like = *like_holder; @@ -254,7 +272,7 @@ TEST_F(TestLikeHolder, TestEmptyEscapeChar) { TEST_F(TestLikeHolder, TestMultipleEscapeChar) { std::shared_ptr like_holder; - auto status = LikeHolder::Make("ab\\_", "\\\\", &like_holder); + auto status = LikeHolder::Make("ab\\_", "\\\\", &like_holder, regex_op); EXPECT_EQ(status.ok(), false) << status.message(); }