Skip to content

Commit

Permalink
[KYUUBI #5452][AUTHZ] Support Compaction table commands for Hudi
Browse files Browse the repository at this point in the history
### _Why are the changes needed?_
To close #5452
Support Compaction table/path related command. The SQL grammar is https://github.com/apache/hudi/blob/release-0.14.0/hudi-spark-datasource/hudi-spark/src/main/antlr4/org/apache/hudi/spark/sql/parser/HoodieSqlCommon.g4

- CompactionHoodieTableCommand :https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionHoodieTableCommand.scala
- CompactionShowHoodiePathCommand: https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionShowHoodiePathCommand.scala
- CompactionShowHoodieTableCommand: https://github.com/apache/hudi/blob/master/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/spark/sql/hudi/command/CompactionShowHoodieTableCommand.scala

### _How was this patch tested?_
- [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible

- [ ] Add screenshots for manual tests if appropriate

- [ ] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request

### _Was this patch authored or co-authored using generative AI tooling?_
No

Closes #5454 from AngersZhuuuu/KYUUBI-5452.

Closes #5452

e42200f [Angerszhuuuu] follow comment
4d5139e [Angerszhuuuu] Update HudiCatalogRangerSparkExtensionSuite.scala
0e7cb92 [Angerszhuuuu] follow comment
e14dc41 [Angerszhuuuu] [KYUUBI #5452][AUTHZ] Support Compaction table/path related command

Authored-by: Angerszhuuuu <[email protected]>
Signed-off-by: Cheng Pan <[email protected]>
  • Loading branch information
AngersZhuuuu authored and pan3793 committed Oct 18, 2023
1 parent 8d2c8d1 commit facfe57
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -1513,6 +1513,43 @@
} ],
"opType" : "ALTERTABLE_PROPERTIES",
"queryDescs" : [ ]
}, {
"classname" : "org.apache.spark.sql.hudi.command.CompactionHoodieTableCommand",
"tableDescs" : [ {
"fieldName" : "table",
"fieldExtractor" : "CatalogTableTableExtractor",
"columnDesc" : null,
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : false,
"setCurrentDatabaseIfMissing" : false
}, {
"fieldName" : "table",
"fieldExtractor" : "CatalogTableTableExtractor",
"columnDesc" : null,
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
"setCurrentDatabaseIfMissing" : false
} ],
"opType" : "CREATETABLE",
"queryDescs" : [ ]
}, {
"classname" : "org.apache.spark.sql.hudi.command.CompactionShowHoodieTableCommand",
"tableDescs" : [ {
"fieldName" : "table",
"fieldExtractor" : "CatalogTableTableExtractor",
"columnDesc" : null,
"actionTypeDesc" : null,
"tableTypeDesc" : null,
"catalogDesc" : null,
"isInput" : true,
"setCurrentDatabaseIfMissing" : false
} ],
"opType" : "SHOW_TBLPROPERTIES",
"queryDescs" : [ ]
}, {
"classname" : "org.apache.spark.sql.hudi.command.CreateHoodieTableAsSelectCommand",
"tableDescs" : [ {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,16 +127,30 @@ object HudiCommands {
TableCommandSpec(cmd, Seq(tableDesc), TRUNCATETABLE)
}

val CompactionHoodieTableCommand = {
val cmd = "org.apache.spark.sql.hudi.command.CompactionHoodieTableCommand"
val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor])
TableCommandSpec(cmd, Seq(tableDesc, tableDesc.copy(isInput = true)), CREATETABLE)
}

val CompactionShowHoodieTableCommand = {
val cmd = "org.apache.spark.sql.hudi.command.CompactionShowHoodieTableCommand"
val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor], isInput = true)
TableCommandSpec(cmd, Seq(tableDesc), SHOW_TBLPROPERTIES)
}

val data: Array[TableCommandSpec] = Array(
AlterHoodieTableAddColumnsCommand,
AlterHoodieTableChangeColumnCommand,
AlterHoodieTableDropPartitionCommand,
AlterHoodieTableRenameCommand,
AlterTableCommand,
Spark31AlterTableCommand,
CreateHoodieTableCommand,
CreateHoodieTableAsSelectCommand,
CreateHoodieTableCommand,
CreateHoodieTableLikeCommand,
CompactionHoodieTableCommand,
CompactionShowHoodieTableCommand,
DropHoodieTableCommand,
TruncateHoodieTableCommand)
TruncateHoodieTableCommand,
Spark31AlterTableCommand)
}
Original file line number Diff line number Diff line change
Expand Up @@ -264,4 +264,35 @@ class HudiCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite {
doAs(admin, sql(truncateTableSql))
}
}

test("CompactionHoodieTableCommand / CompactionShowHoodieTableCommand") {
withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) {
doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1"))
doAs(
admin,
sql(
s"""
|CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string)
|USING HUDI
|OPTIONS (
| type = 'mor',
| primaryKey = 'id',
| 'hoodie.datasource.hive_sync.enable' = 'false'
|)
|PARTITIONED BY(city)
|""".stripMargin))

val compactionTable = s"RUN COMPACTION ON $namespace1.$table1"
interceptContains[AccessControlException] {
doAs(someone, sql(compactionTable))
}(s"does not have [select] privilege on [$namespace1/$table1]")
doAs(admin, sql(compactionTable))

val showCompactionTable = s"SHOW COMPACTION ON $namespace1.$table1"
interceptContains[AccessControlException] {
doAs(someone, sql(showCompactionTable))
}(s"does not have [select] privilege on [$namespace1/$table1]")
doAs(admin, sql(showCompactionTable))
}
}
}

0 comments on commit facfe57

Please sign in to comment.