From 0210d5479629497a06b4848a4d9588a25a01c2ac Mon Sep 17 00:00:00 2001 From: Angerszhuuuu Date: Wed, 8 Nov 2023 17:01:18 +0800 Subject: [PATCH] [KYUUBI #5628][AUTHZ] Support path privilege check for resource command ### _Why are the changes needed?_ To close #5628 Support path privilege check for resource command - AddJarsCommand - AddFilesCommand - AddArchivesCommand ### _How was this patch tested?_ - [x] Add some test cases that check the changes thoroughly including negative and positive cases if possible - [ ] Add screenshots for manual tests if appropriate - [ ] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request ### _Was this patch authored or co-authored using generative AI tooling?_ No Closes #5629 from AngersZhuuuu/SPDI-5628. Closes #5628 77ea99328 [Angerszhuuuu] update 8116045c8 [Angerszhuuuu] Update table_command_spec.json 00194c257 [Angerszhuuuu] Merge branch 'master' into SPDI-5628 04805651e [Angerszhuuuu] Update RangerSparkExtensionSuite.scala 888360bc7 [Angerszhuuuu] Merge branch 'master' into SPDI-5628 af0dba06b [Angerszhuuuu] Update RangerSparkExtensionSuite.scala 980b5512d [Angerszhuuuu] update da12dc2a1 [Angerszhuuuu] Merge branch 'master' into SPDI-5628 70f4330d7 [Angerszhuuuu] Update TableCommands.scala b03addc5b [Angerszhuuuu] [KYUUBI #5628][AUTHZ] Support path privilege check for resource command Authored-by: Angerszhuuuu Signed-off-by: Cheng Pan --- ...uubi.plugin.spark.authz.serde.URIExtractor | 1 + .../main/resources/table_command_spec.json | 50 +++++++++++++++++++ .../plugin/spark/authz/OperationType.scala | 2 +- .../spark/authz/ranger/AccessType.scala | 4 +- .../spark/authz/serde/uriExtractors.scala | 6 +++ .../spark/authz/gen/TableCommands.scala | 18 +++++++ .../ranger/RangerSparkExtensionSuite.scala | 17 +++++++ 7 files changed, 96 insertions(+), 2 deletions(-) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor index f5b1c6e6db3..733994176d0 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor @@ -19,3 +19,4 @@ org.apache.kyuubi.plugin.spark.authz.serde.CatalogStorageFormatURIExtractor org.apache.kyuubi.plugin.spark.authz.serde.BaseRelationFileIndexURIExtractor org.apache.kyuubi.plugin.spark.authz.serde.OptionsUriExtractor org.apache.kyuubi.plugin.spark.authz.serde.StringURIExtractor +org.apache.kyuubi.plugin.spark.authz.serde.StringSeqURIExtractor diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json index 31de180bfef..640bff7eec8 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json @@ -626,6 +626,56 @@ "opType" : "QUERY", "queryDescs" : [ ], "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.execution.command.AddArchivesCommand", + "tableDescs" : [ ], + "opType" : "ADD", + "queryDescs" : [ ], + "uriDescs" : [ { + "fieldName" : "paths", + "fieldExtractor" : "StringSeqURIExtractor", + "isInput" : true + } ] +}, { + "classname" : "org.apache.spark.sql.execution.command.AddFileCommand", + "tableDescs" : [ ], + "opType" : "ADD", + "queryDescs" : [ ], + "uriDescs" : [ { + "fieldName" : "path", + "fieldExtractor" : "StringURIExtractor", + "isInput" : true + } ] +}, { + "classname" : "org.apache.spark.sql.execution.command.AddFilesCommand", + "tableDescs" : [ ], + "opType" : "ADD", + "queryDescs" : [ ], + "uriDescs" : [ { + "fieldName" : "paths", + "fieldExtractor" : "StringSeqURIExtractor", + "isInput" : true + } ] +}, { + "classname" : "org.apache.spark.sql.execution.command.AddJarCommand", + "tableDescs" : [ ], + "opType" : "ADD", + "queryDescs" : [ ], + "uriDescs" : [ { + "fieldName" : "path", + "fieldExtractor" : "StringURIExtractor", + "isInput" : true + } ] +}, { + "classname" : "org.apache.spark.sql.execution.command.AddJarsCommand", + "tableDescs" : [ ], + "opType" : "ADD", + "queryDescs" : [ ], + "uriDescs" : [ { + "fieldName" : "paths", + "fieldExtractor" : "StringSeqURIExtractor", + "isInput" : true + } ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableAddColumnsCommand", "tableDescs" : [ { diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/OperationType.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/OperationType.scala index 3f2062b20a0..63252c1352e 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/OperationType.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/OperationType.scala @@ -22,7 +22,7 @@ object OperationType extends Enumeration { type OperationType = Value // According to https://scalameta.org/scalafmt/docs/known-issues.html // format: off - val ALTERDATABASE, ALTERDATABASE_LOCATION, ALTERTABLE_ADDCOLS, ALTERTABLE_ADDPARTS, + val ADD, ALTERDATABASE, ALTERDATABASE_LOCATION, ALTERTABLE_ADDCOLS, ALTERTABLE_ADDPARTS, ALTERTABLE_RENAMECOL, ALTERTABLE_REPLACECOLS, ALTERTABLE_DROPPARTS, ALTERTABLE_RENAMEPART, ALTERTABLE_RENAME, ALTERTABLE_PROPERTIES, ALTERTABLE_SERDEPROPERTIES, ALTERTABLE_LOCATION, ALTERVIEW_AS, ALTERVIEW_RENAME, ANALYZE_TABLE, CREATEDATABASE, CREATETABLE, diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessType.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessType.scala index ca509154bf1..1c57912e211 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessType.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessType.scala @@ -25,7 +25,8 @@ object AccessType extends Enumeration { type AccessType = Value - val NONE, CREATE, ALTER, DROP, SELECT, UPDATE, USE, READ, WRITE, ALL, ADMIN, INDEX = Value + val NONE, CREATE, ALTER, DROP, SELECT, UPDATE, USE, READ, WRITE, ALL, ADMIN, INDEX, TEMPUDFADMIN = + Value def apply(obj: PrivilegeObject, opType: OperationType, isInput: Boolean): AccessType = { if (obj.privilegeObjectType == DFS_URI || obj.privilegeObjectType == LOCAL_URI) { @@ -35,6 +36,7 @@ object AccessType extends Enumeration { obj.actionType match { case PrivilegeObjectActionType.OTHER => opType match { + case ADD => TEMPUDFADMIN case CREATEDATABASE if obj.privilegeObjectType == DATABASE => CREATE case CREATEFUNCTION if obj.privilegeObjectType == FUNCTION => CREATE case CREATETABLE | CREATEVIEW | CREATETABLE_AS_SELECT diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/uriExtractors.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/uriExtractors.scala index 3283c501912..46fd4d87ab2 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/uriExtractors.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/uriExtractors.scala @@ -37,6 +37,12 @@ class StringURIExtractor extends URIExtractor { } } +class StringSeqURIExtractor extends URIExtractor { + override def apply(v1: AnyRef): Seq[Uri] = { + v1.asInstanceOf[Seq[String]].map(Uri) + } +} + class CatalogStorageFormatURIExtractor extends URIExtractor { override def apply(v1: AnyRef): Seq[Uri] = { v1.asInstanceOf[CatalogStorageFormat].locationUri.map(uri => Uri(uri.getPath)).toSeq diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala index 3ccf3392167..3bf863e75c8 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala @@ -609,7 +609,25 @@ object TableCommands extends CommandSpecs[TableCommandSpec] { TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_PROPERTIES) } + val AddArchivesCommand = { + val cmd = "org.apache.spark.sql.execution.command.AddArchivesCommand" + val uriDesc = UriDesc("paths", classOf[StringSeqURIExtractor], isInput = true) + TableCommandSpec(cmd, Nil, ADD, uriDescs = Seq(uriDesc)) + } + + // For spark-3.1 + val AddFileCommand = { + val cmd = "org.apache.spark.sql.execution.command.AddFileCommand" + val uriDesc = UriDesc("path", classOf[StringURIExtractor], isInput = true) + TableCommandSpec(cmd, Nil, ADD, uriDescs = Seq(uriDesc)) + } + override def specs: Seq[TableCommandSpec] = Seq( + AddArchivesCommand, + AddArchivesCommand.copy(classname = "org.apache.spark.sql.execution.command.AddFilesCommand"), + AddArchivesCommand.copy(classname = "org.apache.spark.sql.execution.command.AddJarsCommand"), + AddFileCommand, + AddFileCommand.copy(classname = "org.apache.spark.sql.execution.command.AddJarCommand"), AddPartitions, DropPartitions, RenamePartitions, diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala index 291745a0d82..ed89f329864 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala @@ -1155,4 +1155,21 @@ class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { } } } + + test("Add resource command") { + withTempDir { path => + withSingleCallEnabled { + val supportedCommand = if (isSparkV32OrGreater) { + Seq("JAR", "FILE", "ARCHIVE") + } else { + Seq("JAR", "FILE") + } + supportedCommand.foreach { cmd => + interceptContains[AccessControlException]( + doAs(someone, sql(s"ADD $cmd $path")))( + s"does not have [read] privilege on [[$path, $path/]]") + } + } + } + } }