From f41180d337e48471f799deb359461c2c1cb56fc3 Mon Sep 17 00:00:00 2001 From: Andy Grove Date: Sat, 4 Jan 2025 08:50:37 -0700 Subject: [PATCH] rename compression level config --- common/src/main/scala/org/apache/comet/CometConf.scala | 2 +- docs/source/user-guide/configs.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/src/main/scala/org/apache/comet/CometConf.scala b/common/src/main/scala/org/apache/comet/CometConf.scala index 2b42b50e46..e32777be3c 100644 --- a/common/src/main/scala/org/apache/comet/CometConf.scala +++ b/common/src/main/scala/org/apache/comet/CometConf.scala @@ -283,7 +283,7 @@ object CometConf extends ShimCometConf { .createWithDefault("lz4") val COMET_EXEC_SHUFFLE_COMPRESSION_LEVEL: ConfigEntry[Int] = - conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.compression.level") + conf(s"$COMET_EXEC_CONFIG_PREFIX.shuffle.compression.zstd.level") .doc("The compression level to use when compressing shuffle files with zstd.") .intConf .createWithDefault(1) diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index d0afebafb1..dc0e4d08c7 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -52,7 +52,7 @@ Comet provides the following configuration settings. | spark.comet.exec.project.enabled | Whether to enable project by default. | true | | spark.comet.exec.replaceSortMergeJoin | Experimental feature to force Spark to replace SortMergeJoin with ShuffledHashJoin for improved performance. This feature is not stable yet. For more information, refer to the Comet Tuning Guide (https://datafusion.apache.org/comet/user-guide/tuning.html). | false | | spark.comet.exec.shuffle.compression.codec | The codec of Comet native shuffle used to compress shuffle data. lz4, zstd, and snappy are supported. Compression can be disabled by setting spark.shuffle.compress=false. | lz4 | -| spark.comet.exec.shuffle.compression.level | The compression level to use when compressing shuffle files with zstd. | 1 | +| spark.comet.exec.shuffle.compression.zstd.level | The compression level to use when compressing shuffle files with zstd. | 1 | | spark.comet.exec.shuffle.enabled | Whether to enable Comet native shuffle. Note that this requires setting 'spark.shuffle.manager' to 'org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager'. 'spark.shuffle.manager' must be set before starting the Spark application and cannot be changed during the application. | true | | spark.comet.exec.sort.enabled | Whether to enable sort by default. | true | | spark.comet.exec.sortMergeJoin.enabled | Whether to enable sortMergeJoin by default. | true |