From 337f980f0073c8605ed2738186d2089a362b7f66 Mon Sep 17 00:00:00 2001
From: Wenchen Fan <wenchen@databricks.com>
Date: Wed, 8 May 2024 16:47:01 -0700
Subject: [PATCH] [SPARK-48204][INFRA] Fix release script for Spark 4.0+

### What changes were proposed in this pull request?

Before Spark 4.0, Scala 2.12 was primary and Scala 2.13 was secondary. The release scripts build more packages (hadoop3, without-hadoop, pyspark, sparkr) for the primary Scala version but only one package for the secondary. However, Spark 4.0 removes Scala 2.12 support and the release script needs to be updated accordingly.

### Why are the changes needed?

to make the release scripts work

### Does this PR introduce _any_ user-facing change?

no

### How was this patch tested?

manual.

### Was this patch authored or co-authored using generative AI tooling?

no

Closes #46484 from cloud-fan/re.

Authored-by: Wenchen Fan <wenchen@databricks.com>
Signed-off-by: Dongjoon Hyun <dhyun@apple.com>
---
 dev/create-release/release-build.sh | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index 75ec98464f3ec..b720a8fc93861 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -194,6 +194,8 @@ fi
 PUBLISH_SCALA_2_12=1
 if [[ $SPARK_VERSION > "3.5.99" ]]; then
   PUBLISH_SCALA_2_12=0
+  # There is no longer scala-2.13 profile since 4.0.0
+  SCALA_2_13_PROFILES=""
 fi
 SCALA_2_12_PROFILES="-Pscala-2.12"
 
@@ -345,21 +347,25 @@ if [[ "$1" == "package" ]]; then
   declare -A BINARY_PKGS_EXTRA
   BINARY_PKGS_EXTRA["hadoop3"]="withpip,withr"
 
-  if [[ $PUBLISH_SCALA_2_13 = 1 ]]; then
-    key="hadoop3-scala2.13"
+  # This is dead code as Scala 2.12 is no longer supported, but we keep it as a template for
+  # adding new Scala version support in the future. This secondary Scala version only has one
+  # binary package to avoid doubling the number of final packages. It doesn't build PySpark and
+  # SparkR as the primary Scala version will build them.
+  if [[ $PUBLISH_SCALA_2_12 = 1 ]]; then
+    key="hadoop3-scala2.12"
     args="-Phadoop-3 $HIVE_PROFILES"
     extra=""
-    if ! make_binary_release "$key" "$SCALA_2_13_PROFILES $args" "$extra" "2.13"; then
+    if ! make_binary_release "$key" "$SCALA_2_12_PROFILES $args" "$extra" "2.12"; then
       error "Failed to build $key package. Check logs for details."
     fi
   fi
 
-  if [[ $PUBLISH_SCALA_2_12 = 1 ]]; then
+  if [[ $PUBLISH_SCALA_2_13 = 1 ]]; then
     echo "Packages to build: ${!BINARY_PKGS_ARGS[@]}"
     for key in ${!BINARY_PKGS_ARGS[@]}; do
       args=${BINARY_PKGS_ARGS[$key]}
       extra=${BINARY_PKGS_EXTRA[$key]}
-      if ! make_binary_release "$key" "$SCALA_2_12_PROFILES $args" "$extra" "2.12"; then
+      if ! make_binary_release "$key" "$SCALA_2_13_PROFILES $args" "$extra" "2.13"; then
         error "Failed to build $key package. Check logs for details."
       fi
     done