[Build] Changes to avoid making too many requests to central maven re…

…po (#4087)

## Description
* Google maven repo URL is changed from
`https://maven-central.storage-download.googleapis.com/repos/central/data/`
to `https://maven-central.storage-download.googleapis.com/maven2/`. This
caused more load on the central maven repo as the google maven repo is
wrong.
* Currently we hard coded the URL to central maven repo for downloading
the `sbt-launch-x.x.x.jar` as part of the build start. Change it to
google maven repo which seems have better rate limits than the central
maven repo. Also add the central maven repo as a backup so that we have
two options. Example retrying:
  ```
   (base) delta2 % ./build/sbt clean package test:package
Attempting to fetch sbt from
https://eemaven-central.storage-download.googleapis.com/maven2/org/scala-sbt/sbt-launch/1.9.9/sbt-launch-1.9.9.jar
Download from
https://eemaven-central.storage-download.googleapis.com/maven2/org/scala-sbt/sbt-launch/1.9.9/sbt-launch-1.9.9.jar
failed. Retrying from
https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/1.9.9/sbt-launch-1.9.9.jar
  Launching sbt from build/sbt-launch-1.9.9.jar
   ```
* Add Google repo to iceberg build so that it is attempted first before
the central maven repo and also as additional backup.
   * Testing
      *   Without any repo changes: 
           ```
           (base) iceberg_src % rm -rf $HOME/.gradle/caches/
(base) iceberg_src % ./gradlew clean --refresh-dependencies
:iceberg-core:build --info | grep Downloading | grep
"mockito-core-4.0.0.jar"
Downloading
https://repo.maven.apache.org/maven2/org/mockito/mockito-core/4.0.0/mockito-core-4.0.0.jar
to
/Users/venkateshwar.korukanti/.gradle/.tmp/gradle_download6315884371983156241bin
           ```
      * By adding google repo as a first in the list
         ```
         (base) iceberg_src % git diff
         diff --git a/build.gradle b/build.gradle
         index 12ed701..c80bbd3 100644
         --- a/build.gradle
         +++ b/build.gradle
         @@ -97,6 +97,9 @@ allprojects {
            group = "org.apache.iceberg"
            version = projectVersion
            repositories {
         +    maven {
+ url "https://maven-central.storage-download.googleapis.com/maven2"
         +    }
              mavenCentral()
              mavenLocal()
            }
         (base) iceberg_src % rm -rf $HOME/.gradle/caches/
(base) iceberg_src % ./gradlew clean --refresh-dependencies
:iceberg-core:build --info | grep Downloading | grep
"mockito-core-4.0.0.jar"
Downloading
https://maven-central.storage-download.googleapis.com/maven2/org/mockito/mockito-core/4.0.0/mockito-core-4.0.0.jar
to
/Users/venkateshwar.korukanti/.gradle/.tmp/gradle_download7065461132604582238bin
         ```

build/sbt-config/repositories

@@ -2,7 +2,7 @@
   local
   local-preloaded-ivy: file:///${sbt.preloaded-${sbt.global.base-${user.home}/.sbt}/preloaded/}, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext]
   local-preloaded: file:///${sbt.preloaded-${sbt.global.base-${user.home}/.sbt}/preloaded/}
-  gcs-maven-central-mirror: https://maven-central.storage-download.googleapis.com/repos/central/data/
+  gcs-maven-central-mirror: https://maven-central.storage-download.googleapis.com/maven2/
   maven-central
   typesafe-ivy-releases: https://repo.typesafe.com/typesafe/ivy-releases/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly
   sbt-ivy-snapshots: https://repo.scala-sbt.org/scalasbt/ivy-snapshots/, [organization]/[module]/[revision]/[type]s/[artifact](-[classifier]).[ext], bootOnly

build/sbt-launch-lib.bash

@@ -36,42 +36,52 @@ dlog () {
   [[ $debug ]] && echoerr "$@"
 }
 
+download_sbt () {
+  local url=$1
+  local output=$2
+  local temp_file="${output}.part"
+
+  if [ $(command -v curl) ]; then
+    curl --fail --location --silent ${url} > "${temp_file}" &&\
+      mv "${temp_file}" "${output}"
+  elif [ $(command -v wget) ]; then
+    wget --quiet ${url} -O "${temp_file}" &&\
+      mv "${temp_file}" "${output}"
+  else
+    printf "You do not have curl or wget installed, unable to downlaod ${url}\n"
+    exit -1
+  fi
+}
+
+
 acquire_sbt_jar () {
   SBT_VERSION=`awk -F "=" '/sbt\.version/ {print $2}' ./project/build.properties`
 
-  # Download sbt from mirror URL if the environment variable is provided
+  # Set primary and fallback URLs
   if [[ "${SBT_VERSION}" == "0.13.18" ]] && [[ -n "${SBT_MIRROR_JAR_URL}" ]]; then
     URL1="${SBT_MIRROR_JAR_URL}"
   elif [[ "${SBT_VERSION}" == "1.5.5" ]] && [[ -n "${SBT_1_5_5_MIRROR_JAR_URL}" ]]; then
     URL1="${SBT_1_5_5_MIRROR_JAR_URL}"
   else
-    URL1=${DEFAULT_ARTIFACT_REPOSITORY:-https://repo1.maven.org/maven2/}org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar
+    URL1=${DEFAULT_ARTIFACT_REPOSITORY:-https://maven-central.storage-download.googleapis.com/maven2/}org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar
   fi
+  BACKUP_URL="https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/${SBT_VERSION}/sbt-launch-${SBT_VERSION}.jar"
 
   JAR=build/sbt-launch-${SBT_VERSION}.jar
   sbt_jar=$JAR
 
   if [[ ! -f "$sbt_jar" ]]; then
-    # Download sbt launch jar if it hasn't been downloaded yet
-    if [ ! -f "${JAR}" ]; then
-    # Download
     printf 'Attempting to fetch sbt from %s\n' "${URL1}"
-    JAR_DL="${JAR}.part"
-    if [ $(command -v curl) ]; then
-      curl --fail --location --silent ${URL1} > "${JAR_DL}" &&\
-        mv "${JAR_DL}" "${JAR}"
-    elif [ $(command -v wget) ]; then
-      wget --quiet ${URL1} -O "${JAR_DL}" &&\
-        mv "${JAR_DL}" "${JAR}"
-    else
-      printf "You do not have curl or wget installed, please install sbt manually from https://www.scala-sbt.org/\n"
-      exit -1
-    fi
+    download_sbt "${URL1}" "${JAR}"
+
+    if [[ ! -f "${JAR}" ]]; then
+      printf 'Download from %s failed. Retrying from %s\n' "${URL1}" "${BACKUP_URL}"
+      download_sbt "${BACKUP_URL}" "${JAR}"
     fi
-    if [ ! -f "${JAR}" ]; then
-    # We failed to download
-    printf "Our attempt to download sbt locally to ${JAR} failed. Please install sbt manually from https://www.scala-sbt.org/\n"
-    exit -1
+
+    if [[ ! -f "${JAR}" ]]; then
+      printf "Failed to download sbt. Please install sbt manually from https://www.scala-sbt.org/\n"
+      exit 1
     fi
     printf "Launching sbt from ${JAR}\n"
   fi

icebergShaded/generate_iceberg_jars.py

@@ -19,6 +19,7 @@
 import argparse
 import os
 import glob
+import re
 import subprocess
 import shlex
 import shutil
@@ -65,6 +66,27 @@ def iceberg_jars_exists():
     return True
 
 
+def add_google_maven_repo_to_gradle_config():
+  with WorkingDirectory(iceberg_src_dir):
+    file_path = 'build.gradle'
+
+    with open(file_path, 'r') as file:
+      content = file.read()
+
+    # Define the old and new configurations
+    old_config = r'repositories {\n'
+
+    new_config = 'repositories {\n    maven {\n      ' + \
+      'url "https://maven-central.storage-download.googleapis.com/maven2"\n    }\n'
+
+    # Replace the old configuration with the new one
+    updated_content = re.sub(old_config, new_config, content, flags=re.DOTALL)
+
+    # Write the updated content back to the file
+    with open(file_path, 'w') as file:
+      file.write(updated_content)
+
+
 def prepare_iceberg_source():
     with WorkingDirectory(iceberg_root_dir):
         print(">>> Cloning Iceberg repo")
@@ -92,6 +114,8 @@ def prepare_iceberg_source():
             run_cmd("git add .")
             run_cmd("git commit -a -m 'applied %s'" % path.basename(patch_file))
 
+        add_google_maven_repo_to_gradle_config()
+
 
 def generate_iceberg_jars():
     print(">>> Compiling JARs")