Merge remote-tracking branch 'upstream/main' into SOLR-17442

dev-tools/scripts/smokeTestRelease.py

@@ -628,7 +628,7 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
     expected_src_root_folders = ['buildSrc', 'dev-docs', 'dev-tools', 'gradle', 'help', 'solr']
     expected_src_root_files = ['build.gradle', 'gradlew', 'gradlew.bat', 'settings.gradle', 'versions.lock', 'versions.props']
     expected_src_solr_files = ['build.gradle']
-    expected_src_solr_folders = ['benchmark',  'bin', 'modules', 'api', 'core', 'docker', 'documentation', 'example', 'licenses', 'packaging', 'distribution', 'prometheus-exporter', 'server', 'solr-ref-guide', 'solrj', 'solrj-streaming', 'solrj-zookeeper', 'test-framework', 'webapp', '.gitignore', '.gitattributes']
+    expected_src_solr_folders = ['benchmark',  'bin', 'modules', 'api', 'core', 'cross-dc-manager', 'docker', 'documentation', 'example', 'licenses', 'packaging', 'distribution', 'prometheus-exporter', 'server', 'solr-ref-guide', 'solrj', 'solrj-streaming', 'solrj-zookeeper', 'test-framework', 'webapp', '.gitignore', '.gitattributes']
     is_in_list(in_root_folder, expected_src_root_folders)
     is_in_list(in_root_folder, expected_src_root_files)
     is_in_list(in_solr_folder, expected_src_solr_folders)
@@ -638,7 +638,7 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
   elif isSlim:
     is_in_list(in_root_folder, ['bin', 'docker', 'docs', 'example', 'licenses', 'server', 'lib'])
   else:
-    is_in_list(in_root_folder, ['bin', 'modules', 'docker', 'prometheus-exporter', 'docs', 'example', 'licenses', 'server', 'lib'])
+    is_in_list(in_root_folder, ['bin', 'modules', 'cross-dc-manager', 'docker', 'prometheus-exporter', 'docs', 'example', 'licenses', 'server', 'lib'])
 
   if len(in_root_folder) > 0:
     raise RuntimeError('solr: unexpected files/dirs in artifact %s: %s' % (artifact, in_root_folder))

gradle/maven/defaults-maven.gradle

@@ -25,6 +25,7 @@ configure(rootProject) {
   ext {
     published = [
         ":solr:api",
+        ":solr:cross-dc-manager",
         ":solr:core",
         ":solr:solrj",
         ":solr:solrj-streaming",

gradle/solr/packaging.gradle

@@ -36,11 +36,11 @@
 // I don't know how to untie these two cleanly.
 //
 
-configure(allprojects.findAll {project -> project.path.startsWith(":solr:modules:") || project.path.startsWith(":solr:prometheus-exporter") }) {
+configure(allprojects.findAll {project -> project.path.startsWith(":solr:modules:") || project.path == ":solr:prometheus-exporter" || project.path == ":solr:cross-dc-manager" }) {
   plugins.withType(JavaPlugin) {
     ext {
       packagingDir = file("${buildDir}/packaging")
-      if (project.path.startsWith(":solr:prometheus-exporter")) {
+      if (project.path.startsWith(":solr:prometheus-exporter") || project.path.startsWith(":solr:cross-dc-manager")) {
         deps = packagingDir
       } else {
         deps = file("${packagingDir}/${project.name}")
@@ -62,6 +62,12 @@ configure(allprojects.findAll {project -> project.path.startsWith(":solr:modules
       solrPlatformLibs project(":solr:solrj-zookeeper")
       // libExt has logging libs, which we don't want.  Lets users decide what they want.
       solrPlatformLibs project(path: ":solr:server", configuration: 'libExt')
+
+      // The cross-dc-manager uses the cross-dc Solr module libraries as well as the Jetty server jars
+      if (project.path == ":solr:cross-dc-manager") {
+        solrPlatformLibs project(":solr:modules:cross-dc")
+        solrPlatformLibs project(path: ":solr:server", configuration: 'serverLib')
+      }
     }
 
     // An aggregate that configures lib and test-lib in a temporary location.

gradle/testing/randomization.gradle

@@ -199,7 +199,12 @@ allprojects {
 
         // Enable security manager, if requested. We could move the selection of security manager and security policy
         // to each project's build/ configuration but it seems compact enough to keep it here for now.
-        if (Boolean.parseBoolean(testOptionsResolved["tests.useSecurityManager"])) {
+        def useSecurityManager = Boolean.parseBoolean(testOptionsResolved["tests.useSecurityManager"]);
+        // Allow the project to override this
+        if (project.ext.has("useSecurityManager")) {
+          useSecurityManager = project.ext.get("useSecurityManager")
+        }
+        if (useSecurityManager) {
           def commonSolrDir = project(':solr').layout.projectDirectory
           def javaSecurityPolicy = layout.projectDirectory.file("${resources}/policies/solr-tests.policy")
           jvmArgumentProviders.add(

gradle/testing/randomization/policies/solr-tests.policy

@@ -85,10 +85,12 @@ grant {
   // needed by bytebuddy
   permission java.lang.RuntimePermission "defineClass";
   permission java.lang.RuntimePermission "net.bytebuddy.createJavaDispatcher";
+  permission java.lang.RuntimePermission "net.bytebuddy.agent.getInstrumentation";
   permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.description.method";
   permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.description.type";
   permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.dynamic.loading";
   permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.utility";
+
   // needed by mockito
   permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect";
   permission java.lang.RuntimePermission "reflectionFactoryAccess";
@@ -153,6 +155,12 @@ grant {
   // needed by s3mock
   permission java.lang.RuntimePermission "accessClassInPackage.sun.nio.fs";
 
+  // needed for kafka mockito
+  permission java.lang.RuntimePermission "manageProcess";
+  permission java.io.FilePermission "${/}proc${/}self${/}io", "read";
+  permission java.io.FilePermission "${java.home}${/}bin${/}java", "execute";
+  permission java.io.FilePermission "${java.home}${/}bin${/}java.exe", "execute";
+
   // SSL related properties for Solr tests
   permission javax.net.ssl.SSLPermission "setDefaultSSLContext";
 

gradle/validation/validate-log-calls.gradle

@@ -118,7 +118,7 @@ class ValidateLogCallsTask extends DefaultTask {
 
     if (hasPlus) {
       cause = "hasPlus: " + hasPlus
-      violation = true
+      violation = level != "error"
     }
     if (violation == false) {
       def m = stripped =~ "\\(.*?\\)"

settings.gradle

@@ -38,9 +38,11 @@ include "solr:solrj"
 include "solr:solrj-zookeeper"
 include "solr:solrj-streaming"
 include "solr:core"
+include "solr:cross-dc-manager"
 include "solr:server"
 include "solr:modules:analysis-extras"
 include "solr:modules:clustering"
+include "solr:modules:cross-dc"
 include "solr:modules:opentelemetry"
 include "solr:modules:extraction"
 include "solr:modules:gcs-repository"

solr/CHANGES.txt

@@ -105,7 +105,9 @@ Other Changes
 ==================  9.8.0 ==================
 New Features
 ---------------------
-(No changes)
+* SOLR-17065: The Solr Cross-DC Project has graduated from the sandbox repository.
+  It will now be released as a fully supported Solr feature.
+  This feature closes SIP-13: Cross Data Center Replication. (Mark Miller, Andrzej Bialecki, Jason Gerlowski, Houston Putman)
 
 Improvements
 ---------------------
@@ -140,6 +142,8 @@ Optimizations
 
 * SOLR-3913: Optimize PostTool to call just optimize when both commit and optimize requested. (Eric Pugh)
 
+* SOLR-17441: Improve system metrics collection by skipping unreadable MXBean properties, making /admin/info/system calls faster (Haythem Khiri)
+
 Bug Fixes
 ---------------------
 * SOLR-12429: Uploading a configset with a symbolic link produces a IOException. Now a error message to user generated instead. (Eric Pugh)
@@ -163,6 +167,10 @@ Other Changes
 
 * SOLR-17142: Fix Gradle build sometimes gives spurious "unreferenced license file" warnings. (Uwe Schindler)
 
+* SOLR-17448: Fixed inadvertent suppression of exceptions in the background tasks across the codebase. For certain
+tasks that were scheduled via ExecutorService#submit, the results of the task execution were never examined which
+led to the suppression of exceptions. (Andrey Bozhko)
+
 * SOLR-11318: Introduce unit testing for AssertTool. (Eric Pugh, Jason Gerlowski)
 
 ==================  9.7.0 ==================
@@ -2643,6 +2651,18 @@ Bug Fixes
 
 * SOLR-16164: ConfigSet API returns error if untrusted user creates from _default configset (Eric Pugh, Kevin Risden)
 
+==================  8.11.4 ==================
+
+Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
+
+Bug Fixes
+---------------------
+* SOLR-17168: Add netty-transport-classes-epoll dependency (Colvin Cowie)
+
+* SOLR-17417: Remove unnecessary code in PKIAuthPlugin and HttpSolrCall  (Houston Putman, janhoy, Liu Huajin)
+
+* SOLR-17418: Streamline ConfigSet modification logic.  (Houston Putman, Liu Huajin)
+
 ==================  8.11.3 ==================
 
 Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

solr/README.adoc

@@ -26,6 +26,9 @@ Benchmarking module for Solr.
 link:bin/[]::
 Scripts to start up, manage and interact with Solr instances.
 
+link:cross-dc-manager/[]::
+Contains a separate application to manage Cross-Datacenter indexing.
+
 link:core/[]::
 Base Solr code.
 

solr/benchmark/src/java/org/apache/solr/bench/Docs.java

@@ -107,10 +107,7 @@ public Iterator<SolrInputDocument> preGenerate(int numDocs) throws InterruptedEx
             new SolrNamedThreadFactory("SolrJMH DocMaker"));
 
     for (int i = 0; i < numDocs; i++) {
-      executorService.submit(
-          () -> {
-            docs.add(Docs.this.inputDocument());
-          });
+      executorService.execute(() -> docs.add(Docs.this.inputDocument()));
     }
 
     executorService.shutdown();

solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java

@@ -410,7 +410,7 @@ private void indexParallel(String collection, Docs docs, int docCount)
           10,
           TimeUnit.SECONDS);
       for (int i = 0; i < docCount; i++) {
-        executorService.submit(
+        executorService.execute(
             new Runnable() {
               final SplittableRandom threadRandom = random.split();
 

solr/core/src/java/org/apache/solr/cli/ExportTool.java

@@ -539,7 +539,7 @@ void exportDocs() throws Exception {
         CountDownLatch producerLatch = new CountDownLatch(corehandlers.size());
         corehandlers.forEach(
             (s, coreHandler) ->
-                producerThreadpool.submit(
+                producerThreadpool.execute(
                     () -> {
                       try {
                         coreHandler.exportDocsFromCore();
@@ -585,7 +585,7 @@ private void addProducers(Map<String, Slice> m) {
     }
 
     private void addConsumer(CountDownLatch consumerlatch) {
-      consumerThreadpool.submit(
+      consumerThreadpool.execute(
           () -> {
             while (true) {
               SolrDocument doc;

solr/core/src/java/org/apache/solr/cli/StatusTool.java

@@ -56,7 +56,7 @@ public String getName() {
     return "status";
   }
 
-  public static final Option OPTION_MAXWAITSECS =
+  private static final Option OPTION_MAXWAITSECS =
       Option.builder()
           .longOpt("max-wait-secs")
           .argName("SECS")