Merge remote-tracking branch 'upstream/main' into SOLR-6962

dev-docs/FAQ.adoc

@@ -97,3 +97,7 @@ If you don't yet have an account, you have to ask for one in the 'users' or 'dev
 * http://fucit.org/solr-jenkins-reports/failure-report.html
 * https://ge.apache.org/scans/tests?search.relativeStartTime=P90D&search.rootProjectNames=solr*
 * https://lists.apache.org[Solr mailing list archives especially builds]
+
+=== How can I build the JavaDoc's and the Reference Guide?
+
+Run `./gradlew buildLocalSite` to build the Javadoc's and the Reference Guide site.

dev-tools/scripts/smokeTestRelease.py

@@ -628,7 +628,7 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
     expected_src_root_folders = ['buildSrc', 'dev-docs', 'dev-tools', 'gradle', 'help', 'solr']
     expected_src_root_files = ['build.gradle', 'gradlew', 'gradlew.bat', 'settings.gradle', 'versions.lock', 'versions.props']
     expected_src_solr_files = ['build.gradle']
-    expected_src_solr_folders = ['benchmark',  'bin', 'modules', 'api', 'core', 'docker', 'documentation', 'example', 'licenses', 'packaging', 'distribution', 'prometheus-exporter', 'server', 'solr-ref-guide', 'solrj', 'solrj-streaming', 'solrj-zookeeper', 'test-framework', 'webapp', '.gitignore', '.gitattributes']
+    expected_src_solr_folders = ['benchmark',  'bin', 'modules', 'api', 'core', 'cross-dc-manager', 'docker', 'documentation', 'example', 'licenses', 'packaging', 'distribution', 'prometheus-exporter', 'server', 'solr-ref-guide', 'solrj', 'solrj-streaming', 'solrj-zookeeper', 'test-framework', 'webapp', '.gitignore', '.gitattributes']
     is_in_list(in_root_folder, expected_src_root_folders)
     is_in_list(in_root_folder, expected_src_root_files)
     is_in_list(in_solr_folder, expected_src_solr_folders)
@@ -638,7 +638,7 @@ def verifyUnpacked(java, artifact, unpackPath, gitRevision, version, testArgs):
   elif isSlim:
     is_in_list(in_root_folder, ['bin', 'docker', 'docs', 'example', 'licenses', 'server', 'lib'])
   else:
-    is_in_list(in_root_folder, ['bin', 'modules', 'docker', 'prometheus-exporter', 'docs', 'example', 'licenses', 'server', 'lib'])
+    is_in_list(in_root_folder, ['bin', 'modules', 'cross-dc-manager', 'docker', 'prometheus-exporter', 'docs', 'example', 'licenses', 'server', 'lib'])
 
   if len(in_root_folder) > 0:
     raise RuntimeError('solr: unexpected files/dirs in artifact %s: %s' % (artifact, in_root_folder))

gradle/maven/defaults-maven.gradle

@@ -25,6 +25,7 @@ configure(rootProject) {
   ext {
     published = [
         ":solr:api",
+        ":solr:cross-dc-manager",
         ":solr:core",
         ":solr:solrj",
         ":solr:solrj-streaming",

gradle/solr/packaging.gradle

@@ -36,11 +36,11 @@
 // I don't know how to untie these two cleanly.
 //
 
-configure(allprojects.findAll {project -> project.path.startsWith(":solr:modules:") || project.path.startsWith(":solr:prometheus-exporter") }) {
+configure(allprojects.findAll {project -> project.path.startsWith(":solr:modules:") || project.path == ":solr:prometheus-exporter" || project.path == ":solr:cross-dc-manager" }) {
   plugins.withType(JavaPlugin) {
     ext {
       packagingDir = file("${buildDir}/packaging")
-      if (project.path.startsWith(":solr:prometheus-exporter")) {
+      if (project.path.startsWith(":solr:prometheus-exporter") || project.path.startsWith(":solr:cross-dc-manager")) {
         deps = packagingDir
       } else {
         deps = file("${packagingDir}/${project.name}")
@@ -62,6 +62,12 @@ configure(allprojects.findAll {project -> project.path.startsWith(":solr:modules
       solrPlatformLibs project(":solr:solrj-zookeeper")
       // libExt has logging libs, which we don't want.  Lets users decide what they want.
       solrPlatformLibs project(path: ":solr:server", configuration: 'libExt')
+
+      // The cross-dc-manager uses the cross-dc Solr module libraries as well as the Jetty server jars
+      if (project.path == ":solr:cross-dc-manager") {
+        solrPlatformLibs project(":solr:modules:cross-dc")
+        solrPlatformLibs project(path: ":solr:server", configuration: 'serverLib')
+      }
     }
 
     // An aggregate that configures lib and test-lib in a temporary location.

gradle/testing/randomization.gradle

@@ -199,7 +199,12 @@ allprojects {
 
         // Enable security manager, if requested. We could move the selection of security manager and security policy
         // to each project's build/ configuration but it seems compact enough to keep it here for now.
-        if (Boolean.parseBoolean(testOptionsResolved["tests.useSecurityManager"])) {
+        def useSecurityManager = Boolean.parseBoolean(testOptionsResolved["tests.useSecurityManager"]);
+        // Allow the project to override this
+        if (project.ext.has("useSecurityManager")) {
+          useSecurityManager = project.ext.get("useSecurityManager")
+        }
+        if (useSecurityManager) {
           def commonSolrDir = project(':solr').layout.projectDirectory
           def javaSecurityPolicy = layout.projectDirectory.file("${resources}/policies/solr-tests.policy")
           jvmArgumentProviders.add(

gradle/testing/randomization/policies/solr-tests.policy

@@ -85,10 +85,12 @@ grant {
   // needed by bytebuddy
   permission java.lang.RuntimePermission "defineClass";
   permission java.lang.RuntimePermission "net.bytebuddy.createJavaDispatcher";
+  permission java.lang.RuntimePermission "net.bytebuddy.agent.getInstrumentation";
   permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.description.method";
   permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.description.type";
   permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.dynamic.loading";
   permission java.lang.reflect.ReflectPermission "newProxyInPackage.net.bytebuddy.utility";
+
   // needed by mockito
   permission java.lang.RuntimePermission "accessClassInPackage.sun.reflect";
   permission java.lang.RuntimePermission "reflectionFactoryAccess";
@@ -153,6 +155,12 @@ grant {
   // needed by s3mock
   permission java.lang.RuntimePermission "accessClassInPackage.sun.nio.fs";
 
+  // needed for kafka mockito
+  permission java.lang.RuntimePermission "manageProcess";
+  permission java.io.FilePermission "${/}proc${/}self${/}io", "read";
+  permission java.io.FilePermission "${java.home}${/}bin${/}java", "execute";
+  permission java.io.FilePermission "${java.home}${/}bin${/}java.exe", "execute";
+
   // SSL related properties for Solr tests
   permission javax.net.ssl.SSLPermission "setDefaultSSLContext";
 

gradle/validation/validate-log-calls.gradle

@@ -118,7 +118,7 @@ class ValidateLogCallsTask extends DefaultTask {
 
     if (hasPlus) {
       cause = "hasPlus: " + hasPlus
-      violation = true
+      violation = level != "error"
     }
     if (violation == false) {
       def m = stripped =~ "\\(.*?\\)"

settings.gradle

@@ -38,9 +38,11 @@ include "solr:solrj"
 include "solr:solrj-zookeeper"
 include "solr:solrj-streaming"
 include "solr:core"
+include "solr:cross-dc-manager"
 include "solr:server"
 include "solr:modules:analysis-extras"
 include "solr:modules:clustering"
+include "solr:modules:cross-dc"
 include "solr:modules:opentelemetry"
 include "solr:modules:extraction"
 include "solr:modules:gcs-repository"

solr/CHANGES.txt

@@ -105,7 +105,9 @@ Other Changes
 ==================  9.8.0 ==================
 New Features
 ---------------------
-(No changes)
+* SOLR-17065: The Solr Cross-DC Project has graduated from the sandbox repository.
+  It will now be released as a fully supported Solr feature.
+  This feature closes SIP-13: Cross Data Center Replication. (Mark Miller, Andrzej Bialecki, Jason Gerlowski, Houston Putman)
 
 Improvements
 ---------------------
@@ -119,6 +121,8 @@ Improvements
 
 * SOLR-17382: Deprecate -a and -addlopts in favour of --jvm-opts for passing options into the JVM in bin/solr.   (Eric Pugh, Christos Malliaridis)
 
+* SOLR-17431: Deprecate -p parameter where it doesn't refer to a port in bin/solr. (Eric Pugh, Christos Malliaridis)
+
 Optimizations
 ---------------------
 * SOLR-14985: Solrj CloudSolrClient with Solr URLs had serious performance regressions (since the
@@ -138,6 +142,8 @@ Optimizations
 
 * SOLR-3913: Optimize PostTool to call just optimize when both commit and optimize requested. (Eric Pugh)
 
+* SOLR-17441: Improve system metrics collection by skipping unreadable MXBean properties, making /admin/info/system calls faster (Haythem Khiri)
+
 Bug Fixes
 ---------------------
 * SOLR-12429: Uploading a configset with a symbolic link produces a IOException. Now a error message to user generated instead. (Eric Pugh)
@@ -147,6 +153,8 @@ Bug Fixes
 
 * SOLR-17416: Fixed ExportHandler bug that silently suppressed errors and returned partial results in some situations (hossman)
 
+* SOLR-16254: Clarify when a bin/solr create needs to be run on the same server as Solr. (Eric Pugh)
+
 Dependency Upgrades
 ---------------------
 (No changes)
@@ -159,6 +167,10 @@ Other Changes
 
 * SOLR-17142: Fix Gradle build sometimes gives spurious "unreferenced license file" warnings. (Uwe Schindler)
 
+* SOLR-17448: Fixed inadvertent suppression of exceptions in the background tasks across the codebase. For certain
+tasks that were scheduled via ExecutorService#submit, the results of the task execution were never examined which
+led to the suppression of exceptions. (Andrey Bozhko)
+
 * SOLR-11318: Introduce unit testing for AssertTool. (Eric Pugh, Jason Gerlowski)
 
 ==================  9.7.0 ==================
@@ -2639,6 +2651,18 @@ Bug Fixes
 
 * SOLR-16164: ConfigSet API returns error if untrusted user creates from _default configset (Eric Pugh, Kevin Risden)
 
+==================  8.11.4 ==================
+
+Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
+
+Bug Fixes
+---------------------
+* SOLR-17168: Add netty-transport-classes-epoll dependency (Colvin Cowie)
+
+* SOLR-17417: Remove unnecessary code in PKIAuthPlugin and HttpSolrCall  (Houston Putman, janhoy, Liu Huajin)
+
+* SOLR-17418: Streamline ConfigSet modification logic.  (Houston Putman, Liu Huajin)
+
 ==================  8.11.3 ==================
 
 Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.

solr/README.adoc

@@ -26,6 +26,9 @@ Benchmarking module for Solr.
 link:bin/[]::
 Scripts to start up, manage and interact with Solr instances.
 
+link:cross-dc-manager/[]::
+Contains a separate application to manage Cross-Datacenter indexing.
+
 link:core/[]::
 Base Solr code.
 

solr/benchmark/src/java/org/apache/solr/bench/Docs.java

@@ -107,10 +107,7 @@ public Iterator<SolrInputDocument> preGenerate(int numDocs) throws InterruptedEx
             new SolrNamedThreadFactory("SolrJMH DocMaker"));
 
     for (int i = 0; i < numDocs; i++) {
-      executorService.submit(
-          () -> {
-            docs.add(Docs.this.inputDocument());
-          });
+      executorService.execute(() -> docs.add(Docs.this.inputDocument()));
     }
 
     executorService.shutdown();

solr/benchmark/src/java/org/apache/solr/bench/MiniClusterState.java

@@ -410,7 +410,7 @@ private void indexParallel(String collection, Docs docs, int docCount)
           10,
           TimeUnit.SECONDS);
       for (int i = 0; i < docCount; i++) {
-        executorService.submit(
+        executorService.execute(
             new Runnable() {
               final SplittableRandom threadRandom = random.split();
 

solr/core/src/java/org/apache/solr/cli/AuthTool.java

@@ -75,7 +75,7 @@ public String getUsage() {
 
   @Override
   public String getHeader() {
-    return "\nUpdates or enables/disables authentication.  Must be run on the machine hosting Solr.\n"
+    return "Updates or enables/disables authentication.  Must be run on the Solr server itself.\n"
         + "\n"
         + "List of options:";
   }

solr/core/src/java/org/apache/solr/cli/ConfigTool.java

@@ -22,6 +22,8 @@
 import java.util.List;
 import java.util.Map;
 import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DeprecatedAttributes;
+import org.apache.commons.cli.MissingArgumentException;
 import org.apache.commons.cli.Option;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.common.util.NamedList;
@@ -66,11 +68,26 @@ public List<Option> getOptions() {
             .desc(
                 "Config API action, one of: set-property, unset-property, set-user-property, unset-user-property; default is 'set-property'.")
             .build(),
-        Option.builder("p")
+        Option.builder()
             .longOpt("property")
             .argName("PROP")
             .hasArg()
-            .required(true)
+            .required(
+                false) // Should be TRUE but have a deprecated option to deal with first, so we
+            // enforce in code
+            .desc(
+                "Name of the Config API property to apply the action to, such as: 'updateHandler.autoSoftCommit.maxTime'.")
+            .build(),
+        Option.builder("p")
+            .deprecated(
+                DeprecatedAttributes.builder()
+                    .setForRemoval(true)
+                    .setSince("9.8")
+                    .setDescription("Use --property instead")
+                    .get())
+            .hasArg()
+            .argName("PROP")
+            .required(false)
             .desc(
                 "Name of the Config API property to apply the action to, such as: 'updateHandler.autoSoftCommit.maxTime'.")
             .build(),
@@ -93,9 +110,13 @@ public void runImpl(CommandLine cli) throws Exception {
     String solrUrl = SolrCLI.normalizeSolrUrl(cli);
     String action = cli.getOptionValue("action", "set-property");
     String collection = cli.getOptionValue("name");
-    String property = cli.getOptionValue("property");
+    String property = SolrCLI.getOptionWithDeprecatedAndDefault(cli, "property", "p", null);
     String value = cli.getOptionValue("value");
 
+    if (property == null) {
+      throw new MissingArgumentException("'property' is a required option.");
+    }
+
     Map<String, Object> jsonObj = new HashMap<>();
     if (value != null) {
       Map<String, String> setMap = new HashMap<>();

solr/core/src/java/org/apache/solr/cli/CreateTool.java

@@ -66,6 +66,14 @@ public String getName() {
     return "create";
   }
 
+  @Override
+  public String getHeader() {
+    return "Creates a core or collection depending on whether Solr is running in standalone (core) or SolrCloud mode (collection).\n"
+        + "If you are using standalone mode you must run this command on the Solr server itself.\n"
+        + "\n"
+        + "List of options:";
+  }
+
   @Override
   public List<Option> getOptions() {
     return List.of(

solr/core/src/java/org/apache/solr/cli/DeleteTool.java

@@ -61,7 +61,7 @@ public String getName() {
 
   @Override
   public String getHeader() {
-    return "\nDeletes a core or collection depending on whether Solr is running in standalone (core) or SolrCloud"
+    return "Deletes a core or collection depending on whether Solr is running in standalone (core) or SolrCloud"
         + " mode (collection). If you're deleting a collection in SolrCloud mode, the default behavior is to also"
         + " delete the configuration directory from Zookeeper so long as it is not being used by another collection.\n"
         + " You can override this behavior by passing -deleteConfig false when running this command.\n"

solr/core/src/java/org/apache/solr/cli/ExportTool.java

@@ -539,7 +539,7 @@ void exportDocs() throws Exception {
         CountDownLatch producerLatch = new CountDownLatch(corehandlers.size());
         corehandlers.forEach(
             (s, coreHandler) ->
-                producerThreadpool.submit(
+                producerThreadpool.execute(
                     () -> {
                       try {
                         coreHandler.exportDocsFromCore();
@@ -585,7 +585,7 @@ private void addProducers(Map<String, Slice> m) {
     }
 
     private void addConsumer(CountDownLatch consumerlatch) {
-      consumerThreadpool.submit(
+      consumerThreadpool.execute(
           () -> {
             while (true) {
               SolrDocument doc;

solr/core/src/java/org/apache/solr/cli/PackageTool.java

@@ -27,6 +27,7 @@
 import java.util.List;
 import java.util.Map;
 import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DeprecatedAttributes;
 import org.apache.commons.cli.Option;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.core.config.Configurator;
@@ -124,7 +125,7 @@ public void runImpl(CommandLine cli) throws Exception {
                   printGreen("\t" + packages.get(packageName));
                 }
               } else {
-                // nuance that we use a arg here instead of requiring a --package parameter with a
+                // nuance that we use an arg here instead of requiring a --package parameter with a
                 // value
                 // in this code path
                 String packageName = cli.getArgs()[1];
@@ -166,21 +167,25 @@ public void runImpl(CommandLine cli) throws Exception {
                   Pair<String, String> parsedVersion = parsePackageVersion(cli.getArgList().get(1));
                   String packageName = parsedVersion.first();
                   String version = parsedVersion.second();
-                  boolean noprompt = cli.hasOption("no-prompt");
+                  boolean noPrompt = cli.hasOption("no-prompt");
                   boolean isUpdate = cli.hasOption("update");
                   String[] collections =
                       cli.hasOption("collections")
                           ? PackageUtils.validateCollections(
                               cli.getOptionValue("collections").split(","))
                           : new String[] {};
+                  String[] parameters =
+                      cli.hasOption("param")
+                          ? cli.getOptionValues("param")
+                          : cli.getOptionValues("p");
                   packageManager.deploy(
                       packageName,
                       version,
                       collections,
                       cli.hasOption("cluster"),
-                      cli.getOptionValues("param"),
+                      parameters,
                       isUpdate,
-                      noprompt);
+                      noPrompt);
                 } else {
                   printRed(
                       "Either specify --cluster to deploy cluster level plugins or --collections <list-of-collections> to deploy collection level plugins");
@@ -330,12 +335,24 @@ public List<Option> getOptions() {
             .longOpt("cluster")
             .desc("Specifies that this action should affect cluster-level plugins only.")
             .build(),
-        Option.builder("p")
+        Option.builder()
             .longOpt("param")
             .hasArgs()
             .argName("PARAMS")
             .desc("List of parameters to be used with deploy command.")
             .build(),
+        Option.builder("p")
+            .deprecated(
+                DeprecatedAttributes.builder()
+                    .setForRemoval(true)
+                    .setSince("9.8")
+                    .setDescription("Use --param instead")
+                    .get())
+            .hasArg()
+            .argName("PARAMS")
+            .required(false)
+            .desc("List of parameters to be used with deploy command.")
+            .build(),
         Option.builder()
             .longOpt("update")
             .desc("If a deployment is an update over a previous deployment.")