diff --git a/gradle/testing/randomization/policies/solr-tests.policy b/gradle/testing/randomization/policies/solr-tests.policy index e5c37b4c912..0a7fea95ad6 100644 --- a/gradle/testing/randomization/policies/solr-tests.policy +++ b/gradle/testing/randomization/policies/solr-tests.policy @@ -109,6 +109,8 @@ grant { permission java.lang.RuntimePermission "writeFileDescriptor"; // needed by hadoop http permission java.lang.RuntimePermission "getProtectionDomain"; + // SolrProcessMgr to list processes + permission java.lang.RuntimePermission "manageProcess"; // These two *have* to be spelled out a separate permission java.lang.management.ManagementPermission "control"; @@ -250,6 +252,10 @@ grant { // expanded to a wildcard if set, allows all networking everywhere permission java.net.SocketPermission "${solr.internal.network.permission}", "accept,listen,connect,resolve"; + + // Run java + permission java.io.FilePermission "${java.home}${/}-", "execute"; + permission java.io.FilePermission "C:\\Windows\\*\\wmic.exe", "execute"; }; // Grant all permissions to Gradle test runner classes. diff --git a/solr/bin/solr b/solr/bin/solr index 6439f180cc8..dbc23b18ff8 100755 --- a/solr/bin/solr +++ b/solr/bin/solr @@ -512,54 +512,13 @@ function run_tool() { # shellcheck disable=SC2086 "$JAVA" $SOLR_SSL_OPTS $AUTHC_OPTS ${SOLR_ZK_CREDS_AND_ACLS:-} ${SOLR_TOOL_OPTS:-} -Dsolr.install.dir="$SOLR_TIP" \ - -Dlog4j.configurationFile="$DEFAULT_SERVER_DIR/resources/log4j2-console.xml" \ + -Dlog4j.configurationFile="$DEFAULT_SERVER_DIR/resources/log4j2-console.xml" -Dsolr.pid.dir="$SOLR_PID_DIR" \ -classpath "$DEFAULT_SERVER_DIR/solr-webapp/webapp/WEB-INF/lib/*:$DEFAULT_SERVER_DIR/lib/ext/*:$DEFAULT_SERVER_DIR/lib/*" \ org.apache.solr.cli.SolrCLI "$@" return $? } # end run_tool function -# get status about any Solr nodes running on this host -function get_status() { - # first, see if Solr is running - numSolrs=$(find "$SOLR_PID_DIR" -name "solr-*.pid" -type f | wc -l | tr -d ' ') - if [ "$numSolrs" != "0" ]; then - echo -e "\nFound $numSolrs Solr nodes: " - while read PIDF - do - ID=$(cat "$PIDF") - port=$(jetty_port "$ID") - if [ "$port" != "" ]; then - echo -e "\nSolr process $ID running on port $port" - run_tool status --solr-url "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$port/solr" "$@" - echo "" - else - echo -e "\nSolr process $ID from $PIDF not found." - fi - done < <(find "$SOLR_PID_DIR" -name "solr-*.pid" -type f) - else - # no pid files but check using ps just to be sure - numSolrs=$(ps auxww | grep start\.jar | grep solr\.solr\.home | grep -v grep | wc -l | sed -e 's/^[ \t]*//') - if [ "$numSolrs" != "0" ]; then - echo -e "\nFound $numSolrs Solr nodes: " - PROCESSES=$(ps auxww | grep start\.jar | grep solr\.solr\.home | grep -v grep | awk '{print $2}' | sort -r) - for ID in $PROCESSES - do - port=$(jetty_port "$ID") - if [ "$port" != "" ]; then - echo "" - echo "Solr process $ID running on port $port" - run_tool status --solr-url "$SOLR_URL_SCHEME://$SOLR_TOOL_HOST:$port/solr" "$@" - echo "" - fi - done - else - echo -e "\nNo Solr nodes are running.\n" - fi - fi - -} # end get_status - # tries to gracefully stop Solr using the Jetty # stop command and if that fails, then uses kill -9 # (will attempt to thread dump before killing) @@ -659,16 +618,6 @@ else exit fi -# status tool -if [ "$SCRIPT_CMD" == "status" ]; then - if [ "$1" == "-h" ] || [ "$1" == "--help" ] || [ "$1" == "-help" ]; then - print_usage status - else - get_status - fi - exit $? -fi - # configure authentication if [[ "$SCRIPT_CMD" == "auth" ]]; then : "${SOLR_SERVER_DIR:=$DEFAULT_SERVER_DIR}" @@ -1201,7 +1150,7 @@ fi if [ "${SOLR_MODE:-}" == 'solrcloud' ]; then : "${ZK_CLIENT_TIMEOUT:=30000}" CLOUD_MODE_OPTS=("-DzkClientTimeout=$ZK_CLIENT_TIMEOUT") - + if [ -n "${ZK_HOST:-}" ]; then CLOUD_MODE_OPTS+=("-DzkHost=$ZK_HOST") else diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index 0455d7c0a7d..e649d7b3260 100755 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -252,7 +252,7 @@ IF "%1"=="-usage" goto run_solrcli IF "%1"=="-h" goto run_solrcli IF "%1"=="--help" goto run_solrcli IF "%1"=="/?" goto run_solrcli -IF "%1"=="status" goto get_status +IF "%1"=="status" goto run_solrcli IF "%1"=="version" goto run_solrcli IF "%1"=="-v" goto run_solrcli IF "%1"=="-version" goto run_solrcli @@ -1268,34 +1268,6 @@ REM Run the requested example REM End of run_example goto done -:get_status -REM Find all Java processes, correlate with those listening on a port -REM and then try to contact via that port using the status tool -for /f "usebackq" %%i in (`dir /b "%SOLR_TIP%\bin" ^| findstr /i "^solr-.*\.port$"`) do ( - set SOME_SOLR_PORT= - For /F "Delims=" %%J In ('type "%SOLR_TIP%\bin\%%i"') do set SOME_SOLR_PORT=%%~J - if NOT "!SOME_SOLR_PORT!"=="" ( - for /f "tokens=2,5" %%j in ('netstat -aon ^| find "TCP " ^| find ":0 " ^| find ":!SOME_SOLR_PORT! "') do ( - IF NOT "%%k"=="0" ( - if "%%j"=="%SOLR_JETTY_HOST%:!SOME_SOLR_PORT!" ( - @echo. - set has_info=1 - echo Found Solr process %%k running on port !SOME_SOLR_PORT! - REM Passing in %2 (-h or --help) directly is captured by a custom help path for usage output - "%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^ - -Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^ - -classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^ - org.apache.solr.cli.SolrCLI status --solr-url !SOLR_URL_SCHEME!://%SOLR_TOOL_HOST%:!SOME_SOLR_PORT! %2 - @echo. - ) - ) - ) - ) -) -if NOT "!has_info!"=="1" echo No running Solr nodes found. -set has_info= -goto done - :parse_healthcheck_args IF [%1]==[] goto run_healthcheck IF "%1"=="-V" goto set_healthcheck_verbose diff --git a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java index 56abba7c4d2..5b54b9dc9bf 100755 --- a/solr/core/src/java/org/apache/solr/cli/SolrCLI.java +++ b/solr/core/src/java/org/apache/solr/cli/SolrCLI.java @@ -451,7 +451,8 @@ public static String getOptionWithDeprecatedAndDefault( // TODO: SOLR-17429 - remove the custom logic when Commons CLI is upgraded and // makes stderr the default, or makes Option.toDeprecatedString() public. private static void deprecatedHandlerStdErr(Option o) { - if (o.isDeprecated()) { + // Deprecated options without a description act as "stealth" options + if (o.isDeprecated() && !o.getDeprecated().getDescription().isBlank()) { final StringBuilder buf = new StringBuilder().append("Option '-").append(o.getOpt()).append('\''); if (o.getLongOpt() != null) { diff --git a/solr/core/src/java/org/apache/solr/cli/SolrProcessManager.java b/solr/core/src/java/org/apache/solr/cli/SolrProcessManager.java new file mode 100644 index 00000000000..42b38b2e7a8 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/cli/SolrProcessManager.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cli; + +import static org.apache.solr.servlet.SolrDispatchFilter.SOLR_INSTALL_DIR_ATTRIBUTE; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.lang.invoke.MethodHandles; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.lucene.util.Constants; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.EnvUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Class to interact with Solr OS processes */ +public class SolrProcessManager { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private final Map pidProcessMap; + private final Map portProcessMap; + private final Path pidDir; + private static final Pattern pidFilePattern = Pattern.compile("^solr-([0-9]+)\\.(pid|port)$"); + // Set this to true during testing to allow the SolrProcessManager to find only mock Solr + // processes + public static boolean enableTestingMode = false; + + public SolrProcessManager() { + pidProcessMap = + ProcessHandle.allProcesses() + .filter(p -> p.info().command().orElse("").contains("java")) + .filter(p -> commandLine(p).orElse("").contains("-Djetty.port=")) + .filter( + p -> !enableTestingMode || commandLine(p).orElse("").contains("-DmockSolr=true")) + .collect( + Collectors.toUnmodifiableMap( + ProcessHandle::pid, + ph -> + new SolrProcess( + ph.pid(), parsePortFromProcess(ph).orElseThrow(), isProcessSsl(ph)))); + portProcessMap = + pidProcessMap.values().stream().collect(Collectors.toUnmodifiableMap(p -> p.port, p -> p)); + String solrInstallDir = EnvUtils.getProperty(SOLR_INSTALL_DIR_ATTRIBUTE); + pidDir = + Paths.get( + EnvUtils.getProperty( + "solr.pid.dir", + solrInstallDir != null + ? solrInstallDir + "/bin" + : System.getProperty("java.io.tmpdir"))); + } + + public boolean isRunningWithPort(Integer port) { + return portProcessMap.containsKey(port); + } + + public boolean isRunningWithPid(Long pid) { + return pidProcessMap.containsKey(pid); + } + + public Optional processForPort(Integer port) { + return portProcessMap.containsKey(port) + ? Optional.of(portProcessMap.get(port)) + : Optional.empty(); + } + + /** Return the SolrProcess for a given PID, if it is running */ + public Optional getProcessForPid(Long pid) { + return pidProcessMap.containsKey(pid) ? Optional.of(pidProcessMap.get(pid)) : Optional.empty(); + } + + /** + * Scans the PID directory for Solr PID files and returns a list of SolrProcesses for each running + * Solr instance. If a PID file is found but no process is running, the PID file is deleted. On + * Windows, the file is a 'PORT' file containing the port number. + * + * @return a list of SolrProcesses for each running Solr instance + */ + public Collection scanSolrPidFiles() throws IOException { + List processes = new ArrayList<>(); + try (Stream pidFiles = + Files.list(pidDir) + .filter(p -> pidFilePattern.matcher(p.getFileName().toString()).matches())) { + for (Path p : pidFiles.collect(Collectors.toList())) { + Optional process; + if (p.toString().endsWith(".port")) { + // On Windows, the file is a 'PORT' file containing the port number. + Integer port = Integer.valueOf(Files.readAllLines(p).get(0)); + process = processForPort(port); + } else { + // On Linux, the file is a 'PID' file containing the process ID. + Long pid = Long.valueOf(Files.readAllLines(p).get(0)); + process = getProcessForPid(pid); + } + if (process.isPresent()) { + processes.add(process.get()); + } else { + log.warn("PID file {} found, but no process running. Deleting PID file", p.getFileName()); + Files.deleteIfExists(p); + } + } + return processes; + } + } + + public Collection getAllRunning() { + return pidProcessMap.values(); + } + + private Optional parsePortFromProcess(ProcessHandle ph) { + Optional portStr = + arguments(ph).stream() + .filter(a -> a.contains("-Djetty.port=")) + .map(s -> s.split("=")[1]) + .findFirst(); + return portStr.isPresent() ? portStr.map(Integer::parseInt) : Optional.empty(); + } + + private boolean isProcessSsl(ProcessHandle ph) { + return arguments(ph).stream() + .anyMatch( + arg -> List.of("--module=https", "--module=ssl", "--module=ssl-reload").contains(arg)); + } + + /** + * Gets the command line of a process as a string. This is a workaround for the fact that + * ProcessHandle.info().command() is not (yet) implemented on Windows. + * + * @param ph the process handle + * @return the command line of the process + */ + private static Optional commandLine(ProcessHandle ph) { + if (!Constants.WINDOWS) { + return ph.info().commandLine(); + } else { + long desiredProcessid = ph.pid(); + try { + Process process = + new ProcessBuilder( + "wmic", + "process", + "where", + "ProcessID=" + desiredProcessid, + "get", + "commandline", + "/format:list") + .redirectErrorStream(true) + .start(); + try (InputStreamReader inputStreamReader = + new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8); + BufferedReader reader = new BufferedReader(inputStreamReader)) { + while (true) { + String line = reader.readLine(); + if (line == null) { + return Optional.empty(); + } + if (!line.startsWith("CommandLine=")) { + continue; + } + return Optional.of(line.substring("CommandLine=".length())); + } + } + } catch (IOException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Error getting command line for process " + desiredProcessid, + e); + } + } + } + + /** + * Gets the arguments of a process as a list of strings. With workaround for Windows. + * + * @param ph the process handle + * @return the arguments of the process + */ + private static List arguments(ProcessHandle ph) { + if (!Constants.WINDOWS) { + return Arrays.asList(ph.info().arguments().orElse(new String[] {})); + } else { + return Arrays.asList(commandLine(ph).orElse("").split("\\s+")); + } + } + + /** Represents a running Solr process */ + public static class SolrProcess { + private final long pid; + private final int port; + private final boolean isHttps; + + public SolrProcess(long pid, int port, boolean isHttps) { + this.pid = pid; + this.port = port; + this.isHttps = isHttps; + } + + public long getPid() { + return pid; + } + + public int getPort() { + return port; + } + + public boolean isHttps() { + return isHttps; + } + + public String getLocalUrl() { + return String.format(Locale.ROOT, "%s://localhost:%s/solr", isHttps ? "https" : "http", port); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/cli/StatusTool.java b/solr/core/src/java/org/apache/solr/cli/StatusTool.java index f69c82debdc..4a8322b1c54 100644 --- a/solr/core/src/java/org/apache/solr/cli/StatusTool.java +++ b/solr/core/src/java/org/apache/solr/cli/StatusTool.java @@ -17,35 +17,43 @@ package org.apache.solr.cli; +import static org.apache.solr.cli.SolrCLI.OPTION_SOLRURL; + import java.io.PrintStream; import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.DeprecatedAttributes; -import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; -import org.apache.commons.cli.Options; +import org.apache.solr.cli.SolrProcessManager.SolrProcess; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.URLUtil; import org.noggit.CharArr; import org.noggit.JSONWriter; public class StatusTool extends ToolBase { - /** Get the status of a Solr server. */ + private final SolrProcessManager processMgr; + public StatusTool() { this(CLIO.getOutStream()); } public StatusTool(PrintStream stdout) { super(stdout); + processMgr = new SolrProcessManager(); } @Override @@ -59,6 +67,7 @@ public String getName() { .argName("SECS") .hasArg() .required(false) + .deprecated() // Will make it a stealth option, not printed or complained about .desc("Wait up to the specified number of seconds to see Solr running.") .build(); @@ -76,73 +85,212 @@ public String getName() { .get()) .build(); + public static final Option OPTION_PORT = + Option.builder("p") + .longOpt("port") + .argName("PORT") + .required(false) + .hasArg() + .desc("Port on localhost to check status for") + .build(); + + public static final Option OPTION_SHORT = + Option.builder() + .longOpt("short") + .argName("SHORT") + .required(false) + .desc("Short format. Prints one URL per line for running instances") + .build(); + @Override public List