Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bin/solr in branch_9_7 has commands that fail #2710

Merged
merged 16 commits into from
Sep 28, 2024
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion solr/bin/solr
Original file line number Diff line number Diff line change
Expand Up @@ -946,7 +946,7 @@ if [ "$SCRIPT_CMD" == "healthcheck" ]; then
exit 1
fi

run_tool healthcheck -zkHost "$ZK_HOST" -collection "$HEALTHCHECK_COLLECTION" $VERBOSE
run_tool healthcheck -z "$ZK_HOST" -c "$HEALTHCHECK_COLLECTION" $VERBOSE

exit $?
fi
Expand Down
2 changes: 1 addition & 1 deletion solr/bin/solr.cmd
Original file line number Diff line number Diff line change
Expand Up @@ -1548,7 +1548,7 @@ echo ZK_HOST: !ZK_HOST!
"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% -Dsolr.install.dir="%SOLR_TIP%" ^
-Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^
-classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^
org.apache.solr.cli.SolrCLI healthcheck -collection !HEALTHCHECK_COLLECTION! -zkHost !ZK_HOST! %HEALTHCHECK_VERBOSE%
org.apache.solr.cli.SolrCLI healthcheck -c !HEALTHCHECK_COLLECTION! -z !ZK_HOST! %HEALTHCHECK_VERBOSE%
goto done

:run_solrcli
Expand Down
55 changes: 38 additions & 17 deletions solr/core/src/java/org/apache/solr/cli/CreateTool.java
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.EnvUtils;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.ConfigSetService;
import org.noggit.CharArr;
Expand Down Expand Up @@ -148,16 +149,36 @@ public List<Option> getOptions() {
.required(false)
.desc("Configuration name; default is the collection name.")
.build(),
Option.builder("p")
.longOpt("port")
.deprecated(
DeprecatedAttributes.builder()
.setForRemoval(true)
.setSince("9.7")
.setDescription("Use --solr-url instead")
.get())
.argName("PORT")
.hasArg()
.required(false)
.desc("Port of a local Solr instance where you want to create the new core.")
.build(),
SolrCLI.OPTION_SOLRURL,
SolrCLI.OPTION_SOLRURL_DEPRECATED,
SolrCLI.OPTION_ZKHOST,
SolrCLI.OPTION_ZKHOST_DEPRECATED);
SolrCLI.OPTION_ZKHOST_DEPRECATED,
SolrCLI.OPTION_VERBOSE_DEPRECATED);
Comment on lines +168 to +169
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be eventually

Suggested change
SolrCLI.OPTION_ZKHOST_DEPRECATED,
SolrCLI.OPTION_VERBOSE_DEPRECATED);
SolrCLI.OPTION_ZKHOST_DEPRECATED,
SolrCLI.OPTION_VERBOSE,
SolrCLI.OPTION_VERBOSE_DEPRECATED);

}

@Override
public void runImpl(CommandLine cli) throws Exception {
SolrCLI.raiseLogLevelUnlessVerbose(cli);

// If a port is provided, inject it into the environment variables so
// that {@link SolrCLI#getDefaultSolrUrl()} can use it instead of the default 8983.
if (cli.hasOption("p")) {
EnvUtils.setProperty("jetty.port", cli.getOptionValue("p"));
}

try (var solrClient = SolrCLI.getSolrClient(cli)) {
if (SolrCLI.isCloudMode(solrClient)) {
createCollection(cli);
Expand Down Expand Up @@ -224,12 +245,12 @@ protected void createCore(CommandLine cli, SolrClient solrClient) throws Excepti

try {
CoreAdminResponse res = CoreAdminRequest.createCore(coreName, coreName, solrClient);
if (cli.hasOption(SolrCLI.OPTION_VERBOSE.getOpt())) {
if (isVerbose()) {
echo(res.jsonStr());
echo("\n");
} else {
echo(String.format(Locale.ROOT, "\nCreated new core '%s'", coreName));
}
echo(String.format(Locale.ROOT, "\nCreated new core '%s'", coreName));

} catch (Exception e) {
/* create-core failed, cleanup the copied configset before propagating the error. */
PathUtils.deleteDirectory(coreInstanceDir);
Expand Down Expand Up @@ -349,25 +370,25 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli
"Failed to create collection '" + collectionName + "' due to: " + sse.getMessage());
}

if (cli.hasOption(SolrCLI.OPTION_VERBOSE.getOpt())) {
if (isVerbose()) {
// pretty-print the response to stdout
CharArr arr = new CharArr();
new JSONWriter(arr, 2).write(response.asMap());
echo(arr.toString());
} else {
String endMessage =
String.format(
Locale.ROOT,
"Created collection '%s' with %d shard(s), %d replica(s)",
collectionName,
numShards,
replicationFactor);
if (confName != null && !confName.trim().isEmpty()) {
endMessage += String.format(Locale.ROOT, " with config-set '%s'", confName);
}
}

echo(endMessage);
String endMessage =
String.format(
Locale.ROOT,
"Created collection '%s' with %d shard(s), %d replica(s)",
collectionName,
numShards,
replicationFactor);
if (confName != null && !confName.trim().isEmpty()) {
endMessage += String.format(Locale.ROOT, " with config-set '%s'", confName);
}

echo(endMessage);
}

private Path getFullConfDir(Path solrInstallDir, Path confDirName) {
Expand Down
42 changes: 33 additions & 9 deletions solr/core/src/java/org/apache/solr/cli/DeleteTool.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.io.PrintStream;
import java.lang.invoke.MethodHandles;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
Expand All @@ -40,6 +41,7 @@
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.EnvUtils;
import org.apache.solr.common.util.NamedList;
import org.noggit.CharArr;
import org.noggit.JSONWriter;
Expand Down Expand Up @@ -122,16 +124,36 @@ public List<Option> getOptions() {
.desc(
"Skip safety checks when deleting the configuration directory used by a collection.")
.build(),
Option.builder("p")
.longOpt("port")
.deprecated(
DeprecatedAttributes.builder()
.setForRemoval(true)
.setSince("9.7")
.setDescription("Use --solr-url instead")
.get())
.argName("PORT")
.hasArg()
.required(false)
.desc("Port of a local Solr instance where you want to create the new core.")
.build(),
SolrCLI.OPTION_SOLRURL,
SolrCLI.OPTION_SOLRURL_DEPRECATED,
SolrCLI.OPTION_ZKHOST,
SolrCLI.OPTION_ZKHOST_DEPRECATED);
SolrCLI.OPTION_ZKHOST_DEPRECATED,
SolrCLI.OPTION_VERBOSE_DEPRECATED);
Comment on lines +143 to +144
Copy link
Contributor

@malliaridis malliaridis Sep 28, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same here:

Suggested change
SolrCLI.OPTION_ZKHOST_DEPRECATED,
SolrCLI.OPTION_VERBOSE_DEPRECATED);
SolrCLI.OPTION_ZKHOST_DEPRECATED,
SolrCLI.OPTION_VERBOSE,
SolrCLI.OPTION_VERBOSE_DEPRECATED);

}

@Override
public void runImpl(CommandLine cli) throws Exception {
SolrCLI.raiseLogLevelUnlessVerbose(cli);

// If a port is provided, inject it into the environment variables so
// that {@link SolrCLI#getDefaultSolrUrl()} can use it instead of the default 8983.
if (cli.hasOption("p")) {
EnvUtils.setProperty("jetty.port", cli.getOptionValue("p"));
}

try (var solrClient = SolrCLI.getSolrClient(cli)) {
Map<String, Object> systemInfo =
solrClient
Expand Down Expand Up @@ -248,21 +270,22 @@ protected void deleteCollection(CloudSolrClient cloudSolrClient, CommandLine cli
}
}

if (response != null) {
// pretty-print the response to stdout
CharArr arr = new CharArr();
new JSONWriter(arr, 2).write(response.asMap());
echo(arr.toString());
echo("\n");
if (isVerbose()) {
if (response != null) {
// pretty-print the response to stdout
CharArr arr = new CharArr();
new JSONWriter(arr, 2).write(response.asMap());
echo(arr.toString());
}
}

echo("Deleted collection '" + collectionName + "' using CollectionAdminRequest");
echo(String.format(Locale.ROOT, "\nDeleted collection '%s'", collectionName));
}

protected void deleteCore(CommandLine cli, SolrClient solrClient) throws Exception {
String coreName = cli.getOptionValue(NAME);

echo("\nDeleting core '" + coreName + "' using CoreAdminRequest\n");
echoIfVerbose("\nDeleting core '" + coreName + "'", cli);

NamedList<Object> response;
try {
Expand All @@ -281,5 +304,6 @@ protected void deleteCore(CommandLine cli, SolrClient solrClient) throws Excepti
echoIfVerbose((String) response.get("response"), cli);
echoIfVerbose("\n", cli);
}
echo(String.format(Locale.ROOT, "\nDeleted core '%s'", coreName));
}
}
14 changes: 12 additions & 2 deletions solr/core/src/java/org/apache/solr/cli/HealthcheckTool.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,14 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.cloud.ClusterState;
Expand Down Expand Up @@ -92,8 +94,8 @@ public void runImpl(CommandLine cli) throws Exception {
CLIO.err("Healthcheck tool only works in Solr Cloud mode.");
System.exit(1);
}
echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli);
try (CloudHttp2SolrClient cloudSolrClient = SolrCLI.getCloudHttp2SolrClient(zkHost)) {
echoIfVerbose("\nConnecting to ZooKeeper at " + zkHost + " ...", cli);
cloudSolrClient.connect();
runCloudTool(cloudSolrClient, cli);
}
Expand Down Expand Up @@ -170,7 +172,15 @@ protected void runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) th
q = new SolrQuery("*:*");
q.setRows(0);
q.set(DISTRIB, "false");
try (var solrClientForCollection = SolrCLI.getSolrClient(coreUrl)) {

// SolrCLI.getSolrClient converts the coreUrl back into a root Solr url, but we need to
// talk to specific core so manually creating the client.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm, would you say this is a bug, or is it well documented in the javadoc?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Humm..... I read it and it says:

  /**
   * Get a SolrClient for the given URL. The URL can be with or without /solr suffix, the method
   * will normalize it and add '/solr'. To support certain unit tests, the hostContext can be
   * overridden via the system property 'hostContext'.
   *
   * <p>Note that from Solr 10.0 Solr will use / as base URL to support /api and other contexts, so
   * handling of hostContext differs between Solr 9.x and 10.x.
   *
   * @param solrUrl the base URL of Solr
   * @return a SolrClient initialized with correct hostContext, default '/solr'
   */
  public static SolrClient getSolrClient(String solrUrl) {

I think this is the only place we need a core specific url... Do you think it should support that use case as well?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I looked a bit more at the code, and decided to go with the javadoc approach!

// try (var solrClientForCollection = SolrCLI.getSolrClient(coreUrl)) {
try (var solrClientForCollection =
new Http2SolrClient.Builder(coreUrl)
.withMaxConnectionsPerHost(32)
.withKeyStoreReloadInterval(-1, TimeUnit.SECONDS)
.build()) {
qr = solrClientForCollection.query(q);
numDocs = qr.getResults().getNumFound();
try (var solrClient = SolrCLI.getSolrClient(replicaCoreProps.getBaseUrl())) {
Expand Down
18 changes: 17 additions & 1 deletion solr/core/src/java/org/apache/solr/cli/SolrCLI.java
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,17 @@ public class SolrCLI implements CLIO {
.required(false)
.desc("Enable verbose command output.")
.build();
public static final Option OPTION_VERBOSE_DEPRECATED =
Option.builder("V")
.deprecated(
DeprecatedAttributes.builder()
.setForRemoval(true)
.setSince("9.7")
.setDescription("Use -v instead")
.get())
.required(false)
.desc("Enable verbose command output.")
.build();
public static final Option OPTION_HELP =
Option.builder("h").longOpt("help").required(false).desc("Print this message.").build();

Expand Down Expand Up @@ -294,7 +305,8 @@ protected static void checkSslStoreSysProp(String solrInstallDir, String key) {
}

public static void raiseLogLevelUnlessVerbose(CommandLine cli) {
if (!cli.hasOption(OPTION_VERBOSE.getOpt())) {
if (!(cli.hasOption(OPTION_VERBOSE.getOpt())
|| cli.hasOption(OPTION_VERBOSE_DEPRECATED.getOpt()))) {
StartupLoggingUtils.changeLogLevel("WARN");
}
}
Expand Down Expand Up @@ -519,6 +531,10 @@ public static boolean exceptionIsAuthRelated(Exception exc) {
* <p>Note that from Solr 10.0 Solr will use / as base URL to support /api and other contexts, so
* handling of hostContext differs between Solr 9.x and 10.x.
*
* <p>Note that this should not be used with individual core specific urls such as
* "http://127.0.0.1:51428/solr/bob_shard1_replica_n1/" as it removes the core name by converting
* the url to "http://127.0.0.1:51428/" before creating the client.
*
* @param solrUrl the base URL of Solr
* @return a SolrClient initialized with correct hostContext, default '/solr'
*/
Expand Down
11 changes: 9 additions & 2 deletions solr/core/src/java/org/apache/solr/cli/ToolBase.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,18 +34,25 @@ protected ToolBase(PrintStream stdout) {
}

protected void echoIfVerbose(final String msg, CommandLine cli) {
if (cli.hasOption(SolrCLI.OPTION_VERBOSE.getOpt())) {
if (verbose) {
echo(msg);
}
}

/** Is this tool being run in a verbose mode? */
protected boolean isVerbose() {
return verbose;
}

protected void echo(final String msg) {
stdout.println(msg);
}

@Override
public int runTool(CommandLine cli) throws Exception {
verbose = cli.hasOption(SolrCLI.OPTION_VERBOSE.getOpt());
verbose =
cli.hasOption(SolrCLI.OPTION_VERBOSE.getOpt())
|| cli.hasOption(SolrCLI.OPTION_VERBOSE_DEPRECATED);

int toolExitStatus = 0;
try {
Expand Down
23 changes: 23 additions & 0 deletions solr/packaging/test/test_create.bats
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,31 @@ teardown() {
assert_output --partial "Created new core 'COLL_NAME'"
}

@test "create_core works" {
run solr start
run solr create_core -c COLL_NAME
assert_output --partial "Created new core 'COLL_NAME'"
}

@test "create for cloud mode" {
run solr start -c
run solr create -c COLL_NAME
assert_output --partial "Created collection 'COLL_NAME'"
}

@test "ensure -p port parameter is supported" {
solr start -p ${SOLR2_PORT}
solr assert --not-started http://localhost:${SOLR_PORT} --timeout 5000
solr assert --started http://localhost:${SOLR2_PORT} --timeout 5000

run solr create -c COLL_NAME -p ${SOLR2_PORT}
assert_output --partial "Created new core 'COLL_NAME'"
}

@test "ensure -V verbose parameter is supported" {
solr start
solr assert --started http://localhost:${SOLR_PORT} --timeout 5000

run solr create -c COLL_NAME -V
assert_output --partial "Created new core 'COLL_NAME'"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should not this assert a string that is only printed in verbose mode?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good question... I think for core, you got this message, and for collection, you got a different dump (that was more like the api response). I changed it so that both core and collection output a short message like this, and then if you add the -v then you ALSO see the api responses... Does this work for you.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sure, I just expected the assert to fail without the -V, which it won't now since the text asserted is always printed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I finally understand what you mean, and am looking at it and then will merge.

}
8 changes: 8 additions & 0 deletions solr/packaging/test/test_delete_collection.bats
Original file line number Diff line number Diff line change
Expand Up @@ -77,3 +77,11 @@ teardown() {
solr delete -c "COLL_NAME" --delete-config false
assert config_exists "COLL_NAME"
}

@test "ensure -p port parameter is supported" {
run solr create -c COLL_NAME -p ${SOLR_PORT}
assert_output --partial "Created collection 'COLL_NAME'"

run solr delete -c COLL_NAME --delete-config false -p ${SOLR_PORT} -V
assert_output --partial "Deleted collection 'COLL_NAME'"
}
5 changes: 3 additions & 2 deletions solr/packaging/test/test_healthcheck.bats
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,9 @@ teardown() {
}

@test "healthcheck on cloud solr" {
solr start -c -e films
run solr healthcheck -c films
solr start -c -e films -p ${SOLR_PORT}
run solr healthcheck -c films -zkhost localhost:${ZK_PORT} -V
refute_output --partial 'error'
janhoy marked this conversation as resolved.
Show resolved Hide resolved
assert_output --partial '"collection":"films"'

}
2 changes: 1 addition & 1 deletion solr/packaging/test/test_zk.bats
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ teardown() {
touch myfile2.txt
run solr zk cp myfile2.txt zk:myfile2.txt -z localhost:${ZK_PORT}
sleep 1
run solr zk ls / -z localhost:${ZK_PORT}
run solr zk ls / -z localhost:${ZK_PORT} -V
assert_output --partial "myfile2.txt"

touch myfile3.txt
Expand Down
Loading