Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
@essiembre
essiembre committed Feb 28, 2016
2 parents 19cb2f1 + 8d20a15 commit ea8ae45
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 45 deletions.
71 changes: 35 additions & 36 deletions norconex-committer-solr/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>com.norconex.collectors</groupId>
<artifactId>norconex-committer-solr</artifactId>
<version>2.0.2</version>
<version>2.1.0</version>
<name>Norconex Committer Solr</name>


Expand Down Expand Up @@ -50,25 +50,18 @@
<enabled>true</enabled>
</snapshots>
</repository>
<!--
<repository>
<id>maven-restlet</id>
<name>Public online Restlet repository</name>
<url>http://maven.restlet.org</url>
</repository>
-->
</repositories>

<dependencies>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>${solr.version}</version>
<groupId>org.apache.solr</groupId>
<artifactId>solr-solrj</artifactId>
<version>${solr.version}</version>
</dependency>
<dependency>
<groupId>com.norconex.collectors</groupId>
<artifactId>norconex-committer-core</artifactId>
<version>2.0.1</version>
<groupId>com.norconex.collectors</groupId>
<artifactId>norconex-committer-core</artifactId>
<version>2.0.3</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
Expand All @@ -81,37 +74,43 @@
<version>1.7.12</version>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.10</version>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.10</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-test-framework</artifactId>
<version>${solr.version}</version>
<scope>test</scope>
<exclusions>
<exclusion>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
</exclusion>
</exclusions>
</dependency>

<!-- The following dependencies are only present here to ensure their
proper version. They are otherwise provided by
transitive dependencies. They can be removed from here once
transitive versions match and don't cause conflicts. -->
<dependency>
</dependencies>

<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.4.1</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.1</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.4.1</version>
</dependency>
</dependencies>
<version>4.5.1</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
<version>4.4.3</version>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpmime</artifactId>
<version>4.5.1</version>
</dependency>
</dependencies>
</dependencyManagement>

<description>Solr implementation of Norconex Committer. Should also work with any Solr-based products, such as LucidWorks.</description>
<build>
Expand Down
12 changes: 12 additions & 0 deletions norconex-committer-solr/src/changes/changes.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,18 @@
</properties>
<body>

<release version="2.1.0" date="2016-02-28" description="Minor release">
<action dev="martinfou" type="add">
New SolrCommitter#setCommitDisabled(boolean) method (and
"commitDisabled" flag for XML configuration) to decide
if the Solr commits should be perfommed by this committer, or leave
it to the server.
</action>
<action dev="essiembre" type="update">
Maven dependency updates: Norconex Committer Core 2.0.3.
</action>
</release>

<release version="2.0.2" date="2015-04-08" description="Bug fix release">
<action dev="essiembre" type="update">
Library updates: Apache Http Components 4.4.1.
Expand Down
41 changes: 36 additions & 5 deletions norconex-committer-solr/src/main/java/com/norconex/committer/solr/SolrCommitter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
* &lt;param name="(parameter name)"&gt;(parameter value)&lt;/param&gt;
* &lt;-- multiple param tags allowed --&gt;
* &lt;/solrUpdateURLParams&gt;
* &lt;commitDisabled&gt;[false|true]&lt;/commitDisabled&gt;
* &lt;sourceReferenceField keep="[false|true]"&gt;
* (Optional name of field that contains the document reference, when
* the default document reference is not used. The reference value
Expand Down Expand Up @@ -102,6 +103,7 @@ public class SolrCommitter extends AbstractMappedCommitter {
public static final String DEFAULT_SOLR_CONTENT_FIELD = "content";

private String solrURL;
private boolean commitDisabled;

private final Map<String, String> updateUrlParams =
new HashMap<String, String>();
Expand Down Expand Up @@ -166,7 +168,24 @@ public String getUpdateUrlParam(String name) {
public Set<String> getUpdateUrlParamNames() {
return updateUrlParams.keySet();
}

/**
* Set the variable to let the client know if it should commit or
* let the server auto-commit.
* @param commitDisabled <code>true</code> if commit is disabled
* @since 2.1.0
*/
public void setCommitDisabled(boolean commitDisabled) {
this.commitDisabled = commitDisabled;
}
/**
* Gets the commitDisabled variable to find out if the client should
* commit or let the server auto-commit.
* @return <code>true</code> if commit is disabled.
* @since 2.1.0
*/
public boolean isCommitDisabled() {
return commitDisabled;
}
@Override
protected void commitBatch(List<ICommitOperation> batch) {

Expand All @@ -191,15 +210,19 @@ protected void commitBatch(List<ICommitOperation> batch) {
previousWasAddition = true;
} else if (op instanceof IDeleteOperation) {
if (previousWasAddition) {
server.commit();
if (!isCommitDisabled()) {
server.commit();
}
}
server.deleteById(((IDeleteOperation) op).getReference());
previousWasAddition = false;
} else {
throw new CommitterException("Unsupported operation:" + op);
}
}
server.commit();
if (!isCommitDisabled()) {
server.commit();
}
} catch (Exception e) {
throw new CommitterException(
"Cannot index document batch to Solr.", e);
Expand Down Expand Up @@ -231,13 +254,19 @@ protected void saveToXML(XMLStreamWriter writer) throws XMLStreamException {
writer.writeCharacters(updateUrlParams.get(name));
writer.writeEndElement();
}

writer.writeStartElement("commitDisabled");
writer.writeCharacters(Boolean.toString(isCommitDisabled()));
writer.writeEndElement();

writer.writeEndElement();
}

@Override
@SuppressWarnings("unchecked")
protected void loadFromXml(XMLConfiguration xml) {
setSolrURL(xml.getString("solrURL", null));
setSolrURL(xml.getString("solrURL", getSolrURL()));
setCommitDisabled(xml.getBoolean("commitDisabled", isCommitDisabled()));

List<HierarchicalConfiguration> uparams =
xml.configurationsAt("solrUpdateURLParams.param");
Expand All @@ -246,14 +275,14 @@ protected void loadFromXml(XMLConfiguration xml) {
}
}


@Override
public int hashCode() {
return new HashCodeBuilder()
.appendSuper(super.hashCode())
.append(solrServerFactory)
.append(solrURL)
.append(updateUrlParams)
.append(commitDisabled)
.toHashCode();
}

Expand All @@ -274,13 +303,15 @@ public boolean equals(Object obj) {
.append(solrServerFactory, other.solrServerFactory)
.append(solrURL, other.solrURL)
.append(updateUrlParams, other.updateUrlParams)
.append(commitDisabled, other.commitDisabled)
.isEquals();
}

@Override
public String toString() {
return "SolrCommitter [solrURL=" + solrURL + ", updateUrlParams="
+ updateUrlParams + ", solrServerFactory=" + solrServerFactory
+ ", commitDisabled=" + commitDisabled
+ ", " + super.toString() + "]";
}

Expand Down
7 changes: 5 additions & 2 deletions ...tter-solr/src/test/java/com/norconex/committer/solr/SolrCommitterSolrIntegrationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,8 @@ public void testCommitDelete() throws Exception {
assertEquals(0, results.getNumFound());
}

private SolrDocumentList queryId(String id) throws SolrServerException {
private SolrDocumentList queryId(String id)
throws SolrServerException, IOException {
ModifiableSolrParams solrParams = new ModifiableSolrParams();
solrParams.set("q", String.format("%s:%s",
SolrCommitter.DEFAULT_SOLR_ID_FIELD, id));
Expand All @@ -295,7 +296,8 @@ private SolrDocumentList queryId(String id) throws SolrServerException {
return results;
}

private SolrDocumentList getAllDocs() throws SolrServerException{
private SolrDocumentList getAllDocs()
throws SolrServerException, IOException{
ModifiableSolrParams solrParams = new ModifiableSolrParams();
solrParams.set("q", "*:*");
QueryResponse response = server.query(solrParams);
Expand All @@ -317,6 +319,7 @@ public void testWriteRead() throws IOException {
outCommitter.setQueueSize(100);
outCommitter.setCommitBatchSize(50);
outCommitter.setSolrURL("http://solrurl.com/test");
outCommitter.setCommitDisabled(false);
outCommitter.setUpdateUrlParam("uparam1", "uvalue1");
outCommitter.setUpdateUrlParam("uparam2", "uvalue2");
// outCommitter.setDeleteUrlParam("dparam1", "dvalue1");
Expand Down
7 changes: 5 additions & 2 deletions norconex-committer-solr/src/test/java/com/norconex/committer/solr/SolrCommitterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,7 +281,8 @@ public void testCommitDelete() throws Exception {
assertEquals(0, results.getNumFound());
}

private SolrDocumentList queryId(String id) throws SolrServerException {
private SolrDocumentList queryId(String id)
throws SolrServerException, IOException {
ModifiableSolrParams solrParams = new ModifiableSolrParams();
solrParams.set("q", String.format("%s:%s",
SolrCommitter.DEFAULT_SOLR_ID_FIELD, id));
Expand All @@ -290,7 +291,8 @@ private SolrDocumentList queryId(String id) throws SolrServerException {
return results;
}

private SolrDocumentList getAllDocs() throws SolrServerException{
private SolrDocumentList getAllDocs()
throws SolrServerException, IOException{
ModifiableSolrParams solrParams = new ModifiableSolrParams();
solrParams.set("q", "*:*");
QueryResponse response = server.query(solrParams);
Expand All @@ -312,6 +314,7 @@ public void testWriteRead() throws IOException {
outCommitter.setQueueSize(100);
outCommitter.setCommitBatchSize(50);
outCommitter.setSolrURL("http://solrurl.com/test");
outCommitter.setCommitDisabled(false);
outCommitter.setUpdateUrlParam("uparam1", "uvalue1");
outCommitter.setUpdateUrlParam("uparam2", "uvalue2");
// outCommitter.setDeleteUrlParam("dparam1", "dvalue1");
Expand Down

0 comments on commit ea8ae45

Please sign in to comment.