18 search improvements

pom.xml

@@ -3,7 +3,7 @@
 
 	<groupId>org.icatproject</groupId>
 	<artifactId>icat.lucene</artifactId>
-	<version>1.1.2</version>
+	<version>2.0.0-SNAPSHOT</version>
 	<packaging>war</packaging>
 	<name>ICAT Lucene</name>
 
@@ -14,7 +14,7 @@
 		<repoUrl>https://repo.icatproject.org/repo</repoUrl>
 		<project.scm.id>github</project.scm.id>
 		<gitUrl>https://github.com/icatproject/icat.lucene</gitUrl>
-		<luceneVersion>5.3.0</luceneVersion>
+		<luceneVersion>8.6.0</luceneVersion>
 	</properties>
 
 	<repositories>
@@ -86,6 +86,12 @@
 			<version>${luceneVersion}</version>
 		</dependency>
 
+		<dependency>
+			<groupId>org.apache.lucene</groupId>
+			<artifactId>lucene-facet</artifactId>
+			<version>${luceneVersion}</version>
+		</dependency>
+
 		<dependency>
 			<groupId>javax</groupId>
 			<artifactId>javaee-api</artifactId>
@@ -95,7 +101,7 @@
 		<dependency>
 			<groupId>org.icatproject</groupId>
 			<artifactId>icat.utils</artifactId>
-			<version>4.16.1</version>
+			<version>4.17.0-SNAPSHOT</version>
 		</dependency>
 
 		<dependency>
@@ -327,6 +333,3 @@
 
 	<description>Exposes lucene calls to an icat server</description>
 </project>
-
-
-

src/main/config/run.properties.example

@@ -1,6 +1,8 @@
 # Real comments in this file are marked with '#' whereas commented out lines
 # are marked with '!'
 
-directory     = ${HOME}/data/lucene
-commitSeconds = 5
-ip            = 127.0.0.1/32
+directory      = ${HOME}/data/search
+commitSeconds  = 5
+maxShardSize   = 2147483648
+ip             = 127.0.0.1/32
+aggregateFiles = false

src/main/java/org/icatproject/lucene/DocumentMapping.java

@@ -0,0 +1,133 @@
+package org.icatproject.lucene;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser;
+
+public class DocumentMapping {
+
+	/**
+	 * Represents the parent child relationship between two ICAT entities.
+	 */
+	public static class ParentRelationship {
+		public String parentName;
+		public String joiningField;
+		public Set<String> fields;
+
+		/**
+		 * @param parentName   Name of the parent entity.
+		 * @param joiningField Field that joins the child to its parent.
+		 * @param fields       Fields that should be updated by this relationship.
+		 */
+		public ParentRelationship(String parentName, String joiningField, String... fields) {
+			this.parentName = parentName;
+			this.joiningField = joiningField;
+			this.fields = new HashSet<>(Arrays.asList(fields));
+		}
+	}
+
+	public static final Set<String> doubleFields = new HashSet<>();
+	public static final Set<String> facetFields = new HashSet<>();
+	public static final Set<String> longFields = new HashSet<>();
+	public static final Set<String> sortFields = new HashSet<>();
+	public static final Set<String> textFields = new HashSet<>();
+	public static final Set<String> indexedEntities = new HashSet<>();
+	public static final Map<String, ParentRelationship[]> relationships = new HashMap<>();
+
+	public static final IcatAnalyzer analyzer = new IcatAnalyzer();
+	public static final StandardQueryParser genericParser = new StandardQueryParser();
+	public static final StandardQueryParser datafileParser = new StandardQueryParser();
+	public static final StandardQueryParser datasetParser = new StandardQueryParser();
+	public static final StandardQueryParser investigationParser = new StandardQueryParser();
+	public static final StandardQueryParser sampleParser = new StandardQueryParser();
+
+	static {
+		doubleFields.addAll(Arrays.asList("numericValue", "numericValueSI", "rangeTop", "rangeTopSI", "rangeBottom",
+				"rangeBottomSI"));
+		facetFields.addAll(Arrays.asList("type.name", "datafileFormat.name", "stringValue", "technique.name"));
+		longFields.addAll(
+				Arrays.asList("date", "startDate", "endDate", "dateTimeValue", "investigation.startDate", "fileSize",
+						"fileCount"));
+		sortFields.addAll(
+				Arrays.asList("datafile.id", "dataset.id", "investigation.id", "instrument.id", "id", "sample.id",
+						"sample.investigation.id", "date", "name", "stringValue", "dateTimeValue", "numericValue",
+						"numericValueSI", "fileSize", "fileCount"));
+		textFields.addAll(Arrays.asList("name", "visitId", "description", "location", "dataset.name",
+				"investigation.name", "instrument.name", "instrument.fullName", "datafileFormat.name", "sample.name",
+				"sample.type.name", "technique.name", "technique.description", "technique.pid", "title", "summary",
+				"facility.name", "user.fullName", "type.name", "doi"));
+
+		indexedEntities.addAll(Arrays.asList("Datafile", "Dataset", "Investigation", "DatafileParameter",
+				"DatasetParameter", "DatasetTechnique", "InstrumentScientist", "InvestigationInstrument",
+				"InvestigationParameter", "InvestigationUser", "Sample", "SampleParameter"));
+
+		relationships.put("Instrument",
+				new ParentRelationship[] { new ParentRelationship("InvestigationInstrument", "instrument.id",
+						"instrument.name", "instrument.fullName") });
+		relationships.put("User",
+				new ParentRelationship[] {
+						new ParentRelationship("InvestigationUser", "user.id", "user.name", "user.fullName"),
+						new ParentRelationship("InstrumentScientist", "user.id", "user.name", "user.fullName") });
+		relationships.put("Sample", new ParentRelationship[] {
+				new ParentRelationship("Dataset", "sample.id", "sample.name", "sample.investigation.id"),
+				new ParentRelationship("Datafile", "sample.id", "sample.name", "sample.investigation.id") });
+		relationships.put("SampleType",
+				new ParentRelationship[] { new ParentRelationship("Sample", "type.id", "type.name"),
+						new ParentRelationship("Dataset", "sample.type.id", "sample.type.name"),
+						new ParentRelationship("Datafile", "sample.type.id", "sample.type.name") });
+		relationships.put("InvestigationType",
+				new ParentRelationship[] { new ParentRelationship("Investigation", "type.id", "type.name") });
+		relationships.put("DatasetType",
+				new ParentRelationship[] { new ParentRelationship("Dataset", "type.id", "type.name") });
+		relationships.put("DatafileFormat",
+				new ParentRelationship[] {
+						new ParentRelationship("Datafile", "datafileFormat.id", "datafileFormat.name") });
+		relationships.put("Facility",
+				new ParentRelationship[] { new ParentRelationship("Investigation", "facility.id", "facility.name") });
+		relationships.put("ParameterType",
+				new ParentRelationship[] { new ParentRelationship("DatafileParameter", "type.id", "type.name"),
+						new ParentRelationship("DatasetParameter", "type.id", "type.name"),
+						new ParentRelationship("InvestigationParameter", "type.id", "type.name"),
+						new ParentRelationship("SampleParameter", "type.id", "type.name") });
+		relationships.put("Technique",
+				new ParentRelationship[] { new ParentRelationship("DatasetTechnique", "technique.id", "technique.name",
+						"technique.description", "technique.pid") });
+		relationships.put("Investigation",
+				new ParentRelationship[] {
+						new ParentRelationship("Dataset", "investigation.id", "investigation.name",
+								"investigation.title", "investigation.startDate", "visitId"),
+						new ParentRelationship("datafile", "investigation.id", "investigation.name", "visitId") });
+		relationships.put("Dataset",
+				new ParentRelationship[] { new ParentRelationship("Datafile", "dataset.id", "dataset.name") });
+
+		genericParser.setAllowLeadingWildcard(true);
+		genericParser.setAnalyzer(analyzer);
+
+		CharSequence[] datafileFields = { "name", "description", "location", "datafileFormat.name", "visitId",
+				"sample.name", "sample.type.name", "doi" };
+		datafileParser.setAllowLeadingWildcard(true);
+		datafileParser.setAnalyzer(analyzer);
+		datafileParser.setMultiFields(datafileFields);
+
+		CharSequence[] datasetFields = { "name", "description", "sample.name", "sample.type.name", "type.name",
+				"visitId", "doi" };
+		datasetParser.setAllowLeadingWildcard(true);
+		datasetParser.setAnalyzer(analyzer);
+		datasetParser.setMultiFields(datasetFields);
+
+		CharSequence[] investigationFields = { "name", "visitId", "title", "summary", "facility.name",
+				"type.name", "doi" };
+		investigationParser.setAllowLeadingWildcard(true);
+		investigationParser.setAnalyzer(analyzer);
+		investigationParser.setMultiFields(investigationFields);
+
+		CharSequence[] sampleFields = { "sample.name", "sample.type.name" };
+		sampleParser.setAllowLeadingWildcard(true);
+		sampleParser.setAnalyzer(analyzer);
+		sampleParser.setMultiFields(sampleFields);
+	}
+}

src/main/java/org/icatproject/lucene/FacetedDimension.java

@@ -0,0 +1,105 @@
+package org.icatproject.lucene;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.json.Json;
+import javax.json.JsonObjectBuilder;
+
+import org.apache.lucene.facet.FacetResult;
+import org.apache.lucene.facet.LabelAndValue;
+import org.apache.lucene.facet.range.DoubleRange;
+import org.apache.lucene.facet.range.LongRange;
+import org.apache.lucene.facet.range.Range;
+
+/**
+ * For a single dimension (field), stores labels (the unique values or ranges of
+ * values for that field in the index) and their respective counts (the number
+ * of times that label appears in different documents).
+ * 
+ * For example, a dimension might be "colour", the label "red", and the count 5.
+ */
+public class FacetedDimension {
+
+	private String dimension;
+	private List<Range> ranges;
+	private List<String> labels;
+	private List<Long> counts;
+
+	/**
+	 * Creates an "empty" FacetedDimension. The dimension (field) is set but ranges,
+	 * labels and counts are not.
+	 * 
+	 * @param dimension The dimension, or field, to be faceted
+	 */
+	public FacetedDimension(String dimension) {
+		this.dimension = dimension;
+		this.ranges = new ArrayList<>();
+		this.labels = new ArrayList<>();
+		this.counts = new ArrayList<>();
+	}
+
+	/**
+	 * Extracts the count for each label in the FacetResult. If the label has
+	 * already been encountered, the count is incremented rather than being
+	 * overridden. Essentially, this allows faceting to be performed across multiple
+	 * shards.
+	 * 
+	 * @param facetResult A Lucene FacetResult object corresponding the relevant
+	 *                    dimension
+	 */
+	public void addResult(FacetResult facetResult) {
+		for (LabelAndValue labelAndValue : facetResult.labelValues) {
+			String label = labelAndValue.label;
+			int labelIndex = labels.indexOf(label);
+			if (labelIndex == -1) {
+				labels.add(label);
+				counts.add(labelAndValue.value.longValue());
+			} else {
+				counts.set(labelIndex, counts.get(labelIndex) + labelAndValue.value.longValue());
+			}
+		}
+	}
+
+	/**
+	 * Formats the labels and counts into Json.
+	 * 
+	 * @param aggregationsBuilder The JsonObjectBuilder to add the facets for this
+	 *                            dimension to.
+	 */
+	public void buildResponse(JsonObjectBuilder aggregationsBuilder) {
+		JsonObjectBuilder bucketsBuilder = Json.createObjectBuilder();
+		for (int i = 0; i < labels.size(); i++) {
+			JsonObjectBuilder bucketBuilder = Json.createObjectBuilder();
+			bucketBuilder.add("doc_count", counts.get(i));
+			if (ranges.size() > i) {
+				Range range = ranges.get(i);
+				if (range.getClass().getSimpleName().equals("LongRange")) {
+					bucketBuilder.add("from", ((LongRange) range).min);
+					bucketBuilder.add("to", ((LongRange) range).max);
+				} else if (range.getClass().getSimpleName().equals("DoubleRange")) {
+					bucketBuilder.add("from", ((DoubleRange) range).min);
+					bucketBuilder.add("to", ((DoubleRange) range).max);
+				}
+			}
+			bucketsBuilder.add(labels.get(i), bucketBuilder);
+		}
+		aggregationsBuilder.add(dimension, Json.createObjectBuilder().add("buckets", bucketsBuilder));
+	}
+
+	/**
+	 * @return The list of Lucene Range Objects for use with numerical facets.
+	 *         For String faceting, this will be empty.
+	 */
+	public List<Range> getRanges() {
+		return ranges;
+	}
+
+	/**
+	 * @return The dimension that these labels and counts correspond to.
+	 */
+	public String getDimension() {
+		return dimension;
+	}
+
+}