Update Lucene to 8.11.2 and remove search caching #18

pom.xml

@@ -14,7 +14,7 @@
 		<repoUrl>https://repo.icatproject.org/repo</repoUrl>
 		<project.scm.id>github</project.scm.id>
 		<gitUrl>https://github.com/icatproject/icat.lucene</gitUrl>
-		<luceneVersion>8.6.0</luceneVersion>
+		<luceneVersion>8.11.2</luceneVersion>
 	</properties>
 
 	<repositories>
@@ -92,6 +92,12 @@
 			<version>${luceneVersion}</version>
 		</dependency>
 
+		<dependency>
+			<groupId>org.apache.lucene</groupId>
+			<artifactId>lucene-backward-codecs</artifactId>
+			<version>${luceneVersion}</version>
+		</dependency>
+
 		<dependency>
 			<groupId>javax</groupId>
 			<artifactId>javaee-api</artifactId>

src/main/java/org/icatproject/lucene/Lucene.java

@@ -153,8 +153,10 @@ public ShardBucket(java.nio.file.Path shardPath) throws IOException {
 		 */
 		public int commit() throws IOException {
 			int cached = indexWriter.numRamDocs();
-			indexWriter.commit();
-			searcherManager.maybeRefreshBlocking();
+			if (cached > 0) {
+				indexWriter.commit();
+				searcherManager.maybeRefreshBlocking();
+			}
 			return cached;
 		}
 	}
@@ -344,13 +346,9 @@ public void releaseSearchers(List<IndexSearcher> subSearchers) throws IOExceptio
 	private long luceneMaxShardSize;
 	private long maxSearchTimeSeconds;
 	private boolean aggregateFiles;
-
-	private AtomicLong bucketNum = new AtomicLong();
 	private Map<String, IndexBucket> indexBuckets = new ConcurrentHashMap<>();
-
 	private Timer timer;
 
-	private Map<Long, SearchBucket> searches = new ConcurrentHashMap<>();
 	public IcatUnits icatUnits;
 
 	/**
@@ -440,11 +438,7 @@ public void clear() throws LuceneException {
 		logger.info("Requesting clear");
 
 		exit();
-		timer = new Timer("LuceneCommitTimer");
-
-		bucketNum.set(0);
 		indexBuckets.clear();
-		searches.clear();
 
 		try {
 			Files.walk(luceneDirectory, FileVisitOption.FOLLOW_LINKS).sorted(Comparator.reverseOrder())
@@ -453,7 +447,7 @@ public void clear() throws LuceneException {
 			throw new LuceneException(HttpURLConnection.HTTP_INTERNAL_ERROR, e.getMessage());
 		}
 
-		timer.schedule(new CommitTimerTask(), luceneCommitMillis, luceneCommitMillis);
+		initTimer();
 		logger.info("clear complete - ready to go again");
 
 	}
@@ -464,11 +458,12 @@ public void clear() throws LuceneException {
 	@POST
 	@Path("commit")
 	public void commit() throws LuceneException {
-		logger.debug("Requesting commit");
+		logger.debug("Requesting commit for {} IndexBuckets", indexBuckets.size());
 		try {
 			for (Entry<String, IndexBucket> entry : indexBuckets.entrySet()) {
 				IndexBucket bucket = entry.getValue();
 				if (!bucket.locked.get()) {
+					logger.info("{} is unlocked", entry.getKey());
 					bucket.commit("Synch", entry.getKey());
 				}
 			}
@@ -819,29 +814,27 @@ private void exit() {
 	public String facet(@PathParam("entityName") String entityName, @Context HttpServletRequest request,
 			@QueryParam("search_after") String searchAfter, @QueryParam("maxResults") int maxResults,
 			@QueryParam("maxLabels") int maxLabels, @QueryParam("sort") String sort) throws LuceneException {
-		Long uid = null;
+		SearchBucket search = null;
 		try {
-			uid = bucketNum.getAndIncrement();
-			SearchBucket search = new SearchBucket(this, SearchType.GENERIC, request, sort, null);
-			searches.put(uid, search);
+			search = new SearchBucket(this, SearchType.GENERIC, request, sort, null);
 			return luceneFacetResult(entityName, search, searchAfter, maxResults, maxLabels);
 		} catch (IOException | QueryNodeException e) {
 			logger.error("Error", e);
-			freeSearcher(uid);
 			throw new LuceneException(HttpURLConnection.HTTP_INTERNAL_ERROR, e.getMessage());
+		} finally {
+			freeSearcher(search);
 		}
 	}
 
 	/**
 	 * Releases all IndexSearchers associated with uid.
 	 * 
-	 * @param uid Unique Identifier for a set of IndexSearcher to be released.
+	 * @param search SearchBucket to be freed.
 	 * @throws LuceneException
 	 */
-	public void freeSearcher(Long uid) throws LuceneException {
-		if (uid != null && searches.containsKey(uid)) { // May not be set for internal calls
-			Map<String, List<IndexSearcher>> search = searches.get(uid).searcherMap;
-			for (Entry<String, List<IndexSearcher>> entry : search.entrySet()) {
+	public void freeSearcher(SearchBucket search) throws LuceneException {
+		if (search != null) {
+			for (Entry<String, List<IndexSearcher>> entry : search.searcherMap.entrySet()) {
 				String name = entry.getKey();
 				List<IndexSearcher> subReaders = entry.getValue();
 				try {
@@ -851,7 +844,6 @@ public void freeSearcher(Long uid) throws LuceneException {
 					throw new LuceneException(HttpURLConnection.HTTP_INTERNAL_ERROR, e.getMessage());
 				}
 			}
-			searches.remove(uid);
 		}
 	}
 
@@ -924,8 +916,7 @@ private void init() {
 					: 5;
 			aggregateFiles = props.getBoolean("aggregateFiles", false);
 
-			timer = new Timer("LuceneCommitTimer");
-			timer.schedule(new CommitTimerTask(), luceneCommitMillis, luceneCommitMillis);
+			initTimer();
 
 			icatUnits = new IcatUnits(props.getString("units", ""));
 
@@ -940,6 +931,14 @@ private void init() {
 				aggregateFiles);
 	}
 
+	/**
+	 * Starts a timer and schedules regular commits of the IndexWriter.
+	 */
+	private void initTimer() {
+		timer = new Timer("LuceneCommitTimer");
+		timer.schedule(new CommitTimerTask(), luceneCommitMillis, luceneCommitMillis);
+	}
+
 	class CommitTimerTask extends TimerTask {
 		@Override
 		public void run() {
@@ -1235,18 +1234,16 @@ private void addFacetResults(int maxLabels, Map<String, FacetedDimension> facete
 	 * @throws LuceneException
 	 */
 	private String searchEntity(HttpServletRequest request, String searchAfter, int maxResults, String sort,
-			SearchType searchType)
-			throws LuceneException {
-		Long uid = null;
+			SearchType searchType) throws LuceneException {
+		SearchBucket search = null;
 		try {
-			uid = bucketNum.getAndIncrement();
-			SearchBucket search = new SearchBucket(this, searchType, request, sort, searchAfter);
-			searches.put(uid, search);
+			search = new SearchBucket(this, searchType, request, sort, searchAfter);
 			return luceneSearchResult(searchType.toString(), search, searchAfter, maxResults);
 		} catch (IOException | QueryNodeException e) {
 			logger.error("Error", e);
-			freeSearcher(uid);
 			throw new LuceneException(HttpURLConnection.HTTP_INTERNAL_ERROR, e.getMessage());
+		} finally {
+			freeSearcher(search);
 		}
 	}