From a578608204b0b60bb4fe55de86cdd95c3820e5c3 Mon Sep 17 00:00:00 2001 From: Tanguy Leroux Date: Mon, 20 Jan 2025 14:37:41 +0100 Subject: [PATCH] Delete persisted cold cache index if created in the future (#120465) The Lucene index used to store cold cache files information (to persist the cold cache across node restarts) is upgraded at node startup, before the cluster is formed. This upgrade can become an issue if the node cannot join the cluster: an attempt to downgrade the node back to the previous version will likely fail due to this Lucene index being already upgraded. Ideally we would prefer to upgrade the Lucene index after the node joined the cluster, but it requires more work as this Lucene index can be queried concurrently during shard allocation (see TransportSearchableSnapshotCacheStoresAction). Instead of doing a more involved fix, this change deletes the persistent cache index from disk if it detects that the Lucene index has been upgraded. It assumes that we are already in a best effort downgrading procedure, so losing persistent cache is acceptable in order to allow downgrading the node. --- .../cache/full/PersistentCache.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java index a7fb5571995b3..92b15cd9851db 100644 --- a/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java +++ b/x-pack/plugin/searchable-snapshots/src/main/java/org/elasticsearch/xpack/searchablesnapshots/cache/full/PersistentCache.java @@ -437,6 +437,16 @@ static Map loadDocuments(Path directoryPath) throws IOExceptio } catch (IndexNotFoundException e) { logger.debug("persistent cache index does not exist yet", e); } + } catch (Exception e) { + if (e instanceof IllegalArgumentException iae) { + final var message = iae.getMessage(); + if (message != null && message.startsWith("indexCreatedVersionMajor is in the future:")) { + logger.warn("Deleting persistent cache index created in the future [message: {}]", message); + IOUtils.rm(directoryPath); + return Map.of(); + } + } + throw e; } return documents; }