Skip to content

Commit

Permalink
Add test for system index migration using reindexing script (#120667)
Browse files Browse the repository at this point in the history
  • Loading branch information
@alexey-ivanov-es
alexey-ivanov-es authored Jan 27, 2025
1 parent bac3b14 commit 529ad04
Show file tree
Hide file tree
Showing 5 changed files with 118 additions and 32 deletions.
3 changes: 3 additions & 0 deletions modules/reindex/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ dependencies {
clusterModules project(':modules:lang-painless')
clusterModules project(':modules:parent-join')
clusterModules project(":modules:rest-root")

internalClusterTestImplementation project(':modules:lang-painless')
internalClusterTestImplementation project(':modules:lang-painless:spi')
}

restResources {
Expand Down
12 changes: 8 additions & 4 deletions ...ternalClusterTest/java/org/elasticsearch/migration/AbstractFeatureMigrationIntegTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ public <T extends Plugin> T getPlugin(Class<T> type) {
return pluginsService.filterPlugins(type).findFirst().get();
}

public void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) throws InterruptedException {
protected void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) {
assertThat(
"the strategy used below to create index names for descriptors without a primary index name only works for simple patterns",
descriptor.getIndexPattern(),
Expand Down Expand Up @@ -180,9 +180,13 @@ public void createSystemIndexForDescriptor(SystemIndexDescriptor descriptor) thr
CreateIndexResponse response = createRequest.get();
Assert.assertTrue(response.isShardsAcknowledged());

indexDocs(indexName);
}

protected void indexDocs(String indexName) {
List<IndexRequestBuilder> docs = new ArrayList<>(INDEX_DOC_COUNT);
for (int i = 0; i < INDEX_DOC_COUNT; i++) {
docs.add(ESIntegTestCase.prepareIndex(indexName).setId(Integer.toString(i)).setSource("some_field", "words words"));
docs.add(ESIntegTestCase.prepareIndex(indexName).setId(Integer.toString(i)).setSource(FIELD_NAME, "words words"));
}
indexRandom(true, docs);
IndicesStatsResponse indexStats = ESIntegTestCase.indicesAdmin().prepareStats(indexName).setDocs(true).get();
Expand All @@ -207,7 +211,7 @@ static String createMapping(boolean descriptorManaged, boolean descriptorInterna
builder.field("dynamic", "strict");
builder.startObject("properties");
{
builder.startObject("some_field");
builder.startObject(FIELD_NAME);
builder.field("type", "keyword");
builder.endObject();
}
Expand All @@ -221,7 +225,7 @@ static String createMapping(boolean descriptorManaged, boolean descriptorInterna
}
}

public void assertIndexHasCorrectProperties(
protected void assertIndexHasCorrectProperties(
Metadata metadata,
String indexName,
int settingsFlagValue,
Expand Down
130 changes: 102 additions & 28 deletions .../reindex/src/internalClusterTest/java/org/elasticsearch/migration/FeatureMigrationIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.template.put.PutComponentTemplateAction;
import org.elasticsearch.action.admin.indices.template.put.TransportPutComposableIndexTemplateAction;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.support.ActiveShardCount;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.ClusterStateUpdateTask;
Expand All @@ -33,8 +34,11 @@
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.indices.SystemIndexDescriptor;
import org.elasticsearch.painless.PainlessPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.plugins.SystemIndexPlugin;
import org.elasticsearch.reindex.ReindexPlugin;
import org.elasticsearch.upgrades.FeatureMigrationResults;
import org.elasticsearch.upgrades.SingleFeatureMigrationResult;
Expand All @@ -51,6 +55,7 @@
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCountAndNoFailures;
import static org.hamcrest.Matchers.aMapWithSize;
import static org.hamcrest.Matchers.allOf;
import static org.hamcrest.Matchers.equalTo;
Expand All @@ -62,6 +67,25 @@
import static org.hamcrest.Matchers.nullValue;

public class FeatureMigrationIT extends AbstractFeatureMigrationIntegTest {
private static final String INTERNAL_MANAGED_WITH_SCRIPT_INDEX_NAME = ".int-mans-old";
private static final String SCRIPTED_INDEX_FEATURE_NAME = "B-test-feature";
private static final SystemIndexDescriptor INTERNAL_MANAGED_WITH_SCRIPT = SystemIndexDescriptor.builder()
.setIndexPattern(".int-mans-*")
.setAliasName(".internal-managed-with-script-alias")
.setPrimaryIndex(INTERNAL_MANAGED_WITH_SCRIPT_INDEX_NAME)
.setType(SystemIndexDescriptor.Type.INTERNAL_MANAGED)
.setSettings(createSettings(NEEDS_UPGRADE_INDEX_VERSION, INTERNAL_MANAGED_FLAG_VALUE))
.setMappings(createMapping(true, true))
.setOrigin(ORIGIN)
.setAllowedElasticProductOrigins(Collections.emptyList())
.setPriorSystemIndexDescriptors(Collections.emptyList())
.setMigrationScript("""
if (ctx._source.some_field != null) {
ctx._source.some_field = 'migrated';
}
""")
.build();

@Override
protected Settings nodeSettings(int nodeOrdinal, Settings otherSettings) {
return Settings.builder().put(super.nodeSettings(nodeOrdinal, otherSettings)).build();
Expand All @@ -77,7 +101,9 @@ protected boolean forbidPrivateIndexSettings() {
protected Collection<Class<? extends Plugin>> nodePlugins() {
List<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());
plugins.add(TestPlugin.class);
plugins.add(SecondTestPlugin.class);
plugins.add(ReindexPlugin.class);
plugins.add(PainlessPlugin.class);
return plugins;
}

Expand Down Expand Up @@ -115,7 +141,7 @@ public void testStartMigrationAndImmediatelyCheckStatus() throws Exception {
});
}

public void testMigrateInternalManagedSystemIndex() throws Exception {
public void testMigrateSystemIndex() throws Exception {
createSystemIndexForDescriptor(INTERNAL_MANAGED);
createSystemIndexForDescriptor(INTERNAL_UNMANAGED);
createSystemIndexForDescriptor(EXTERNAL_MANAGED);
Expand Down Expand Up @@ -171,40 +197,15 @@ public void testMigrateInternalManagedSystemIndex() throws Exception {
postUpgradeHookCalled.set(true);
});

PostFeatureUpgradeRequest migrationRequest = new PostFeatureUpgradeRequest(TEST_REQUEST_TIMEOUT);
PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get();
assertThat(migrationResponse.getReason(), nullValue());
assertThat(migrationResponse.getElasticsearchException(), nullValue());
final Set<String> migratingFeatures = migrationResponse.getFeatures()
.stream()
.map(PostFeatureUpgradeResponse.Feature::getFeatureName)
.collect(Collectors.toSet());
assertThat(migratingFeatures, hasItem(FEATURE_NAME));

GetFeatureUpgradeStatusRequest getStatusRequest = new GetFeatureUpgradeStatusRequest(TEST_REQUEST_TIMEOUT);
// The feature upgrade may take longer than ten seconds when tests are running
// in parallel, so we give assertBusy a sixty-second timeout.
assertBusy(() -> {
GetFeatureUpgradeStatusResponse statusResponse = client().execute(GetFeatureUpgradeStatusAction.INSTANCE, getStatusRequest)
.get();
logger.info(Strings.toString(statusResponse));
assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED));
}, 60, TimeUnit.SECONDS);
executeMigration(FEATURE_NAME);

// Waiting for shards to stabilize if indices were moved around
ensureGreen();

assertTrue("the pre-migration hook wasn't actually called", preUpgradeHookCalled.get());
assertTrue("the post-migration hook wasn't actually called", postUpgradeHookCalled.get());

Metadata finalMetadata = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState().metadata();
// Check that the results metadata is what we expect.
FeatureMigrationResults currentResults = finalMetadata.custom(FeatureMigrationResults.TYPE);
assertThat(currentResults, notNullValue());
assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(1), hasKey(FEATURE_NAME)));
assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).succeeded(), is(true));
assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getFailedIndexName(), nullValue());
assertThat(currentResults.getFeatureStatuses().get(FEATURE_NAME).getException(), nullValue());
Metadata finalMetadata = assertMetadataAfterMigration(FEATURE_NAME);

assertIndexHasCorrectProperties(
finalMetadata,
Expand Down Expand Up @@ -240,6 +241,18 @@ public void testMigrateInternalManagedSystemIndex() throws Exception {
);
}

private static Metadata assertMetadataAfterMigration(String featureName) {
Metadata finalMetadata = clusterAdmin().prepareState(TEST_REQUEST_TIMEOUT).get().getState().metadata();
// Check that the results metadata is what we expect.
FeatureMigrationResults currentResults = finalMetadata.custom(FeatureMigrationResults.TYPE);
assertThat(currentResults, notNullValue());
assertThat(currentResults.getFeatureStatuses(), allOf(aMapWithSize(1), hasKey(featureName)));
assertThat(currentResults.getFeatureStatuses().get(featureName).succeeded(), is(true));
assertThat(currentResults.getFeatureStatuses().get(featureName).getFailedIndexName(), nullValue());
assertThat(currentResults.getFeatureStatuses().get(featureName).getException(), nullValue());
return finalMetadata;
}

public void testMigrateIndexWithWriteBlock() throws Exception {
createSystemIndexForDescriptor(INTERNAL_UNMANAGED);

Expand Down Expand Up @@ -317,6 +330,50 @@ public void onFailure(Exception e) {
});
}

private void executeMigration(String featureName) throws Exception {
PostFeatureUpgradeRequest migrationRequest = new PostFeatureUpgradeRequest(TEST_REQUEST_TIMEOUT);
PostFeatureUpgradeResponse migrationResponse = client().execute(PostFeatureUpgradeAction.INSTANCE, migrationRequest).get();
assertThat(migrationResponse.getReason(), nullValue());
assertThat(migrationResponse.getElasticsearchException(), nullValue());
final Set<String> migratingFeatures = migrationResponse.getFeatures()
.stream()
.map(PostFeatureUpgradeResponse.Feature::getFeatureName)
.collect(Collectors.toSet());
assertThat(migratingFeatures, hasItem(featureName));

GetFeatureUpgradeStatusRequest getStatusRequest = new GetFeatureUpgradeStatusRequest(TEST_REQUEST_TIMEOUT);
// The feature upgrade may take longer than ten seconds when tests are running
// in parallel, so we give assertBusy a sixty-second timeout.
assertBusy(() -> {
GetFeatureUpgradeStatusResponse statusResponse = client().execute(GetFeatureUpgradeStatusAction.INSTANCE, getStatusRequest)
.get();
logger.info(Strings.toString(statusResponse));
assertThat(statusResponse.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED));
}, 60, TimeUnit.SECONDS);
}

public void testMigrateUsingScript() throws Exception {
createSystemIndexForDescriptor(INTERNAL_MANAGED_WITH_SCRIPT);

executeMigration(SCRIPTED_INDEX_FEATURE_NAME);
ensureGreen();

Metadata metadata = assertMetadataAfterMigration(SCRIPTED_INDEX_FEATURE_NAME);
String newIndexName = ".int-mans-old-reindexed-for-" + UPGRADED_TO_VERSION;
assertIndexHasCorrectProperties(
metadata,
newIndexName,
INTERNAL_MANAGED_FLAG_VALUE,
true,
true,
Arrays.asList(".int-mans-old", ".internal-managed-with-script-alias")
);

SearchRequestBuilder searchRequestBuilder = prepareSearch(newIndexName).setQuery(QueryBuilders.termsQuery(FIELD_NAME, "migrated"))
.setSize(0);
assertHitCountAndNoFailures(searchRequestBuilder, INDEX_DOC_COUNT);
}

private String featureUpgradeErrorResponse(GetFeatureUpgradeStatusResponse statusResp) {
return statusResp.getFeatureUpgradeStatuses()
.stream()
Expand Down Expand Up @@ -463,4 +520,21 @@ public void testMigrateWithTemplatesV2() throws Exception {
assertThat(statusResp.getUpgradeStatus(), equalTo(GetFeatureUpgradeStatusResponse.UpgradeStatus.NO_MIGRATION_NEEDED));
});
}

public static class SecondTestPlugin extends Plugin implements SystemIndexPlugin {
@Override
public String getFeatureName() {
return SCRIPTED_INDEX_FEATURE_NAME;
}

@Override
public String getFeatureDescription() {
return "a plugin for testing system index migration";
}

@Override
public Collection<SystemIndexDescriptor> getSystemIndexDescriptors(Settings settings) {
return Collections.singletonList(INTERNAL_MANAGED_WITH_SCRIPT);
}
}
}
2 changes: 2 additions & 0 deletions modules/reindex/src/main/plugin-metadata/entitlement-policy.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
ALL-UNNAMED:
- outbound_network
org.elasticsearch.painless:
- create_class_loader
3 changes: 3 additions & 0 deletions modules/reindex/src/main/plugin-metadata/plugin-security.policy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
grant {
// reindex opens socket connections using the rest client
permission java.net.SocketPermission "*", "connect";

// needed for Painless to generate runtime classes
permission java.lang.RuntimePermission "createClassLoader";
};

grant codeBase "${codebase.elasticsearch-rest-client}" {
Expand Down

0 comments on commit 529ad04

Please sign in to comment.