Use CLDR locale provider on JDK 23+ (elastic#110222)

JDK 23 removes the COMPAT locale provider, leaving CLDR as the only option. This commit configures Elasticsearch
to use the CLDR provider when on JDK 23, but still use the existing COMPAT provider when on JDK 22 and below.

This causes some differences in locale behaviour; this also adapts various tests to still work whether run on COMPAT or CLDR.

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -167,7 +167,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
             vmParameters = [
               '-ea',
               '-Djava.security.manager=allow',
-              '-Djava.locale.providers=SPI,COMPAT',
+              '-Djava.locale.providers=SPI,CLDR',
               '-Des.nativelibs.path="' + testLibraryPath + '"',
               // TODO: only open these for mockito when it is modularized
               '--add-opens=java.base/java.security.cert=ALL-UNNAMED',

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchTestBasePlugin.java

@@ -92,7 +92,7 @@ public void execute(Task t) {
                     mkdirs(test.getWorkingDir().toPath().resolve("temp").toFile());
 
                     // TODO remove once jvm.options are added to test system properties
-                    test.systemProperty("java.locale.providers", "SPI,COMPAT");
+                    test.systemProperty("java.locale.providers", "SPI,CLDR");
                 }
             });
             test.getJvmArgumentProviders().add(nonInputProperties);

distribution/tools/server-cli/src/main/java/org/elasticsearch/server/cli/SystemJvmOptions.java

@@ -10,6 +10,7 @@
 
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
+import org.elasticsearch.core.UpdateForV9;
 
 import java.util.List;
 import java.util.Map;
@@ -59,11 +60,7 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
                 "-Dlog4j.shutdownHookEnabled=false",
                 "-Dlog4j2.disable.jmx=true",
                 "-Dlog4j2.formatMsgNoLookups=true",
-                /*
-                 * Due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise time/date
-                 * parsing will break in an incompatible way for some date patterns and locales.
-                 */
-                "-Djava.locale.providers=SPI,COMPAT",
+                "-Djava.locale.providers=" + getLocaleProviders(),
                 maybeEnableNativeAccess(),
                 maybeOverrideDockerCgroup(distroType),
                 maybeSetActiveProcessorCount(nodeSettings),
@@ -75,6 +72,16 @@ static List<String> systemJvmOptions(Settings nodeSettings, final Map<String, St
         ).filter(e -> e.isEmpty() == false).collect(Collectors.toList());
     }
 
+    @UpdateForV9    // only use CLDR in v9+
+    private static String getLocaleProviders() {
+        /*
+         * Specify SPI to load IsoCalendarDataProvider (see #48209), specifying the first day of week as Monday.
+         * When on pre-23, use COMPAT instead to maintain existing date formats as much as we can.
+         * When on JDK 23+, use the default CLDR locale database, as COMPAT was removed in JDK 23.
+         */
+        return Runtime.version().feature() >= 23 ? "SPI,CLDR" : "SPI,COMPAT";
+    }
+
     /*
      * The virtual file /proc/self/cgroup should list the current cgroup
      * membership. For each hierarchy, you can follow the cgroup path from

docs/painless/painless-contexts/painless-field-context.asciidoc

@@ -64,14 +64,14 @@ actors that appear in each play:
 ----
 GET seats/_search
 {
-  "size": 2, 
+  "size": 2,
   "query": {
     "match_all": {}
   },
   "script_fields": {
     "day-of-week": {
       "script": {
-        "source": "doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ROOT)"
+        "source": "doc['datetime'].value.getDayOfWeekEnum().getDisplayName(TextStyle.FULL, Locale.ENGLISH)"
       }
     },
     "number-of-actors": {
@@ -132,4 +132,4 @@ GET seats/_search
   }
 }
 ----
-// TESTRESPONSE[s/"took" : 68/"took" : "$body.took"/]
+// TESTRESPONSE[s/"took" : 68/"took" : "$body.took"/]

docs/painless/painless-guide/painless-execute-script.asciidoc

@@ -749,7 +749,7 @@ POST /_scripts/painless/_execute
 {
   "script": {
     "source": """
-      emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+      emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
     """
   },
   "context": "keyword_field",

docs/reference/aggregations/bucket/composite-aggregation.asciidoc

@@ -156,7 +156,7 @@ GET /_search
       "type": "keyword",
       "script": """
         emit(doc['timestamp'].value.dayOfWeekEnum
-          .getDisplayName(TextStyle.FULL, Locale.ROOT))
+          .getDisplayName(TextStyle.FULL, Locale.ENGLISH))
       """
     }
   },

docs/reference/aggregations/bucket/datehistogram-aggregation.asciidoc

@@ -582,7 +582,7 @@ For example, the offset of `+19d` will result in buckets with names like `2022-0
 
 Increasing the offset to `+20d`, each document will appear in a bucket for the previous month,
 with all bucket keys ending with the same day of the month, as normal.
-However, further increasing to `+28d`, 
+However, further increasing to `+28d`,
 what used to be a February bucket has now become `"2022-03-01"`.
 
 [source,console,id=datehistogram-aggregation-offset-example-28d]
@@ -819,7 +819,7 @@ POST /sales/_search?size=0
   "runtime_mappings": {
     "date.day_of_week": {
       "type": "keyword",
-      "script": "emit(doc['date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+      "script": "emit(doc['date'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
     }
   },
   "aggs": {

docs/reference/indices/index-templates.asciidoc

@@ -102,7 +102,7 @@ PUT _component_template/runtime_component_template
         "day_of_week": {
           "type": "keyword",
           "script": {
-            "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+            "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
           }
         }
       }

docs/reference/mapping/runtime.asciidoc

@@ -135,7 +135,7 @@ PUT my-index-000001/
       "day_of_week": {
         "type": "keyword",
         "script": {
-          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
         }
       }
     },
@@ -291,7 +291,7 @@ GET my-index-000001/_search
     "day_of_week": {
       "type": "keyword",
       "script": {
-        "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+        "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
       }
     }
   },
@@ -667,7 +667,7 @@ PUT my-index-000001/
       "day_of_week": {
         "type": "keyword",
         "script": {
-          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+          "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
         }
       }
     },

docs/reference/ml/ml-shared.asciidoc

@@ -951,7 +951,7 @@ For example:
   "day_of_week": {
     "type": "keyword",
     "script": {
-      "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+      "source": "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
     }
   }
 }

docs/reference/search/search-your-data/search-api.asciidoc

@@ -173,7 +173,7 @@ GET /my-index-000001/_search
       "script": {
         "source":
         """emit(doc['@timestamp'].value.dayOfWeekEnum
-        .getDisplayName(TextStyle.FULL, Locale.ROOT))"""
+        .getDisplayName(TextStyle.FULL, Locale.ENGLISH))"""
       }
     }
   },

modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/65_runtime_doc_values.yml

@@ -12,7 +12,7 @@ setup:
                 script:
                   source: |
                     for (date in field('date')) {
-                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                     }
               total_value_double:
                 type: double
@@ -55,7 +55,7 @@ setup:
                   source: |
                     if (doc.containsKey('date')) {
                       for (date in doc['date']) {
-                        emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                        emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                       }
                     }
               doc_total_value_double:
@@ -737,7 +737,7 @@ setup:
               script:
                 source: |
                   for (date in field('date')) {
-                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                   }
           sort: [ { rank: asc } ]
           script_fields:
@@ -758,7 +758,7 @@ setup:
               script:
                 source: |
                   for (date in field('date')) {
-                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                   }
           sort: [ { rank: asc } ]
           script_fields:
@@ -924,7 +924,7 @@ setup:
                 source: |
                   if (doc.containsKey('date')) {
                     for (date in doc['date']) {
-                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                     }
                   }
           sort: [ { rank: asc } ]
@@ -947,7 +947,7 @@ setup:
                 source: |
                   if (doc.containsKey('date')) {
                     for (date in doc['date']) {
-                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                     }
                   }
           sort: [ { rank: asc } ]
@@ -1133,7 +1133,7 @@ setup:
               script:
                 source: |
                   for (date in field('date')) {
-                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                   }
           sort: [ { rank: asc } ]
           script_fields:
@@ -1156,7 +1156,7 @@ setup:
               script:
                 source: |
                   for (date in field('date')) {
-                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                    emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                   }
           sort: [ { rank: asc } ]
           script_fields:
@@ -1337,7 +1337,7 @@ setup:
                 source: |
                   if (doc.containsKey('date')) {
                     for (date in doc['date']) {
-                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                     }
                   }
           sort: [ { rank: asc } ]
@@ -1362,7 +1362,7 @@ setup:
                 source: |
                   if (doc.containsKey('date')) {
                     for (date in doc['date']) {
-                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                      emit(date.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
                     }
                   }
           sort: [ { rank: asc } ]

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/10_keyword.yml

@@ -12,7 +12,7 @@ setup:
               day_of_week:
                 type: keyword
                 script: |
-                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
               # Test fetching from _source
               day_of_week_from_source:
                 type: keyword
@@ -75,7 +75,7 @@ setup:
   - match: {sensor.mappings.runtime.day_of_week.type: keyword }
   - match:
       sensor.mappings.runtime.day_of_week.script.source: |
-        emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+        emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
   - match: {sensor.mappings.runtime.day_of_week.script.lang: painless }
 
 # --- TODO get field mappings needs to be adapted
@@ -90,7 +90,7 @@ setup:
 #        type: keyword
 #        script:
 #          source: |
-#            emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+#            emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
 #          lang: painless
 #        meta: {}
 #

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml

@@ -21,7 +21,7 @@ setup:
               day_of_week:
                 type: keyword
                 script: |
-                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
               # Test fetching from _source
               day_of_week_from_source:
                 type: keyword
@@ -74,7 +74,7 @@ setup:
   - match: {sensor.mappings.properties.day_of_week.type: keyword }
   - match:
       sensor.mappings.properties.day_of_week.script.source: |
-        emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+        emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
   - match: {sensor.mappings.properties.day_of_week.script.lang: painless }
 
 ---

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/40_runtime_mappings.yml

@@ -34,7 +34,7 @@ setup:
             day_of_week:
               type: keyword
               script:
-                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
 
   - match: {indices:                             ["test-1"]}
   - length: {fields.timestamp:                            1}
@@ -78,7 +78,7 @@ setup:
             day_of_week:
               type: keyword
               script:
-                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+                source: "emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
 
   - match: {indices:                    ["test-1", "test-2"]}
   - length: {fields.day_of_week:                          1}

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/80_multiple_indices.yml

@@ -12,7 +12,7 @@ setup:
               day_of_week:
                 type: keyword
                 script: |
-                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT));
+                  emit(doc['timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH));
               tomorrow:
                 type: date
                 script:

muted-tests.yml

@@ -1,25 +1,10 @@
 tests:
-- class: "org.elasticsearch.xpack.textstructure.structurefinder.TimestampFormatFinderTests"
-  issue: "https://github.com/elastic/elasticsearch/issues/108855"
-  method: "testGuessIsDayFirstFromLocale"
-- class: "org.elasticsearch.test.rest.ClientYamlTestSuiteIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/108857"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
 - class: "org.elasticsearch.upgrades.SearchStatesIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108991"
   method: "testCanMatch"
 - class: "org.elasticsearch.upgrades.MlTrainedModelsUpgradeIT"
   issue: "https://github.com/elastic/elasticsearch/issues/108993"
   method: "testTrainedModelInference"
-- class: "org.elasticsearch.xpack.security.CoreWithSecurityClientYamlTestSuiteIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109188"
-  method: "test {yaml=search/180_locale_dependent_mapping/Test Index and Search locale dependent mappings / dates}"
-- class: "org.elasticsearch.xpack.esql.qa.mixed.EsqlClientYamlIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109189"
-  method: "test {p0=esql/70_locale/Date format with Italian locale}"
-- class: "org.elasticsearch.xpack.test.rest.XPackRestIT"
-  issue: "https://github.com/elastic/elasticsearch/issues/109200"
-  method: "test {p0=esql/70_locale/Date format with Italian locale}"
 - class: org.elasticsearch.smoketest.DocsClientYamlTestSuiteIT
   method: test {yaml=reference/esql/esql-async-query-api/line_17}
   issue: https://github.com/elastic/elasticsearch/issues/109260

qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml

@@ -13,7 +13,7 @@ setup:
                 day_of_week:
                   type: keyword
                   script:
-                    source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+                    source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
   - do:
       bulk:
         refresh: true

qa/ccs-common-rest/src/yamlRestTest/resources/rest-api-spec/test/eql/20_runtime_mappings.yml

@@ -9,7 +9,7 @@ setup:
                 day_of_week:
                   type: keyword
                   script:
-                    source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
+                    source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ENGLISH))"
   - do:
       bulk:
         refresh: true