From c9ba5554bfd4107d184f8de43c09b4c1775040bc Mon Sep 17 00:00:00 2001 From: Frank Yung-Fong Tang Date: Thu, 27 May 2021 02:47:37 +0000 Subject: [PATCH 1/3] ICU-21569 Fix Java7 breakage in LSTM & add Java7 CI See #1734 --- .github/workflows/icu_ci.yml | 31 ++++++ .../dev/test/rbbi/LSTMBreakEngineTest.java | 80 +++++++-------- .../ibm/icu/dev/test/rbbi/RBBILSTMTest.java | 98 +++++++++---------- 3 files changed, 120 insertions(+), 89 deletions(-) diff --git a/.github/workflows/icu_ci.yml b/.github/workflows/icu_ci.yml index 628d5db04b7c..9bf6235857b0 100644 --- a/.github/workflows/icu_ci.yml +++ b/.github/workflows/icu_ci.yml @@ -30,6 +30,37 @@ jobs: # Regex note: (?! ... ) is a negative lookahead. Succeed if the pattern is not present. set +o pipefail && make doc 2>&1 | tee doxygen.log && ( ! grep -P 'warning:(?! .* file .?Doxyfile)' doxygen.log ) + # Java7 ICU4J build and unit test + java7-icu4j-build-and-test: + runs-on: ubuntu-latest + steps: + - name: Checkout and setup + uses: actions/checkout@v2 + with: + lfs: true + - name: Checkout lfs objects + run: git lfs pull + - uses: actions/setup-java@v1 + with: + java-version: '8' + - name: Download Java7 + run: | + download_url="https://download.java.net/openjdk/jdk7u75/ri/openjdk-7u75-b13-linux-x64-18_dec_2014.tar.gz" + wget -O $RUNNER_TEMP/java_package.tar.gz $download_url + pushd $RUNNER_TEMP ; gunzip java_package.tar.gz ; tar xvf java_package.tar; popd + echo "java7.bootclasspath" > $RUNNER_TEMP/draft ; ls $RUNNER_TEMP/java-se-7u75-ri/jre/lib/*.jar|paste -sd ":" - >> $RUNNER_TEMP/draft + cat $RUNNER_TEMP/draft | paste -sd "=" - > icu4j/build-local.properties + - name: ICU4J + run: | + cd icu4j; + ant init; + ant check; + - name: List failures (if any) + run: | + [ -d icu4j/out/junit-results ] && cd icu4j && cat `find out/junit-results -name "*.txt" -exec grep -l FAILED {} \;`; + if: ${{ failure() }} + + # ICU4J build and unit test under Java11 java11-icu4j-build-and-test: runs-on: ubuntu-latest diff --git a/icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/LSTMBreakEngineTest.java b/icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/LSTMBreakEngineTest.java index 0208e68fe294..8d248a551a2c 100644 --- a/icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/LSTMBreakEngineTest.java +++ b/icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/LSTMBreakEngineTest.java @@ -9,8 +9,6 @@ import java.nio.charset.StandardCharsets; import java.text.CharacterIterator; import java.text.StringCharacterIterator; -import java.util.Iterator; -import java.util.stream.Stream; import org.junit.Test; import org.junit.runner.RunWith; @@ -66,52 +64,54 @@ private void runTestFromFile(String filename, int script) { errln("Could not open test data file " + filename); return; } - Stream lines = (new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))).lines(); - Iterator iterator = lines.iterator(); + BufferedReader br = (new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))); int caseNum = 0; String expected = ""; String actual = ""; LSTMBreakEngine engine = null; - while (iterator.hasNext()) { - String line = iterator.next(); - String fields[] = line.split("\t"); - if (fields[0].equals("Model:")) { - engine = createEngineFromTestData(fields[1], script); - } else if (fields[0].equals("Input:")) { - caseNum++; - int length = fields[1].length(); - CharacterIterator input = new StringCharacterIterator(fields[1]); - DictionaryBreakEngine.DequeI foundBreaks = new DictionaryBreakEngine.DequeI(); - int ret = engine.findBreaks(input, 0, length, foundBreaks); - StringBuilder sb = new StringBuilder(); - sb.append('{'); - for (int i = 0; i < foundBreaks.size(); i++) { - sb.append(foundBreaks.elementAt(i)).append(", "); - } - sb.append(length).append('}'); - actual = sb.toString(); - } else if (fields[0].equals("Output:")) { - StringBuilder sb = new StringBuilder(); - int sep; - int start = 0; - int curr = 0; - sb.append('{'); - while ((sep = fields[1].indexOf('|', start)) >= 0) { - int len = sep - start; - if (len > 0) { - if (curr > 0) { - sb.append(", "); + String line; + try { + while ((line = br.readLine()) != null) { + String fields[] = line.split("\t"); + if (fields[0].equals("Model:")) { + engine = createEngineFromTestData(fields[1], script); + } else if (fields[0].equals("Input:")) { + caseNum++; + int length = fields[1].length(); + CharacterIterator input = new StringCharacterIterator(fields[1]); + DictionaryBreakEngine.DequeI foundBreaks = new DictionaryBreakEngine.DequeI(); + int ret = engine.findBreaks(input, 0, length, foundBreaks); + StringBuilder sb = new StringBuilder(); + sb.append('{'); + for (int i = 0; i < foundBreaks.size(); i++) { + sb.append(foundBreaks.elementAt(i)).append(", "); + } + sb.append(length).append('}'); + actual = sb.toString(); + } else if (fields[0].equals("Output:")) { + StringBuilder sb = new StringBuilder(); + int sep; + int start = 0; + int curr = 0; + sb.append('{'); + while ((sep = fields[1].indexOf('|', start)) >= 0) { + int len = sep - start; + if (len > 0) { + if (curr > 0) { + sb.append(", "); + } + curr += len; + sb.append(curr); } - curr += len; - sb.append(curr); + start = sep + 1; } - start = sep + 1; + sb.append('}'); + expected = sb.toString(); + assertEquals(line + " Test Case#" + caseNum , expected, actual); } - sb.append('}'); - expected = sb.toString(); - - assertEquals(line + " Test Case#" + caseNum , expected, actual); } + } catch (IOException e) { + errln("Exception while reading lines of test data file " + filename + e.toString()); } } } diff --git a/icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/RBBILSTMTest.java b/icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/RBBILSTMTest.java index 06eb2f40ae88..441ec7d6fda8 100644 --- a/icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/RBBILSTMTest.java +++ b/icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/RBBILSTMTest.java @@ -9,8 +9,6 @@ import java.nio.charset.StandardCharsets; import java.text.CharacterIterator; import java.text.StringCharacterIterator; -import java.util.Iterator; -import java.util.stream.Stream; import org.junit.Test; import org.junit.runner.RunWith; @@ -56,63 +54,65 @@ private void runTestFromFile(String filename, int script) { errln("Could not open test data file " + filename); return; } - Stream lines = (new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))).lines(); - Iterator iterator = lines.iterator(); + BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8)); int caseNum = 0; String expected = ""; String actual = ""; LSTMBreakEngine engine = null; - while (iterator.hasNext()) { - String line = iterator.next(); - String fields[] = line.split("\t"); - if (fields[0].equals("Model:")) { - String actualModelName = LSTMBreakEngine.createData(script).fName; - if (!actualModelName.equals(fields[1])) { - errln("The name of the built in model " + actualModelName + - " does not match the model (" + fields[1] + ") expected for this test"); - return; - } - } else if (fields[0].equals("Input:")) { - caseNum++; - int length = fields[1].length(); - String input = "prefix " + fields[1] + " suffix"; - bi.setText(input); - System.out.println("Input = " + input); - StringBuilder sb = new StringBuilder(); - sb.append('{'); - for (int bp = bi.first(); bp != BreakIterator.DONE; bp = bi.next()) { - sb.append(bp); - if (bp != input.length()) { - sb.append(", "); + String line; + try { + while ((line = br.readLine()) != null) { + String fields[] = line.split("\t"); + if (fields[0].equals("Model:")) { + String actualModelName = LSTMBreakEngine.createData(script).fName; + if (!actualModelName.equals(fields[1])) { + errln("The name of the built in model " + actualModelName + + " does not match the model (" + fields[1] + ") expected for this test"); + return; } - } - sb.append('}'); - actual = sb.toString(); - } else if (fields[0].equals("Output:")) { - StringBuilder sb = new StringBuilder(); - int sep; - int start = 0; - int curr = 0; - sb.append("{0, "); - String input = "prefix| |" + fields[1] + "| |suffix"; - while ((sep = input.indexOf('|', start)) >= 0) { - int len = sep - start; - if (len > 0) { - if (curr > 0) { + } else if (fields[0].equals("Input:")) { + caseNum++; + int length = fields[1].length(); + String input = "prefix " + fields[1] + " suffix"; + bi.setText(input); + System.out.println("Input = " + input); + StringBuilder sb = new StringBuilder(); + sb.append('{'); + for (int bp = bi.first(); bp != BreakIterator.DONE; bp = bi.next()) { + sb.append(bp); + if (bp != input.length()) { sb.append(", "); } - curr += len; - sb.append(curr); } - start = sep + 1; + sb.append('}'); + actual = sb.toString(); + } else if (fields[0].equals("Output:")) { + StringBuilder sb = new StringBuilder(); + int sep; + int start = 0; + int curr = 0; + sb.append("{0, "); + String input = "prefix| |" + fields[1] + "| |suffix"; + while ((sep = input.indexOf('|', start)) >= 0) { + int len = sep - start; + if (len > 0) { + if (curr > 0) { + sb.append(", "); + } + curr += len; + sb.append(curr); + } + start = sep + 1; + } + sb.append(", ").append(curr + input.length() - start); + sb.append('}'); + expected = sb.toString(); + assertEquals(input + " Test Case#" + caseNum , expected, actual); + actual = ""; } - sb.append(", ").append(curr + input.length() - start); - sb.append('}'); - expected = sb.toString(); - - assertEquals(input + " Test Case#" + caseNum , expected, actual); - actual = ""; } + } catch (IOException e) { + errln("Exception while reading lines of test data file " + filename + e.toString()); } } } From 1de938b33bc6e7d1c17a05b867d63533d1da51b0 Mon Sep 17 00:00:00 2001 From: Elango Cheran Date: Thu, 27 May 2021 14:21:00 -0700 Subject: [PATCH 2/3] Add caching to the download of Java 7, separate out step to use it --- .github/workflows/icu_ci.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/icu_ci.yml b/.github/workflows/icu_ci.yml index 9bf6235857b0..2e0342dab23f 100644 --- a/.github/workflows/icu_ci.yml +++ b/.github/workflows/icu_ci.yml @@ -43,12 +43,22 @@ jobs: - uses: actions/setup-java@v1 with: java-version: '8' - - name: Download Java7 + - name: Cache for Java 7 tarball + id: cache-java7 + uses: actions/cache@v2 + with: + path: java7-tarball + key: ${{ runner.os }}-java7-tarball + - name: Download Java 7 + if: steps.cache-java7.outputs.cache-hit != 'true' run: | + mkdir -p java7-tarball download_url="https://download.java.net/openjdk/jdk7u75/ri/openjdk-7u75-b13-linux-x64-18_dec_2014.tar.gz" - wget -O $RUNNER_TEMP/java_package.tar.gz $download_url - pushd $RUNNER_TEMP ; gunzip java_package.tar.gz ; tar xvf java_package.tar; popd - echo "java7.bootclasspath" > $RUNNER_TEMP/draft ; ls $RUNNER_TEMP/java-se-7u75-ri/jre/lib/*.jar|paste -sd ":" - >> $RUNNER_TEMP/draft + wget -O java7-tarball/java_package.tar.gz $download_url + pushd java7-tarball ; gunzip java_package.tar.gz ; tar xvf java_package.tar; popd + - name: Configure Ant build to use Java 7 JRE + run: | + echo "java7.bootclasspath" > $RUNNER_TEMP/draft ; ls $java7-tarball/java-se-7u75-ri/jre/lib/*.jar|paste -sd ":" - >> $RUNNER_TEMP/draft cat $RUNNER_TEMP/draft | paste -sd "=" - > icu4j/build-local.properties - name: ICU4J run: | From 7dc33d4b036a0d315fa837adc363a7a24c76d058 Mon Sep 17 00:00:00 2001 From: Elango Cheran Date: Thu, 27 May 2021 14:23:54 -0700 Subject: [PATCH 3/3] Fix typo --- .github/workflows/icu_ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/icu_ci.yml b/.github/workflows/icu_ci.yml index 2e0342dab23f..e9ebb0bfa3d0 100644 --- a/.github/workflows/icu_ci.yml +++ b/.github/workflows/icu_ci.yml @@ -58,7 +58,7 @@ jobs: pushd java7-tarball ; gunzip java_package.tar.gz ; tar xvf java_package.tar; popd - name: Configure Ant build to use Java 7 JRE run: | - echo "java7.bootclasspath" > $RUNNER_TEMP/draft ; ls $java7-tarball/java-se-7u75-ri/jre/lib/*.jar|paste -sd ":" - >> $RUNNER_TEMP/draft + echo "java7.bootclasspath" > $RUNNER_TEMP/draft ; ls java7-tarball/java-se-7u75-ri/jre/lib/*.jar|paste -sd ":" - >> $RUNNER_TEMP/draft cat $RUNNER_TEMP/draft | paste -sd "=" - > icu4j/build-local.properties - name: ICU4J run: |