diff --git a/.github/linters/.gitleaks.toml b/.github/linters/.gitleaks.toml new file mode 100644 index 000000000..6012aaeb6 --- /dev/null +++ b/.github/linters/.gitleaks.toml @@ -0,0 +1,5 @@ +title = "gitleaks config" +[allowlist] +files = [ + "cyclonedx-lib/dependency_data/dependency_data.properties" +] diff --git a/.github/linters/suppressed-java.xml b/.github/linters/suppressed-java.xml index b9c1a256e..0acb11e80 100644 --- a/.github/linters/suppressed-java.xml +++ b/.github/linters/suppressed-java.xml @@ -28,4 +28,5 @@ - \ No newline at end of file + + diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 11a7ffd93..b5fb32a3f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,10 +35,6 @@ jobs: build_linux: name: Linux runs-on: ubuntu-latest - env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: 'true' - container: - image: ${{ matrix.image }} strategy: fail-fast: false matrix: @@ -80,24 +76,23 @@ jobs: variant: bisheng image: adoptopenjdk/centos7_build_image steps: - # pinned at v3 to as Node.js 20.x is not supported on Centos 7 - - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: actions/checkout@v4 - - name: Build Linux - run: ./build-farm/make-adopt-build-farm.sh - env: - JAVA_TO_BUILD: ${{ matrix.version }} - ARCHITECTURE: x64 - VARIANT: ${{ matrix.variant }} - TARGET_OS: ${{ matrix.os }} - FILENAME: OpenJDK.tar.gz - # Don't set the OS as we use both linux and alpine-linux - PLATFORM_CONFIG_LOCATION: adoptium/temurin-build/master/build-farm/platform-specific-configurations - BUILD_ARGS: --create-sbom - CONFIGURE_ARGS: --with-native-debug-symbols=none - - # pinned at v3 to as Node.js 20.x is not supported on Centos 7 - - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + - name: Build Linux within container image "${{ matrix.image }}" + run: | + docker run --rm -w /home/jenkins -v "$PWD":"/home/jenkins" \ + -e "JAVA_TO_BUILD=${{ matrix.version }}" \ + -e "ARCHITECTURE=x64" \ + -e "VARIANT=${{ matrix.variant }}" \ + -e "TARGET_OS=${{ matrix.os }}" \ + -e "FILENAME=OpenJDK.tar.gz" \ + -e "PLATFORM_CONFIG_LOCATION=adoptium/temurin-build/master/build-farm/platform-specific-configurations" \ + -e "BUILD_ARGS=--create-sbom" \ + -e "CONFIGURE_ARGS=--with-native-debug-symbols=none" \ + "${{ matrix.image }}" \ + ./build-farm/make-adopt-build-farm.sh + + - uses: actions/upload-artifact@v4 name: Collect and Archive Artifacts with: name: ${{matrix.version}}-${{matrix.os}}-${{matrix.variant}} @@ -110,17 +105,33 @@ jobs: - name: Set root of jdk image dir run: | imageroot=$(find "${HOME}/JDK" -name release -type f) - echo "TEST_JDK_HOME=$(dirname "${imageroot}")" >> "$GITHUB_ENV" - - name: Smoke test - uses: adoptium/run-aqa@6bacb4e732ad546eda1b09665b9067cdc87651f4 # v2 + # TEST_JDK_HOME needs to be mapped to the docker container /home/jenkins mapping + echo "TEST_JDK_HOME=$(dirname "${imageroot}")" | sed "s,${HOME},/home/jenkins," >> "$GITHUB_ENV" + + - name: Checkout aqa-tests repo + uses: actions/checkout@v4 with: - build_list: 'functional/buildAndPackage' - target: '_extended.functional' - vendor_testRepos: "${{ github.event.pull_request.head.repo.html_url }}.git" - vendor_testBranches: "${{ github.head_ref }}" - vendor_testDirs: "/test/functional" - # pinned at v3 to as Node.js 20.x is not supported on Centos 7 - - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + repository: adoptium/aqa-tests + path: aqa-tests + - name: Run Smoke test within container image "${{ matrix.image }}" + env: + VENDOR_REPOS: ${{ github.event.pull_request.head.repo.html_url }}.git + VENDOR_BRANCH: ${{ github.head_ref }} + run: | + WORK_DIR="${PWD//${HOME}//home/jenkins}" + docker run --rm -w /home/jenkins -v "$HOME":"/home/jenkins" \ + -e "TEST_JDK_HOME=${TEST_JDK_HOME}" \ + -e "BUILD_LIST=functional/buildAndPackage" \ + "${{ matrix.image }}" \ + sh -c "cd ${WORK_DIR}/aqa-tests && \ + ./get.sh --vendor_repos ${VENDOR_REPOS} \ + --vendor_branches ${VENDOR_BRANCH} \ + --vendor_dirs /test/functional && \ + cd TKG && \ + make compile && \ + make _extended.functional" + + - uses: actions/upload-artifact@v4 name: Collect and Archive SmokeTest Results if: failure() with: diff --git a/.github/workflows/ca-cert-updater.yml b/.github/workflows/ca-cert-updater.yml index 71419dc9f..cfc616c39 100644 --- a/.github/workflows/ca-cert-updater.yml +++ b/.github/workflows/ca-cert-updater.yml @@ -35,7 +35,7 @@ jobs: working-directory: ./security run: "./mk-ca-bundle.pl" - - uses: gr2m/create-or-update-pull-request-action@488876a65a2ca38b7eb05e9086166337087f5323 # v1.10.0 + - uses: gr2m/create-or-update-pull-request-action@b65137ca591da0b9f43bad7b24df13050ea45d1b # v1.10.1 env: GITHUB_TOKEN: ${{ secrets.ADOPTIUM_TEMURIN_BOT_TOKEN }} with: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8670bb98a..a23b92de7 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -58,7 +58,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 + uses: github/codeql-action/init@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -68,7 +68,7 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 + uses: github/codeql-action/autobuild@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5 # ℹī¸ Command-line programs to run using the OS shell. # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun @@ -81,6 +81,6 @@ jobs: # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@662472033e021d55d94146f66f6058822b0b39fd # v3.27.0 + uses: github/codeql-action/analyze@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v3.27.5 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/ossf-scorecard.yml b/.github/workflows/ossf-scorecard.yml index 6fefd074e..bcb2d44cb 100644 --- a/.github/workflows/ossf-scorecard.yml +++ b/.github/workflows/ossf-scorecard.yml @@ -46,6 +46,6 @@ jobs: name: SARIF file path: results.sarif retention-days: 5 - - uses: github/codeql-action/upload-sarif@662472033e021d55d94146f66f6058822b0b39fd # v2.13.4 + - uses: github/codeql-action/upload-sarif@f09c1c0a94de965c15400f5634aa42fac8fb8f88 # v2.13.4 with: sarif_file: results.sarif diff --git a/.github/workflows/testsbom.yml b/.github/workflows/testcyclonedx.yml similarity index 52% rename from .github/workflows/testsbom.yml rename to .github/workflows/testcyclonedx.yml index 073411098..f4f5703d9 100644 --- a/.github/workflows/testsbom.yml +++ b/.github/workflows/testcyclonedx.yml @@ -1,5 +1,5 @@ # ******************************************************************************** -# Copyright (c) 2023 Contributors to the Eclipse Foundation +# Copyright (c) 2023, 2024 Contributors to the Eclipse Foundation # # See the NOTICE file(s) with this work for additional # information regarding copyright ownership. @@ -12,7 +12,7 @@ # ******************************************************************************** --- -name: TestSBOM +name: TestCycloneDX on: pull_request: @@ -30,21 +30,21 @@ permissions: contents: read jobs: - test_sbom_gen: - name: gen_sbom + test_cyclonedx_gen: + name: gen_cyclonedx runs-on: ubuntu-latest steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - # Build with jdk8 to ensure TemurinGenSBOM meets min compatibility + # Build with jdk8 to ensure TemurinGen* meets min compatibility - uses: actions/setup-java@8df1039502a15bceb9433410b1a100fbe190c53b # v4.5.0 id: setup-java with: java-version: 8 distribution: 'temurin' - - name: Build TemurinGenSBOM.java + - name: Build TemurinGenSBOM.java and TemurinGenCDXA.java run: | ant -noinput -buildfile cyclonedx-lib/build.xml clean ant -noinput -buildfile cyclonedx-lib/build.xml build @@ -52,8 +52,27 @@ jobs: - name: Run TemurinGenSBOM Unit test run: ant -noinput -buildfile cyclonedx-lib/build.xml run + - name: Run TemurinGenCDXA Unit test + run: ant -noinput -buildfile cyclonedx-lib/build.xml runCDXA + + - name: Validate generated SBOM and CDXA documents using cyclonedx-cli validate + run: | + curl -L -O https://github.com/CycloneDX/cyclonedx-cli/releases/latest/download/cyclonedx-linux-x64 + chmod +x cyclonedx-linux-x64 + ./cyclonedx-linux-x64 validate --input-file cyclonedx-lib/build/testSBOM.json --fail-on-errors --input-version v1_6 + ./cyclonedx-linux-x64 validate --input-file cyclonedx-lib/build/testSBOM.xml --fail-on-errors --input-version v1_6 + ./cyclonedx-linux-x64 validate --input-file cyclonedx-lib/build/testCDXA.json --fail-on-errors --input-version v1_6 + ./cyclonedx-linux-x64 validate --input-file cyclonedx-lib/build/testCDXA.xml --fail-on-errors --input-version v1_6 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 name: Collect and Archive TemurinGenSBOM Artifacts with: name: testSBOM - path: cyclonedx-lib/build/testSBOM.json + path: cyclonedx-lib/build/testSBOM.* + + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + name: Collect and Archive TemurinGenCDXA Artifacts + with: + name: testCDXA + path: cyclonedx-lib/build/testCDXA.* + diff --git a/.licenserc.yaml b/.licenserc.yaml index 125b90a34..ae1a2dd8e 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -46,6 +46,7 @@ header: - 'security/certdata.txt' - 'sbin/*.template' - '.github/linters/*' + - '.github/workflows/dependabot-auto-merge.yml' - 'cyclonedx-lib/getDependencies' - 'cyclonedx-lib/dependency_data/**' - 'makejdk-any-platform.1' diff --git a/README.md b/README.md index 11f38584a..e628bb60e 100644 --- a/README.md +++ b/README.md @@ -185,6 +185,13 @@ the one you are trying to build. -k, --keep if using docker, keep the container after the build. +--local-dependency-cache-dir +specify the location of a local cache of required build dependency jars. If not specified +the following default locations are searched +Windows: c:/dependency_cache +MacOS: ${HOME}/dependency_cache +Unix: /usr/local/dependency_cache + --make-exploded-image creates an exploded image (useful for codesigning jmods). Use --assemble-exploded-image once you have signed the jmods to complete the packaging steps. diff --git a/RELEASING.md b/RELEASING.md index 5c6ec42db..c17e36ea6 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -149,7 +149,7 @@ flowchart TD -Disable nightly testing so the release builds aren't delayed by any nightly test runs (set `enableTests : false` in [defaults.json](https://github.com/adoptium/ci-jenkins-pipelines/blob/master/pipelines/defaults.json)). Ensure the build pipeline generator job runs successfully (), and the flag is disabled by bringing up the Build pipeline job and check the `enableTests` checkbox is unticked. +Scheduled pipeline Testing is automatically disabled from the Saturday prior to "release Tuesday", to the Sunday after, see: https://github.com/adoptium/ci-jenkins-pipelines/blob/5bd79eb1d95a033c4ee364a8f9fcc270ad653178/pipelines/build/common/trigger_beta_build.groovy#L51 Add a banner to the website to indicate that the releases are coming in the near future ([Example Changes](https://github.com/adoptium/adoptium.net/blob/main/src/components/Banner.tsx)). diff --git a/SmokeTesting.md b/SmokeTesting.md index f3179a09e..a57b0f26a 100644 --- a/SmokeTesting.md +++ b/SmokeTesting.md @@ -7,8 +7,9 @@ These are the general steps to execute the Smoke Tests found in[/test/functional 1. export TEST_JDK_HOME=/someLocation // set test JDK home. On windows, the windows path format is expected. (i.e., TEST_JDK_HOME=C:\someLocation ) 1. git clone [https://github.com/adoptium/aqa-tests.git](https://github.com/adoptium/aqa-tests) to /testLocation 1. cd aqa-tests -1. ./get.sh +1. ./get.sh --vendor_repos https://github.com/adoptium/temurin-build --vendor_branches master --vendor_dirs /test/functional +1. ( When running get.sh ensure the vendor parameters are passed correctly, the above example shows how to run the smoke tests contained within the temurin-build repository ) 1. cd TKG -1. Export environment variables suitable for the SDK under test and for the test materials being used (i.e., export BUILD_LIST=functional/buildAndPackage, VENDOR_TEST_REPOS=https://github.com/adoptium/temurin-build, VENDOR_TEST_BRANCHES=master, VENDOR_TEST_DIRS=/test/functional ) +1. Export environment variables suitable for the SDK under test and for the test materials being used (i.e., export BUILD_LIST=functional/buildAndPackage, this value details which test material that should be compiled. 1. make compile // fetches test material and compiles it, based on build.xml files in the test directories 1. make _extended.functional // executes the test target (can be test group, level, level.group or specific test). i.e., openjdk (all tests in openjdk group), sanity.functional (all functional tests labelled at sanity level), or in the case of smoke tests which are all tagged to belong to level=extended and group=functional, we use `_extended.functional` and because we have limited BUILD_LIST to the directory where the smoke test material lives, we will only run tests from that directory tagged as extended.functional. diff --git a/build-farm/platform-specific-configurations/alpine-linux.sh b/build-farm/platform-specific-configurations/alpine-linux.sh index 16cefc7ff..07775006b 100644 --- a/build-farm/platform-specific-configurations/alpine-linux.sh +++ b/build-farm/platform-specific-configurations/alpine-linux.sh @@ -49,7 +49,7 @@ export GNUPGHOME BOOT_JDK_VARIABLE="JDK${JDK_BOOT_VERSION}_BOOT_DIR" if [ ! -d "$(eval echo "\$$BOOT_JDK_VARIABLE")" ]; then - bootDir="$PWD/jdk-$JDK_BOOT_VERSION" + bootDir="$PWD/jdk$JDK_BOOT_VERSION" # Note we export $BOOT_JDK_VARIABLE (i.e. JDKXX_BOOT_DIR) here # instead of BOOT_JDK_VARIABLE (no '$'). export "${BOOT_JDK_VARIABLE}"="$bootDir" diff --git a/build-farm/platform-specific-configurations/linux.sh b/build-farm/platform-specific-configurations/linux.sh index 482d51f84..e51e3bc3b 100755 --- a/build-farm/platform-specific-configurations/linux.sh +++ b/build-farm/platform-specific-configurations/linux.sh @@ -51,11 +51,11 @@ function locateDragonwell8BootJDK() export "${BOOT_JDK_VARIABLE}"=/usr/lib/jvm/dragonwell8 else echo Dragonwell 8 requires a Dragonwell boot JDK - downloading one ... - mkdir -p "$PWD/jdk-8" + mkdir -p "$PWD/jdk8" # if [ "$(uname -m)" = "x86_64" ]; then - # curl -L "https://github.com/alibaba/dragonwell8/releases/download/dragonwell-8.11.12_jdk8u332-ga/Alibaba_Dragonwell_8.11.12_x64_linux.tar.gz" | tar xpzf - --strip-components=1 -C "$PWD/jdk-8" + # curl -L "https://github.com/alibaba/dragonwell8/releases/download/dragonwell-8.11.12_jdk8u332-ga/Alibaba_Dragonwell_8.11.12_x64_linux.tar.gz" | tar xpzf - --strip-components=1 -C "$PWD/jdk8" # elif [ "$(uname -m)" = "aarch64" ]; then - # curl -L "https://github.com/alibaba/dragonwell8/releases/download/dragonwell-8.8.9_jdk8u302-ga/Alibaba_Dragonwell_8.8.9_aarch64_linux.tar.gz" | tar xpzf - --strip-components=1 -C "$PWD/jdk-8" + # curl -L "https://github.com/alibaba/dragonwell8/releases/download/dragonwell-8.8.9_jdk8u302-ga/Alibaba_Dragonwell_8.8.9_aarch64_linux.tar.gz" | tar xpzf - --strip-components=1 -C "$PWD/jdk8" # else # echo "Unknown architecture $(uname -m) for building Dragonwell - cannot download boot JDK" # exit 1 @@ -86,11 +86,11 @@ function locateDragonwell8BootJDK() fi # Extract the downloaded file - tar xpzf "$TMP_FILE" --strip-components=1 -C "$PWD/jdk-8" + tar xpzf "$TMP_FILE" --strip-components=1 -C "$PWD/jdk8" # Clean up the temporary file rm "$TMP_FILE" - export "${BOOT_JDK_VARIABLE}"="$PWD/jdk-8" + export "${BOOT_JDK_VARIABLE}"="$PWD/jdk8" fi } @@ -273,16 +273,16 @@ if [ "${VARIANT}" == "${BUILD_VARIANT_DRAGONWELL}" ] && [ "$JAVA_FEATURE_VERSION fi if [ ! -d "$(eval echo "\$$BOOT_JDK_VARIABLE")" ]; then - bootDir="$PWD/jdk-$JDK_BOOT_VERSION" + bootDir="$PWD/jdk$JDK_BOOT_VERSION" # Note we export $BOOT_JDK_VARIABLE (i.e. JDKXX_BOOT_DIR) here # instead of BOOT_JDK_VARIABLE (no '$'). export "${BOOT_JDK_VARIABLE}"="$bootDir" if [ ! -x "$bootDir/bin/javac" ]; then # Set to a default location as linked in the ansible playbooks - if [ -x "/usr/lib/jvm/jdk-${JDK_BOOT_VERSION}/bin/javac" ]; then - echo "Could not use ${BOOT_JDK_VARIABLE} - using /usr/lib/jvm/jdk-${JDK_BOOT_VERSION}" + if [ -x "/usr/lib/jvm/jdk${JDK_BOOT_VERSION}/bin/javac" ]; then + echo "Could not use ${BOOT_JDK_VARIABLE} - using /usr/lib/jvm/jdk${JDK_BOOT_VERSION}" # shellcheck disable=SC2140 - export "${BOOT_JDK_VARIABLE}"="/usr/lib/jvm/jdk-${JDK_BOOT_VERSION}" + export "${BOOT_JDK_VARIABLE}"="/usr/lib/jvm/jdk${JDK_BOOT_VERSION}" elif [ "$JDK_BOOT_VERSION" -ge 8 ]; then # Adoptium has no build pre-8 downloadLinuxBootJDK "${ARCHITECTURE}" "${JDK_BOOT_VERSION}" "$bootDir" fi diff --git a/build-farm/platform-specific-configurations/mac.sh b/build-farm/platform-specific-configurations/mac.sh index 248842196..2d0f6241c 100755 --- a/build-farm/platform-specific-configurations/mac.sh +++ b/build-farm/platform-specific-configurations/mac.sh @@ -105,13 +105,18 @@ if [ "${JDK_BOOT_VERSION}" == "7" ]; then fi BOOT_JDK_VARIABLE="JDK${JDK_BOOT_VERSION}_BOOT_DIR" if [ ! -d "$(eval echo "\$$BOOT_JDK_VARIABLE")" ]; then - bootDir="$PWD/jdk-$JDK_BOOT_VERSION" + bootDir="$PWD/jdk$JDK_BOOT_VERSION" # Note we export $BOOT_JDK_VARIABLE (i.e. JDKXX_BOOT_DIR) here # instead of BOOT_JDK_VARIABLE (no '$'). export "${BOOT_JDK_VARIABLE}"="$bootDir/Contents/Home" if [ ! -x "$bootDir/Contents/Home/bin/javac" ]; then - # To support multiple vendor names we set a jdk-* symlink pointing to the actual boot JDK - if [ -x "/Library/Java/JavaVirtualMachines/jdk-${JDK_BOOT_VERSION}/Contents/Home/bin/javac" ]; then + # To support multiple vendor names we set a jdk* symlink pointing to the actual boot JDK + if [ -x "/Library/Java/JavaVirtualMachines/jdk${JDK_BOOT_VERSION}/Contents/Home/bin/javac" ]; then + echo "Could not use ${BOOT_JDK_VARIABLE} - using /Library/Java/JavaVirtualMachines/jdk${JDK_BOOT_VERSION}/Contents/Home" + export "${BOOT_JDK_VARIABLE}"="/Library/Java/JavaVirtualMachines/jdk${JDK_BOOT_VERSION}/Contents/Home" + elif [ -x "/Library/Java/JavaVirtualMachines/jdk-${JDK_BOOT_VERSION}/Contents/Home/bin/javac" ]; then + # TODO: This temporary ELIF allows us to accomodate undesired dashes that may be present + # in boot directory names (e.g. jdk-10) on Orka node images (fix pending). echo "Could not use ${BOOT_JDK_VARIABLE} - using /Library/Java/JavaVirtualMachines/jdk-${JDK_BOOT_VERSION}/Contents/Home" export "${BOOT_JDK_VARIABLE}"="/Library/Java/JavaVirtualMachines/jdk-${JDK_BOOT_VERSION}/Contents/Home" elif [ "$JDK_BOOT_VERSION" -ge 8 ]; then # Adoptium has no build pre-8 diff --git a/cyclonedx-lib/build.xml b/cyclonedx-lib/build.xml index 788c5cc5f..53a2cde4f 100644 --- a/cyclonedx-lib/build.xml +++ b/cyclonedx-lib/build.xml @@ -1,7 +1,7 @@ - - + - + + + + + - - + + - + @@ -35,21 +38,23 @@ - + + + + + - - - + + - - + @@ -65,7 +70,7 @@ - + @@ -85,41 +90,50 @@ - - + - - + - - + - - - + + - - + - - + + + + + + + + + - - + + + + + - + + + + + @@ -135,11 +149,16 @@ - + + + + + + @@ -180,9 +199,70 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -194,6 +274,7 @@ + @@ -334,7 +415,7 @@ - + @@ -354,7 +435,7 @@ - + @@ -418,7 +499,7 @@ - + @@ -430,7 +511,7 @@ - + @@ -477,19 +558,308 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - + + + + + + + + + + diff --git a/cyclonedx-lib/dependency_data/dependency_data.properties b/cyclonedx-lib/dependency_data/dependency_data.properties new file mode 100644 index 000000000..cb2d15487 --- /dev/null +++ b/cyclonedx-lib/dependency_data/dependency_data.properties @@ -0,0 +1,72 @@ +# ******************************************************************************** +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made +# available under the terms of the Apache Software License 2.0 +# which is available at https://www.apache.org/licenses/LICENSE-2.0. +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************** + +# Repositories +maven.central.repo=https://repo1.maven.org/maven2 + +# Component versions, SHAs and jar names +commons-codec.version=1.17.1 +commons-codec.sha256=f9f6cb103f2ddc3c99a9d80ada2ae7bf0685111fd6bffccb72033d1da4e6ff23 +commons-codec.jar=commons-codec-${commons-codec.version}.jar +commons-collections4.version=4.4 +commons-collections4.sha256=1df8b9430b5c8ed143d7815e403e33ef5371b2400aadbe9bda0883762e0846d1 +commons-collections4.jar=commons-collections4-${commons-collections4.version}.jar +commons-lang3.version=3.17.0 +commons-lang3.sha256=6ee731df5c8e5a2976a1ca023b6bb320ea8d3539fbe64c8a1d5cb765127c33b4 +commons-lang3.jar=commons-lang3-${commons-lang3.version}.jar +commons-io.version=2.16.1 +commons-io.sha256=f41f7baacd716896447ace9758621f62c1c6b0a91d89acee488da26fc477c84f +commons-io.jar=commons-io-${commons-io.version}.jar +cyclonedx-core-java.version=9.1.0 +cyclonedx-core-java.sha256=a911ee5e5ebdabc2b2c08d08f9c92c673c21965ee1b982f40fc166d80f1eb088 +cyclonedx-core-java.jar=cyclonedx-core-java-${cyclonedx-core-java.version}.jar +github-package-url.version=1.5.0 +github-package-url.sha256=e45551727707acc0c56ac62d56964332ea0f138d6cc3656d988b9369150f5247 +github-package-url.jar=packageurl-java-${github-package-url.version}.jar +jackson-annotations.version=2.17.2 +jackson-annotations.sha256=873a606e23507969f9bbbea939d5e19274a88775ea5a169ba7e2d795aa5156e1 +jackson-annotations.jar=jackson-annotations-${jackson-annotations.version}.jar +jackson-core.version=2.17.2 +jackson-core.sha256=721a189241dab0525d9e858e5cb604d3ecc0ede081e2de77d6f34fa5779a5b46 +jackson-core.jar=jackson-core-${jackson-core.version}.jar +jackson-databind.version=2.17.2 +jackson-databind.sha256=c04993f33c0f845342653784f14f38373d005280e6359db5f808701cfae73c0c +jackson-databind.jar=jackson-databind-${jackson-databind.version}.jar +jackson-dataformat-xml.version=2.17.2 +jackson-dataformat-xml.sha256=517add5f3848517894b319a93a7ebfc1c21737b2c17c9acccd38fea97d6adc6f +jackson-dataformat-xml.jar=jackson-dataformat-xml-${jackson-dataformat-xml.version}.jar +json-schema-validator.version=1.5.1 +json-schema-validator.sha256=de015f79d4a63d22c002bad76bb30c039cafa205465eef8770e2c6b85880ded7 +json-schema-validator.jar=json-schema-validator-${json-schema-validator.version}.jar +stax2-api.version=4.2.2 +stax2-api.sha256=a61c48d553efad78bc01fffc4ac528bebbae64cbaec170b2a5e39cf61eb51abe +stax2-api.jar=stax2-api-${stax2-api.version}.jar +woodstox-core.version=7.1.0 +woodstox-core.sha256=81266920a1cdc47306a8a2b4726c99ec89b3fbf31c2470e4f5e477d9d857ca9f +woodstox-core.jar=woodstox-core-${woodstox-core.version}.jar + +# Download URLs +commons-codec.url=${maven.central.repo}/commons-codec/commons-codec/${commons-codec.version}/${commons-codec.jar} +commons-collections4.url=${maven.central.repo}/org/apache/commons/commons-collections4/${commons-collections4.version}/${commons-collections4.jar} +commons-lang3.url=${maven.central.repo}/org/apache/commons/commons-lang3/${commons-lang3.version}/${commons-lang3.jar} +commons-io.url=${maven.central.repo}/commons-io/commons-io/${commons-io.version}/${commons-io.jar} +cyclonedx-core-java.url=${maven.central.repo}/org/cyclonedx/cyclonedx-core-java/${cyclonedx-core-java.version}/${cyclonedx-core-java.jar} +github-package-url.url=${maven.central.repo}/com/github/package-url/packageurl-java/${github-package-url.version}/${github-package-url.jar} +jackson-annotations.url=${maven.central.repo}/com/fasterxml/jackson/core/jackson-annotations/${jackson-annotations.version}/${jackson-annotations.jar} +jackson-core.url=${maven.central.repo}/com/fasterxml/jackson/core/jackson-core/${jackson-core.version}/${jackson-core.jar} +jackson-databind.url=${maven.central.repo}/com/fasterxml/jackson/core/jackson-databind/${jackson-databind.version}/${jackson-databind.jar} +jackson-dataformat-xml.url=${maven.central.repo}/com/fasterxml/jackson/dataformat/jackson-dataformat-xml/${jackson-dataformat-xml.version}/${jackson-dataformat-xml.jar} +json-schema-validator.url=${maven.central.repo}/com/networknt/json-schema-validator/${json-schema-validator.version}/${json-schema-validator.jar} +stax2-api.url=${maven.central.repo}/org/codehaus/woodstox/stax2-api/${stax2-api.version}/${stax2-api.jar} +woodstox-core.url=${maven.central.repo}/com/fasterxml/woodstox/woodstox-core/${woodstox-core.version}/${woodstox-core.jar} + diff --git a/cyclonedx-lib/dependency_data/shas/commons-codec.jar.sha256 b/cyclonedx-lib/dependency_data/shas/commons-codec.jar.sha256 deleted file mode 100644 index a5f27b9cb..000000000 --- a/cyclonedx-lib/dependency_data/shas/commons-codec.jar.sha256 +++ /dev/null @@ -1 +0,0 @@ -b3e9f6d63a790109bf0d056611fbed1cf69055826defeb9894a71369d246ed63 \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/shas/commons-io.jar.sha256 b/cyclonedx-lib/dependency_data/shas/commons-io.jar.sha256 deleted file mode 100644 index 854ca3d3e..000000000 --- a/cyclonedx-lib/dependency_data/shas/commons-io.jar.sha256 +++ /dev/null @@ -1 +0,0 @@ -961b2f6d87dbacc5d54abf45ab7a6e2495f89b75598962d8c723cea9bc210908 \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/shas/cyclonedx-core-java.jar.sha256 b/cyclonedx-lib/dependency_data/shas/cyclonedx-core-java.jar.sha256 deleted file mode 100644 index 44090c667..000000000 --- a/cyclonedx-lib/dependency_data/shas/cyclonedx-core-java.jar.sha256 +++ /dev/null @@ -1 +0,0 @@ -ecc371d12808dfe76047f87f8235665d74dd6cf8ec12c41d052715a3fd79e0b5 \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/shas/github-package-url.jar.sha256 b/cyclonedx-lib/dependency_data/shas/github-package-url.jar.sha256 deleted file mode 100644 index b9d1f60ef..000000000 --- a/cyclonedx-lib/dependency_data/shas/github-package-url.jar.sha256 +++ /dev/null @@ -1 +0,0 @@ -8e23280221afd1e6561d433dfb133252cd287167acb0eca5a991667118ff10a2 \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/shas/jackson-annotations.jar.sha256 b/cyclonedx-lib/dependency_data/shas/jackson-annotations.jar.sha256 deleted file mode 100644 index 95f7e9a2e..000000000 --- a/cyclonedx-lib/dependency_data/shas/jackson-annotations.jar.sha256 +++ /dev/null @@ -1 +0,0 @@ -2c6869d505cf60dc066734b7d50339f975bd3adc635e26a78abb71acb4473c0d \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/shas/jackson-core.jar.sha256 b/cyclonedx-lib/dependency_data/shas/jackson-core.jar.sha256 deleted file mode 100644 index c75a5db5e..000000000 --- a/cyclonedx-lib/dependency_data/shas/jackson-core.jar.sha256 +++ /dev/null @@ -1 +0,0 @@ -b5d37a77c88277b97e3593c8740925216c06df8e4172bbde058528df04ad3e7a \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/shas/jackson-databind.jar.sha256 b/cyclonedx-lib/dependency_data/shas/jackson-databind.jar.sha256 deleted file mode 100644 index 49bdf8a3a..000000000 --- a/cyclonedx-lib/dependency_data/shas/jackson-databind.jar.sha256 +++ /dev/null @@ -1 +0,0 @@ -501d3abce4d18dcc381058ec593c5b94477906bba6efbac14dae40a642f77424 \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/shas/jackson-dataformat-xml.jar.sha256 b/cyclonedx-lib/dependency_data/shas/jackson-dataformat-xml.jar.sha256 deleted file mode 100644 index bdf90ce8c..000000000 --- a/cyclonedx-lib/dependency_data/shas/jackson-dataformat-xml.jar.sha256 +++ /dev/null @@ -1 +0,0 @@ -edbda6c775a36049cf0088b111ab958cca0dc70cb9326918d6cf153cb3fa426b \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/shas/json-schema.jar.sha256 b/cyclonedx-lib/dependency_data/shas/json-schema.jar.sha256 deleted file mode 100644 index 492cd69b8..000000000 --- a/cyclonedx-lib/dependency_data/shas/json-schema.jar.sha256 +++ /dev/null @@ -1 +0,0 @@ -968991e5718520cdd7b224770f790cf2c241cddf64d10a36c21f9f8b4a15e79c \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/versions/commons-codec.jar.version b/cyclonedx-lib/dependency_data/versions/commons-codec.jar.version deleted file mode 100644 index 07fe6f6c9..000000000 --- a/cyclonedx-lib/dependency_data/versions/commons-codec.jar.version +++ /dev/null @@ -1 +0,0 @@ -1.15 \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/versions/commons-io.jar.version b/cyclonedx-lib/dependency_data/versions/commons-io.jar.version deleted file mode 100644 index ed0edc885..000000000 --- a/cyclonedx-lib/dependency_data/versions/commons-io.jar.version +++ /dev/null @@ -1 +0,0 @@ -2.11.0 \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/versions/cyclonedx-core-java.jar.version b/cyclonedx-lib/dependency_data/versions/cyclonedx-core-java.jar.version deleted file mode 100644 index 24afbc91d..000000000 --- a/cyclonedx-lib/dependency_data/versions/cyclonedx-core-java.jar.version +++ /dev/null @@ -1 +0,0 @@ -8.0.3 \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/versions/github-package-url.jar.version b/cyclonedx-lib/dependency_data/versions/github-package-url.jar.version deleted file mode 100644 index 13175fdc4..000000000 --- a/cyclonedx-lib/dependency_data/versions/github-package-url.jar.version +++ /dev/null @@ -1 +0,0 @@ -1.4.1 \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/versions/jackson-annotations.jar.version b/cyclonedx-lib/dependency_data/versions/jackson-annotations.jar.version deleted file mode 100644 index fb71e071a..000000000 --- a/cyclonedx-lib/dependency_data/versions/jackson-annotations.jar.version +++ /dev/null @@ -1 +0,0 @@ -2.14.2 \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/versions/jackson-core.jar.version b/cyclonedx-lib/dependency_data/versions/jackson-core.jar.version deleted file mode 100644 index fb71e071a..000000000 --- a/cyclonedx-lib/dependency_data/versions/jackson-core.jar.version +++ /dev/null @@ -1 +0,0 @@ -2.14.2 \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/versions/jackson-databind.jar.version b/cyclonedx-lib/dependency_data/versions/jackson-databind.jar.version deleted file mode 100644 index fb71e071a..000000000 --- a/cyclonedx-lib/dependency_data/versions/jackson-databind.jar.version +++ /dev/null @@ -1 +0,0 @@ -2.14.2 \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/versions/jackson-dataformat-xml.jar.version b/cyclonedx-lib/dependency_data/versions/jackson-dataformat-xml.jar.version deleted file mode 100644 index fb71e071a..000000000 --- a/cyclonedx-lib/dependency_data/versions/jackson-dataformat-xml.jar.version +++ /dev/null @@ -1 +0,0 @@ -2.14.2 \ No newline at end of file diff --git a/cyclonedx-lib/dependency_data/versions/json-schema.jar.version b/cyclonedx-lib/dependency_data/versions/json-schema.jar.version deleted file mode 100644 index 0c59751cf..000000000 --- a/cyclonedx-lib/dependency_data/versions/json-schema.jar.version +++ /dev/null @@ -1 +0,0 @@ -1.0.77 \ No newline at end of file diff --git a/cyclonedx-lib/sign_src/TemurinSignSBOM.java b/cyclonedx-lib/sign_src/TemurinSignSBOM.java index afe584d10..d9d35f105 100644 --- a/cyclonedx-lib/sign_src/TemurinSignSBOM.java +++ b/cyclonedx-lib/sign_src/TemurinSignSBOM.java @@ -1,6 +1,6 @@ /* * ******************************************************************************** - * Copyright (c) 2023 Contributors to the Eclipse Foundation + * Copyright (c) 2023, 2024 Contributors to the Eclipse Foundation * * See the NOTICE file(s) with this work for additional * information regarding copyright ownership. @@ -15,11 +15,11 @@ package temurin.sbom; -import org.cyclonedx.BomGeneratorFactory; -import org.cyclonedx.CycloneDxSchema; +import org.cyclonedx.exception.GeneratorException; import org.cyclonedx.generators.json.BomJsonGenerator; import org.cyclonedx.model.Bom; import org.cyclonedx.parsers.JsonParser; +import org.cyclonedx.Version; import org.webpki.json.JSONAsymKeySigner; import org.webpki.json.JSONObjectReader; @@ -60,7 +60,7 @@ private TemurinSignSBOM() { * @param args Arguments for sbom operation. */ public static void main(final String[] args) { - String cmd = null; + String cmd = ""; String privateKeyFile = null; String publicKeyFile = null; String fileName = null; @@ -95,6 +95,8 @@ public static void main(final String[] args) { } else if (cmd.equals("verifySignature")) { success = verifySignature(fileName, publicKeyFile); // set success to the result of verifySignature System.out.println("Signature verification result: " + (success ? "Valid" : "Invalid")); + } else { + System.out.println("Please enter a command."); } // Set success to true only when the operation is completed successfully. @@ -113,7 +115,13 @@ static Bom signSBOM(final String jsonFile, final String pemFile) { if (bom == null) { return null; } - String sbomDataToSign = generateBomJson(bom); + String sbomDataToSign; + try { + sbomDataToSign = generateBomJson(bom); + } catch (GeneratorException e) { + LOGGER.log(Level.SEVERE, "Exception generating BOM", e); + return null; + } // Read the private key KeyPair signingKey = PEMDecoder.getKeyPair(Files.readAllBytes(Paths.get(pemFile))); @@ -132,15 +140,22 @@ static Bom signSBOM(final String jsonFile, final String pemFile) { } } - static String generateBomJson(final Bom bom) { - BomJsonGenerator bomGen = BomGeneratorFactory.createJson(CycloneDxSchema.Version.VERSION_14, bom); + static String generateBomJson(final Bom bom) throws GeneratorException { + BomJsonGenerator bomGen = new BomJsonGenerator(bom, Version.VERSION_16); String json = bomGen.toJsonString(); return json; } static boolean writeJSONfile(final Bom bom, final String fileName) { // Creates testJson.json file - String json = generateBomJson(bom); + String json; + try { + json = generateBomJson(bom); + } catch (GeneratorException e) { + LOGGER.log(Level.SEVERE, "Exception generating BOM", e); + return false; + } + try (FileWriter file = new FileWriter(fileName)) { file.write(json); return true; @@ -164,7 +179,13 @@ static boolean verifySignature(final String jsonFile, final String publicKeyFile try { // Read the JSON file to be verified Bom bom = readJSONfile(jsonFile); - String signedSbomData = generateBomJson(bom); + String signedSbomData; + try { + signedSbomData = generateBomJson(bom); + } catch (GeneratorException e) { + LOGGER.log(Level.SEVERE, "Exception generating BOM", e); + return false; + } // Parse JSON JSONObjectReader reader = JSONParser.parse(signedSbomData); diff --git a/cyclonedx-lib/src/temurin/sbom/TemurinGenCDXA.java b/cyclonedx-lib/src/temurin/sbom/TemurinGenCDXA.java new file mode 100644 index 000000000..a1f02c0f1 --- /dev/null +++ b/cyclonedx-lib/src/temurin/sbom/TemurinGenCDXA.java @@ -0,0 +1,336 @@ +/* + * ******************************************************************************** + * Copyright (c) 2024 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made + * available under the terms of the Apache Software License 2.0 + * which is available at https://www.apache.org/licenses/LICENSE-2.0. + * + * SPDX-License-Identifier: Apache-2.0 + * ******************************************************************************** + */ + +package temurin.sbom; + +import org.cyclonedx.exception.GeneratorException; +import org.cyclonedx.generators.json.BomJsonGenerator; +import org.cyclonedx.generators.xml.BomXmlGenerator; +import org.cyclonedx.model.Bom; +import org.cyclonedx.model.Component; +import org.cyclonedx.model.ExternalReference; +import org.cyclonedx.model.Hash; +import org.cyclonedx.model.OrganizationalEntity; +import org.cyclonedx.model.attestation.Declarations; +import org.cyclonedx.model.attestation.Assessor; +import org.cyclonedx.model.attestation.Attestation; +import org.cyclonedx.model.attestation.AttestationMap; +import org.cyclonedx.model.attestation.Claim; +import org.cyclonedx.model.attestation.affirmation.Affirmation; +import org.cyclonedx.model.attestation.affirmation.Signatory; +import org.cyclonedx.model.attestation.Targets; +import org.cyclonedx.parsers.JsonParser; +import org.cyclonedx.parsers.XmlParser; +import org.cyclonedx.Version; +import java.io.FileReader; +import java.io.FileWriter; +import java.util.List; +import java.util.LinkedList; +import java.util.UUID; + +/** + * Command line tool to construct a CycloneDX CDXA. + */ +public final class TemurinGenCDXA { + + private static boolean verbose = false; + private static boolean useJson = false; + + // Valid predicates + enum CDXAPredicate { + VERIFIED_REPRODUCIBLE_BUILD + }; + + private TemurinGenCDXA() { + } + + /** + * Main entry. + * @param args Arguments for operation. + */ + public static void main(final String[] args) { + String cmd = ""; + String fileName = null; + String attestingOrgName = null; + String predicate = null; + String targetName = null; + String targetUrl = null; + String targetHash = null; + String affirmationStmt = null; + String affirmationWebsite = null; + boolean thirdParty = true; + + for (int i = 0; i < args.length; i++) { + if (args[i].equals("--jsonFile")) { + fileName = args[++i]; + useJson = true; + } else if (args[i].equals("--xmlFile")) { + fileName = args[++i]; + useJson = false; + } else if (args[i].equals("--attesting-org-name")) { + attestingOrgName = args[++i]; + } else if (args[i].equals("--predicate")) { + predicate = args[++i]; + } else if (args[i].equals("--target-name")) { + targetName = args[++i]; + } else if (args[i].equals("--target-url")) { + targetUrl = args[++i]; + } else if (args[i].equals("--target-sha256-hash")) { + targetHash = args[++i]; + } else if (args[i].equals("--affirmation-stmt")) { + affirmationStmt = args[++i]; + } else if (args[i].equals("--affirmation-website")) { + affirmationWebsite = args[++i]; + } else if (args[i].equals("--not-third-party")) { + thirdParty = false; + } else if (args[i].equals("--createNewCDXA")) { + cmd = "createCDXA"; + } else if (args[i].equals("--verbose")) { + verbose = true; + } + } + + switch (cmd) { + case "createCDXA": // Create a new CDXA json file + Bom bom = createCdxa(fileName, attestingOrgName, predicate, targetName, targetUrl, targetHash, affirmationStmt, affirmationWebsite, thirdParty); + if (bom != null) { + writeFile(bom, fileName); + } else { + System.exit(1); + } + break; + + default: + System.out.println("Please enter a command."); + System.exit(1); + } + } + + static Bom createCdxa(final String fileName, final String attestingOrgName, final String predicate, + final String targetName, final String targetUrl, final String targetHash, + final String affirmationStmt, final String affirmationWebsite, final boolean thirdParty) { + // Validate inputs + boolean validInput = true; + if (fileName == null) { + System.out.println("--xmlFile|--jsonFile not specified"); + validInput = false; + } + if (attestingOrgName == null) { + System.out.println("--attesting-org-name not specified"); + validInput = false; + } + if (predicate == null) { + System.out.println("--predicate not specified"); validInput = false; + } else { + boolean validPred = false; + for (CDXAPredicate validPredicate : CDXAPredicate.values()) { + if (validPredicate.name().equals(predicate)) { + validPred = true; + break; + } + } + if (!validPred) { + System.out.println("--predicate " + predicate + " not a valid value"); + validInput = false; + } + } + if (targetName == null) { + System.out.println("--target-name not specified"); + validInput = false; + } + if (targetUrl == null) { + System.out.println("--target-url not specified"); + validInput = false; + } + if (targetHash == null) { + System.out.println("--target-sha256-hash not specified"); + validInput = false; + } + if (affirmationStmt == null) { + System.out.println("--affirmation-stmt not specified"); + validInput = false; + } + if (affirmationWebsite == null) { + System.out.println("--affirmation-website not specified"); + validInput = false; + } + if (!validInput) { + return null; + } + + Declarations declarations = new Declarations(); + Assessor assessor = new Assessor(); + Claim claim = new Claim(); + Targets targets = new Targets(); + Affirmation affirmation = new Affirmation(); + Signatory signatory = new Signatory(); + Attestation attestation = new Attestation(); + AttestationMap attestationMap = new AttestationMap(); + + final String targetJdkBomRef = "target-jdk-1"; + final String assessorBomRef = "assessor-1"; + final String claimBomRef = "claim-1"; + + // External reference to the target JDK + ExternalReference extRef = new ExternalReference(); + Hash hash1 = new Hash(Hash.Algorithm.SHA_256, targetHash); + extRef.addHash(hash1); + extRef.setUrl(targetUrl); + extRef.setType(ExternalReference.Type.DISTRIBUTION); + + // Target JDK Component + Component targetJDK = new Component(); + targetJDK.setType(Component.Type.APPLICATION); + targetJDK.setName(targetName); + targetJDK.addExternalReference(extRef); + targetJDK.setBomRef(targetJdkBomRef); + List components = new LinkedList(); + components.add(targetJDK); + targets.setComponents(components); + declarations.setTargets(targets); + + // Assessor + assessor.setThirdParty(thirdParty); + OrganizationalEntity org = new OrganizationalEntity(); + org.setName(attestingOrgName); + assessor.setOrganization(org); + assessor.setBomRef(assessorBomRef); + List assessors = new LinkedList(); + assessors.add(assessor); + declarations.setAssessors(assessors); + + // Claim + claim.setPredicate(predicate); + claim.setTarget(targetJDK.getBomRef()); + claim.setBomRef(claimBomRef); + List claims = new LinkedList(); + claims.add(claim); + declarations.setClaims(claims); + + // Affirmation + affirmation.setStatement(affirmationStmt); + signatory.setOrganization(org); + ExternalReference orgExtRef = new ExternalReference(); + orgExtRef.setUrl(affirmationWebsite); + orgExtRef.setType(ExternalReference.Type.WEBSITE); + signatory.setExternalReference(orgExtRef); + List signatories = new LinkedList(); + signatories.add(signatory); + affirmation.setSignatories(signatories); + declarations.setAffirmation(affirmation); + + // Construct the Attestation + attestation.setSummary("Eclipse Temurin Attestation"); + attestation.setAssessor(assessor.getBomRef()); + List claimsList = new LinkedList(); + claimsList.add(claim.getBomRef()); + attestationMap.setClaims(claimsList); + List attestationMaps = new LinkedList(); + attestationMaps.add(attestationMap); + attestation.setMap(attestationMaps); + List attestations = new LinkedList(); + attestations.add(attestation); + declarations.setAttestations(attestations); + + // Create CDXA Bom + Bom cdxa = new Bom(); + cdxa.setSerialNumber("urn:uuid:" + UUID.randomUUID()); + cdxa.setDeclarations(declarations); + + return cdxa; + } + + static String generateBomJson(final Bom bom) throws GeneratorException { + // Use schema v16: https://cyclonedx.org/schema/bom-1.6.schema.json + BomJsonGenerator bomGen = new BomJsonGenerator(bom, Version.VERSION_16); + String json = bomGen.toJsonString(); + return json; + } + + static String generateBomXml(final Bom bom) throws GeneratorException { + BomXmlGenerator bomGen = new BomXmlGenerator(bom, Version.VERSION_16); + String xml = bomGen.toXmlString(); + return xml; + } + + // Writes the BOM object to the specified type of file + static void writeFile(final Bom bom, final String fileName) { + if (useJson) { + writeJSONfile(bom, fileName); + } else { + writeXMLfile(bom, fileName); + } + } + + // Writes the BOM object to the specified JSON file. + static void writeJSONfile(final Bom bom, final String fileName) { + FileWriter file; + try { + String json = generateBomJson(bom); + + file = new FileWriter(fileName); + file.write(json); + file.close(); + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } + } + + // Writes the BOM object to the specified XML file. + static void writeXMLfile(final Bom bom, final String fileName) { + FileWriter file; + try { + String xml = generateBomXml(bom); + + file = new FileWriter(fileName); + file.write(xml); + file.close(); + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } + } + + // Returns a parsed BOM object from the specified file. + static Bom readJSONfile(final String fileName) { + Bom bom = null; + try { + FileReader reader = new FileReader(fileName); + JsonParser parser = new JsonParser(); + bom = parser.parse(reader); + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } finally { + return bom; + } + } + + // Returns a parsed BOM object from the specified file. + static Bom readXMLfile(final String fileName) { + Bom bom = null; + try { + FileReader reader = new FileReader(fileName); + XmlParser parser = new XmlParser(); + bom = parser.parse(reader); + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); + } finally { + return bom; + } + } +} diff --git a/cyclonedx-lib/src/temurin/sbom/TemurinGenSBOM.java b/cyclonedx-lib/src/temurin/sbom/TemurinGenSBOM.java index 36a60cc0a..4533a9c37 100644 --- a/cyclonedx-lib/src/temurin/sbom/TemurinGenSBOM.java +++ b/cyclonedx-lib/src/temurin/sbom/TemurinGenSBOM.java @@ -1,6 +1,6 @@ /* * ******************************************************************************** - * Copyright (c) 2021 Contributors to the Eclipse Foundation + * Copyright (c) 2021, 2024 Contributors to the Eclipse Foundation * * See the NOTICE file(s) with this work for additional * information regarding copyright ownership. @@ -15,12 +15,11 @@ package temurin.sbom; -import org.cyclonedx.BomGeneratorFactory; -import org.cyclonedx.CycloneDxSchema; +import org.cyclonedx.exception.GeneratorException; import org.cyclonedx.generators.json.BomJsonGenerator; +import org.cyclonedx.generators.xml.BomXmlGenerator; import org.cyclonedx.model.Bom; import org.cyclonedx.model.Component; -import org.cyclonedx.model.ExternalReference; import org.cyclonedx.model.formulation.Formula; import org.cyclonedx.model.Hash; import org.cyclonedx.model.Metadata; @@ -29,11 +28,14 @@ import org.cyclonedx.model.Property; import org.cyclonedx.model.Tool; import org.cyclonedx.parsers.JsonParser; +import org.cyclonedx.parsers.XmlParser; +import org.cyclonedx.Version; import java.io.FileReader; import java.io.FileWriter; import java.util.Collections; import java.util.List; import java.util.LinkedList; +import java.util.UUID; /** * Command line tool to construct a CycloneDX SBOM. @@ -41,6 +43,7 @@ public final class TemurinGenSBOM { private static boolean verbose = false; + private static boolean useJson = false; private TemurinGenSBOM() { } @@ -50,7 +53,7 @@ private TemurinGenSBOM() { * @param args Arguments for sbom operation. */ public static void main(final String[] args) { - String cmd = null; + String cmd = ""; String comment = null; String compName = null; String formulaName = null; @@ -67,6 +70,10 @@ public static void main(final String[] args) { for (int i = 0; i < args.length; i++) { if (args[i].equals("--jsonFile")) { fileName = args[++i]; + useJson = true; + } else if (args[i].equals("--xmlFile")) { + fileName = args[++i]; + useJson = false; } else if (args[i].equals("--version")) { version = args[++i]; } else if (args[i].equals("--name")) { @@ -120,69 +127,60 @@ public static void main(final String[] args) { } } switch (cmd) { - case "createNewSBOM": // Creates JSON file + case "createNewSBOM": // Creates new SBOM Bom bom = createBom(); - writeJSONfile(bom, fileName); + writeFile(bom, fileName); break; case "addMetadata": // Adds Metadata --> name bom = addMetadata(fileName); - writeJSONfile(bom, fileName); + writeFile(bom, fileName); break; case "addMetadataComponent": // Adds Metadata --> Component --> name bom = addMetadataComponent(fileName, name, type, version, description); - writeJSONfile(bom, fileName); + writeFile(bom, fileName); break; case "addMetadataProperty": // Adds MetaData --> Property --> name-value: bom = addMetadataProperty(fileName, name, value); - writeJSONfile(bom, fileName); + writeFile(bom, fileName); break; case "addFormulation": // Adds Formulation --> name bom = addFormulation(fileName, formulaName); - writeJSONfile(bom, fileName); + writeFile(bom, fileName); break; case "addFormulationComp": // Adds Formulation --> Component--> name bom = addFormulationComp(fileName, formulaName, name, type); - writeJSONfile(bom, fileName); + writeFile(bom, fileName); break; case "addFormulationCompProp": // Adds Formulation --> Component -> name-value: bom = addFormulationCompProp(fileName, formulaName, compName, name, value); - writeJSONfile(bom, fileName); + writeFile(bom, fileName); break; case "addMetadataTools": bom = addMetadataTools(fileName, tool, version); - writeJSONfile(bom, fileName); + writeFile(bom, fileName); break; case "addComponent": // Adds Components --> Component --> name bom = addComponent(fileName, compName, version, description); - writeJSONfile(bom, fileName); + writeFile(bom, fileName); break; case "addComponentHash": // Adds Components --> Component --> hash bom = addComponentHash(fileName, compName, hash); - writeJSONfile(bom, fileName); + writeFile(bom, fileName); break; case "addComponentProp": // Adds Components --> Component --> name-value pairs bom = addComponentProperty(fileName, compName, name, value); - writeJSONfile(bom, fileName); - break; - - case "addExternalReference": // Adds external Reference - bom = addExternalReference(fileName, hash, url, comment); - writeJSONfile(bom, fileName); + writeFile(bom, fileName); break; - case "addComponentExternalReference": // Adds external Reference to component - bom = addComponentExternalReference(fileName, hash, url, comment); - writeJSONfile(bom, fileName); - break; default: System.out.println("Please enter a command."); } @@ -194,12 +192,13 @@ public static void main(final String[] args) { */ static Bom createBom() { Bom bom = new Bom(); + bom.setSerialNumber("urn:uuid:" + UUID.randomUUID()); return bom; } // Method to store Metadata --> name. static Bom addMetadata(final String fileName) { - Bom bom = readJSONfile(fileName); + Bom bom = readFile(fileName); Metadata meta = new Metadata(); OrganizationalEntity org = new OrganizationalEntity(); org.setName("Eclipse Foundation"); @@ -213,7 +212,7 @@ static Bom addMetadata(final String fileName) { } static Bom addMetadataComponent(final String fileName, final String name, final String type, final String version, final String description) { - Bom bom = readJSONfile(fileName); + Bom bom = readFile(fileName); Metadata meta = new Metadata(); Component comp = new Component(); Component.Type compType = Component.Type.FRAMEWORK; @@ -235,7 +234,7 @@ static Bom addMetadataComponent(final String fileName, final String name, final // Method to store Metadata --> Properties List --> name-values. static Bom addMetadataProperty(final String fileName, final String name, final String value) { - Bom bom = readJSONfile(fileName); + Bom bom = readFile(fileName); Metadata meta = new Metadata(); Property prop1 = new Property(); meta = bom.getMetadata(); @@ -247,7 +246,7 @@ static Bom addMetadataProperty(final String fileName, final String name, final S } static Bom addMetadataTools(final String fileName, final String toolName, final String version) { - Bom bom = readJSONfile(fileName); + Bom bom = readFile(fileName); Metadata meta = new Metadata(); Tool tool = new Tool(); meta = bom.getMetadata(); @@ -260,7 +259,7 @@ static Bom addMetadataTools(final String fileName, final String toolName, final // Method to store Component --> name & single name-value pair. static Bom addComponent(final String fileName, final String compName, final String version, final String description) { - Bom bom = readJSONfile(fileName); + Bom bom = readFile(fileName); Component comp = new Component(); comp.setName(compName); comp.setVersion(version); @@ -274,7 +273,7 @@ static Bom addComponent(final String fileName, final String compName, final Stri } static Bom addComponentHash(final String fileName, final String compName, final String hash) { - Bom bom = readJSONfile(fileName); + Bom bom = readFile(fileName); List componentArrayList = bom.getComponents(); for (Component item : componentArrayList) { if (item.getName().equals(compName)) { @@ -287,7 +286,7 @@ static Bom addComponentHash(final String fileName, final String compName, final // Method to add Component --> Property --> name-value pairs. static Bom addComponentProperty(final String fileName, final String compName, final String name, final String value) { - Bom bom = readJSONfile(fileName); + Bom bom = readFile(fileName); List componentArrayList = bom.getComponents(); for (Component item : componentArrayList) { if (item.getName().equals(compName)) { @@ -300,36 +299,8 @@ static Bom addComponentProperty(final String fileName, final String compName, fi return bom; } - // Method to store externalReferences: dependency_version_alsa. - static Bom addExternalReference(final String fileName, final String hash, final String url, final String comment) { - Bom bom = readJSONfile(fileName); - ExternalReference extRef = new ExternalReference(); - Hash hash1 = new Hash(Hash.Algorithm.SHA3_256, hash); - extRef.setType(ExternalReference.Type.BUILD_SYSTEM); //required - extRef.setUrl(url); // required must be a valid URL with protocol - extRef.setComment(comment); - extRef.addHash(hash1); - bom.addExternalReference(extRef); - return bom; - } - - // Method to store externalReferences to store: openjdk_source. - static Bom addComponentExternalReference(final String fileName, final String hash, final String url, final String comment) { - Bom bom = readJSONfile(fileName); - ExternalReference extRef = new ExternalReference(); - Hash hash1 = new Hash(Hash.Algorithm.SHA3_256, hash); - Component comp = new Component(); - extRef.addHash(hash1); - extRef.setUrl(url); - extRef.setComment(comment); //"openjdk_source" - extRef.setType(ExternalReference.Type.BUILD_SYSTEM); - comp.addExternalReference(extRef); - bom.addComponent(comp); - return bom; - } - static Bom addFormulation(final String fileName, final String name) { - Bom bom = readJSONfile(fileName); + Bom bom = readFile(fileName); List formulation = bom.getFormulation(); if (formulation == null) { formulation = new LinkedList(); @@ -343,7 +314,7 @@ static Bom addFormulation(final String fileName, final String name) { } static Bom addFormulationComp(final String fileName, final String formulaName, final String name, final String type) { - Bom bom = readJSONfile(fileName); + Bom bom = readFile(fileName); if (formulaName == null) { System.out.println("addFormulationComp: formulaName is null"); return bom; @@ -376,7 +347,7 @@ static Bom addFormulationComp(final String fileName, final String formulaName, f } static Bom addFormulationCompProp(final String fileName, final String formulaName, final String componentName, final String name, final String value) { - Bom bom = readJSONfile(fileName); + Bom bom = readFile(fileName); boolean foundFormula = false; boolean foundComponent = false; List formulation = bom.getFormulation(); @@ -410,23 +381,66 @@ static Bom addFormulationCompProp(final String fileName, final String formulaNam return bom; } - static String generateBomJson(final Bom bom) { - // Use schema v15: https://cyclonedx.org/schema/bom-1.5.schema.json - BomJsonGenerator bomGen = BomGeneratorFactory.createJson(CycloneDxSchema.Version.VERSION_15, bom); + static String generateBomJson(final Bom bom) throws GeneratorException { + // Use schema v16: https://cyclonedx.org/schema/bom-1.6.schema.json + BomJsonGenerator bomGen = new BomJsonGenerator(bom, Version.VERSION_16); String json = bomGen.toJsonString(); return json; } + static String generateBomXml(final Bom bom) throws GeneratorException { + BomXmlGenerator bomGen = new BomXmlGenerator(bom, Version.VERSION_16); + String xml = bomGen.toXmlString(); + return xml; + } + + // Writes the BOM object to the specified type of file + static void writeFile(final Bom bom, final String fileName) { + if (useJson) { + writeJSONfile(bom, fileName); + } else { + writeXMLfile(bom, fileName); + } + } + + // Read the BOM object from the specified type of file + static Bom readFile(final String fileName) { + Bom bom; + if (useJson) { + bom = readJSONfile(fileName); + } else { + bom = readXMLfile(fileName); + } + return bom; + } + // Writes the BOM object to the specified file. static void writeJSONfile(final Bom bom, final String fileName) { FileWriter file; - String json = generateBomJson(bom); try { + String json = generateBomJson(bom); + file = new FileWriter(fileName); file.write(json); file.close(); } catch (Exception e) { e.printStackTrace(); + System.exit(1); + } + } + + // Writes the BOM object to the specified XML file. + static void writeXMLfile(final Bom bom, final String fileName) { + FileWriter file; + try { + String xml = generateBomXml(bom); + + file = new FileWriter(fileName); + file.write(xml); + file.close(); + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); } } @@ -439,6 +453,22 @@ static Bom readJSONfile(final String fileName) { bom = parser.parse(reader); } catch (Exception e) { e.printStackTrace(); + System.exit(1); + } finally { + return bom; + } + } + + // Returns a parsed BOM object from the specified file. + static Bom readXMLfile(final String fileName) { + Bom bom = null; + try { + FileReader reader = new FileReader(fileName); + XmlParser parser = new XmlParser(); + bom = parser.parse(reader); + } catch (Exception e) { + e.printStackTrace(); + System.exit(1); } finally { return bom; } diff --git a/sbin/build.sh b/sbin/build.sh index a51c4947c..46dded89e 100755 --- a/sbin/build.sh +++ b/sbin/build.sh @@ -1,7 +1,7 @@ #!/bin/bash # shellcheck disable=SC2155,SC2153,SC2038,SC1091,SC2116,SC2086 # ******************************************************************************** -# Copyright (c) 2017 Contributors to the Eclipse Foundation +# Copyright (c) 2017, 2024 Contributors to the Eclipse Foundation # # See the NOTICE file(s) with this work for additional # information regarding copyright ownership. @@ -79,12 +79,18 @@ configureDevKitConfigureParameter() { if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then # Windows DevKit, currently only Redist DLLs + # Default to build architecture unless target ARCHITECTURE variable is set + local target_arch="${BUILD_CONFIG[OS_ARCHITECTURE]}" + if [ ${ARCHITECTURE+x} ] && [ -n "${ARCHITECTURE}" ]; then + target_arch="${ARCHITECTURE}" + fi + echo "Target architecture for Windows devkit: ${target_arch}" + # This is TARGET Architecture for the Redist DLLs to use - # ARCHITECTURE is set to the "target" architecture by caller, or defaults to build architecture if not set local dll_arch - if [[ "${ARCHITECTURE}" == "x86-32" ]]; then + if [[ "${target_arch}" == "x86-32" ]]; then dll_arch="x86" - elif [[ "${ARCHITECTURE}" == "aarch64" ]]; then + elif [[ "${target_arch}" == "aarch64" ]]; then dll_arch="arm64" else dll_arch="x64" @@ -168,6 +174,17 @@ configureMacOSCodesignParameter() { fi } +# JDK 24+ includes JEP 493 which allows for the JDK to enable +# linking from the run-time image (instead of only from JMODs). Enable +# this option. This has the effect, that no 'jmods' directory will be +# produced in the resulting build. Thus, the tarball and, especially the +# extracted tarball will be smaller in terms of disk space size. +configureLinkableRuntimeParameter() { + if [[ "${BUILD_CONFIG[OPENJDK_FEATURE_NUMBER]}" -ge 24 ]]; then + addConfigureArg "--enable-linkable-runtime" "" + fi +} + # Get the OpenJDK update version and build version getOpenJDKUpdateAndBuildVersion() { cd "${BUILD_CONFIG[WORKSPACE_DIR]}/${BUILD_CONFIG[WORKING_DIR]}" @@ -568,6 +585,7 @@ configureCommandParameters() { configureVersionStringParameter configureBootJDKConfigureParameter configureDevKitConfigureParameter + configureLinkableRuntimeParameter configureShenandoahBuildParameter configureMacOSCodesignParameter configureDebugParameters @@ -881,8 +899,28 @@ buildCyclonedxLib() { else ANTBUILDFILE="${CYCLONEDB_DIR}/build.xml" fi + + # Has the user specified their own local cache for the dependency jars? + local localJarCacheOption="" + if [[ -n "${BUILD_CONFIG[LOCAL_DEPENDENCY_CACHE_DIR]}" ]]; then + localJarCacheOption="-Dlocal.deps.cache.dir=${BUILD_CONFIG[LOCAL_DEPENDENCY_CACHE_DIR]}" + else + # Select a suitable default location that users may use + if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then + # Windows + localJarCacheOption="-Dlocal.deps.cache.dir=c:/dependency_cache" + elif [[ "${BUILD_CONFIG[OS_KERNEL_NAME]}" == "darwin" ]]; then + # MacOS + localJarCacheOption="-Dlocal.deps.cache.dir=${HOME}/dependency_cache" + else + # Assume unix based path + localJarCacheOption="-Dlocal.deps.cache.dir=/usr/local/dependency_cache" + fi + fi + echo "Using CycloneDX local jar cache build option: ${localJarCacheOption}" + JAVA_HOME=${javaHome} ant -f "${ANTBUILDFILE}" clean - JAVA_HOME=${javaHome} ant -f "${ANTBUILDFILE}" build + JAVA_HOME=${javaHome} ant -f "${ANTBUILDFILE}" build "${localJarCacheOption}" } # get the classpath to run the CycloneDX java app TemurinGenSBOM @@ -890,14 +928,14 @@ getCyclonedxClasspath() { local CYCLONEDB_JAR_DIR="${CYCLONEDB_DIR}/build/jar" - local classpath="${CYCLONEDB_JAR_DIR}/temurin-gen-sbom.jar:${CYCLONEDB_JAR_DIR}/cyclonedx-core-java.jar:${CYCLONEDB_JAR_DIR}/jackson-core.jar:${CYCLONEDB_JAR_DIR}/jackson-dataformat-xml.jar:${CYCLONEDB_JAR_DIR}/jackson-databind.jar:${CYCLONEDB_JAR_DIR}/jackson-annotations.jar:${CYCLONEDB_JAR_DIR}/json-schema.jar:${CYCLONEDB_JAR_DIR}/commons-codec.jar:${CYCLONEDB_JAR_DIR}/commons-io.jar:${CYCLONEDB_JAR_DIR}/github-package-url.jar" + local classpath="${CYCLONEDB_JAR_DIR}/temurin-gen-sbom.jar:${CYCLONEDB_JAR_DIR}/cyclonedx-core-java.jar:${CYCLONEDB_JAR_DIR}/jackson-core.jar:${CYCLONEDB_JAR_DIR}/jackson-dataformat-xml.jar:${CYCLONEDB_JAR_DIR}/jackson-databind.jar:${CYCLONEDB_JAR_DIR}/jackson-annotations.jar:${CYCLONEDB_JAR_DIR}/json-schema-validator.jar:${CYCLONEDB_JAR_DIR}/commons-codec.jar:${CYCLONEDB_JAR_DIR}/commons-io.jar:${CYCLONEDB_JAR_DIR}/github-package-url.jar:${CYCLONEDB_JAR_DIR}/commons-collections4.jar" if [[ "$OSTYPE" == "cygwin" ]] || [[ "$OSTYPE" == "msys" ]]; then classpath="" for jarfile in "${CYCLONEDB_JAR_DIR}/temurin-gen-sbom.jar" "${CYCLONEDB_JAR_DIR}/cyclonedx-core-java.jar" \ "${CYCLONEDB_JAR_DIR}/jackson-core.jar" "${CYCLONEDB_JAR_DIR}/jackson-dataformat-xml.jar" \ "${CYCLONEDB_JAR_DIR}/jackson-databind.jar" "${CYCLONEDB_JAR_DIR}/jackson-annotations.jar" \ - "${CYCLONEDB_JAR_DIR}/json-schema.jar" "${CYCLONEDB_JAR_DIR}/commons-codec.jar" "${CYCLONEDB_JAR_DIR}/commons-io.jar" \ - "${CYCLONEDB_JAR_DIR}/github-package-url.jar" ; + "${CYCLONEDB_JAR_DIR}/json-schema-validator.jar" "${CYCLONEDB_JAR_DIR}/commons-codec.jar" "${CYCLONEDB_JAR_DIR}/commons-io.jar" \ + "${CYCLONEDB_JAR_DIR}/github-package-url.jar" "${CYCLONEDB_JAR_DIR}/commons-collections4.jar"; do classpath+=$(cygpath -w "${jarfile}")";" done @@ -1205,21 +1243,20 @@ addCycloneDXVersions() { else # Should we do something special if the sha256sum fails? for JAR in "${CYCLONEDB_DIR}/build/jar"/*.jar; do - JarName=$(basename "$JAR") + JarName=$(basename "$JAR" | cut -d'.' -f1) if [ "$(uname)" = "Darwin" ]; then JarSha=$(shasum -a 256 "$JAR" | cut -d' ' -f1) else JarSha=$(sha256sum "$JAR" | cut -d' ' -f1) fi - addSBOMFormulationComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "CycloneDX" "CycloneDX jar SHAs" "${JarName}" "${JarSha}" + addSBOMFormulationComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "CycloneDX" "CycloneDX jar SHAs" "${JarName}.jar" "${JarSha}" # Now the jar's SHA has been added, we add the version string. - JarVersionFile="$(joinPath ${CYCLONEDB_DIR} dependency_data versions ${JarName}.version)" - if [ -f "${JarVersionFile}" ]; then - JarVersionString=$(cat "${JarVersionFile}") - addSBOMFormulationComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "CycloneDX" "CycloneDX jar versions" "${JarName}" "${JarVersionString}" - elif [ "${JarName}" != "temurin-gen-sbom.jar" ]; then - echo "ERROR: Cannot find jar version file for SBOM creation dependency ${JarName}." - echo "ERROR: Expected location: ${JarVersionFile}" + JarDepsFile="$(joinPath ${CYCLONEDB_DIR} dependency_data/dependency_data.properties)" + JarVersionString=$(grep "${JarName}\.version=" "${JarDepsFile}" | cut -d'=' -f2) + if [ -n "${JarVersionString}" ]; then + addSBOMFormulationComponentProperty "${javaHome}" "${classpath}" "${sbomJson}" "CycloneDX" "CycloneDX jar versions" "${JarName}.jar" "${JarVersionString}" + elif [ "${JarName}" != "temurin-gen-sbom" ]; then + echo "ERROR: Cannot determine jar version from ${JarDepsFile} for SBOM creation dependency ${JarName}.jar." fi done fi @@ -2069,7 +2106,10 @@ getTargetFileNameForComponent() { echo "${target_file_name}" | sed "s/-jdk/-${component}/" else # if no pattern is found, append the component name right before the extension. - echo "${target_file_name}" | sed -r "s/(.+)(\.tar\.gz|\.zip)/\1-${component}\2/" + # Stopped using -r here and split this in 2 as -r not a standard sed argument + echo "${target_file_name}" | sed \ + -e "s/\(.*\)\(\.tar\.gz\)/\1-$component\2/" \ + -e "s/\(.*\)\(\.zip\)/\1-${component}\2/" fi } diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index d2bccb4b8..1a649a805 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -75,6 +75,7 @@ JRE_PATH TEST_IMAGE_PATH STATIC_LIBS_IMAGE_PATH JVM_VARIANT +LOCAL_DEPENDENCY_CACHE_DIR MACOSX_CODESIGN_IDENTITY MAKE_ARGS_FOR_ANY_PLATFORM MAKE_EXPLODED @@ -381,6 +382,9 @@ function parseConfigurationArguments() { "--use-adoptium-devkit") BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]="$1"; shift;; + "--local-dependency-cache-dir") + BUILD_CONFIG[LOCAL_DEPENDENCY_CACHE_DIR]="$1"; shift;; + "--user-openjdk-build-root-directory" ) BUILD_CONFIG[USER_OPENJDK_BUILD_ROOT_DIRECTORY]="$1"; shift;; @@ -651,6 +655,9 @@ function configDefaults() { BUILD_CONFIG[USE_ADOPTIUM_DEVKIT]="" BUILD_CONFIG[ADOPTIUM_DEVKIT_LOCATION]="" + # Default to no local dependency cache + BUILD_CONFIG[LOCAL_DEPENDENCY_CACHE_DIR]="" + # By default dont backport JEP318 certs to < Java 10 BUILD_CONFIG[USE_JEP319_CERTS]=false diff --git a/sbin/common/sbom.sh b/sbin/common/sbom.sh index 945389b68..7fd7869aa 100755 --- a/sbin/common/sbom.sh +++ b/sbin/common/sbom.sh @@ -205,18 +205,3 @@ addSBOMComponentPropertyFromFile() { fi } -# Function not in use -# Ref: https://cyclonedx.org/docs/1.4/json/#externalReferences -addExternalReference() { - local javaHome="${1}" - local classpath="${2}" - local jsonFile="${3}" - local url="${4}" # required - local comment="${5}" - local hash="${6}" - if [ -z "${hash}" ]; then - "${javaHome}"/bin/java -cp "${classpath}" temurin.sbom.TemurinGenSBOM --addExternalReference --jsonFile "${jsonFile}" --url "${url}" --comment "${comment}" --hash "${hash}" - else - "${javaHome}"/bin/java -cp "${classpath}" temurin.sbom.TemurinGenSBOM --addExternalReference --jsonFile "${jsonFile}" --url "${url}" --comment "${comment}" - fi -} diff --git a/test/functional/buildAndPackage/src/net/adoptium/test/FeatureTests.java b/test/functional/buildAndPackage/src/net/adoptium/test/FeatureTests.java index 6c22f83e6..976e92b74 100644 --- a/test/functional/buildAndPackage/src/net/adoptium/test/FeatureTests.java +++ b/test/functional/buildAndPackage/src/net/adoptium/test/FeatureTests.java @@ -15,12 +15,15 @@ package net.adoptium.test; +import org.testng.annotations.BeforeTest; import org.testng.annotations.Test; import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.logging.Logger; +import java.util.regex.Pattern; import static net.adoptium.test.JdkPlatform.Architecture; import static net.adoptium.test.JdkPlatform.OperatingSystem; @@ -40,6 +43,71 @@ public class FeatureTests { private final JdkPlatform jdkPlatform = new JdkPlatform(); + private String testJdkHome = null; + + /** + * Ensure TEST_JDK_HOME environment variable is set for every test in this class. + */ + @BeforeTest + public final void ensureTestJDKSet() { + String tmpJdkHome = System.getenv("TEST_JDK_HOME"); + if (tmpJdkHome == null) { + throw new AssertionError("TEST_JDK_HOME is not set"); + } + this.testJdkHome = tmpJdkHome; + } + + /** + * Tests whether JEP 493 is enabled for Eclipse Temurin builds. + * + * @see JEP 493: Linking Run-Time Images without JMODs + */ + @Test + public void testLinkableRuntimeJDK24Plus() { + // Only JDK 24 and better and temurin builds have this enabled + if (jdkVersion.isNewerOrEqual(24) && isVendorAdoptium()) { + List command = new ArrayList<>(); + command.add(String.format("%s/bin/jlink", testJdkHome)); + command.add("-J-Duser.lang=en"); + command.add("--help"); + + try { + ProcessBuilder processBuilder = new ProcessBuilder(command); + Process process = processBuilder.start(); + + String stdout = StreamUtils.consumeStream(process.getInputStream()); + if (process.waitFor() != 0) { + throw new AssertionError("Could not run jlink --help"); + } + String[] lines = stdout.split(Pattern.quote(System.lineSeparator())); + boolean seenCapabilities = false; + String capLine = ""; + for (int i = 0; i < lines.length; i++) { + if (lines[i].trim().startsWith("Capabilities:")) { + seenCapabilities = true; + continue; // skip Capabilities line + } + if (!seenCapabilities) { + continue; + } + if (seenCapabilities) { + capLine = lines[i].trim(); + break; + } + } + LOGGER.info(String.format("Matched 'Capabilities:' line: %s", capLine)); + assertEquals(capLine, "Linking from run-time image enabled", + "jlink should have enabled run-time image link capability"); + } catch (InterruptedException | IOException e) { + throw new RuntimeException("Failed to launch JVM", e); + } + } + } + + private boolean isVendorAdoptium() { + return System.getProperty("java.vendor", "").toLowerCase(Locale.US).contains("adoptium"); + } + /** * Tests whether Shenandoah GC is available. *

@@ -53,10 +121,6 @@ public class FeatureTests { */ @Test public void testShenandoahAvailable() { - String testJdkHome = System.getenv("TEST_JDK_HOME"); - if (testJdkHome == null) { - throw new AssertionError("TEST_JDK_HOME is not set"); - } boolean shouldBePresent = false; if ((jdkVersion.isNewerOrEqual(15) || jdkVersion.isNewerOrEqualSameFeature(11, 0, 9))) { @@ -73,7 +137,7 @@ public void testShenandoahAvailable() { } } if (jdkVersion.isNewerOrEqual(17) && jdkPlatform.runsOn(OperatingSystem.LINUX, Architecture.PPC64LE)) { - shouldBePresent = true; + shouldBePresent = true; } if (jdkVersion.isNewerOrEqual(19) || jdkVersion.isNewerOrEqualSameFeature(17, 0, 9) @@ -116,10 +180,6 @@ public void testShenandoahAvailable() { */ @Test public void testZGCAvailable() { - String testJdkHome = System.getenv("TEST_JDK_HOME"); - if (testJdkHome == null) { - throw new AssertionError("TEST_JDK_HOME is not set"); - } boolean shouldBePresent = false; if (jdkVersion.isNewerOrEqual(15)) { @@ -184,10 +244,6 @@ public void testZGCAvailable() { */ @Test public void testJFRAvailable() { - String testJdkHome = System.getenv("TEST_JDK_HOME"); - if (testJdkHome == null) { - throw new AssertionError("TEST_JDK_HOME is not set"); - } boolean shouldBePresent = false; if (jdkVersion.isNewerOrEqual(11) || jdkVersion.isNewerOrEqualSameFeature(8, 0, 262)) { if (!jdkPlatform.runsOn(OperatingSystem.AIX) || jdkVersion.isNewerOrEqual(20)) { @@ -213,4 +269,5 @@ public void testJFRAvailable() { throw new RuntimeException("Failed to launch JVM", e); } } + } diff --git a/test/system/reproducibleCompare/playlist.xml b/test/system/reproducibleCompare/playlist.xml index 9fd60ce40..36e3877b8 100644 --- a/test/system/reproducibleCompare/playlist.xml +++ b/test/system/reproducibleCompare/playlist.xml @@ -19,7 +19,7 @@ Rebuild_Same_JDK_Reproducibility_Test docker container rm -f reproducibleCompare; \ - docker run -v "$(TEST_RESROOT):/home/jenkins/test" -w "/home/jenkins/" -v "$(TEST_JDK_HOME):/home/jenkins/jdkbinary/" --name reproducibleCompare centos:7 /bin/bash /home/jenkins/test/linux_repro_build_compare.sh $(SBOM_FILE) /home/jenkins/jdkbinary; \ + docker run --sysctl net.ipv6.conf.all.disable_ipv6=0 --sysctl net.ipv6.conf.default.disable_ipv6=0 -v "$(TEST_RESROOT):/home/jenkins/test" -w "/home/jenkins/" -v "$(TEST_JDK_HOME):/home/jenkins/jdkbinary/" --name reproducibleCompare centos:7 /bin/bash /home/jenkins/test/linux_repro_build_compare.sh $(SBOM_FILE) /home/jenkins/jdkbinary; \ $(TEST_STATUS); \ docker cp reproducibleCompare:/home/jenkins/reprotest.diff ./; \ docker cp reproducibleCompare:/home/jenkins/reproJDK.tar.gz ./; \ @@ -47,7 +47,7 @@ $(TEST_STATUS) - dev + special system @@ -68,7 +68,7 @@ $(TEST_STATUS) - dev + special system @@ -79,6 +79,6 @@ 21+ - os.osx,arch.aarch64 + os.osx \ No newline at end of file diff --git a/tooling/build_autotriage/build_autotriage.sh b/tooling/build_autotriage/build_autotriage.sh index 27bd40772..5445f050f 100644 --- a/tooling/build_autotriage/build_autotriage.sh +++ b/tooling/build_autotriage/build_autotriage.sh @@ -59,6 +59,7 @@ temurinPlatforms+=("mac-aarch64"); platformStart+=(11); platformEnd+=(9 temurinPlatforms+=("mac-x64"); platformStart+=(8); platformEnd+=(99) temurinPlatforms+=("solaris-sparcv9"); platformStart+=(8); platformEnd+=(8) temurinPlatforms+=("solaris-x64"); platformStart+=(8); platformEnd+=(8) +temurinPlatforms+=("windows-aarch64"); platformStart+=(21); platformEnd+=(99) temurinPlatforms+=("windows-x64"); platformStart+=(8); platformEnd+=(99) temurinPlatforms+=("windows-x86-32"); platformStart+=(8); platformEnd+=(17) diff --git a/tooling/reproducible/ReproducibleBuilds.md b/tooling/reproducible/ReproducibleBuilds.md index a43b5b551..cf5627e40 100755 --- a/tooling/reproducible/ReproducibleBuilds.md +++ b/tooling/reproducible/ReproducibleBuilds.md @@ -63,11 +63,15 @@ before the comparable_patch.sh can be run. - Ensure VS2022 SDK is installed and on PATH - Compile: - cd tooling/src/c + - run vcvarsall.bat as your arch needs. Eg: vcvars64.bat on x64 windows + - You can set up INCLUDES manually but it is not worthy + - vcvarsall.bat creates a subshell, if you do not want it to create a subshell, use `call` eg `call vcvars64.bat` instead of the direct execution. - cl WindowsUpdateVsVersionInfo.c version.lib 3. Compile [src/java/temurin/tools/BinRepl.java](https://github.com/adoptium/temurin-build/blob/master/tooling/src/java/temurin/tools/BinRepl.java) : - Ensure suitable JDK on PATH + - **do not** use JDK you are just patching, as that JDK gets **broken** by the process of patching - cd tooling/src/java - javac temurin/tools/BinRepl.java @@ -79,9 +83,47 @@ before the comparable_patch.sh can be run. - For BinRepl.class : export CLASSPATH=/tooling/src/java:$CLASSPATH - A JDK for running BinRepl java : export PATH=/bin:$PATH +##### Cygwin treacherousness + +- It is extremely difficult (maybe impossible) to invoke `vcvarsall.bat+cl` in cygwin directly +- Thus, it is recommended to launch this via `cmd -c` or preferably by an executable `.bat` file such as: + +```bash + pushd "$MSVSC/BUILD/TOOLS" + rm -f WindowsUpdateVsVersionInfo.obj + echo " + call vcvars64.bat + cl $(cygpath -m $YOUR_WORKDIR/temurin-build/tooling/src/c/WindowsUpdateVsVersionInfo.c) version.lib + " > setupEnvAndCompile.bat + chmod 755 setupEnvAndCompile.bat + ./setupEnvAndCompile.bat + # copy it to any dir on path or add this dir to path + mv WindowsUpdateVsVersionInfo.exe "$FUTURE_PATH_ADDITIONS" + rm WindowsUpdateVsVersionInfo.obj setupEnvAndCompile.bat + popd +``` + +- NOTE: The default paths should work fine. In Cygwin, they are usually at: + - MSVSC cl/bat files in `/cygdrive/c/Program Files/Microsoft Visual Studio/` under `Hostx64/x64` (or similar) and `Auxiliary/Build` dirs + - the signtool.exe then in `/cygdrive/c/Program Files (x86)/Windows Kits`. Again in arch-specific dir like `x64` + +- NOTE: Using `cygpath` is sometimes necessary. However, Java *binaries* can have issues with it: + - e.g., Use `cygpath` for `$CLASSPATH`, + - or javac it is mandatory: + +```bash + ftureDir="$(pwd)/classes" + if uname | grep CYGWIN ; then + ftureDir=$(cygpath -m "${ftureDir}") + fi + $AQA_DIR/$jdkName/bin/javac -d "${ftureDir}" "../../tooling/src/java/temurin/tools/BinRepl.java" +``` + #### Running comparable_patch.sh: 1. Unzip your JDK archive into a directory (eg.jdk1) + - Note, that jdk will be modified, so the location must be writable + - if it is in admin/root location, `cp -rL` it to some temp directory. 2. Run comparable_patch.sh @@ -89,7 +131,7 @@ before the comparable_patch.sh can be run. bash comparable_patch.sh --jdk-dir "" --version-string "" --vendor-name "" --vendor_url "" --vendor-bug-url "" --vendor-vm-bug-url "" [--patch-vs-version-info] ``` -The Vendor strings and urls can be found by running your jdk's "java -XshowSettings": +The Vendor strings and URLs can be found by running your jdk's "java -XshowSettings": ```java java -XshowSettings: @@ -101,7 +143,9 @@ java -XshowSettings: ... ``` -eg. +In cygwin, you must handle the trailing `\r` otherwise it will fail later. sed `\r` away as eg: `sed 's/\r.*//'` is usually enough. + +e.g.,: ```bash bash ./comparable_patch.sh --jdk-dir "jdk1/jdk-21.0.1+12" --version-string "Temurin-21.0.1+12" --vendor-name "Eclipse Adoptium" --vendor_url "https://adoptium.net/" --vendor-bug-url "https://github.com/adoptium/adoptium-support/issues" --vendor-vm-bug-url "https://github.com/adoptium/adoptium-support/issues" diff --git a/tooling/reproducible/repro_common.sh b/tooling/reproducible/repro_common.sh index 5648fef81..eb1284b3d 100755 --- a/tooling/reproducible/repro_common.sh +++ b/tooling/reproducible/repro_common.sh @@ -25,10 +25,11 @@ function expandJDK() { JDK_ROOT=$(realpath ${JDK_DIR}/../../) JDK_BIN_DIR="${JDK_ROOT}_CP/Contents/Home/bin" fi - + echo "$(date +%T) : Expanding various components to enable comparisons ${JDK_DIR} (original files will be removed):" mkdir "${JDK_ROOT}_CP" cp -R ${JDK_ROOT}/* ${JDK_ROOT}_CP - echo "Expanding the 'modules' Image to compare extracted files" + + echo "$(date +%T) : Using 'jimage extract' to expand lib/modules image into lib/modules_extracted" modulesFile="${JDK_DIR}/lib/modules" mkdir "${JDK_DIR}/lib/modules_extracted" extractedDir="${JDK_DIR}/lib/modules_extracted" @@ -38,11 +39,11 @@ function expandJDK() { fi "${JDK_BIN_DIR}/jimage" extract --dir "${extractedDir}" "${modulesFile}" rm "${JDK_DIR}/lib/modules" - echo "Expanding the 'src.zip' to normalize file permissions" - unzip "${JDK_DIR}/lib/src.zip" -d "${JDK_DIR}/lib/src_zip_expanded" 1> /dev/null + echo "$(date +%T) : Unzipping lib/src.zip to normalize file permissions, then removing src.zip" + unzip -q "${JDK_DIR}/lib/src.zip" -d "${JDK_DIR}/lib/src_zip_expanded" rm "${JDK_DIR}/lib/src.zip" - echo "Expanding jmods to process binaries within" + echo "$(date +%T) : Using 'jmod extract' to expand all jmods to jmods/expanded_ directories" FILES=$(find "${JDK_DIR}" -type f -path '*.jmod') for f in $FILES do @@ -58,21 +59,21 @@ function expandJDK() { rm "$f" done - echo "Expanding the 'jrt-fs.jar' to remove signatures from within.." + echo "$(date +%T) : Expanding lib/jrt-fs.jar to lib/jrt-fs-exanded to remove signatures from within.." mkdir "${JDK_DIR}/lib/jrt-fs-expanded" - unzip -d "${JDK_DIR}/lib/jrt-fs-expanded" "${JDK_DIR}/lib/jrt-fs.jar" 1> /dev/null + unzip -qd "${JDK_DIR}/lib/jrt-fs-expanded" "${JDK_DIR}/lib/jrt-fs.jar" rm "${JDK_DIR}/lib/jrt-fs.jar" mkdir -p "${JDK_DIR}/jmods/expanded_java.base.jmod/lib/jrt-fs-expanded" - unzip -d "${JDK_DIR}/jmods/expanded_java.base.jmod/lib/jrt-fs-expanded" "${JDK_DIR}/jmods/expanded_java.base.jmod/lib/jrt-fs.jar" 1> /dev/null + unzip -qd "${JDK_DIR}/jmods/expanded_java.base.jmod/lib/jrt-fs-expanded" "${JDK_DIR}/jmods/expanded_java.base.jmod/lib/jrt-fs.jar" rm "${JDK_DIR}/jmods/expanded_java.base.jmod/lib/jrt-fs.jar" - echo "Expanding 'ct.sym' to workaround zip timestamp differences (https://bugs.openjdk.org/browse/JDK-8327466)" + echo "$(date +%T) : Expanding lib/ct.sym to workaround zip timestamp differences (https://bugs.openjdk.org/browse/JDK-8327466)" mkdir "${JDK_DIR}/lib/ct-sym-expanded" - unzip -d "${JDK_DIR}/lib/ct-sym-expanded" "${JDK_DIR}/lib/ct.sym" 1> /dev/null + unzip -qd "${JDK_DIR}/lib/ct-sym-expanded" "${JDK_DIR}/lib/ct.sym" rm "${JDK_DIR}/lib/ct.sym" mkdir -p "${JDK_DIR}/jmods/expanded_jdk.compiler.jmod/lib/ct-sym-expanded" - unzip -d "${JDK_DIR}/jmods/expanded_jdk.compiler.jmod/lib/ct-sym-expanded" "${JDK_DIR}/jmods/expanded_jdk.compiler.jmod/lib/ct.sym" 1> /dev/null + unzip -qd "${JDK_DIR}/jmods/expanded_jdk.compiler.jmod/lib/ct-sym-expanded" "${JDK_DIR}/jmods/expanded_jdk.compiler.jmod/lib/ct.sym" rm "${JDK_DIR}/jmods/expanded_jdk.compiler.jmod/lib/ct.sym" rm -rf "${JDK_ROOT}_CP" @@ -141,7 +142,7 @@ function removeSystemModulesHashBuilderParams() { done done - echo "Successfully removed all SystemModules jdk.jpackage hash differences from ${JDK_DIR}" + echo "$(date +%T) : Successfully removed all SystemModules jdk.jpackage hash differences from ${JDK_DIR}" } # Required for Vendor "Comparable Builds" @@ -153,7 +154,7 @@ function removeSystemModulesHashBuilderParams() { # checksum - A checksum value of the binary # reprohex - A hex UUID to identify the binary version, again generated from binary content function removeWindowsNonComparableData() { - echo "Removing EXE/DLL timestamps, CRC and debug repro hex from ${JDK_DIR}" + echo "$(date +%T) : Removing EXE/DLL timestamps, CRC and debug repro hex from ${JDK_DIR}" # We need to do this for all executables if patching VS_VERSION_INFO if [[ "$PATCH_VS_VERSION_INFO" = true ]]; then FILES=$(find "${JDK_DIR}" -type f -path '*.exe' && find "${JDK_DIR}" -type f -path '*.dll') @@ -163,13 +164,13 @@ function removeWindowsNonComparableData() { for ff in $FILES do f=$(cygpath -w $ff) - echo "Removing EXE/DLL non-comparable timestamp, CRC, debug repro hex from $f" + echo "$(date +%T) : Removing EXE/DLL non-comparable timestamp, CRC, debug repro hex from $f" # Determine non-comparable data using dumpbin dmpfile="$ff.dumpbin.tmp" rm -f "$dmpfile" if ! dumpbin "$f" /HEADERS > "$dmpfile"; then - echo " FAILED == > dumpbin \"$f\" /ALL > $dmpfile" + echo "$(date +%T) : FAILED == > dumpbin \"$f\" /ALL > $dmpfile" exit 1 fi @@ -254,8 +255,8 @@ function processModuleInfo() { local JDK_DIR="$1" local OS="$2" local work_JDK="$3" - echo "process Module Info from ${JDK_DIR}" - echo "Normalizing ModuleAttributes order in module-info.class, converting to javap" + echo "$(date +%T) : Process Module Info from ${JDK_DIR}" + echo "$(date +%T) : Normalizing ModuleAttributes order in module-info.class, converting to javap" moduleAttr="ModuleResolution ModuleTarget" FILES=$(find "${JDK_DIR}" -type f -name "module-info.class") for f in $FILES @@ -343,7 +344,7 @@ function removeSignatures() { if [[ "$OS" =~ CYGWIN* ]]; then # signtool should be on PATH signToolPath="signtool" - echo "Removing all Signatures from ${JDK_DIR}" + echo "$(date +%T) : Removing all signatures from exe and dll files in ${JDK_DIR}" FILES=$(find "${JDK_DIR}" -type f -name '*.exe' -o -name '*.dll') for f in $FILES do @@ -384,10 +385,16 @@ function tempSign() { if [[ "$OS" =~ CYGWIN* ]]; then # signtool should be on PATH signToolPath="signtool" - echo "Adding temp Signatures for ${JDK_DIR}" + echo "Generating temp signatures with openssl and adding them to exe/dll files in ${JDK_DIR}" selfCert="test" - openssl req -x509 -newkey rsa:4096 -sha256 -days 3650 -nodes -keyout $selfCert.key -out $selfCert.crt -subj "/CN=example.com" -addext "subjectAltName=DNS:example.com,DNS:*.example.com,IP:10.0.0.1" - openssl pkcs12 -export -passout pass:test -out $selfCert.pfx -inkey $selfCert.key -in $selfCert.crt + + # semgrep needs to ignore this as it objects to the password, but that + # is only used for generating a temporary dummy signature required for + # the comparison and not used for validating anything + # nosemgrep + openssl req -x509 -quiet -newkey rsa:4096 -sha256 -days 3650 -passout pass:test -keyout $selfCert.key -out $selfCert.crt -subj "/CN=example.com" -addext "subjectAltName=DNS:example.com,DNS:*.example.com,IP:10.0.0.1" + # nosemgrep + openssl pkcs12 -export -passout pass:test -passin pass:test -out $selfCert.pfx -inkey $selfCert.key -in $selfCert.crt FILES=$(find "${JDK_DIR}" -type f -name '*.exe' -o -name '*.dll') for f in $FILES do @@ -416,8 +423,7 @@ function tempSign() { function cleanTemurinFiles() { local DIR="$1" - echo "Cleaning Temurin build-scripts specific files and metadata from ${DIR}" - echo "Removing Temurin NOTICE file from $DIR" + echo "$(date +%T): Cleaning Temurin NOTICE file and build-scripts specific files and metadata from ${DIR}" rm "${DIR}"/NOTICE if [[ $(uname) =~ Darwin* ]]; then diff --git a/tooling/reproducible/repro_compare.sh b/tooling/reproducible/repro_compare.sh index 2ee8616fb..e6d1f3ae5 100755 --- a/tooling/reproducible/repro_compare.sh +++ b/tooling/reproducible/repro_compare.sh @@ -34,11 +34,11 @@ do exit 1 fi - echo "Pre-processing ${JDK_DIR}" + echo "$(date +%T) : Pre-processing ${JDK_DIR}" rc=0 source "$(dirname "$0")"/repro_process.sh "${JDK_DIR}" "${OS}" || rc=$? if [ $rc != 0 ]; then - echo "Pre-process of ${JDK_DIR} ${OS} failed" + echo "$(date +%T): Pre-processing of ${JDK_DIR} ${OS} failed" exit 1 fi @@ -59,11 +59,19 @@ files1=$(find "${JDK_DIR1}" -type f | wc -l) echo "Number of files: ${files1}" rc=0 output="reprotest.diff" -echo "Comparing ${JDK_DIR1} with ${JDK_DIR2} ... output to file: ${output}" +echo "$(date +%T) : Comparing expanded JDKs from ${JDK_DIR1} with ${JDK_DIR2} ..." diff -r -q "${JDK_DIR1}" "${JDK_DIR2}" > "${output}" || rc=$? +echo "$(date +%T) : diff complete - rc=$rc. Output written to file: ${output}" cat "${output}" +grep "Files .*" "${output}" | while read -r line; do + FILE1=$(echo "$line" | awk '{print $2}') + FILE2=$(echo "$line" | awk '{print $4}') + echo "diff -c on $FILE1 and $FILE2" + diff -c "$FILE1" "$FILE2" +done + num_differences=$(wc -l < "${output}") echo "Number of differences: ${num_differences}" repro_pc100=$(( (files1-num_differences)*100*100/files1 )) diff --git a/tooling/reproducible/repro_process.sh b/tooling/reproducible/repro_process.sh index d44f9ac8b..2bc8dcc2a 100755 --- a/tooling/reproducible/repro_process.sh +++ b/tooling/reproducible/repro_process.sh @@ -38,7 +38,5 @@ fi patchManifests "${JDK_DIR}" -echo "***********" -echo " Preprocess ${JDK_DIR} SUCCESS :-)" -echo "***********" - +echo "$(date +%T) : Pre-processing of ${JDK_DIR} SUCCESSFUL :-)" +echo "" # blank line separator in log file diff --git a/tooling/reproducible/windows_repro_build_compare.sh b/tooling/reproducible/windows_repro_build_compare.sh index 9796f17b2..e8b5b2400 100755 --- a/tooling/reproducible/windows_repro_build_compare.sh +++ b/tooling/reproducible/windows_repro_build_compare.sh @@ -684,10 +684,12 @@ Check_Architecture echo "---------------------------------------------" Check_VS_Versions echo "---------------------------------------------" -Get_SRC_UCRT_Version -echo "---------------------------------------------" -Check_UCRT_Location -echo "---------------------------------------------" +if [[ "${buildArgs}" != *"--use-adoptium-devkit"* ]]; then + Get_SRC_UCRT_Version + echo "---------------------------------------------" + Check_UCRT_Location + echo "---------------------------------------------" +fi echo "All Validation Checks Passed - Proceeding To Build" echo "---------------------------------------------" Check_And_Install_Ant