From e1df877bfe4d84b352dff0d84c86b98c36cf3ebc Mon Sep 17 00:00:00 2001 From: Yuting Jiang Date: Thu, 27 Jul 2023 10:42:31 +0800 Subject: [PATCH] Release - SuperBench v0.9.0 (#558) **Description** Cherry-pick bug fixes from v0.9.0 to main. **Major Revision** - CI/CD: pipeline - clean more disk space to fix rocm building image pipeline(#555 ) - Benchmarks: bug fix - use absolute path for input file in DirectXEncodingLatency(#554) - CI/CD - add push win docker image on release branch in pipeline (#552) - Docs - Upgrade version and release note(#557) --- .github/workflows/build-image.yml | 11 ++++++ .github/workflows/build-win.yml | 29 ++++++++++++-- README.md | 2 +- docs/getting-started/installation.mdx | 2 +- docs/getting-started/run-superbench.md | 2 +- docs/superbench-config.mdx | 2 +- docs/user-tutorial/container-images.mdx | 14 +++++++ docs/user-tutorial/data-diagnosis.md | 2 +- docs/user-tutorial/result-summary.md | 2 +- superbench/__init__.py | 2 +- .../directx_gpu_encoding_latency.py | 4 +- superbench/config/amd_mi100_hpe.yaml | 2 +- superbench/config/amd_mi100_z53.yaml | 2 +- .../inference/standard_nc64as_t4_v3.yaml | 2 +- .../inference/standard_nc96ads_a100_v4.yaml | 2 +- .../inference/standard_nv18ads_a10_v5.yaml | 2 +- superbench/config/azure_ndmv4.yaml | 2 +- superbench/config/azure_ndv4.yaml | 2 +- superbench/config/default.yaml | 2 +- third_party/Makefile | 2 +- website/blog/2023-07-25-release-0-9.md | 38 +++++++++++++++++++ website/docusaurus.config.js | 2 +- website/package-lock.json | 2 +- website/package.json | 2 +- 24 files changed, 109 insertions(+), 25 deletions(-) create mode 100644 website/blog/2023-07-25-release-0-9.md diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 824418a6f..6b796830a 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -64,6 +64,17 @@ jobs: sudo apt-get clean sudo docker rmi $(sudo docker images --format "{{.Repository}}:{{.Tag}}" --filter=reference="node" --filter=reference="buildpack-deps") df -h + - name: Free Up GitHub Actions Ubuntu Runner Disk Space 🔧 + uses: jlumbroso/free-disk-space@main + with: + # This might remove tools that are actually needed, if set to "true" but frees about 6 GB + tool-cache: false + # All of these default to true, but feel free to set to "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + swap-storage: true - name: Prepare metadata id: metadata run: | diff --git a/.github/workflows/build-win.yml b/.github/workflows/build-win.yml index d1b9a1c8d..252783421 100644 --- a/.github/workflows/build-win.yml +++ b/.github/workflows/build-win.yml @@ -12,7 +12,7 @@ on: jobs: docker: - name: Docker build win2004 + name: Docker build win directx12 runs-on: [self-hosted, windows, x64, win2004] steps: - name: Checkout @@ -24,6 +24,25 @@ jobs: docker system prune -a -f docker volume prune -a -f shell: pwsh + - name: Set TAG variable based on the branch + run: | + if ($env:GITHUB_EVENT_NAME -match "release") { + $version = $env:GITHUB_REF.Substring($env:GITHUB_REF.LastIndexOf('/') + 1) + echo "TAG=superbench/superbench:$version-directx12" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + } elseif ($env:GITHUB_REF -match "refs/heads/release/(.*)") { + $version = $Matches[1] + echo "TAG=superbench/release:$version-directx12" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + } elseif ($env:GITHUB_BASEREF -match "release/(.*)"){ + $version = $Matches[1] + echo "TAG=superbench/release:$version-directx12" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + } else { + echo "TAG=superbench/main:directx12" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + } + shell: pwsh + env: + GITHUB_REF: ${{ github.ref }} + GITHUB_BASEREF: ${{ github.base_ref }} + GITHUB_EVENT_NAME: ${{ github.event_name }} - name: Build Docker image working-directory: . shell: pwsh @@ -37,7 +56,7 @@ jobs: --isolation=process ` --tag $env:TAG . env: - TAG: superbench/main:win2004 + TAG: ${{ env.TAG }} - name: Push Docker image if: ${{ github.event_name != 'pull_request' }} shell: pwsh @@ -46,7 +65,7 @@ jobs: docker push $env:TAG docker logout env: - TAG: superbench/main:win2004 + TAG: ${{ env.TAG }} USER: ${{ secrets.DOCKERHUB_USERNAME }} PASS: ${{ secrets.DOCKERHUB_TOKEN }} - name: Add bash to PATH @@ -64,7 +83,9 @@ jobs: docker run --rm ` --isolation process ` --device class/5B45201D-F2F2-4F3B-85BB-30FF1F953599 ` - -e CI=true $ci_env -e SB_TEST_CUDA="0" -e SB_TEST_ROCM="0" -e SB_TEST_PYTORCH="0" -e SB_TEST_DIRECTX="1" -e CODECOV_TOKEN superbench/main:win2004 cmd /c $command + -v C:/Windows/System32/DriverStore:C:/Windows/System32/DriverStore ` + -e CI=true $ci_env -e SB_TEST_CUDA="0" -e SB_TEST_ROCM="0" -e SB_TEST_PYTORCH="0" -e SB_TEST_DIRECTX="1" -e CODECOV_TOKEN --entrypoint "cmd" $env:TAG "/c python dockerfile/directx/enable-graphics-apis.py && cmd /c $command" shell: pwsh env: + TAG: ${{ env.TAG }} CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/README.md b/README.md index ffcd51960..cfcd4b6b3 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ __SuperBench__ is a validation and profiling tool for AI infrastructure. -📢 [v0.8.0](https://github.com/microsoft/superbenchmark/releases/tag/v0.8.0) has been released! +📢 [v0.9.0](https://github.com/microsoft/superbenchmark/releases/tag/v0.9.0) has been released! ## _Check [aka.ms/superbench](https://aka.ms/superbench) for more details._ diff --git a/docs/getting-started/installation.mdx b/docs/getting-started/installation.mdx index 82c1fc9c3..8570306c9 100644 --- a/docs/getting-started/installation.mdx +++ b/docs/getting-started/installation.mdx @@ -61,7 +61,7 @@ You can clone the source from GitHub and build it. :::note Note You should checkout corresponding tag to use release version, for example, -`git clone -b v0.8.0 https://github.com/microsoft/superbenchmark` +`git clone -b v0.9.0 https://github.com/microsoft/superbenchmark` ::: ```bash diff --git a/docs/getting-started/run-superbench.md b/docs/getting-started/run-superbench.md index 32a8c6d80..16c6d7a21 100644 --- a/docs/getting-started/run-superbench.md +++ b/docs/getting-started/run-superbench.md @@ -27,7 +27,7 @@ sb deploy -f remote.ini --host-password [password] :::note Note You should deploy corresponding Docker image to use release version, for example, -`sb deploy -f local.ini -i superbench/superbench:v0.8.0-cuda12.1` +`sb deploy -f local.ini -i superbench/superbench:v0.9.0-cuda12.1` You should note that version of git repo only determines version of sb CLI, and not the sb container. You should define the container version even if you specified a release version for the git clone. diff --git a/docs/superbench-config.mdx b/docs/superbench-config.mdx index 5720a8125..8893c46b9 100644 --- a/docs/superbench-config.mdx +++ b/docs/superbench-config.mdx @@ -70,7 +70,7 @@ superbench: ```yaml -version: v0.8 +version: v0.9 superbench: enable: benchmark_1 monitor: diff --git a/docs/user-tutorial/container-images.mdx b/docs/user-tutorial/container-images.mdx index 27cf8da6f..5fd11502c 100644 --- a/docs/user-tutorial/container-images.mdx +++ b/docs/user-tutorial/container-images.mdx @@ -23,12 +23,15 @@ available tags are listed below for all stable versions. values={[ {label: 'CUDA', value: 'cuda'}, {label: 'ROCm', value: 'rocm'}, + {label: 'DirectX', value: 'directx'}, ] }> | Tag | Description | |-------------------|------------------------------------| +| v0.9.0-cuda12.1 | SuperBench v0.9.0 with CUDA 12.1 | +| v0.9.0-cuda11.1.1 | SuperBench v0.9.0 with CUDA 11.1.1 | | v0.8.0-cuda12.1 | SuperBench v0.8.0 with CUDA 12.1 | | v0.8.0-cuda11.1.1 | SuperBench v0.8.0 with CUDA 11.1.1 | | v0.7.0-cuda11.8 | SuperBench v0.7.0 with CUDA 11.8 | @@ -45,6 +48,10 @@ available tags are listed below for all stable versions. | Tag | Description | |-------------------------------|--------------------------------------------------| +| v0.9.0-rocm5.1.3 | SuperBench v0.9.0 with ROCm 5.1.3 | +| v0.9.0-rocm5.1.1 | SuperBench v0.9.0 with ROCm 5.1.1 | +| v0.9.0-rocm5.0.1 | SuperBench v0.9.0 with ROCm 5.0.1 | +| v0.9.0-rocm5.0 | SuperBench v0.9.0 with ROCm 5.0 | | v0.8.0-rocm5.1.3 | SuperBench v0.8.0 with ROCm 5.1.3 | | v0.8.0-rocm5.1.1 | SuperBench v0.8.0 with ROCm 5.1.1 | | v0.8.0-rocm5.0.1 | SuperBench v0.8.0 with ROCm 5.0.1 | @@ -66,5 +73,12 @@ available tags are listed below for all stable versions. | v0.3.0-rocm4.2-pytorch1.7.0 | SuperBench v0.3.0 with ROCm 4.2, PyTorch 1.7.0 | | v0.3.0-rocm4.0-pytorch1.7.0 | SuperBench v0.3.0 with ROCm 4.0, PyTorch 1.7.0 | + + + +| Tag | Description | +|-------------------------------|--------------------------------------------------| +| v0.9.0-directx12 | SuperBench v0.9.0 with DirectX12, Windows10-2004 | + diff --git a/docs/user-tutorial/data-diagnosis.md b/docs/user-tutorial/data-diagnosis.md index 94a2a025d..a0bd99640 100644 --- a/docs/user-tutorial/data-diagnosis.md +++ b/docs/user-tutorial/data-diagnosis.md @@ -65,7 +65,7 @@ superbench: example: ```yaml # SuperBench rules -version: v0.8 +version: v0.9 superbench: rules: failure-rule: diff --git a/docs/user-tutorial/result-summary.md b/docs/user-tutorial/result-summary.md index e53738ff8..7e393a188 100644 --- a/docs/user-tutorial/result-summary.md +++ b/docs/user-tutorial/result-summary.md @@ -58,7 +58,7 @@ superbench: ```yaml title="Example" # SuperBench rules -version: v0.8 +version: v0.9 superbench: rules: kernel_launch: diff --git a/superbench/__init__.py b/superbench/__init__.py index 5b85c9a9a..bc20aebf9 100644 --- a/superbench/__init__.py +++ b/superbench/__init__.py @@ -6,5 +6,5 @@ Provide hardware and software benchmarks for AI systems. """ -__version__ = '0.8.0' +__version__ = '0.9.0' __author__ = 'Microsoft' diff --git a/superbench/benchmarks/micro_benchmarks/directx_gpu_encoding_latency.py b/superbench/benchmarks/micro_benchmarks/directx_gpu_encoding_latency.py index 70d6c75ad..ed17ea5bd 100644 --- a/superbench/benchmarks/micro_benchmarks/directx_gpu_encoding_latency.py +++ b/superbench/benchmarks/micro_benchmarks/directx_gpu_encoding_latency.py @@ -98,11 +98,11 @@ def _preprocess(self): command += f' -HEIGHT {self._args.height}' command += f' -WIDTH {self._args.width}' if self._args.input_file is not None: - command += f' -INPUT {self._args.input_file}' + command += f' -INPUT {os.path.abspath(self._args.input_file)}' else: if not os.path.exists(f'{self._test_file}'): create_nv12_file(self._test_file, self._args.frames, self._args.width, self._args.height) - command += f' -INPUT {self._test_file}' + command += f' -INPUT {os.path.abspath(self._test_file)}' if self._args.output_file is not None: command += f' -OUTPUT {self._args.output_file}' command += f' -OUTPUT_HEIGHT {self._args.output_height}' diff --git a/superbench/config/amd_mi100_hpe.yaml b/superbench/config/amd_mi100_hpe.yaml index 150424c0f..718224531 100644 --- a/superbench/config/amd_mi100_hpe.yaml +++ b/superbench/config/amd_mi100_hpe.yaml @@ -3,7 +3,7 @@ # Server: # - Product: HPE Apollo 6500 -version: v0.8 +version: v0.9 superbench: enable: null var: diff --git a/superbench/config/amd_mi100_z53.yaml b/superbench/config/amd_mi100_z53.yaml index 188c93547..8aa8fd85e 100644 --- a/superbench/config/amd_mi100_z53.yaml +++ b/superbench/config/amd_mi100_z53.yaml @@ -4,7 +4,7 @@ # - Product: G482-Z53 # - Link: https://www.gigabyte.cn/FileUpload/Global/MicroSite/553/G482-Z53.html -version: v0.8 +version: v0.9 superbench: enable: null var: diff --git a/superbench/config/azure/inference/standard_nc64as_t4_v3.yaml b/superbench/config/azure/inference/standard_nc64as_t4_v3.yaml index 62e0d6586..5ffa26311 100644 --- a/superbench/config/azure/inference/standard_nc64as_t4_v3.yaml +++ b/superbench/config/azure/inference/standard_nc64as_t4_v3.yaml @@ -1,4 +1,4 @@ -version: v0.8 +version: v0.9 superbench: enable: null monitor: diff --git a/superbench/config/azure/inference/standard_nc96ads_a100_v4.yaml b/superbench/config/azure/inference/standard_nc96ads_a100_v4.yaml index 337affacf..5c78d866d 100644 --- a/superbench/config/azure/inference/standard_nc96ads_a100_v4.yaml +++ b/superbench/config/azure/inference/standard_nc96ads_a100_v4.yaml @@ -1,4 +1,4 @@ -version: v0.8 +version: v0.9 superbench: enable: null monitor: diff --git a/superbench/config/azure/inference/standard_nv18ads_a10_v5.yaml b/superbench/config/azure/inference/standard_nv18ads_a10_v5.yaml index f95469cb0..75375cd79 100644 --- a/superbench/config/azure/inference/standard_nv18ads_a10_v5.yaml +++ b/superbench/config/azure/inference/standard_nv18ads_a10_v5.yaml @@ -1,4 +1,4 @@ -version: v0.8 +version: v0.9 superbench: enable: null monitor: diff --git a/superbench/config/azure_ndmv4.yaml b/superbench/config/azure_ndmv4.yaml index e482d6ed0..8aabb65f7 100644 --- a/superbench/config/azure_ndmv4.yaml +++ b/superbench/config/azure_ndmv4.yaml @@ -3,7 +3,7 @@ # Azure NDm A100 v4 # reference: https://docs.microsoft.com/en-us/azure/virtual-machines/ndm-a100-v4-series -version: v0.8 +version: v0.9 superbench: enable: null monitor: diff --git a/superbench/config/azure_ndv4.yaml b/superbench/config/azure_ndv4.yaml index cb9a93ddc..274556842 100644 --- a/superbench/config/azure_ndv4.yaml +++ b/superbench/config/azure_ndv4.yaml @@ -1,5 +1,5 @@ # SuperBench Config -version: v0.8 +version: v0.9 superbench: enable: null monitor: diff --git a/superbench/config/default.yaml b/superbench/config/default.yaml index 60d6be7b0..1a6af7dc5 100644 --- a/superbench/config/default.yaml +++ b/superbench/config/default.yaml @@ -1,5 +1,5 @@ # SuperBench Config -version: v0.8 +version: v0.9 superbench: enable: null monitor: diff --git a/third_party/Makefile b/third_party/Makefile index b0c01d453..ec72ccae9 100755 --- a/third_party/Makefile +++ b/third_party/Makefile @@ -157,5 +157,5 @@ directx_amf_encoding_latency: curl -L -o vs_buildtools.exe https://aka.ms/vs/16/release/vs_buildtools.exe && echo "Downloaded vs_buildtools.exe" && \ start /wait vs_buildtools.exe --quiet --wait --norestart --nocache --installPath C:/temp/BuildTools --add Microsoft.VisualStudio.Workload.VCTools --add Microsoft.VisualStudio.Component.VC.ATLMFC --includeRecommended && echo "Installed VS Build Tools" && \ del vs_buildtools.exe && echo "Deleted vs_buildtools.exe" && \ - "C:\temp\BuildTools\MSBuild\Current\Bin\MSBuild.exe" "AMF\amf\public\samples\CPPSamples_vs2019.sln" /t:EncoderLatency /p:Configuration=Release /p:OutDir="%SB_MICRO_PATH%\bin" \ + "C:\temp\BuildTools\MSBuild\Current\Bin\MSBuild.exe" "AMF\amf\public\samples\CPPSamples_vs2019.sln" /t:EncoderLatency /p:Platform=x64 /p:Configuration=Release /p:OutDir="%SB_MICRO_PATH%\bin" \ ) diff --git a/website/blog/2023-07-25-release-0-9.md b/website/blog/2023-07-25-release-0-9.md new file mode 100644 index 000000000..59e931103 --- /dev/null +++ b/website/blog/2023-07-25-release-0-9.md @@ -0,0 +1,38 @@ +--- +slug: release-sb-v0.9 +title: Releasing SuperBench v0.9 +author: Peng Cheng +author_title: SuperBench Team +author_url: https://github.com/cp5555 +author_image_url: https://github.com/cp5555.png +tags: [superbench, announcement, release] +--- + +We are very happy to announce that **SuperBench 0.9.0 version** is officially released today! + +You can install and try superbench by following [Getting Started Tutorial](https://microsoft.github.io/superbenchmark/docs/getting-started/installation). + +## SuperBench 0.9.0 Release Notes + +### SuperBench Improvement +- Support Ctrl+C and interrupt to stop all SuperBench testing. +- Support Windows Docker for VDI/Gaming GPU. +- Support DirectX platform for Nvidia and AMD GPU. +- Add System Config Info feature in SB runner to support distributed collection. +- Support DirectX test pipeline. + +### Micro-benchmark Improvement +- Add DirectXGPUCopyBw Benchmark to measure HtoD/DtoH bandwidth by DirectX. +- Add DirectXGPUCoreFLops Benchmark to measure peak FLOPS by DirectX.. +- Add DirectXGPUMemBw Benchmark to measure GPU memory bandwidth by DirectX.. +- Add DirectXVCNEncodingLatency Benchmark to measure the VCN hardware encoding latency on AMD graphic GPUs. +- Support best algorithm selection in cudnn-function microbenchmark. +- Revise step time collection in distributed inference benchmark. + +### Model Benchmark Improvement +- Fix early stop logic due to num_steps in model benchmarks. +- Support TensorRT models on Nvidia H100. + +### Documentation +- Improve documentation for System Config Info. +- Update outdate references. diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index cc583913d..c1d83edfa 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -101,7 +101,7 @@ module.exports = { announcementBar: { id: 'supportus', content: - '📢 v0.8.0 has been released! ' + + '📢 v0.9.0 has been released! ' + '⭐️ If you like SuperBench, give it a star on GitHub! ⭐️', }, algolia: { diff --git a/website/package-lock.json b/website/package-lock.json index 80c139a56..a2e3b219d 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -1,6 +1,6 @@ { "name": "superbench-website", - "version": "0.8.0", + "version": "0.9.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/website/package.json b/website/package.json index c761f26d8..38ca1f75a 100644 --- a/website/package.json +++ b/website/package.json @@ -1,6 +1,6 @@ { "name": "superbench-website", - "version": "0.8.0", + "version": "0.9.0", "private": true, "scripts": { "docusaurus": "docusaurus",