Add an option to not encode sentencepiece during training/decoding al… #658
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Windows | |
| on: | |
| push: | |
| branches: [ master ] | |
| pull_request: | |
| branches: [ master ] | |
| env: | |
| MKL_URL: "https://data.statmt.org/romang/marian-regression-tests/ci/mkl-2020.1-windows-static.zip" | |
| BOOST_ROOT: "C:/hostedtoolcache/windows/Boost/1.72.0/x86_64" | |
| BOOST_URL: "https://sourceforge.net/projects/boost/files/boost-binaries/1.72.0/boost_1_72_0-msvc-14.2-64.exe" | |
| jobs: | |
| build-windows: | |
| strategy: | |
| matrix: | |
| include: | |
| # Windows CPU-only build | |
| - name: "Windows CPU-only" | |
| cuda: "" | |
| gpu: false | |
| # Windows CPU+GPU build | |
| - name: "Windows CPU+CUDA" | |
| cuda: "10.2" | |
| gpu: true | |
| runs-on: windows-2019 | |
| name: ${{ matrix.name }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - name: Download MKL | |
| run: | | |
| # Wget retries downloading files and is faster than Invoke-WebRequest | |
| C:\msys64\usr\bin\wget.exe -nv ${{ env.MKL_URL }} -O mkl.zip | |
| Expand-Archive -Force mkl.zip ${{ github.workspace }}\mkl | |
| # Set MKLROOT environment variable so that CMake can find MKL | |
| echo "MKLROOT=${{ github.workspace }}\mkl" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| shell: powershell | |
| - name: Install CUDA | |
| run: | | |
| .\scripts\ci\install_cuda_windows.ps1 "10.2" | |
| # Set CUDA_PATH environment variable so that CMake can find CUDA | |
| echo "CUDA_PATH=$env:CUDA_PATH" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | |
| echo "$env:CUDA_PATH/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append | |
| shell: powershell | |
| if: matrix.gpu == true | |
| # Boost is no longer pre-installed on GitHub-hosted Windows runners | |
| - name: Download Boost | |
| run: | | |
| Write-Host "Downloading Boost to ${{ env.BOOST_ROOT }}" | |
| C:\msys64\usr\bin\wget.exe -nv "${{ env.BOOST_URL }}" -O "${{ github.workspace }}/boost.exe" | |
| Start-Process -Wait -FilePath "${{ github.workspace }}/boost.exe" "/SILENT","/SP-","/SUPPRESSMSGBOXES","/DIR=${{ env.BOOST_ROOT }}" | |
| shell: powershell | |
| - name: Prepare vcpkg | |
| uses: lukka/run-vcpkg@v4 | |
| with: | |
| vcpkgArguments: protobuf | |
| vcpkgGitCommitId: 6185aa76504a5025f36754324abf307cc776f3da | |
| vcpkgDirectory: ${{ github.workspace }}/vcpkg/ | |
| vcpkgTriplet: x64-windows-static | |
| # Windows CUDA builds use USE_NCCL=off due to compilation errors. | |
| - name: Build Debug | |
| uses: lukka/run-cmake@v3 | |
| with: | |
| buildDirectory: ${{ github.workspace }}/build/Debug | |
| cmakeAppendedArgs: '-G Ninja | |
| -DCMAKE_BUILD_TYPE="Debug" | |
| -DOPENSSL_USE_STATIC_LIBS="TRUE" | |
| -DOPENSSL_MSVC_STATIC_RT="TRUE" | |
| -DCOMPILE_CPU="TRUE" | |
| -DCOMPILE_CUDA="${{ matrix.gpu }}" | |
| -DCOMPILE_SERVER="FALSE" | |
| -DCOMPILE_TESTS="TRUE" | |
| -DDETERMINISTIC="TRUE" | |
| -DUSE_FBGEMM="TRUE" | |
| -DUSE_MPI="FALSE" | |
| -DUSE_NCCL="FALSE" | |
| -DUSE_SENTENCEPIECE="TRUE" | |
| -DUSE_STATIC_LIBS="TRUE"' | |
| cmakeListsOrSettingsJson: CMakeListsTxtAdvanced | |
| cmakeListsTxtPath: ${{ github.workspace }}/CMakeLists.txt | |
| useVcpkgToolchainFile: true | |
| # Building in Debug is sufficient for the all-in CPU+GPU compilation; | |
| # its main purpose is to detect warnings that the Release build is not | |
| # able to find sometimes. | |
| if: matrix.gpu == true | |
| # Windows CUDA builds use USE_NCCL=off due to compilation errors | |
| - name: Build Release | |
| uses: lukka/run-cmake@v3 | |
| with: | |
| buildDirectory: ${{ github.workspace }}/build/ | |
| cmakeAppendedArgs: '-G Ninja | |
| -DBOOST_ROOT="${{ env.BOOST_ROOT }}" | |
| -DBOOST_INCLUDEDIR="${{ env.BOOST_ROOT }}/include" | |
| -DBOOST_LIBRARYDIR="${{ env.BOOST_ROOT }}/lib" | |
| -DCMAKE_BUILD_TYPE="Release" | |
| -DOPENSSL_USE_STATIC_LIBS="TRUE" | |
| -DOPENSSL_MSVC_STATIC_RT="TRUE" | |
| -DCOMPILE_CPU="TRUE" | |
| -DCOMPILE_CUDA="${{ matrix.gpu }}" | |
| -DCOMPILE_SERVER="FALSE" | |
| -DCOMPILE_TESTS="TRUE" | |
| -DDETERMINISTIC="TRUE" | |
| -DUSE_FBGEMM="TRUE" | |
| -DUSE_MPI="FALSE" | |
| -DUSE_NCCL="FALSE" | |
| -DUSE_SENTENCEPIECE="TRUE" | |
| -DUSE_STATIC_LIBS="TRUE"' | |
| cmakeListsOrSettingsJson: CMakeListsTxtAdvanced | |
| cmakeListsTxtPath: ${{ github.workspace }}/CMakeLists.txt | |
| useVcpkgToolchainFile: true | |
| - name: Run unit tests | |
| working-directory: build/ | |
| run: ctest | |
| # Not run in GPU builds because GitHub-hosted VMs do not have GPUs | |
| if: matrix.gpu == false | |
| - name: Print versions | |
| working-directory: build/ | |
| run: | | |
| .\marian.exe --version | |
| .\marian-decoder.exe --version | |
| .\marian-scorer.exe --version | |
| dir *.exe | |
| shell: cmd |