fix downloading of models in the python binding (#472) #378
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Build" | |
| 'on': | |
| push: | |
| branches: | |
| - main | |
| - ci-sandbox | |
| tags: | |
| - "v*.*.*" | |
| pull_request: | |
| branches: | |
| - '**' | |
| env: | |
| qt_version: "6.2.1" # only used by build-macos | |
| emsdk_version: 3.1.8 # For use in emscripten build | |
| ccache_basedir: ${{ github.workspace }} | |
| ccache_dir: "${{ github.workspace }}/.ccache" | |
| ccache_compilercheck: content | |
| ccache_compress: 'true' | |
| ccache_compresslevel: 9 | |
| ccache_maxsize: 200M | |
| ccache_cmake: -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_C_COMPILER_LAUNCHER=ccache | |
| jobs: | |
| build-wheels: | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macos-latest] | |
| fail-fast: false | |
| name: "cibuildwheel / ${{ matrix.os }}" | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - name: Generate ccache_vars for ccache based on machine | |
| shell: bash | |
| id: ccache_vars | |
| run: |- | |
| echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})" | |
| echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')" | |
| - name: Cache-op for build-cache through ccache | |
| uses: actions/cache@v2 | |
| with: | |
| path: ${{ env.ccache_dir }} | |
| key: ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }} | |
| restore-keys: |- | |
| ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }} | |
| ccache-cibuildwheel-${{ matrix.os }}-${{ steps.ccache_vars.outputs.hash }} | |
| ccache-cibuildwheel-${{ matrix.os }} | |
| - name: ccache environment setup | |
| run: |- | |
| mkdir -p ${{ env.ccache_dir }} | |
| - name: Inject local version identifier for non tag builds | |
| if: ${{ !startsWith(github.ref, 'refs/tags/v') }} | |
| run: |- | |
| echo "PYTHON_LOCAL_VERSION_IDENTIFIER=$(git rev-parse --short HEAD)" >> $GITHUB_ENV | |
| - name: Apply MacOS patch | |
| if: ${{ startsWith(runner.os, 'mac') }} | |
| run: | | |
| patch -p1 < patches/01-marian-fstream-for-macos.patch | |
| - name: Build wheels | |
| uses: pypa/[email protected] | |
| # to supply options, put them in 'env', like: | |
| env: | |
| CIBW_ENVIRONMENT_LINUX: | |
| BUILD_ARCH=core-avx-i | |
| USE_CCACHE=1 | |
| CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }} | |
| CCACHE_COMPRESS=${{ env.ccache_compress }} | |
| CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }} | |
| CCACHE_MAXSIZE=${{ env.ccache_maxsize }} | |
| PYTHON_LOCAL_VERSION_IDENTIFIER=${{ env.PYTHON_LOCAL_VERSION_IDENTIFIER }} | |
| CCACHE_DIR=/host/${{ env.ccache_dir }} | |
| CCACHE_BASEDIR=/host/${{ env.ccache_basedir }} | |
| CIBW_ENVIRONMENT_MACOS: | |
| BUILD_ARCH=core-avx-i | |
| USE_CCACHE=1 | |
| CCACHE_COMPILER_CHECK=${{ env.ccache_compilercheck }} | |
| CCACHE_COMPRESS=${{ env.ccache_compress }} | |
| CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }} | |
| CCACHE_MAXSIZE=${{ env.ccache_maxsize }} | |
| PYTHON_LOCAL_VERSION_IDENTIFIER=${{ env.PYTHON_LOCAL_VERSION_IDENTIFIER }} | |
| CCACHE_DIR=${{ env.ccache_dir }} | |
| CCACHE_BASEDIR=${{ env.ccache_basedir }} | |
| MACOSX_DEPLOYMENT_TARGET=10.9 | |
| CIBW_BEFORE_BUILD_LINUX: | | |
| yum install -y ccache | |
| # Install Intel MKL. | |
| yum-config-manager -y --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo | |
| yum install -y intel-mkl | |
| chmod -R a+rwx /host/${{ env.ccache_dir }} | |
| ccache -s # Print current cache stats | |
| ccache -z # Zero cache entry | |
| CIBW_BEFORE_BUILD_MACOS: | | |
| brew install openblas protobuf ccache boost pybind11 | |
| chmod -R a+rwx ${{ env.ccache_dir }} | |
| ccache -s # Print current cache stats | |
| ccache -z # Zero cache entry | |
| CIBW_BUILD: "cp{36,37,38,39,310}-*manylinux_x86_64 cp{36,37,38,39,310}-macosx_x86_64" | |
| CIBW_BEFORE_TEST: | | |
| ccache -s # Print current ccache stats | |
| CIBW_TEST_COMMAND: | | |
| # The wheels are installed automatically and available. | |
| # Fetch models from translateLocally repository. | |
| python3 -m bergamot download -m en-de-tiny | |
| python3 -m bergamot download -m de-en-tiny | |
| python3 -m bergamot ls | |
| # Fetch models from opus repository. | |
| python3 -m bergamot download -m eng-fin-tiny -r opus | |
| python3 -m bergamot ls -r opus | |
| # Run the sample python script shipped with module | |
| python3 -m bergamot translate --model en-de-tiny <<< "Hello World" | |
| python3 -m bergamot translate --model en-de-tiny de-en-tiny <<< "Hello World" | |
| python3 -m bergamot translate --model eng-fin-tiny --repository opus <<< "Hello World" | |
| - uses: actions/upload-artifact@v2 | |
| with: | |
| name: wheels | |
| path: ./wheelhouse/*.whl | |
| upload-wheels: | |
| name: "Upload wheels to PyPI" | |
| runs-on: ubuntu-latest | |
| if: ${{ startsWith(github.ref, 'refs/tags/v') }} | |
| needs: [build-wheels] | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: wheels | |
| - name: Publish wheels to PyPI | |
| env: | |
| TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} | |
| run: | | |
| python3 -m pip install twine | |
| twine upload *.whl | |
| build-wasm: | |
| name: "emscripten" | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - name: Set ccache environment for emcc | |
| run: | | |
| # We are hardcoding this to mtime instead of env pickup. Rest use content. | |
| echo "CCACHE_COMPILER_CHECK=mtime" >> $GITHUB_ENV | |
| echo "CCACHE_BASEDIR=${{ env.ccache_basedir }}" >> $GITHUB_ENV | |
| echo "CCACHE_COMPRESS=${{ env.ccache_compress }}" >> $GITHUB_ENV | |
| echo "CCACHE_COMPRESSLEVEL=${{ env.ccache_compresslevel }}" >> $GITHUB_ENV | |
| echo "CCACHE_DIR=${{ env.ccache_dir }}" >> $GITHUB_ENV | |
| echo "CCACHE_MAXSIZE=${{ env.ccache_maxsize }}" >> $GITHUB_ENV | |
| # https://emscripten.org/docs/compiling/Building-Projects.html#using-a-compiler-wrapper | |
| echo "EM_COMPILER_WRAPPER=ccache" >> $GITHUB_ENV | |
| # This need to be run before setup, so ccache build caching doesn't complain. | |
| - name: Obtain emsdk sources | |
| run: | | |
| git clone --depth 1 https://github.com/emscripten-core/emsdk.git | |
| - name: Cache-op for build-cache through ccache | |
| uses: actions/cache@v2 | |
| with: | |
| path: | | |
| ${{ env.ccache_dir }} | |
| ${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/ | |
| key: ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }}-${{ steps.ccache_vars.outputs.timestamp }} | |
| restore-keys: |- | |
| ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }}-${{ github.ref }} | |
| ccache-${{ github.job }}-${{ env.emsdk_version }}-${{ steps.ccache_vars.outputs.hash }} | |
| ccache-${{ github.job }}-${{ env.emsdk_version }} | |
| - name: Setup Emscripten toolchain | |
| run: | | |
| (cd emsdk && ./emsdk install ${{ env.emsdk_version }} ccache-git-emscripten-64bit) | |
| (cd emsdk && ./emsdk activate ${{ env.emsdk_version }} ccache-git-emscripten-64bit) | |
| # mtime of this file is checked by ccache, we set it to avoid cache misses. | |
| touch -m -d '1 Jan 2021 12:00' emsdk/.emscripten | |
| # These needs to be done in the activated shell. | |
| eval $(./emsdk/emsdk construct_env \ | |
| | sed 's/export PATH=\(.*\);/echo \1 >> $GITHUB_PATH;/' \ | |
| | sed 's/export \(.*\);/echo \1 >> $GITHUB_ENV;/' ); | |
| # This looks more permanent than version pinned, so keeping temporarily to avoid failures. | |
| echo "${{ github.workspace }}/emsdk/ccache/git-emscripten_64bit/bin" >> $GITHUB_PATH | |
| - name: Generate ccache_vars for ccache based on machine | |
| shell: bash | |
| id: ccache_vars | |
| run: |- | |
| echo "::set-output name=hash::$(echo ${{ env.ccache_compilercheck }})" | |
| echo "::set-output name=timestamp::$(date '+%Y-%m-%dT%H.%M.%S')" | |
| - name: Verify Emscripten setup | |
| run: | | |
| emcc --version | |
| emcmake cmake --version | |
| emmake make --version | |
| - name: ccache prolog | |
| run: |- | |
| ccache -s # Print current cache stats | |
| ccache -z # Zero cache entry | |
| - name: "Configure builds" | |
| run: | | |
| mkdir -p build-wasm | |
| cd build-wasm | |
| emcmake cmake -DCOMPILE_WASM=on .. | |
| - name: "Compile" | |
| working-directory: build-wasm | |
| run: | | |
| emmake make -j2 | |
| - name: ccache epilog | |
| run: | | |
| ccache -s # Print current cache stats | |
| - name: Import GEMM library from a separate wasm module | |
| working-directory: build-wasm | |
| run: bash ../wasm/patch-artifacts-import-gemm-module.sh | |
| # Setup nodejs-18, as nodejs-14 provided by emsdk fails when running | |
| # and newer version of node allows us to use fetch(). | |
| - name: Setup nodejs | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version: 18 | |
| - name: Test run | |
| working-directory: wasm | |
| run: | | |
| cp ../build-wasm/bergamot-translator-worker.{js,wasm} ./ | |
| npm install jsdom | |
| # --unhandled-rejections make the script exit with a non-zero code (at least on node-14). | |
| # So leaving this here. | |
| node --unhandled-rejections=strict node-test.js | |
| # Upload both together. | |
| - name: Upload wasm artifact | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: wasm-artefacts | |
| if-no-files-found: error | |
| path: | | |
| ${{github.workspace}}/build-wasm/bergamot-translator-worker.js | |
| ${{github.workspace}}/build-wasm/bergamot-translator-worker.wasm | |
| ${{github.workspace}}/build-wasm/bergamot-translator-worker.js.bak | |
| upload-wasm: | |
| name: "Upload node package to NPM" | |
| runs-on: ubuntu-latest | |
| if: ${{ startsWith(github.ref, 'refs/tags/v') }} | |
| needs: [build-wasm] | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: wasm-artefacts | |
| path: wasm/module/worker | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: '18.x' | |
| registry-url: 'https://registry.npmjs.org' | |
| - run: npm ci | |
| - run: npm publish | |
| env: | |
| NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} | |
| # Try to upload a release using https://github.com/marvinpinto/actions/issues/177#issuecomment-917605585 as a model | |
| release-latest: | |
| name: Release Latest Build | |
| runs-on: ubuntu-latest | |
| needs: [build-wheels, build-wasm] | |
| if: github.ref == 'refs/heads/main' | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| # Leave the below be, it will be useful. | |
| - name: List downloaded assets | |
| run: | | |
| find ./ | |
| - name: Update GitHub prerelease | |
| uses: marvinpinto/action-automatic-releases@latest | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| automatic_release_tag: latest | |
| prerelease: true | |
| title: "Latest Build" | |
| files: | | |
| wheels/*.whl | |
| wasm-artefacts/bergamot-translator-worker.js | |
| wasm-artefacts/bergamot-translator-worker.wasm | |
| release-version: | |
| name: Release version | |
| runs-on: ubuntu-latest | |
| needs: [build-wheels, build-wasm] | |
| permissions: | |
| contents: "write" | |
| packages: "write" | |
| pull-requests: "read" | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| # Leave the below be, it will be useful. | |
| - name: List downloaded assets | |
| run: | | |
| find ./ | |
| - name: Update GitHub release | |
| uses: marvinpinto/action-automatic-releases@latest | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| automatic_release_tag: ${{ github.ref_name }} | |
| prerelease: false | |
| title: "${{ github.ref_name }}" | |
| files: | | |
| wheels/*.whl | |
| wasm-artefacts/bergamot-translator-worker.js | |
| wasm-artefacts/bergamot-translator-worker.wasm | |
| python-checks: | |
| name: "formatting and typechecks" | |
| runs-on: "ubuntu-latest" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| - name: Install Dependencies | |
| run: |- | |
| python3 -m pip install black isort pytype | |
| - name: "Formatting checks: black, isort" | |
| run: | | |
| python3 -m black --diff --check bindings/python/ setup.py doc/conf.py | |
| python3 -m isort --profile black --diff --check bindings/python setup.py doc/conf.py | |
| - name: "Static typing checks: pytype" | |
| run: |- | |
| python3 -m pytype bindings/python | |
| docs: | |
| runs-on: ubuntu-latest | |
| needs: [build-wheels] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: recursive | |
| # Runs javascript to extract push events from both tags and branch (only main, due to workflow trigger) | |
| # converts refs/<>/<name> -> <name> | |
| # eg: | |
| # refs/head/main -> main | |
| # refs/tags/v0.1.0 -> v0.1.0 | |
| # | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| - name: Extract tag name | |
| id: tag | |
| uses: actions/[email protected] | |
| if: ${{ github.event_name == 'push' }} | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| script: | | |
| const args = context.payload.ref.split("/"); | |
| [refs, category, ...rest] = args; | |
| return rest.join("/"); | |
| # Patches the BERGAMOT_VERSION file used by sphinx-docs at run time to | |
| # obtain names like 'main' or 'ci-sandbox' to not confuse with version | |
| # based documentation built separately. | |
| - name: Deploy-time patch version | |
| run: | | |
| echo ${{steps.tag.outputs.result }} > BERGAMOT_VERSION | |
| - name: Set up Doxygen | |
| run: sudo apt-get install -y doxygen | |
| - name: Set up Python | |
| uses: actions/setup-python@v2 | |
| with: | |
| python-version: 3.7 | |
| - name: Set up dependency cache | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('doc/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| working-directory: ./doc | |
| run: | | |
| python3 -m pip install -r requirements.txt | |
| python3 -m pip install --find-links=${{github.workspace}}/wheels bergamot | |
| - name: Build documentation | |
| working-directory: ./doc | |
| run: sphinx-build -b html ./ build/ | |
| - name: Deploy 🚀 | |
| uses: JamesIves/[email protected] | |
| if: ${{ github.event_name == 'push' && github.repository == 'browsermt/bergamot-translator' }} | |
| with: | |
| repository-name: 'browsermt/docs' | |
| branch: gh-pages # The branch the action should deploy to. | |
| folder: './doc/build/' # The folder the action should deploy. | |
| target-folder: '${{ steps.tag.outputs.result }}' | |
| ssh-key: ${{ secrets.BERGAMOT_SSH_PRIVATE_KEY }} | |
| # This artifact contains the HTML output of Sphinx only. | |
| # With index.html at the root of the produced zip file. | |
| # For use for maintainers to download the zip and check render of | |
| # documentation while generated at pull-request. | |
| - name: Upload documentation | |
| uses: actions/upload-artifact@v2 | |
| if: ${{ github.event_name == 'pull_request'}} | |
| with: | |
| name: api-docs | |
| path: ./doc/build/ | |
| if-no-files-found: error |