Generate conda packages #468
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate conda packages | |
| # This action automatically generate conda packages for the packages in the robotology-superbuild | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| upload_conda_binaries: | |
| description: 'If true, upload conda binaries, for any other value only generate them without uploading.' | |
| required: true | |
| default: 'true' | |
| test_metapackages_generation: | |
| description: 'If true, also test metapackages generation. Not compatible with upload_conda_binaries set to true.' | |
| required: true | |
| default: 'false' | |
| schedule: | |
| # Run the job once a week | |
| - cron: '0 0 * * 2' | |
| release: | |
| types: [published] | |
| jobs: | |
| # Regardless of the branch on which this action is trigged, | |
| # the CONDA_BUILD_NUMBER CMake option needs to be read from the | |
| # master branch, to avoid that a conda package generation from the | |
| # master branch and one from a releases/YYYY.MM branch use the same | |
| # CONDA_BUILD_NUMBER | |
| get-conda-build-number: | |
| name: "Read Conda Build number from master branch" | |
| runs-on: ubuntu-latest | |
| # Define outputs (see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-defining-outputs-for-a-job) | |
| outputs: | |
| conda_build_number: ${{ steps.step1.outputs.conda_build_number }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: 'master' | |
| - id: step1 | |
| name: Get CONDA_BUILD_NUMBER and set it as output | |
| shell: bash | |
| run: | | |
| # Get CONDA_BUILD_NUMBER via grep and set it to an environment variable | |
| export CONDA_BUILD_NUMBER=`grep "CONDA_BUILD_NUMBER" ./conda/cmake/CondaGenerationOptions.cmake | grep -oe '\([0-9.]*\)'` | |
| echo "conda_build_number=${CONDA_BUILD_NUMBER}" >> $GITHUB_OUTPUT | |
| generate-conda-packages: | |
| name: "Generate conda packages @${{ matrix.os }}" | |
| runs-on: ${{ matrix.os }} | |
| needs: get-conda-build-number | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - os: ubuntu-20.04 | |
| conda_platform: linux-64 | |
| - os: macos-14 | |
| conda_platform: osx-arm64 | |
| - os: windows-2019 | |
| conda_platform: win-64 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: conda-incubator/setup-miniconda@v3 | |
| with: | |
| miniforge-variant: Miniforge3 | |
| miniforge-version: latest | |
| - name: Install files to enable compilation of mex files [Conda/Linux] | |
| if: contains(matrix.os, 'ubuntu') | |
| run: | | |
| curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2020b_mexa64.zip | |
| unzip msdk_R2020b_mexa64.zip | |
| rm msdk_R2020b_mexa64.zip | |
| echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2020b_mexa64" >> $GITHUB_ENV | |
| echo "GHA_Matlab_MEX_EXTENSION=mexa64" >> $GITHUB_ENV | |
| - name: Install files to enable compilation of mex files [Conda/macOS] | |
| if: contains(matrix.os, 'macos') | |
| run: | | |
| curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2023b_mexmaca64.zip | |
| unzip msdk_R2023b_mexmaca64.zip | |
| rm msdk_R2023b_mexmaca64.zip | |
| echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2023b_mexmaca64" >> $GITHUB_ENV | |
| echo "GHA_Matlab_MEX_EXTENSION=mexmaca64" >> $GITHUB_ENV | |
| - name: Install files to enable compilation of mex files [Conda/Windows] | |
| if: contains(matrix.os, 'windows') | |
| shell: bash | |
| run: | | |
| curl -L -O https://github.com/robotology/robotology-vcpkg-ports/releases/download/storage/msdk_R2020a_mexw64.zip | |
| unzip msdk_R2020a_mexw64.zip | |
| rm msdk_R2020a_mexw64.zip | |
| echo "GHA_Matlab_ROOT_DIR=${GITHUB_WORKSPACE}/msdk_R2020a_mexw64" >> $GITHUB_ENV | |
| echo "GHA_Matlab_MEX_EXTENSION=mexw64" >> $GITHUB_ENV | |
| - name: Dependencies for conda recipes generation and upload | |
| shell: bash -l {0} | |
| run: | | |
| conda install pyyaml jinja2 conda-build ninja anaconda-client conda-forge-pinning mamba boa multisheller pixi | |
| - name: Print used environment | |
| shell: bash -l {0} | |
| run: | | |
| conda list | |
| env | |
| - name: Generate recipes [Linux&macOS] | |
| if: contains(matrix.os, 'macos') || contains(matrix.os, 'ubuntu') | |
| shell: bash -l {0} | |
| run: | | |
| mkdir build | |
| cd build | |
| cmake -GNinja -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}} .. | |
| - name: Generate recipes [Windows] | |
| if: contains(matrix.os, 'windows') | |
| shell: bash -l {0} | |
| run: | | |
| mkdir build | |
| cd build | |
| cmake -G"Visual Studio 16 2019" -C ${GITHUB_WORKSPACE}/.ci/initial-cache.gh.cmake -DYCM_EP_ADDITIONAL_CMAKE_ARGS:STRING="-DMatlab_ROOT_DIR:PATH=${GHA_Matlab_ROOT_DIR} -DMatlab_MEX_EXTENSION:STRING=${GHA_Matlab_MEX_EXTENSION}" -DROBOTOLOGY_USES_MATLAB:BOOL=ON -DROBOTOLOGY_PROJECT_TAGS=LatestRelease -DROBOTOLOGY_GENERATE_CONDA_RECIPES:BOOL=ON -DCONDA_BUILD_NUMBER=${{needs.get-conda-build-number.outputs.conda_build_number}} .. | |
| - name: Specify additional option if we are in a release and we need to generate robotology-distro metapackages | |
| if: github.event_name == 'release' | |
| shell: bash -l {0} | |
| run: | | |
| cd build | |
| cmake -DROBOTOLOGY_PROJECT_TAGS=Custom -DCONDA_GENERATE_ROBOTOLOGY_METAPACKAGES:BOOL=ON . | |
| - name: Specify additional option if we want generate robotology-distro metapackages | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true' | |
| shell: bash -l {0} | |
| run: | | |
| cd build | |
| cmake -DCONDA_GENERATE_ROBOTOLOGY_METAPACKAGES:BOOL=ON . | |
| - name: Fail if both upload_conda_binaries and test_metapackages_generation are passed | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true' && github.event.inputs.upload_conda_binaries == 'true' | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| core.setFailed('Build failed. It is not possible to set to true both upload_conda_binaries and test_metapackages_generation.') | |
| - name: If test_metapackages_generation is enabled set an arbitrary version number | |
| if: github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true' | |
| shell: bash -l {0} | |
| run: | | |
| sed -i -e 's/set(INSTALLER_VERSION "")/set(INSTALLER_VERSION 1234.0)/g' ./packaging/windows/CMakeLists.txt | |
| sed -i -e 's/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION "")/set(CONDA_ROBOTOLOGY_SUPERBUILD_VERSION 1234.0)/g' ./conda/cmake/CondaGenerationOptions.cmake | |
| - name: Set CONDA_BLD_PATH on Windows to avoid long path issues [Windows] | |
| if: contains(matrix.os, 'windows') | |
| shell: bash -l {0} | |
| run: | | |
| echo "CONDA_BLD_PATH=C:\\bld" >> $GITHUB_ENV | |
| - name: Set CONDA_BLD_PATH on non-Windows for consistency [non Windows] | |
| if: contains(matrix.os, 'macos') || contains(matrix.os, 'ubuntu') | |
| shell: bash -l {0} | |
| run: | | |
| echo "CONDA_BLD_PATH=${CONDA_PREFIX}/conda-bld" >> $GITHUB_ENV | |
| - name: Build conda packages | |
| shell: bash -l {0} | |
| run: | | |
| conda config --remove channels defaults | |
| conda config --env --set channel_priority strict | |
| cd build/conda/generated_recipes | |
| # Debug generated recipes | |
| cat */meta.yaml | |
| cat */bld.bat | |
| cat */build.sh | |
| # We use the conda_build_config from conda-forge-pinning, and our local one | |
| # See https://docs.conda.io/projects/conda-build/en/latest/resources/variants.html#creating-conda-build-variant-config-files | |
| # We manually specify the build order as an old workaround for mambabuild, in theory this can be avoided, but it needs to be tested | |
| # see https://github.com/mamba-org/boa/issues/117 | |
| # We also manually delete the recipe directory to avoid that it is built again, | |
| # specifically because we can't use --skip-existing as these packages are part of the robotology channel | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml icub-contrib-common | |
| rm -rf icub-contrib-common | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml robots-configuration | |
| rm -rf robots-configuration | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml robot-log-visualizer | |
| rm -rf robot-log-visualizer | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml idyntree-matlab-bindings | |
| rm -rf idyntree-matlab-bindings | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml blockfactory | |
| rm -rf blockfactory | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml yarp-matlab-bindings | |
| rm -rf yarp-matlab-bindings | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml wb-toolbox | |
| rm -rf wb-toolbox | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml blocktest | |
| rm -rf blocktest | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml . | |
| - name: Build robotology-distro conda metapackage (if necessary) | |
| if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true') | |
| shell: bash -l {0} | |
| run: | | |
| cd build/conda/generated_recipes_metapackages | |
| # Debug generated recipes | |
| cat */meta.yaml | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro | |
| - name: Upload conda packages | |
| shell: bash -l {0} | |
| # Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true' | |
| if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true') | |
| env: | |
| ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} | |
| PREFIX_DEV_TOKEN: ${{ secrets.PREFIX_DEV_TOKEN }} | |
| run: | | |
| cd ${CONDA_BLD_PATH}/${{ matrix.conda_platform}}/ | |
| ls *.tar.bz2 | |
| anaconda upload --skip-existing *.tar.bz2 | |
| pixi auth login https://prefix.dev --token $PREFIX_DEV_TOKEN | |
| for condapackage in *.tar.bz2; do | |
| pixi upload https://prefix.dev/api/v1/upload/robotology "$condapackage" | |
| done | |
| pixi auth logout https://prefix.dev | |
| # Only on linux-64 we upload noarch packages (noarch packages need to be uploaded only once, as they can be reused across platforms) | |
| - name: Upload conda packages (noarch) | |
| shell: bash -l {0} | |
| # Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true' | |
| if: matrix.conda_platform == 'linux-64' && (github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true')) | |
| env: | |
| ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} | |
| PREFIX_DEV_TOKEN: ${{ secrets.PREFIX_DEV_TOKEN }} | |
| run: | | |
| cd ${CONDA_BLD_PATH}/noarch/ | |
| ls *.tar.bz2 | |
| anaconda upload --skip-existing *.tar.bz2 | |
| pixi auth login https://prefix.dev --token $PREFIX_DEV_TOKEN | |
| for condapackage in *.tar.bz2; do | |
| pixi upload https://prefix.dev/api/v1/upload/robotology "$condapackage" | |
| done | |
| pixi auth logout https://prefix.dev | |
| - name: Build robotology-distro-all conda metapackage (if necessary) | |
| if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true') | |
| shell: bash -l {0} | |
| run: | | |
| cd build/conda/generated_recipes_metapackages | |
| # Debug generated recipes | |
| cat */meta.yaml | |
| conda mambabuild -m ${CONDA_PREFIX}/conda_build_config.yaml -m ${GITHUB_WORKSPACE}/conda/conda_build_config.yml -c local -c conda-forge -c robotology robotology-distro-all | |
| - name: Upload robotology-distro-all metapackage | |
| shell: bash -l {0} | |
| # Upload by default on schedule events, and on workflow dispatch only if input upload_conda_binaries is 'true' | |
| if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.test_metapackages_generation == 'true' && github.event.inputs.upload_conda_binaries == 'true') | |
| env: | |
| ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }} | |
| PREFIX_DEV_TOKEN: ${{ secrets.PREFIX_DEV_TOKEN }} | |
| run: | | |
| cd ${CONDA_BLD_PATH}/${{ matrix.conda_platform}}/ | |
| ls robotology-distro-all-*.tar.bz2 | |
| anaconda upload --skip-existing robotology-distro-all-*.tar.bz2 | |
| pixi auth login https://prefix.dev --token $PREFIX_DEV_TOKEN | |
| for condapackage in robotology-distro-all-*.tar.bz2; do | |
| pixi upload https://prefix.dev/api/v1/upload/robotology "$condapackage" | |
| done | |
| pixi auth logout https://prefix.dev | |
| # If the generate-conda-packages completed correctly and binaries are uploaded, | |
| # bump automatically the CONDA_BUILD_NUMBER in conda/cmake/CondaGenerationOptions.cmake | |
| # of master branch for future builds | |
| # the master branch is always used in case the action is triggered by a release on | |
| # a release/vYYYY.MM branch | |
| bump-conda-build-number: | |
| name: "Bump Conda Build number for future builds" | |
| runs-on: ubuntu-latest | |
| needs: generate-conda-packages | |
| if: github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && github.event.inputs.upload_conda_binaries == 'true') | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: 'master' | |
| - name: Bump Conda Build number for future builds | |
| shell: bash | |
| run: | | |
| sh ./scripts/robotologyBumpCondaBuildNumber.sh | |
| - uses: EndBug/[email protected] | |
| with: | |
| default_author: github_actions | |
| message: 'Bump CONDA_BUILD_NUMBER after successful Conda packages build and upload' |