From a148b5ee99b037ac8aff6f41cc67ea57be54db08 Mon Sep 17 00:00:00 2001 From: Krishna Pai Date: Tue, 21 Jan 2025 22:35:40 -0800 Subject: [PATCH 1/6] Print out various metrics. --- .github/workflows/build-metrics.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml index 5e743b34210e..f627830abbd9 100644 --- a/.github/workflows/build-metrics.yml +++ b/.github/workflows/build-metrics.yml @@ -35,7 +35,7 @@ permissions: jobs: metrics: - name: Linux ${{ matrix.type }} with adapters + name: Linux ${{ matrix.link-type }} - ${{ matrix.type }} with adapters if: ${{ github.repository == 'facebookincubator/velox' }} runs-on: ${{ matrix.runner }} container: ghcr.io/facebookincubator/velox-dev:adapters @@ -108,6 +108,9 @@ jobs: find velox -type f -name 'velox_*' -exec ls -l -BB {} \; | awk '{print $5, $9; total += $5} END {print total," total_exec_size"}' >> $sizes_file + + echo 'Size file for ${{ matrix.link-type }} - ${{ matrix.type }} ' + cat $sizes_file - name: Copy ninja_log run: cp _build/${{ matrix.type }}/.ninja_log /tmp/metrics/.ninja_log From 6be2180b5fa5f6f7abb096d2cfdb8764ee2a6001 Mon Sep 17 00:00:00 2001 From: Krishna Pai Date: Tue, 21 Jan 2025 23:32:12 -0800 Subject: [PATCH 2/6] Export extra cmake flags. --- .github/workflows/build-metrics.yml | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml index f627830abbd9..68359c303306 100644 --- a/.github/workflows/build-metrics.yml +++ b/.github/workflows/build-metrics.yml @@ -80,19 +80,8 @@ jobs: env: MAKEFLAGS: 'MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=4' run: | - EXTRA_CMAKE_FLAGS=( - "-DVELOX_ENABLE_BENCHMARKS=ON" - "-DVELOX_ENABLE_ARROW=ON" - "-DVELOX_ENABLE_PARQUET=ON" - "-DVELOX_ENABLE_HDFS=ON" - "-DVELOX_ENABLE_S3=ON" - "-DVELOX_ENABLE_GCS=ON" - "-DVELOX_ENABLE_ABFS=ON" - "-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON" - "-DVELOX_MONO_LIBRARY=ON" - "-DVELOX_BUILD_SHARED=ON" - ) - make '${{ matrix.type }}' + EXTRA_CMAKE_FLAGS=( "-DVELOX_ENABLE_BENCHMARKS=ON -DVELOX_ENABLE_ARROW=ON -DVELOX_ENABLE_PARQUET=ON -DVELOX_ENABLE_HDFS=ON -DVELOX_ENABLE_S3=ON -DVELOX_ENABLE_GCS=ON -DVELOX_ENABLE_ABFS=ON -DVELOX_ENABLE_REMOTE_FUNCTIONS=ON -DVELOX_MONO_LIBRARY=ON -DVELOX_BUILD_SHARED=ON" ) + EXTRA_CMAKE_FLAGS=$EXTRA_CMAKE_FLAGS make '${{ matrix.type }}' - name: Log binary sizes run: | From 6a1ac401ba4ef40625b45af72300b7fde793fdb0 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 22 Jan 2025 17:57:58 +0100 Subject: [PATCH 3/6] unify build step, fix cmake flags and upload size file as artifact --- .github/workflows/build-metrics.yml | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml index 68359c303306..9927ee58d669 100644 --- a/.github/workflows/build-metrics.yml +++ b/.github/workflows/build-metrics.yml @@ -59,7 +59,6 @@ jobs: run: git config --global --add safe.directory ${GITHUB_WORKSPACE} - name: Make ${{ matrix.link-type }} - ${{ matrix.type }} Build - if: ${{ matrix.link-type == 'static' }} env: MAKEFLAGS: 'MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=4' run: | @@ -72,21 +71,19 @@ jobs: "-DVELOX_ENABLE_GCS=ON" "-DVELOX_ENABLE_ABFS=ON" "-DVELOX_ENABLE_REMOTE_FUNCTIONS=ON" + "-DVELOX_MONO_LIBRARY=ON" + "-DVELOX_BUILD_SHARED=${{ matrix.link-type == 'shared' && 'ON' || 'OFF' }}" ) - make '${{ matrix.type }}' - - name: Make ${{ matrix.link-type }} - ${{ matrix.type }} Build - if: ${{ matrix.link-type == 'shared' }} - env: - MAKEFLAGS: 'MAX_HIGH_MEM_JOBS=8 MAX_LINK_JOBS=4' - run: | - EXTRA_CMAKE_FLAGS=( "-DVELOX_ENABLE_BENCHMARKS=ON -DVELOX_ENABLE_ARROW=ON -DVELOX_ENABLE_PARQUET=ON -DVELOX_ENABLE_HDFS=ON -DVELOX_ENABLE_S3=ON -DVELOX_ENABLE_GCS=ON -DVELOX_ENABLE_ABFS=ON -DVELOX_ENABLE_REMOTE_FUNCTIONS=ON -DVELOX_MONO_LIBRARY=ON -DVELOX_BUILD_SHARED=ON" ) - EXTRA_CMAKE_FLAGS=$EXTRA_CMAKE_FLAGS make '${{ matrix.type }}' + echo "CMake Flags: ${EXTRA_CMAKE_FLAGS[*]}" + make '${{ matrix.type }}' EXTRA_CMAKE_FLAGS="${EXTRA_CMAKE_FLAGS[*]}" - name: Log binary sizes run: | mkdir -p /tmp/metrics sizes_file=/tmp/metrics/object_sizes + echo "sizes_file=$sizes_file" >> $GITHUB_ENV + pushd '_build/${{ matrix.type }}' find velox -type f -name '*.so' -o -name '*.a' -exec ls -l -BB {} \; | @@ -97,9 +94,14 @@ jobs: find velox -type f -name 'velox_*' -exec ls -l -BB {} \; | awk '{print $5, $9; total += $5} END {print total," total_exec_size"}' >> $sizes_file - - echo 'Size file for ${{ matrix.link-type }} - ${{ matrix.type }} ' + + echo "::group::Size file for ${{ matrix.link-type }} - ${{ matrix.type }}" cat $sizes_file + echo "::endgroup::" + + - uses: actions/upload-artifact@v4 + with: + path: ${{ env.sizes_file }} - name: Copy ninja_log run: cp _build/${{ matrix.type }}/.ninja_log /tmp/metrics/.ninja_log From b9927b6d2635149f47711dc17271921c3df8619c Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 22 Jan 2025 20:24:41 +0100 Subject: [PATCH 4/6] fix find invocation --- .github/workflows/build-metrics.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml index 7161a4282a42..99b26aabf952 100644 --- a/.github/workflows/build-metrics.yml +++ b/.github/workflows/build-metrics.yml @@ -87,13 +87,13 @@ jobs: pushd '_build/${{ matrix.type }}' # . to also check in lib for mono build - find . -type f -name 'libvelox*.so' -o -name 'libvelox*.a' -exec ls -l -BB {} \; | + find . -type f -name 'libvelox*.so' -o -name 'libvelox*.a' | xargs ls -l -BB | awk '{print $5, $9; total += $5} END {print total," total_lib_size"}' > $sizes_file - find velox -type f -name '*.o' -exec ls -l -BB {} \; | + find velox -type f -name '*.o' | ls -l -BB | awk '{print $5, $9; total += $5} END {print total," total_obj_size"}' >> $sizes_file - find velox -type f -name 'velox_*' -exec ls -l -BB {} \; | + find velox -type f -name 'velox_*' | ls -l -BB | awk '{print $5, $9; total += $5} END {print total," total_exec_size"}' >> $sizes_file echo "::group::Size file for ${{ matrix.link-type }} - ${{ matrix.type }}" From 14090bd8d7d1b9dd77aeea03b72fc4842be7f1c1 Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Wed, 22 Jan 2025 22:49:48 +0100 Subject: [PATCH 5/6] use exec but with grouping --- .github/workflows/build-metrics.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml index 99b26aabf952..b88d8586ca56 100644 --- a/.github/workflows/build-metrics.yml +++ b/.github/workflows/build-metrics.yml @@ -87,13 +87,13 @@ jobs: pushd '_build/${{ matrix.type }}' # . to also check in lib for mono build - find . -type f -name 'libvelox*.so' -o -name 'libvelox*.a' | xargs ls -l -BB | + find . -type f \( -name 'libvelox*.so' -o -name 'libvelox*.a' \) -exec ls -l -BB {} \; | awk '{print $5, $9; total += $5} END {print total," total_lib_size"}' > $sizes_file - find velox -type f -name '*.o' | ls -l -BB | + find velox -type f -name '*.o' -exec ls -l -BB {} \; | awk '{print $5, $9; total += $5} END {print total," total_obj_size"}' >> $sizes_file - find velox -type f -name 'velox_*' | ls -l -BB | + find velox -type f -name 'velox_*' -exec ls -l -BB {} \; | awk '{print $5, $9; total += $5} END {print total," total_exec_size"}' >> $sizes_file echo "::group::Size file for ${{ matrix.link-type }} - ${{ matrix.type }}" From b723889fa95c49ff912e751aac16b4d7b6ffbd8a Mon Sep 17 00:00:00 2001 From: Jacob Wujciak-Jens Date: Thu, 23 Jan 2025 00:20:34 +0100 Subject: [PATCH 6/6] fix artifact name --- .github/workflows/build-metrics.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-metrics.yml b/.github/workflows/build-metrics.yml index b88d8586ca56..409790724fdb 100644 --- a/.github/workflows/build-metrics.yml +++ b/.github/workflows/build-metrics.yml @@ -103,6 +103,7 @@ jobs: - uses: actions/upload-artifact@v4 with: path: ${{ env.sizes_file }} + name: "${{ matrix.type }}-${{ matrix.link-type }}-sizes" - name: Copy ninja_log run: cp _build/${{ matrix.type }}/.ninja_log /tmp/metrics/.ninja_log