From 03f10e99d5f80696107038f3e8da8525aa31d50a Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Wed, 4 Sep 2024 14:25:18 -0700 Subject: [PATCH] Rename matmul/ to linalg_ops/, rebase IREE, add test workflow. (#21) Progress on https://github.com/iree-org/iree-test-suites/issues/2. * Renamed top level `matmul/` folder to `linalg_ops/`. We can put `convolution` (linalg) and `attention` (linalg_ext) tests under here too. Maybe other named ops or generics too. * Rebased on the latest IREE code, including flag renames (though we should drop usage of some flags and prefer to test the main user-facing options) * Added a workflow that goes through the instructions in `linalg_ops/README.md` and configured both workflows to run on `push` events in addition to the current triggers. --- .github/workflows/test_linalg_ops.yml | 77 +++++++++++++++++++ .github/workflows/test_onnx_ops.yml | 21 +++-- README.md | 7 +- {matmul => linalg_ops}/CMakeLists.txt | 13 ++-- {matmul => linalg_ops}/README.md | 13 ++-- .../iree-e2e-matmul-test.cc | 0 .../iree_e2e_generated_runner_test.cmake | 2 +- .../matmul}/CMakeLists.txt | 6 +- .../matmul}/generate_e2e_matmul_tests.py | 0 {matmul => linalg_ops}/requirements-iree.txt | 0 {matmul => linalg_ops}/test_utils.c | 4 +- {matmul => linalg_ops}/test_utils.h | 6 +- 12 files changed, 118 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/test_linalg_ops.yml rename {matmul => linalg_ops}/CMakeLists.txt (91%) rename {matmul => linalg_ops}/README.md (85%) rename {matmul => linalg_ops}/iree-e2e-matmul-test.cc (100%) rename {matmul => linalg_ops}/iree_e2e_generated_runner_test.cmake (99%) rename {matmul/tests => linalg_ops/matmul}/CMakeLists.txt (99%) rename {matmul/tests => linalg_ops/matmul}/generate_e2e_matmul_tests.py (100%) rename {matmul => linalg_ops}/requirements-iree.txt (100%) rename {matmul => linalg_ops}/test_utils.c (99%) rename {matmul => linalg_ops}/test_utils.h (97%) diff --git a/.github/workflows/test_linalg_ops.yml b/.github/workflows/test_linalg_ops.yml new file mode 100644 index 00000000..c4611d45 --- /dev/null +++ b/.github/workflows/test_linalg_ops.yml @@ -0,0 +1,77 @@ +# Copyright 2024 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: Test Linalg Ops +on: + push: + branches: + - main + paths: + - ".github/workflows/test_linalg_ops.yml" + - "linalg_ops/**" + pull_request: + paths: + - ".github/workflows/test_linalg_ops.yml" + - "linalg_ops/**" + workflow_dispatch: + schedule: + # Runs at 3:00 PM UTC, which is 8:00 AM PST + - cron: "0 15 * * *" + +concurrency: + # A PR number if a pull request and otherwise the commit hash. This cancels + # queued and in-progress runs for the same PR (presubmit) or commit + # (postsubmit). The workflow name is prepended to avoid conflicts between + # different workflows. + group: ${{ github.workflow }}-${{ github.event.number || github.sha }} + cancel-in-progress: true + +jobs: + test-linalg-ops: + runs-on: ubuntu-24.04 + env: + BUILD_DIR: build + VENV_DIR: ${{ github.workspace }}/.venv + CC: clang + CXX: clang++ + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Install Python packages. + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Setup Python venv + run: python3 -m venv ${VENV_DIR} + - name: Install IREE nightly release Python packages + run: | + source ${VENV_DIR}/bin/activate + python3 -m pip install -r linalg_ops/requirements-iree.txt + + # Build and run tests. + - name: Install build dependencies + run: sudo apt update && sudo apt install -y ninja-build + - name: CMake - Configure + working-directory: linalg_ops + run: | + source ${VENV_DIR}/bin/activate + cmake -G Ninja -B ${BUILD_DIR} . \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DIREE_HOST_BIN_DIR=${VENV_DIR}/bin + - name: CMake - build + working-directory: linalg_ops + run: | + cmake --build ${BUILD_DIR} + cmake --build ${BUILD_DIR} --target iree-test-suites-linalg-ops-deps + - name: CTest + working-directory: linalg_ops + run: | + ctest \ + --test-dir build/ \ + -R iree-test-suites \ + --label-exclude "^driver=vulkan$|^driver=metal$|^driver=cuda$|^driver=hip$" diff --git a/.github/workflows/test_onnx_ops.yml b/.github/workflows/test_onnx_ops.yml index e4db890f..28f1f45f 100644 --- a/.github/workflows/test_onnx_ops.yml +++ b/.github/workflows/test_onnx_ops.yml @@ -6,9 +6,14 @@ name: Test ONNX Ops on: + push: + branches: + - main + paths: + - ".github/workflows/test_onnx_ops.yml" + - "onnx_ops/**" pull_request: paths: - # This file itself. - ".github/workflows/test_onnx_ops.yml" - "onnx_ops/**" workflow_dispatch: @@ -31,23 +36,23 @@ jobs: VENV_DIR: ${{ github.workspace }}/.venv CONFIG_FILE_PATH: onnx_ops/configs/onnx_ops_cpu_llvm_sync.json steps: - - name: "Checking out repository" + - name: Checkout repository uses: actions/checkout@v4 # Install Python packages. - - name: "Setting up Python" + - name: Setup Python uses: actions/setup-python@v5 with: python-version: "3.11" - - name: "Setup Python venv" + - name: Setup Python venv run: python3 -m venv ${VENV_DIR} - - name: "Installing IREE nightly release Python packages" + - name: Install IREE nightly release Python packages run: | source ${VENV_DIR}/bin/activate python3 -m pip install -r onnx_ops/requirements-iree.txt # Run tests and output new config files as needed. - - name: "Running the ONNX ops test suite" + - name: Run ONNX ops test suite run: | source ${VENV_DIR}/bin/activate pytest onnx_ops/ \ @@ -57,7 +62,7 @@ jobs: --durations=10 \ --report-log=/tmp/onnx_ops_cpu_logs.json \ --config-files=${CONFIG_FILE_PATH} - - name: "Updating config file with latest XFAIL lists" + - name: Update config file with latest XFAIL lists if: failure() run: | source ${VENV_DIR}/bin/activate @@ -65,7 +70,7 @@ jobs: --log-file=/tmp/onnx_ops_cpu_logs.json \ --config-file=${CONFIG_FILE_PATH} cat ${CONFIG_FILE_PATH} - - name: "Uploading new config file" + - name: Upload new config file if: failure() uses: actions/upload-artifact@v4 with: diff --git a/README.md b/README.md index 5611c2b1..8d8cc655 100644 --- a/README.md +++ b/README.md @@ -8,10 +8,13 @@ See https://groups.google.com/g/iree-discuss/c/GIWyj8hmP0k/ for context. ## Test suites -### [matmul/](matmul/) : matrix multiplication +### [linalg_ops/](linalg_ops/) : 'linalg' and related ops + +[![Test Linalg Ops](https://github.com/iree-org/iree-test-suites/actions/workflows/test_linalg_ops.yml/badge.svg?branch=main)](https://github.com/iree-org/iree-test-suites/actions/workflows/test_linalg_ops.yml?query=branch%3Amain) * Generated tests for matrix multiplication using the - [MLIR 'linalg' dialect](https://mlir.llvm.org/docs/Dialects/Linalg/). + [MLIR 'linalg' dialect](https://mlir.llvm.org/docs/Dialects/Linalg/) + (tests for other ops like 'attention' and 'convolution' are planned). * Built with [cmake](https://cmake.org/) and run via [ctest](https://cmake.org/cmake/help/latest/manual/ctest.1.html) (for now?). diff --git a/matmul/CMakeLists.txt b/linalg_ops/CMakeLists.txt similarity index 91% rename from matmul/CMakeLists.txt rename to linalg_ops/CMakeLists.txt index f9a94ec5..969e9e5c 100644 --- a/matmul/CMakeLists.txt +++ b/linalg_ops/CMakeLists.txt @@ -10,7 +10,7 @@ cmake_minimum_required(VERSION 3.21...3.24) # Project configuration #------------------------------------------------------------------------------- -project(iree-test-suites-matmul C CXX) +project(iree-test-suites-linalg C CXX) set(CMAKE_C_STANDARD 11) set(CMAKE_CXX_STANDARD 17) set_property(GLOBAL PROPERTY USE_FOLDERS ON) @@ -47,11 +47,10 @@ else() # see https://gitlab.kitware.com/cmake/cmake/-/issues/18238#note_440475 include(FetchContent) - # TODO(scotttodd): pin to a version from a stable release? FetchContent_Declare( iree GIT_REPOSITORY https://github.com/iree-org/iree.git - GIT_TAG f1319fc3404a2c97bfc76c6ee2268a96954a1b2c # 2024-08-16 + GIT_TAG candidate-20240828.999 GIT_SUBMODULES_RECURSE OFF GIT_SHALLOW OFF GIT_PROGRESS ON @@ -69,11 +68,11 @@ endif() # Test code #------------------------------------------------------------------------------- -enable_testing(iree-test-suites-matmul) +enable_testing(iree-test-suites-linalg-ops) -add_custom_target(iree-test-suites-matmul-deps +add_custom_target(iree-test-suites-linalg-ops-deps COMMENT - "Building matmul test suite deps" + "Building linalg operator test suite deps" ) iree_cc_library( @@ -125,4 +124,4 @@ iree_cc_binary( list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) include(iree_e2e_generated_runner_test) -add_subdirectory(tests) +add_subdirectory(matmul) diff --git a/matmul/README.md b/linalg_ops/README.md similarity index 85% rename from matmul/README.md rename to linalg_ops/README.md index b21cdc05..4b2a984f 100644 --- a/matmul/README.md +++ b/linalg_ops/README.md @@ -1,16 +1,19 @@ -# Matrix Multiplication (Matmul) Tests +# Linalg Operator Tests These are being migrated from -https://github.com/iree-org/iree/tree/main/tests/e2e/matmul. -## Prerequisites +* https://github.com/iree-org/iree/tree/main/tests/e2e/attention +* https://github.com/iree-org/iree/tree/main/tests/e2e/convolution +* https://github.com/iree-org/iree/tree/main/tests/e2e/matmul -## Quickstart +## Prerequisites First ensure you have the prerequisites from https://iree.dev/building-from-source/getting-started/, including CMake, a compiler like clang, and Python. +## Quickstart + 1. Get the IREE compiler tools, either from release packages or a source build: * To use Python packages: @@ -60,7 +63,7 @@ compiler like clang, and Python. ```bash cmake --build build/ - cmake --build build/ --target iree-test-suites-matmul-deps + cmake --build build/ --target iree-test-suites-linalg-ops-deps ``` 4. Run tests: diff --git a/matmul/iree-e2e-matmul-test.cc b/linalg_ops/iree-e2e-matmul-test.cc similarity index 100% rename from matmul/iree-e2e-matmul-test.cc rename to linalg_ops/iree-e2e-matmul-test.cc diff --git a/matmul/iree_e2e_generated_runner_test.cmake b/linalg_ops/iree_e2e_generated_runner_test.cmake similarity index 99% rename from matmul/iree_e2e_generated_runner_test.cmake rename to linalg_ops/iree_e2e_generated_runner_test.cmake index 64346c8e..a99409a5 100644 --- a/matmul/iree_e2e_generated_runner_test.cmake +++ b/linalg_ops/iree_e2e_generated_runner_test.cmake @@ -105,7 +105,7 @@ function(iree_e2e_runner_test) "${_RULE_TEST_RUNNER}" ) - add_dependencies(iree-test-suites-matmul-deps "${_NAME}${_RULE_VARIANT_NAME}") + add_dependencies(iree-test-suites-linalg-ops-deps "${_NAME}${_RULE_VARIANT_NAME}") if(_RULE_TEST_DEFINED) iree_native_test( diff --git a/matmul/tests/CMakeLists.txt b/linalg_ops/matmul/CMakeLists.txt similarity index 99% rename from matmul/tests/CMakeLists.txt rename to linalg_ops/matmul/CMakeLists.txt index adc3bca8..346e251e 100644 --- a/matmul/tests/CMakeLists.txt +++ b/linalg_ops/matmul/CMakeLists.txt @@ -1888,7 +1888,7 @@ iree_generated_e2e_runner_test( DRIVERS "cuda" COMPILER_FLAGS - "--iree-hal-cuda-llvm-target-arch=sm_80" + "--iree-cuda-target=sm_80" LABELS "noasan" "nomsan" @@ -1915,7 +1915,7 @@ iree_generated_e2e_runner_test( DRIVERS "cuda" COMPILER_FLAGS - "--iree-hal-cuda-llvm-target-arch=sm_80" + "--iree-cuda-target=sm_80" LABELS "noasan" "nomsan" @@ -1942,7 +1942,7 @@ iree_generated_e2e_runner_test( DRIVERS "cuda" COMPILER_FLAGS - "--iree-flow-split-matmul-reduction=4" + "--iree-dispatch-creation-split-matmul-reduction=4" LABELS "noasan" "nomsan" diff --git a/matmul/tests/generate_e2e_matmul_tests.py b/linalg_ops/matmul/generate_e2e_matmul_tests.py similarity index 100% rename from matmul/tests/generate_e2e_matmul_tests.py rename to linalg_ops/matmul/generate_e2e_matmul_tests.py diff --git a/matmul/requirements-iree.txt b/linalg_ops/requirements-iree.txt similarity index 100% rename from matmul/requirements-iree.txt rename to linalg_ops/requirements-iree.txt diff --git a/matmul/test_utils.c b/linalg_ops/test_utils.c similarity index 99% rename from matmul/test_utils.c rename to linalg_ops/test_utils.c index d33276df..8b8aecd5 100644 --- a/matmul/test_utils.c +++ b/linalg_ops/test_utils.c @@ -130,8 +130,8 @@ iree_test_utils_e2e_value_t iree_test_utils_read_buffer_element( } else if (result_type == IREE_HAL_ELEMENT_TYPE_FLOAT_32) { return iree_test_utils_value_make_f32(((float*)data)[index]); } - iree_status_abort(iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "unhandled matmul result type")); + iree_status_abort( + iree_make_status(IREE_STATUS_INVALID_ARGUMENT, "unhandled result type")); return iree_test_utils_value_make_none(); } diff --git a/matmul/test_utils.h b/linalg_ops/test_utils.h similarity index 97% rename from matmul/test_utils.h rename to linalg_ops/test_utils.h index 71a350f8..626097ba 100644 --- a/matmul/test_utils.h +++ b/linalg_ops/test_utils.h @@ -4,8 +4,8 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef IREE_TEST_SUITES_MATMUL_TEST_UTILS_H_ -#define IREE_TEST_SUITES_MATMUL_TEST_UTILS_H_ +#ifndef IREE_TEST_SUITES_LINALG_OPS_TEST_UTILS_H_ +#define IREE_TEST_SUITES_LINALG_OPS_TEST_UTILS_H_ #include @@ -146,4 +146,4 @@ iree_status_t iree_test_utils_load_and_run_e2e_tests( } // extern "C" #endif // __cplusplus -#endif // IREE_TEST_SUITES_MATMUL_TEST_UTILS_H_ +#endif // IREE_TEST_SUITES_LINALG_OPS_TEST_UTILS_H_