From f3c3d4d6f561e19b87899f2903bcc46231f86c5c Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 12 Oct 2022 12:25:58 -0700 Subject: [PATCH 1/6] Add testing for OSX --- .github/workflows/test-upstream.yml | 2 +- .github/workflows/test.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-upstream.yml b/.github/workflows/test-upstream.yml index 0e0a8e849..bdfa5d972 100644 --- a/.github/workflows/test-upstream.yml +++ b/.github/workflows/test-upstream.yml @@ -22,7 +22,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest, windows-latest, macos-latest] python: ["3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1072478ba..2c459bdc2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,7 +41,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest, windows-latest, macos-latest] python: ["3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 From acf512f1fbf164e1bac538aea25a00f86c6f1396 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Wed, 12 Oct 2022 13:21:00 -0700 Subject: [PATCH 2/6] Install hive testing dependencies --- .github/workflows/test-upstream.yml | 4 ++-- .github/workflows/test.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-upstream.yml b/.github/workflows/test-upstream.yml index bdfa5d972..9f91c10ca 100644 --- a/.github/workflows/test-upstream.yml +++ b/.github/workflows/test-upstream.yml @@ -41,8 +41,8 @@ jobs: - name: Build the Rust DataFusion bindings run: | python setup.py build install - - name: Install hive testing dependencies for Linux - if: matrix.os == 'ubuntu-latest' + - name: Install hive testing dependencies + if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' run: | mamba install -c conda-forge sasl>=0.3.1 docker pull bde2020/hive:2.3.2-postgresql-metastore diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2c459bdc2..7210173f4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,8 +58,8 @@ jobs: - name: Build the Rust DataFusion bindings run: | python setup.py build install - - name: Install hive testing dependencies for Linux - if: matrix.os == 'ubuntu-latest' + - name: Install hive testing dependencies + if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' run: | mamba install -c conda-forge sasl>=0.3.1 docker pull bde2020/hive:2.3.2-postgresql-metastore From 63442651ca342af9faa72c7e28779c1cea4a2a16 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 13 Oct 2022 06:58:04 -0700 Subject: [PATCH 3/6] Try using colima instead of docker for macOS testing --- .github/workflows/test-upstream.yml | 9 ++++++--- .github/workflows/test.yml | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-upstream.yml b/.github/workflows/test-upstream.yml index 9f91c10ca..27bbcf396 100644 --- a/.github/workflows/test-upstream.yml +++ b/.github/workflows/test-upstream.yml @@ -43,10 +43,13 @@ jobs: python setup.py build install - name: Install hive testing dependencies if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' + env: + # `docker` isn't available on macos-latest, use colima instead + DOCKER_ENTRY: ${{ matrix.os == 'ubuntu-latest' && 'docker' || 'colima' }} run: | - mamba install -c conda-forge sasl>=0.3.1 - docker pull bde2020/hive:2.3.2-postgresql-metastore - docker pull bde2020/hive-metastore-postgresql:2.3.0 + mamba install -c conda-forge "sasl>=0.3.1" + ${{ env.DOCKER_ENTRY }} pull bde2020/hive:2.3.2-postgresql-metastore + ${{ env.DOCKER_ENTRY }} pull bde2020/hive-metastore-postgresql:2.3.0 - name: Install upstream dev Dask / dask-ml run: | mamba update dask diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7210173f4..2f9afa114 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,10 +60,13 @@ jobs: python setup.py build install - name: Install hive testing dependencies if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' + env: + # `docker` isn't available on macos-latest, use colima instead + DOCKER_ENTRY: ${{ matrix.os == 'ubuntu-latest' && 'docker' || 'colima' }} run: | - mamba install -c conda-forge sasl>=0.3.1 - docker pull bde2020/hive:2.3.2-postgresql-metastore - docker pull bde2020/hive-metastore-postgresql:2.3.0 + mamba install -c conda-forge "sasl>=0.3.1" + ${{ env.DOCKER_ENTRY }} pull bde2020/hive:2.3.2-postgresql-metastore + ${{ env.DOCKER_ENTRY }} pull bde2020/hive-metastore-postgresql:2.3.0 - name: Optionally install upstream dev Dask / dask-ml if: needs.detect-ci-trigger.outputs.triggered == 'true' run: | From 0bf8234511e4ff9e4797eac986be9804a0ae1e67 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 13 Oct 2022 14:30:46 -0400 Subject: [PATCH 4/6] Disable tests requiring docker for macOS --- .github/workflows/test-upstream.yml | 9 +++------ .github/workflows/test.yml | 9 +++------ tests/integration/test_hive.py | 4 +++- tests/integration/test_postgres.py | 2 +- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-upstream.yml b/.github/workflows/test-upstream.yml index 27bbcf396..f7b978fca 100644 --- a/.github/workflows/test-upstream.yml +++ b/.github/workflows/test-upstream.yml @@ -42,14 +42,11 @@ jobs: run: | python setup.py build install - name: Install hive testing dependencies - if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' - env: - # `docker` isn't available on macos-latest, use colima instead - DOCKER_ENTRY: ${{ matrix.os == 'ubuntu-latest' && 'docker' || 'colima' }} + if: matrix.os == 'ubuntu-latest' run: | mamba install -c conda-forge "sasl>=0.3.1" - ${{ env.DOCKER_ENTRY }} pull bde2020/hive:2.3.2-postgresql-metastore - ${{ env.DOCKER_ENTRY }} pull bde2020/hive-metastore-postgresql:2.3.0 + docker pull bde2020/hive:2.3.2-postgresql-metastore + docker pull bde2020/hive-metastore-postgresql:2.3.0 - name: Install upstream dev Dask / dask-ml run: | mamba update dask diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2f9afa114..a20ee2b14 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,14 +59,11 @@ jobs: run: | python setup.py build install - name: Install hive testing dependencies - if: matrix.os == 'ubuntu-latest' || matrix.os == 'macos-latest' - env: - # `docker` isn't available on macos-latest, use colima instead - DOCKER_ENTRY: ${{ matrix.os == 'ubuntu-latest' && 'docker' || 'colima' }} + if: matrix.os == 'ubuntu-latest' run: | mamba install -c conda-forge "sasl>=0.3.1" - ${{ env.DOCKER_ENTRY }} pull bde2020/hive:2.3.2-postgresql-metastore - ${{ env.DOCKER_ENTRY }} pull bde2020/hive-metastore-postgresql:2.3.0 + docker pull bde2020/hive:2.3.2-postgresql-metastore + docker pull bde2020/hive-metastore-postgresql:2.3.0 - name: Optionally install upstream dev Dask / dask-ml if: needs.detect-ci-trigger.outputs.triggered == 'true' run: | diff --git a/tests/integration/test_hive.py b/tests/integration/test_hive.py index 131150326..8d0c247cb 100644 --- a/tests/integration/test_hive.py +++ b/tests/integration/test_hive.py @@ -10,7 +10,7 @@ from tests.utils import assert_eq pytestmark = pytest.mark.skipif( - sys.platform == "win32", reason="hive testing not supported on Windows" + sys.platform in ("win32", "darwin"), reason="hive testing not supported on Windows/macOS" ) docker = pytest.importorskip("docker") sqlalchemy = pytest.importorskip("sqlalchemy") @@ -59,6 +59,8 @@ def hive_cursor(): """ client = docker.from_env() + breakpoint() + network = None hive_server = None hive_metastore = None diff --git a/tests/integration/test_postgres.py b/tests/integration/test_postgres.py index 67a639c73..b19e99e08 100644 --- a/tests/integration/test_postgres.py +++ b/tests/integration/test_postgres.py @@ -3,7 +3,7 @@ import pytest pytestmark = pytest.mark.skipif( - sys.platform == "win32", reason="postgres testing not supported on Windows" + sys.platform in ("win32", "darwin"), reason="hive testing not supported on Windows/macOS" ) docker = pytest.importorskip("docker") sqlalchemy = pytest.importorskip("sqlalchemy") From 82829dc693b47f4719da95c4dfd0eb1cbbe5655b Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 13 Oct 2022 14:33:50 -0400 Subject: [PATCH 5/6] Style fix --- tests/integration/test_hive.py | 3 ++- tests/integration/test_postgres.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_hive.py b/tests/integration/test_hive.py index 8d0c247cb..5941229fb 100644 --- a/tests/integration/test_hive.py +++ b/tests/integration/test_hive.py @@ -10,7 +10,8 @@ from tests.utils import assert_eq pytestmark = pytest.mark.skipif( - sys.platform in ("win32", "darwin"), reason="hive testing not supported on Windows/macOS" + sys.platform in ("win32", "darwin"), + reason="hive testing not supported on Windows/macOS", ) docker = pytest.importorskip("docker") sqlalchemy = pytest.importorskip("sqlalchemy") diff --git a/tests/integration/test_postgres.py b/tests/integration/test_postgres.py index b19e99e08..05a249c77 100644 --- a/tests/integration/test_postgres.py +++ b/tests/integration/test_postgres.py @@ -3,7 +3,8 @@ import pytest pytestmark = pytest.mark.skipif( - sys.platform in ("win32", "darwin"), reason="hive testing not supported on Windows/macOS" + sys.platform in ("win32", "darwin"), + reason="hive testing not supported on Windows/macOS", ) docker = pytest.importorskip("docker") sqlalchemy = pytest.importorskip("sqlalchemy") From 885ddd0509a24142e44506ce3f8a8622e3e863f3 Mon Sep 17 00:00:00 2001 From: Charles Blackmon-Luca <20627856+charlesbluca@users.noreply.github.com> Date: Thu, 13 Oct 2022 14:39:47 -0400 Subject: [PATCH 6/6] Remove breakpoint --- tests/integration/test_hive.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/test_hive.py b/tests/integration/test_hive.py index 5941229fb..48be5df94 100644 --- a/tests/integration/test_hive.py +++ b/tests/integration/test_hive.py @@ -60,8 +60,6 @@ def hive_cursor(): """ client = docker.from_env() - breakpoint() - network = None hive_server = None hive_metastore = None