diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000..3536e6b06 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,56 @@ +name: Nightly + +on: + workflow_dispatch: + schedule: + - cron: '0 2 * * *' + + +jobs: + nightly: + name: "Python: ${{ matrix.python-version }} + SQLA: ${{ matrix.sqla-version }} + CrateDB: ${{ matrix.cratedb-version }} + on ${{ matrix.os }}" + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + cratedb-version: ['nightly'] + sqla-version: ['1.1.18', '1.2.19', '1.3.23'] + fail-fast: false + + steps: + - uses: actions/checkout@master + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + + # Workaround needed for Python 3.5 + python -m pip install --upgrade "setuptools>=31,<51" + + pip install zc.buildout==2.13.4 + + # replace SQLAlchemy version + sed -ir 's/SQLAlchemy.*/SQLAlchemy = ${{ matrix.sqla-version }}/g' versions.cfg + + # replace CrateDB version + if [ ${{ matrix.cratedb-version }} = "nightly" ]; then + sed -ir 's/releases/releases\/nightly/g' base.cfg + sed -ir 's/crate_server.*/crate_server = latest/g' versions.cfg + else + sed -ir 's/crate_server.*/crate_server = ${{ matrix.cratedb-version }}/g' versions.cfg + fi + + buildout -n -c base.cfg + + - name: Test + run: | + bin/flake8 + bin/coverage run bin/test -vv1 diff --git a/.github/workflows/main.yml b/.github/workflows/tests.yml similarity index 83% rename from .github/workflows/main.yml rename to .github/workflows/tests.yml index fd1696174..81eaf99dd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/tests.yml @@ -6,23 +6,21 @@ on: pull_request: branches: [ master ] workflow_dispatch: - schedule: - - cron: '0 2 * * *' jobs: test: name: "Python: ${{ matrix.python-version }} SQLA: ${{ matrix.sqla-version }} - CrateDB: ${{ matrix.crate-version }} + CrateDB: ${{ matrix.cratedb-version }} on ${{ matrix.os }}" runs-on: ${{ matrix.os }} strategy: matrix: - crate-version: [nightly] - os: [ubuntu-latest] - sqla-version: ['1.1.18', '1.2.19', '1.3.23'] + os: [ubuntu-latest, macos-latest] python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + cratedb-version: ['4.5.0'] + sqla-version: ['1.1.18', '1.2.19', '1.3.23'] fail-fast: false steps: @@ -45,12 +43,11 @@ jobs: sed -ir 's/SQLAlchemy.*/SQLAlchemy = ${{ matrix.sqla-version }}/g' versions.cfg # replace CrateDB version - if [ ${{ matrix.crate-version }} = "nightly" ]; then + if [ ${{ matrix.cratedb-version }} = "nightly" ]; then sed -ir 's/releases/releases\/nightly/g' base.cfg sed -ir 's/crate_server.*/crate_server = latest/g' versions.cfg else - sed -ir 's/crate-/crate_/g' base.cfg - sed -ir 's/crate_server.*/crate_server = ${{ matrix.crate-version }}/g' versions.cfg + sed -ir 's/crate_server.*/crate_server = ${{ matrix.cratedb-version }}/g' versions.cfg fi buildout -n -c base.cfg diff --git a/base.cfg b/base.cfg index f0df080e0..e470e47bd 100644 --- a/base.cfg +++ b/base.cfg @@ -19,11 +19,21 @@ eggs = crate recipe = zc.recipe.egg eggs = createcoverage -[crate] +[crate:linux] recipe = hexagonit.recipe.download url = https://cdn.crate.io/downloads/releases/crate-${versions:crate_server}.tar.gz strip-top-level-dir = true +[crate:macosx] +recipe = hexagonit.recipe.download +url = https://cdn.crate.io/downloads/releases/cratedb/x64_mac/crate-${versions:crate_server}.tar.gz +strip-top-level-dir = true + +[crate:windows] +recipe = hexagonit.recipe.download +url = https://cdn.crate.io/downloads/releases/cratedb/x64_windows/crate-${versions:crate_server}.zip +strip-top-level-dir = true + [test] relative-paths = true recipe = zc.recipe.testrunner diff --git a/src/crate/testing/doctests/layer.txt b/src/crate/testing/doctests/layer.txt index 6a17dffde..2e4e67b4e 100644 --- a/src/crate/testing/doctests/layer.txt +++ b/src/crate/testing/doctests/layer.txt @@ -225,26 +225,9 @@ We might have to wait a moment before the cluster is finally created:: From Uri -------- -The CrateLayer can also be created by providing a URI that points to a Crate +The CrateLayer can also be created by providing a URI that points to a CrateDB tarball:: - >>> import urllib.request - >>> import json - >>> with urllib.request.urlopen('http://crate.io/versions.json') as response: - ... versions = json.loads(response.read().decode()) - ... version = versions['crate_testing'] - - >>> uri = 'https://cdn.crate.io/downloads/releases/crate-{}.tar.gz'.format(version) - >>> tmpdir = tempfile.mkdtemp() - >>> layer = CrateLayer.from_uri( - ... uri, name='crate-uri', http_port=42203, directory=tmpdir) - >>> layer.setUp() - - >>> work_dir = os.path.join(tmpdir, 'crate-' + version) - >>> os.path.exists(work_dir) - True - - >>> layer.tearDown() - - >>> os.path.exists(work_dir) - False + uri = 'https://cdn.crate.io/downloads/releases/crate-{}.tar.gz'.format(version) + layer = CrateLayer.from_uri( + uri, name='crate-uri', http_port=42203, directory=tmpdir) diff --git a/src/crate/testing/layer.py b/src/crate/testing/layer.py index 47578b828..3bd3fc992 100644 --- a/src/crate/testing/layer.py +++ b/src/crate/testing/layer.py @@ -72,6 +72,8 @@ def prepend_http(host): def _download_and_extract(uri, directory): + sys.stderr.write("\nINFO: Downloading CrateDB archive from {} into {}".format(uri, directory)) + sys.stderr.flush() with io.BytesIO(urlopen(uri).read()) as tmpfile: with tarfile.open(fileobj=tmpfile) as t: t.extractall(directory) @@ -160,7 +162,8 @@ def from_uri(uri, crate_home = os.path.join(directory, crate_dir) if os.path.exists(crate_home): - sys.stderr.write('Not extracting Crate tarball because folder already exists') + sys.stderr.write("\nWARNING: Not extracting Crate tarball because folder already exists") + sys.stderr.flush() else: _download_and_extract(uri, directory) diff --git a/src/crate/testing/test_layer.py b/src/crate/testing/test_layer.py index f2bcfc4b0..ca55a2f92 100644 --- a/src/crate/testing/test_layer.py +++ b/src/crate/testing/test_layer.py @@ -18,11 +18,15 @@ # However, if you have executed another commercial license agreement # with Crate these terms will supersede the license and you may use the # software solely pursuant to the terms of the relevant commercial agreement. - +import json import os import tempfile +import urllib +from distutils.version import LooseVersion from unittest import TestCase, mock from io import BytesIO + +import crate from .layer import CrateLayer, prepend_http, http_url_from_host_port, wait_for_http_url @@ -58,6 +62,22 @@ def test_wait_for_http(self): addr = wait_for_http_url(log=log, timeout=1) self.assertEqual(None, addr) + @mock.patch.object(crate.testing.layer, "_download_and_extract", lambda uri, directory: None) + def test_layer_from_uri(self): + """ + The CrateLayer can also be created by providing an URI that points to + a CrateDB tarball. + """ + with urllib.request.urlopen("https://crate.io/versions.json") as response: + versions = json.loads(response.read().decode()) + version = versions["crate_testing"] + + self.assertGreaterEqual(LooseVersion(version), LooseVersion("4.5.0")) + + uri = "https://cdn.crate.io/downloads/releases/crate-{}.tar.gz".format(version) + layer = CrateLayer.from_uri(uri, name="crate-by-uri", http_port=42203) + self.assertIsInstance(layer, CrateLayer) + @mock.patch.dict('os.environ', {}, clear=True) def test_java_home_env_not_set(self): with tempfile.TemporaryDirectory() as tmpdir: diff --git a/versions.cfg b/versions.cfg index 1ff59bf64..b2a01b5b2 100644 --- a/versions.cfg +++ b/versions.cfg @@ -1,5 +1,5 @@ [versions] -crate_server = 4.4.2 +crate_server = 4.5.0 flake8 = 3.7.9 mccabe = 0.6.1