Skip to content

Commit

Permalink
Merge pull request #1183 from datajoint/docker-img-refactor
Browse files Browse the repository at this point in the history
Refactor Docker image and package builds
  • Loading branch information
dimitri-yatsenko authored Sep 22, 2024
2 parents 467990e + 618a8d5 commit 07338fa
Show file tree
Hide file tree
Showing 19 changed files with 217 additions and 303 deletions.
2 changes: 1 addition & 1 deletion .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ RUN \
pip uninstall datajoint -y

USER root
ENV DJ_HOST fakeservices.datajoint.io
ENV DJ_HOST db
ENV DJ_USER root
ENV DJ_PASS password
4 changes: 2 additions & 2 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
// Update the 'dockerComposeFile' list if you have more compose files or use different names.
// The .devcontainer/docker-compose.yml file contains any overrides you need/want to make.
"dockerComposeFile": [
"../LNX-docker-compose.yml",
"../docker-compose.yaml",
"docker-compose.yml"
],
// The 'service' property is the name of the service for the container that VS Code should
Expand All @@ -28,7 +28,7 @@
// "runServices": [],
// Uncomment the next line if you want to keep your containers running after VS Code shuts down.
"shutdownAction": "stopCompose",
"onCreateCommand": "python3 -m pip install -e .",
"onCreateCommand": "python3 -m pip install -q -e .[test]",
"features": {
"ghcr.io/devcontainers/features/git:1": {},
"ghcr.io/devcontainers/features/docker-in-docker:2": {},
Expand Down
27 changes: 13 additions & 14 deletions .github/workflows/development.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,13 @@ jobs:
COMPOSE_HTTP_TIMEOUT: "120"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{matrix.py_ver}}
- name: Validate version and release notes
run: |
DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py)
RELEASE_BODY=$(python -c \
RELEASE_BODY=$(python3 -c \
'print(open("./CHANGELOG.md").read().split("\n\n")[1].split("\n", 1)[1])' \
)
echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
Expand All @@ -41,8 +44,8 @@ jobs:
echo "EOF" >> $GITHUB_ENV
- name: Build pip artifacts
run: |
export HOST_UID=$(id -u)
docker compose -f docker-compose-build.yaml up --exit-code-from app --build
python3 -m pip install build
python3 -m build .
echo "DJ_VERSION=${DJ_VERSION}" >> $GITHUB_ENV
- if: matrix.py_ver == '3.9' && matrix.distro == 'debian'
name: Add pip artifacts
Expand All @@ -58,6 +61,8 @@ jobs:
py_ver: ["3.9"]
mysql_ver: ["8.0", "5.7"]
include:
- py_ver: "3.12"
mysql_ver: "8.0"
- py_ver: "3.11"
mysql_ver: "8.0"
- py_ver: "3.10"
Expand All @@ -70,12 +75,6 @@ jobs:
uses: actions/setup-python@v5
with:
python-version: ${{matrix.py_ver}}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install flake8 black
- name: Run syntax tests
run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
- name: Run primary tests
env:
PY_VER: ${{matrix.py_ver}}
Expand All @@ -87,7 +86,7 @@ jobs:
COMPOSE_HTTP_TIMEOUT: "120"
run: |
export HOST_UID=$(id -u)
docker compose -f LNX-docker-compose.yml up --build --exit-code-from app
docker compose --profile test up --quiet-pull --build --exit-code-from djtest djtest
lint:
runs-on: ubuntu-latest
strategy:
Expand All @@ -101,8 +100,8 @@ jobs:
python-version: ${{matrix.py_ver}}
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install flake8 black==24.2.0
python3 -m pip install --upgrade pip
python3 -m pip install ".[test]"
- name: Run syntax tests
run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
- name: Run style tests
Expand Down Expand Up @@ -138,7 +137,7 @@ jobs:
export PACKAGE=datajoint
export UPSTREAM_REPO=https://github.com/${GITHUB_REPOSITORY}.git
export HOST_UID=$(id -u)
docker compose -f docs/docker-compose.yaml up --exit-code-from docs --build
docker compose -f docs/docker-compose.yaml up --quiet-pull --exit-code-from docs --build
git push origin gh-pages
publish-release:
if: |
Expand Down Expand Up @@ -219,7 +218,7 @@ jobs:
- name: Publish pip release
run: |
export HOST_UID=$(id -u)
docker compose -f docker-compose-build.yaml run \
docker compose run --build --quiet-pull \
-e TWINE_USERNAME=${TWINE_USERNAME} -e TWINE_PASSWORD=${TWINE_PASSWORD} app \
sh -c "pip install twine && python -m twine upload dist/*"
- name: Login to DockerHub
Expand Down
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
## Release notes

### 0.14.3 -- TBD
### 0.14.3 -- Sep 20, 2024
- Added - `dj.Top` restriction ([#1024](https://github.com/datajoint/datajoint-python/issues/1024)) PR [#1084](https://github.com/datajoint/datajoint-python/pull/1084)
- Fixed - Added encapsulating double quotes to comply with [DOT language](https://graphviz.org/doc/info/lang.html) - PR [#1177](https://github.com/datajoint/datajoint-python/pull/1177)
- Added - Datajoint python CLI ([#940](https://github.com/datajoint/datajoint-python/issues/940)) PR [#1095](https://github.com/datajoint/datajoint-python/pull/1095)
- Added - Ability to set hidden attributes on a table - PR [#1091](https://github.com/datajoint/datajoint-python/pull/1091)
- Added - Ability to specify a list of keys to populate - PR [#989](https://github.com/datajoint/datajoint-python/pull/989)
- Fixed - fixed topological sort [#1057](https://github.com/datajoint/datajoint-python/issues/1057)- PR [#1184](https://github.com/datajoint/datajoint-python/pull/1184)
- Fixed - .parts() not always returning parts [#1103](https://github.com/datajoint/datajoint-python/issues/1103)- PR [#1184](https://github.com/datajoint/datajoint-python/pull/1184)
- Changed - replace `setup.py` with `pyproject.toml` PR [#1183](https://github.com/datajoint/datajoint-python/pull/1183)

### 0.14.2 -- Aug 19, 2024
- Added - Migrate nosetests to pytest - PR [#1142](https://github.com/datajoint/datajoint-python/pull/1142)
Expand Down
27 changes: 21 additions & 6 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,9 +1,24 @@
ARG IMAGE=djbase
ARG IMAGE=mambaorg/micromamba:1.5-bookworm-slim
FROM ${IMAGE}

ARG CONDA_BIN=micromamba
ARG PY_VER=3.9
ARG DISTRO=debian
FROM datajoint/${IMAGE}:py${PY_VER}-${DISTRO}
COPY --chown=anaconda:anaconda ./setup.py ./datajoint.pub ./requirements.txt /main/
COPY --chown=anaconda:anaconda ./datajoint /main/datajoint
ARG HOST_UID=1000

RUN ${CONDA_BIN} install --no-pin -qq -y -n base -c conda-forge \
python=${PY_VER} pip setuptools git graphviz pydot && \
${CONDA_BIN} clean -qq -afy
ENV PATH="$PATH:/home/mambauser/.local/bin"

COPY --chown=${HOST_UID:-1000}:mambauser ./pyproject.toml ./README.md ./LICENSE.txt /main/
COPY --chown=${HOST_UID:-1000}:mambauser ./datajoint /main/datajoint

VOLUME /src
WORKDIR /src
USER root
RUN \
pip install --no-cache-dir /main && \
chown -R ${HOST_UID:-1000}:mambauser /main && \
chown -R ${HOST_UID:-1000}:mambauser /src && \
${CONDA_BIN} run -n base pip install -q --no-cache-dir /main && \
rm -r /main/*
USER ${MAMBA_USER}
101 changes: 0 additions & 101 deletions LNX-docker-compose.yml

This file was deleted.

1 change: 0 additions & 1 deletion MANIFEST.in

This file was deleted.

2 changes: 1 addition & 1 deletion datajoint/fetch.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def __call__(
format=None,
as_dict=None,
squeeze=False,
download_path="."
download_path=".",
):
"""
Fetches the expression results from the database into an np.array or list of dictionaries and
Expand Down
2 changes: 1 addition & 1 deletion datajoint/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ def __init__(
*,
secure=False,
proxy_server=None,
**_
**_,
):
# from https://docs.min.io/docs/python-client-api-reference
self.client = minio.Minio(
Expand Down
24 changes: 0 additions & 24 deletions docker-compose-build.yaml

This file was deleted.

79 changes: 79 additions & 0 deletions docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
# HOST_UID=$(id -u) PY_VER=3.11 DJ_VERSION=$(grep -oP '\d+\.\d+\.\d+' datajoint/version.py) docker compose --profile test up --build --exit-code-from djtest djtest
services:
db:
image: datajoint/mysql:${MYSQL_VER:-8.0}
environment:
- MYSQL_ROOT_PASSWORD=${DJ_PASS:-password}
command: mysqld --default-authentication-plugin=mysql_native_password
# ports:
# - "3306:3306"
# volumes:
# - ./mysql/data:/var/lib/mysql
healthcheck:
test: [ "CMD", "mysqladmin", "ping", "-h", "localhost" ]
timeout: 30s
retries: 5
interval: 15s
minio:
image: minio/minio:${MINIO_VER:-RELEASE.2022-08-11T04-37-28Z}
environment:
- MINIO_ACCESS_KEY=datajoint
- MINIO_SECRET_KEY=datajoint
# ports:
# - "9000:9000"
# volumes:
# - ./minio/config:/root/.minio
# - ./minio/data:/data
command: server --address ":9000" /data
healthcheck:
test:
- "CMD"
- "curl"
- "--fail"
- "http://minio:9000/minio/health/live"
timeout: 30s
retries: 5
interval: 15s
app:
image: datajoint/datajoint:${DJ_VERSION:-latest}
build:
context: .
dockerfile: Dockerfile
args:
PY_VER: ${PY_VER:-3.8}
HOST_UID: ${HOST_UID:-1000}
depends_on:
db:
condition: service_healthy
minio:
condition: service_healthy
environment:
- DJ_HOST=db
- DJ_USER=root
- DJ_PASS=password
- DJ_TEST_HOST=db
- DJ_TEST_USER=datajoint
- DJ_TEST_PASSWORD=datajoint
- S3_ENDPOINT=minio:9000
- S3_ACCESS_KEY=datajoint
- S3_SECRET_KEY=datajoint
- S3_BUCKET=datajoint.test
- PYTHON_USER=dja
- JUPYTER_PASSWORD=datajoint
working_dir: /src
user: ${HOST_UID:-1000}:mambauser
volumes:
- .:/src
djtest:
extends:
service: app
profiles: ["test"]
command:
- sh
- -c
- |
set -e
pip install -q -e ".[test]"
pip freeze | grep datajoint
pytest --cov-report term-missing --cov=datajoint tests
Loading

0 comments on commit 07338fa

Please sign in to comment.