From bcf141c462f554523ef05063879af89851554fb1 Mon Sep 17 00:00:00 2001 From: Aleksandra Lazoroska Date: Thu, 31 Oct 2024 10:22:13 +0100 Subject: [PATCH 01/12] feat: Upgrade CKAN to version 2.11.0 --- sddi-base/Dockerfile | 491 ++++++++++++++++++++++--------- sddi-base/setup/prerun.py | 222 ++++++++++++++ sddi-base/setup/start_ckan.sh | 56 ++++ sddi-base/setup/supervisord.conf | 23 ++ sddi-base/setup/uwsgi.conf | 2 + 5 files changed, 663 insertions(+), 131 deletions(-) create mode 100644 sddi-base/setup/prerun.py create mode 100644 sddi-base/setup/start_ckan.sh create mode 100644 sddi-base/setup/supervisord.conf create mode 100644 sddi-base/setup/uwsgi.conf diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index 5e0614c..9800d40 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -1,16 +1,98 @@ ############################################################################### -# Build stage +# CKAN build stage ############################################################################### -ARG CKAN_VERSION_BUILD_STAGE=2.9.9-dev -ARG CKAN_VERSION_BUILD_SPATIAL=2.9.9-focal -ARG CKAN_VERSION_RUNTIME_STAGE=2.9.9-focal +FROM python:3.9-slim as ckanbuild -FROM ckan/ckan-base:${CKAN_VERSION_BUILD_STAGE} as extbuild +# Used by Github Actions to tag the image with +ENV IMAGE_TAG=2.11.0 + +# Set CKAN version to build +ENV GIT_URL=https://github.com/ckan/ckan.git +ENV GIT_BRANCH=ckan-2.11.0 + +# Set src dirs +ENV SRC_DIR=/srv/app/src +ENV PIP_SRC=${SRC_DIR} + +WORKDIR ${SRC_DIR} + +RUN apt-get update && apt-get install -y \ + git \ + curl \ + libpq-dev \ + gcc \ + make \ + g++ \ + autoconf \ + automake \ + libtool \ + patch \ + musl-dev \ + libpcre3-dev \ + libpcre3 \ + libffi-dev \ + libxml2-dev \ + libxslt-dev + + # Cleanup to reduce image size +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + +# Link python to python3 +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Create the src directory +RUN mkdir -p ${SRC_DIR} + +# Downgrade setuptools so that CKAN requirements can be built +RUN pip3 install setuptools==44.1.0 + +# Fetch and build CKAN and requirements +RUN pip3 install -e git+${GIT_URL}@${GIT_BRANCH}#egg=ckan + +# Apply patches +RUN rm -rf /srv/app/src/ckan/.git + +# Create a constraint file that limits the Cython version to a compatible one, see https://github.com/yaml/pyyaml/issues/736 +RUN echo 'Cython < 3.0' > /tmp/constraint.txt +RUN pip3_CONSTRAINT=/tmp/constraint.txt pip3 wheel --wheel-dir=/wheels PyYAML==5.4.1 + +# RUN pip3-compile ckan/requirements.in +RUN pip3 wheel --wheel-dir=/wheels -r ckan/requirements.txt +RUN pip3 wheel --wheel-dir=/wheels uWSGI==2.0.21 gevent==22.10.2 greenlet==2.0.1 + +############################################################################### +# Extbuild stage +############################################################################### +FROM python:3.9-slim as extbuild USER root +RUN apt-get update && apt-get install -y \ + git \ + curl \ + libpq-dev \ + gcc \ + make \ + g++ \ + autoconf \ + automake \ + libtool \ + patch \ + musl-dev \ + libpcre3-dev \ + libpcre3 \ + libffi-dev \ + libxml2-dev \ + libxslt-dev + +RUN pip install -U markupsafe==2.0.1 + +# Create a constraint file that limits the Cython version to a compatible one, see https://github.com/yaml/pyyaml/issues/736 +RUN echo 'Cython < 3.0' > /tmp/constraint.txt +RUN pip3_CONSTRAINT=/tmp/constraint.txt pip3 wheel --wheel-dir=/wheels PyYAML==5.4.1 + # ckanext-hierarchy ########################################################### -ARG CKANEXT_HIERARCHY_VERSION="v1.2.0" +ARG CKANEXT_HIERARCHY_VERSION="abb4e2d" ENV CKANEXT_HIERARCHY_VERSION=${CKANEXT_HIERARCHY_VERSION} RUN set -ex && \ @@ -23,21 +105,15 @@ RUN set -ex && \ curl -o /wheels/ckanext-hierarchy.txt https://raw.githubusercontent.com/ckan/ckanext-hierarchy/${CKANEXT_HIERARCHY_VERSION}/requirements.txt && \ ls -lah /wheels -# ckanext-grouphierarchy ###################################################### -ARG CKANEXT_SDDI_VERSION="1.1.4" -ENV CKANEXT_SDDI_VERSION=${CKANEXT_SDDI_VERSION} +# ckanext-envvars +ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars +ENV ENVVARS_GIT_BRANCH=0.0.2 RUN set -ex && \ - pip wheel --wheel-dir=/wheels \ - git+https://github.com/tum-gis/ckanext-grouphierarchy-sddi.git@${CKANEXT_SDDI_VERSION}#egg=ckanext-grouphierarchy && \ - pip wheel --wheel-dir=/wheels -r \ - https://raw.githubusercontent.com/tum-gis/ckanext-grouphierarchy-sddi/${CKANEXT_SDDI_VERSION}/requirements.txt && \ - curl -o /wheels/ckanext-grouphierarchy.txt \ - https://raw.githubusercontent.com/tum-gis/ckanext-grouphierarchy-sddi/${CKANEXT_SDDI_VERSION}/requirements.txt && \ - ls -lah /wheels + pip3 wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars # ckanext-relation ############################################################ -ARG CKANEXT_RELATION_VERSION="1.0.3" +ARG CKANEXT_RELATION_VERSION="1.1.0" ENV CKANEXT_RELATION_VERSION=${CKANEXT_RELATION_VERSION} RUN set -ex && \ @@ -50,16 +126,16 @@ RUN set -ex && \ ls -lah /wheels # ckanext-scheming ############################################################ -ARG CKANEXT_SCHEMING_VERSION="f98daec" +ARG CKANEXT_SCHEMING_VERSION="27035f4" ENV CKANEXT_SCHEMING_VERSION=${CKANEXT_SCHEMING_VERSION} -ENV CKANEXT_SCHEMING_GITHUB_URL="https://github.com/MarijaKnezevic/ckanext-scheming" +ENV CKANEXT_SCHEMING_GITHUB_URL="https://github.com//ckan/ckanext-scheming" RUN set -ex && \ pip wheel --wheel-dir=/wheels \ git+${CKANEXT_SCHEMING_GITHUB_URL}.git@${CKANEXT_SCHEMING_VERSION}#egg=ckanext-scheming # ckanext datesearch ########################################################## -ARG CKANEXT_DATESEARCH_VERSION="1.0.2" +ARG CKANEXT_DATESEARCH_VERSION="1.1.0" ENV CKANEXT_DATESEARCH_VERSION=${CKANEXT_DATESEARCH_VERSION} ENV CKANEXT_DATESEARCH_VERSION_GITHUB_URL="https://github.com/MarijaKnezevic/ckanext-datesearch" @@ -67,184 +143,337 @@ RUN set -ex && \ pip wheel --wheel-dir=/wheels \ git+${CKANEXT_DATESEARCH_VERSION_GITHUB_URL}.git@${CKANEXT_DATESEARCH_VERSION}#egg=ckanext-datesearch -# ckanext-composite ########################################################### -ARG CKANEXT_COMPOSITE_VERSION="1e6d7bb" -ENV CKANEXT_COMPOSITE_VERSION=${CKANEXT_COMPOSITE_VERSION} -ENV CKANEXT_COMPOSITE_GITHUB_URL="https://github.com/EnviDat/ckanext-composite" +# ckanext-harvest ########################################################### +ARG CKANEXT_HARVEST_VERSION="v1.5.6" +ENV CKANEXT_HARVEST_VERSION=${CKANEXT_HARVEST_VERSION} +ENV CKANEXT_HARVEST_GITHUB_URL="https://github.com/ckan/ckanext-harvest.git" RUN set -ex && \ - pip install -r \ - https://raw.githubusercontent.com/EnviDat/ckanext-composite/${CKANEXT_COMPOSITE_VERSION}/dev-requirements.txt && \ - pip wheel --wheel-dir=/wheels \ - git+${CKANEXT_COMPOSITE_GITHUB_URL}.git@${CKANEXT_COMPOSITE_VERSION}#egg=ckanext-composite + mkdir -p /wheels && \ + pip install -r https://raw.githubusercontent.com/ckan/ckanext-harvest/${CKANEXT_HARVEST_VERSION}/dev-requirements.txt -# ckanext-repeating ########################################################### -ARG CKANEXT_REPEATING_VERSION="1.0.0" -ENV CKANEXT_REPEATING_VERSION=${CKANEXT_REPEATING_VERSION} -ENV CKANEXT_REPEATING_GITHUB_URL="https://github.com/MarijaKnezevic/ckanext-repeating" +RUN set -ex && \ + pip wheel --wheel-dir=/wheels -r https://raw.githubusercontent.com/ckan/ckanext-harvest/${CKANEXT_HARVEST_VERSION}/requirements.txt && \ + pip wheel --wheel-dir=/wheels git+https://github.com/ckan/ckanext-harvest.git@${CKANEXT_HARVEST_VERSION}#egg=ckanext-harvest && \ + curl -o /wheels/ckanext-harvest.txt https://raw.githubusercontent.com/ckan/ckanext-harvest/${CKANEXT_HARVEST_VERSION}/requirements.txt && \ + ls -lah /wheels + +# ckanext-spatial ############################################################# +ENV CKANEXT_SPATIAL_GITHUB_URL="https://github.com/ckan/ckanext-spatial" +ENV CKANEXT_SPATIAL_VERSION="8a00a2b" RUN set -ex && \ + curl -o /wheels/ckanext-spatial-requirements.txt \ + https://raw.githubusercontent.com/ckan/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements-py2.txt && \ pip wheel --wheel-dir=/wheels \ - git+${CKANEXT_REPEATING_GITHUB_URL}.git@${CKANEXT_REPEATING_VERSION}#egg=ckanext-repeating + git+${CKANEXT_SPATIAL_GITHUB_URL}.git@${CKANEXT_SPATIAL_VERSION}#egg=ckanext-spatial -# ckanext-clamav ############################################################## -ARG CKANEXT_CALMAV_VERSION="master" -ENV CKANEXT_CALMAV_VERSION=${CKANEXT_CALMAV_VERSION} -ENV CKANEXT_CALMAV_GITHUB_URL="https://github.com/mutantsan/ckanext-clamav" +# ckanext-geoview ############################################################# +ARG CKANEXT_GEOVIEW_VERSION="v0.2.2" +ENV CKANEXT_GEOVIEW_VERSION=${CKANEXT_GEOVIEW_VERSION} +ENV CKANEXT_GEOVIEW_GITHUB_URL="https://github.com/ckan/ckanext-geoview" RUN set -ex && \ - pip wheel --wheel-dir=/wheels -r \ - https://raw.githubusercontent.com/mutantsan/ckanext-clamav/${CKANEXT_CALMAV_VERSION}/requirements.txt && \ - curl -o /wheels/ckanext-clamav.txt \ - https://raw.githubusercontent.com/mutantsan/ckanext-clamav/${CKANEXT_CALMAV_VERSION}/requirements.txt && \ - pip wheel --wheel-dir=/wheels \ - git+${CKANEXT_CALMAV_GITHUB_URL}.git@${CKANEXT_CALMAV_VERSION}#egg=ckanext-clamav + curl -o /wheels/ckanext-geoview-dev-requirements.txt \ + ${CKANEXT_GEOVIEW_GITHUB_URL}/raw/${CKANEXT_GEOVIEW_VERSION}/dev-requirements.txt && \ + pip install -r /wheels/ckanext-geoview-dev-requirements.txt && \ + pip wheel --wheel-dir=/wheels \ + git+${CKANEXT_GEOVIEW_GITHUB_URL}.git@${CKANEXT_GEOVIEW_VERSION}#egg=ckanext-geoview -# ckanext-password-policy ##################################################### -ARG CKANEXT_PASSWORD_POLICY_VERSION="5618dc9" -ENV CKANEXT_PASSWORD_POLICY_VERSION=${CKANEXT_PASSWORD_POLICY_VERSION} -ENV CKANEXT_PASSWORD_POLICY_GITHUB_URL="https://github.com/keitaroinc/ckanext-password-policy" +# ckanext-scheme-sddi ############################################################# +ARG CKANEXT_SCHEME_SDDI_VERSION="0.0.1" +ENV CKANEXT_SCHEME_SDDI_VERSION=${CKANEXT_SCHEME_SDDI_VERSION} +ENV CKANEXT_SCHEME_SDDI_GITHUB_URL="https://github.com/MarijaKnezevic/ckanext-scheme-sddi" RUN set -ex && \ - pip install -r \ - https://raw.githubusercontent.com/keitaroinc/ckanext-password-policy/${CKANEXT_PASSWORD_POLICY_VERSION}/requirements.txt && \ - curl -o /wheels/ckanext-password-policy.txt \ - https://raw.githubusercontent.com/keitaroinc/ckanext-password-policy/${CKANEXT_PASSWORD_POLICY_VERSION}/requirements.txt && \ - pip wheel --wheel-dir=/wheels \ - git+${CKANEXT_PASSWORD_POLICY_GITHUB_URL}.git@${CKANEXT_PASSWORD_POLICY_VERSION}#egg=ckanext-password-policy + mkdir -p /wheels && \ + pip install -r ${CKANEXT_SCHEME_SDDI_GITHUB_URL}/raw/${CKANEXT_SCHEME_SDDI_VERSION}/dev-requirements.txt -# ckanext-spatial ############################################################# -FROM ghcr.io/keitaroinc/ckan:${CKAN_VERSION_BUILD_SPATIAL} as extbuild-spatial +RUN set -ex && \ + pip wheel --wheel-dir=/wheels -r ${CKANEXT_SCHEME_SDDI_GITHUB_URL}/raw/${CKANEXT_SCHEME_SDDI_VERSION}/requirements.txt && \ + pip wheel --wheel-dir=/wheels git+${CKANEXT_SCHEME_SDDI_GITHUB_URL}.git@${CKANEXT_SCHEME_SDDI_VERSION}#egg=ckanext-scheme-sddi && \ + curl -o /wheels/ckanext-scheme-sddi.txt ${CKANEXT_SCHEME_SDDI_GITHUB_URL}/raw/${CKANEXT_SCHEME_SDDI_VERSION}/requirements.txt && \ + ls -lah /wheels -ARG CKANEXT_SPATIAL_VERSION="c2118b9" -ENV CKANEXT_SPATIAL_VERSION=${CKANEXT_SPATIAL_VERSION} +# ckanext-theme-sddi ############################################################# +ARG CKANEXT_THEME_SDDI_VERSION="0.0.1" +ENV CKANEXT_THEME_SDDI_VERSION=${CKANEXT_THEME_SDDI_VERSION} +ENV CKANEXT_THEME_SDDI_GITHUB_URL="https://github.com/MarijaKnezevic/ckanext-theme-sddi" -USER root +RUN set -ex && \ + curl -o /wheels/ckanext-theme-sddi-dev-requirements.txt \ + ${CKANEXT_THEME_SDDI_GITHUB_URL}/raw/${CKANEXT_THEME_SDDI_VERSION}/requirements.txt && \ + pip install -r /wheels/ckanext-theme-sddi-dev-requirements.txt && \ + pip wheel --wheel-dir=/wheels \ + git+${CKANEXT_THEME_SDDI_GITHUB_URL}.git@${CKANEXT_THEME_SDDI_VERSION}#egg=ckanext-theme-sddi + +# ckanext-heroslideradmin ############################################################# +ARG CKANEXT_HEROSLIDERADMIN_VERSION="4b60e00" +ENV CKANEXT_HEROSLIDERADMIN_VERSION=${CKANEXT_HEROSLIDERADMIN_VERSION} +ENV CKANEXT_HEROSLIDERADMIN_GITHUB_URL="https://github.com/dathere/ckanext-heroslideradmin" -# Install any system packages necessary to build extensions RUN set -ex && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - python3-dev python3-pip libxml2-dev libxslt1-dev libgeos-c1v5 python-is-python3 && \ - mkdir -p /wheels && \ - pip install -U pip + curl -o /wheels/ckanext-heroslideradmin-requirements.txt \ + ${CKANEXT_HEROSLIDERADMIN_GITHUB_URL}/raw/${CKANEXT_HEROSLIDERADMIN_VERSION}/requirements.txt && \ + pip install -r /wheels/ckanext-heroslideradmin-requirements.txt && \ + pip wheel --wheel-dir=/wheels \ + git+${CKANEXT_HEROSLIDERADMIN_GITHUB_URL}.git@${CKANEXT_HEROSLIDERADMIN_VERSION}#egg=ckanext-heroslideradmin + +# ckanext-clamav ############################################################# +ARG CKANEXT_CLAMAV_VERSION="a1d23ac" +ENV CKANEXT_CLAMAV_VERSION=${CKANEXT_CLAMAV_VERSION} +ENV CKANEXT_CLAMAV_GITHUB_URL="https://github.com/DataShades/ckanext-clamav" RUN set -ex && \ - pip install -r https://raw.githubusercontent.com/MarijaKnezevic/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements.txt && \ - curl -o /wheels/ckanext-spatial.txt \ - https://raw.githubusercontent.com/MarijaKnezevic/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements.txt && \ - pip install -r https://raw.githubusercontent.com/MarijaKnezevic/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements-postgis.txt && \ - curl -o /wheels/ckanext-spatial-postgis.txt \ - https://raw.githubusercontent.com/MarijaKnezevic/ckanext-spatial/${CKANEXT_SPATIAL_VERSION}/requirements-postgis.txt && \ - ls -lah /wheels + curl -o /wheels/ckanext-clamav-requirements.txt \ + ${CKANEXT_CLAMAV_GITHUB_URL}/raw/${CKANEXT_CLAMAV_VERSION}/requirements.txt && \ + pip install -r /wheels/ckanext-clamav-requirements.txt && \ + pip wheel --wheel-dir=/wheels \ + git+${CKANEXT_CLAMAV_GITHUB_URL}.git@${CKANEXT_CLAMAV_VERSION}#egg=ckanext-clamav + +# ckanext-fortify ############################## +ARG CKANEXT_FORTIFY_VERSION="bda5d12" +ENV CKANEXT_FORTIFY_VERSION=${CKANEXT_FORTIFY_VERSION} +ENV CKANEXT_FORTIFY_GITHUB_URL="https://github.com/salsadigitalauorg/ckanext-fortify" RUN set -ex && \ - pip wheel --wheel-dir=/wheels \ - git+https://github.com/MarijaKnezevic/ckanext-spatial.git@${CKANEXT_SPATIAL_VERSION}#egg=ckanext-spatial + pip wheel --wheel-dir=/wheels \ + git+${CKANEXT_FORTIFY_GITHUB_URL}.git@${CKANEXT_FORTIFY_VERSION}#egg=ckanext-fortify + +# ckanext-dcat ########################################################## +ARG CKANEXT_DCAT_VERSION="v2.0.0" +ENV CKANEXT_DCAT_VERSION=${CKANEXT_DCAT_VERSION} +ENV CKANEXT_DCAT_GITHUB_URL="https://github.com/ckan/ckanext-dcat" + +RUN set -ex && \ + curl -o /wheels/ckanext-dcat-requirements.txt \ + https://raw.githubusercontent.com/ckan/ckanext-dcat/${CKANEXT_DCAT_VERSION}/requirements.txt && \ + pip install -r /wheels/ckanext-dcat-requirements.txt && \ + pip wheel --wheel-dir=/wheels \ + git+${CKANEXT_DCAT_GITHUB_URL}.git@${CKANEXT_DCAT_VERSION}#egg=ckanext-dcat + +# ckanext-security ###################################################### +ARG CKANEXT_SECURITY_VERSION="80dc1d6" +ENV CKANEXT_SECURITY_VERSION=${CKANEXT_SECURITY_VERSION} +ENV CKANEXT_SECURITY_GITHUB_URL="https://github.com/MarijaKnezevic/ckanext-security" + +RUN set -ex && \ + curl -o /wheels/ckanext-security-requirements.txt \ + https://raw.githubusercontent.com/MarijaKnezevic/ckanext-security/${CKANEXT_SECURITY_VERSION}/requirements.txt && \ + pip install -r /wheels/ckanext-security-requirements.txt && \ + pip wheel --wheel-dir=/wheels \ + git+${CKANEXT_SECURITY_GITHUB_URL}.git@${CKANEXT_SECURITY_VERSION}#egg=ckanext-security ############################################################################### # Runtime stage ############################################################################### -FROM ghcr.io/keitaroinc/ckan:${CKAN_VERSION_RUNTIME_STAGE} as runtime - -ENV CKAN__PLUGINS "image_view text_view recline_view webpage_view datastore datapusher \ - hierarchy_display hierarchy_form display_group relation \ - spatial_metadata spatial_query datesearch repeating composite scheming_datasets \ - password_policy clamav \ - envvars" - -# Extra env for compatibility with ckan/base Docker images for downstream k8s +FROM python:3.9-slim + +ENV APP_DIR=/srv/app +ENV SRC_DIR=/srv/app/src +ENV CKAN_DIR=${SRC_DIR}/ckan +ENV DATA_DIR=/srv/app/data +ENV PIP_SRC=${SRC_DIR} +ENV GIT_BRANCH=ckan-2.11.0 + +# Setting the locale +ENV LC_ALL="en_US.UTF-8" +RUN apt-get update && apt-get install --no-install-recommends -y locales +RUN sed -i "/$LC_ALL/s/^# //g" /etc/locale.gen +RUN dpkg-reconfigure --frontend=noninteractive locales +RUN update-locale LANG=${LC_ALL} + +# Set timezone +RUN echo "UTC" > /etc/timezone ENV CKAN_INI=${APP_DIR}/production.ini ENV CKAN_STORAGE_PATH=/var/lib/ckan -ENV TZ="UTC" -USER root +# Update the package lists and install required packages +RUN apt-get update && apt-get install -y \ + bash \ + git \ + gettext \ + curl \ + wget \ + unzip \ + postgresql-client \ + libmagic1 \ + libpcre3 \ + libxslt1.1 \ + libxml2 \ + tzdata \ + apache2-utils \ + musl-dev \ + libssl-dev \ + proj-bin \ + libproj-dev \ + proj-data \ + python3-cffi \ + supervisor + +# Cleanup to reduce image size +RUN apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN pip install markupsafe==2.0.1 setuptools wheel + +# Create a constraint file that limits the Cython version to a compatible one, see https://github.com/yaml/pyyaml/issues/736 +RUN echo 'Cython < 3.0' > /tmp/constraint.txt +RUN pip3_CONSTRAINT=/tmp/constraint.txt pip3 wheel --wheel-dir=/wheels PyYAML==5.4.1 + +# Create SRC_DIR +RUN mkdir -p ${SRC_DIR} && \ + # Link python to python3 + ln -s /usr/bin/python3 /usr/bin/python + +# Get artifacts from build stages +COPY --from=ckanbuild /wheels ${APP_DIR}/wheels +COPY --from=extbuild /wheels ${APP_DIR}/ext_wheels +COPY --from=ckanbuild ${APP_DIR}/src/ckan ${CKAN_DIR} -# Install any system packages necessary to build extensions -RUN set -ex && \ - apt-get update && \ - apt-get install -y --no-install-recommends \ - clamav \ - clamav-daemon \ - libxml2-dev libxslt1-dev libgeos-c1v5 && \ - pip install --no-cache-dir -U pip && \ - rm -rf /var/lib/apt/lists/* +# Additional install steps for build stages artifacts +RUN pip3 install --no-index --find-links=${APP_DIR}/wheels uWSGI==2.0.21 gevent==22.10.2 greenlet==2.0.1 -# Copy python wheels from build stage -COPY --from=extbuild /wheels ${APP_DIR}/ext_wheels -COPY --from=extbuild-spatial /wheels ${APP_DIR}/ext_wheels +# Create a local user and group to run the app +# Add a group with a specific GID (92) +RUN groupadd -g 92 ckan +# Add a user with a specific UID (92), home directory, and add to the ckan group +RUN useradd -u 92 -g ckan -M -d ${APP_DIR} -s /bin/bash ckan + +WORKDIR ${CKAN_DIR} + +# Install CKAN +RUN pip3 install -e ${APP_DIR}/src/ckan +RUN pip3 install --no-index --find-links=${APP_DIR}/wheels -r requirements.txt + +# ckanext-harvest ########################################################### +RUN set -ex && \ + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-harvest && \ + pip install --no-index --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-harvest.txt # ckanext-hierarchy ########################################################### RUN set -ex && \ pip install --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-hierarchy.txt && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-hierarchy -# ckanext-grouphierarchy ###################################################### +# ckanext-envvars ############################################################ RUN set -ex && \ - pip install --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-grouphierarchy.txt && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-grouphierarchy + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-envvars -# ckanext-relation ############################################################ +# ckanext-scheming ############################################################ +RUN set -ex && \ + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-scheming + +# ckanext-datesearch ########################################################## RUN set -ex && \ - pip install --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-relation.txt && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-relation + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-datesearch # ckanext-spatial ############################################################# RUN set -ex && \ - pip install -r ${APP_DIR}/ext_wheels/ckanext-spatial.txt && \ - pip install -r ${APP_DIR}/ext_wheels/ckanext-spatial-postgis.txt && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-spatial + pip3 install -e 'git+https://github.com/ckan/ckanext-spatial.git#egg=ckanext-spatial' && \ + pip3 install -r 'https://raw.githubusercontent.com/ckan/ckanext-spatial/master/requirements.txt' -# ckanext-scheming ############################################################ +# ckanext-geoview ############################################################# RUN set -ex && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-scheming + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-geoview -# ckanext-datesearch ########################################################## +# ckanext-scheme-sddi ############################################################# RUN set -ex && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-datesearch + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-scheme-sddi -# ckanext-composite ########################################################### +# ckanext-theme-sddi ############################################################# RUN set -ex && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-composite + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-theme-sddi -# ckanext-repeating ########################################################### +# ckanext-heroslideradmin ############################################################# RUN set -ex && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-repeating - -# ckanext-clamav ############################################################## + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-heroslideradmin + +# ckanext-clamav ############################################################# RUN set -ex && \ - pip install -r ${APP_DIR}/ext_wheels/ckanext-clamav.txt && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-clamav - -# ckanext-password-policy ##################################################### + +# ckanext-fortify ############################## RUN set -ex && \ - pip install -r ${APP_DIR}/ext_wheels/ckanext-password-policy.txt && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-password-policy + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-fortify -# Copy init scripts and additional files -COPY --chown=ckan:ckan initScripts/ ${APP_DIR}/docker-afterinit.d -COPY --chown=ckan:ckan who.ini ${APP_DIR}/who.ini +# ckanext-dcat ########################################################## +RUN set -ex && \ + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-dcat + +# ckanext-security ###################################################### +RUN set -ex && \ + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-security + + +ENV CKAN__PLUGINS "envvars image_view text_view recline_view webpage_view datastore \ + harvest ckan_harvester \ + hierarchy_display hierarchy_form \ + # datapusher Token required \ + # relation ImportError: cannot import name 'Mapping' from 'collections' \ + spatial_metadata spatial_query \ + scheming_datasets \ + datesearch \ + geo_view geojson_view wmts_view shp_view \ + scheme_sddi \ + theme_sddi \ + heroslideradmin \ + clamav \ + fortify \ + dcat \ + security" + # password_policy No module named 'ckan.lib.repoze_plugins' \ + +RUN set -ex && \ + ckan generate config ${APP_DIR}/production.ini RUN set -ex && \ ckan config-tool "${CKAN_INI}" "ckan.plugins = ${CKAN__PLUGINS}" && \ ckan config-tool "${CKAN_INI}" "ckan.spatial.srid = 4326" && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.search_backend = solr-bbox" && \ + ckan config-tool "${CKAN_INI}" "ckanext.spatial.use_postgis_sorting = true" && \ ckan config-tool "${CKAN_INI}" "scheming.dataset_schemas = ckanext.scheming:ckan_dataset.yaml" && \ - ckan config-tool "${CKAN_INI}" "scheming.presets = ckanext.scheming:presets.json ckanext.repeating:presets.json ckanext.composite:presets.json" && \ + ckan config-tool "${CKAN_INI}" "scheming.presets = ckanext.scheming:presets.json ckanext.scheme_sddi:sddi_presets.json" && \ ckan config-tool "${CKAN_INI}" "scheming.dataset_fallback = false" && \ - ckan config-tool "${CKAN_INI}" "licenses_group_url = https://raw.githubusercontent.com/tum-gis/ckanext-grouphierarchy-sddi/main/ckanext/grouphierarchy/licenses_SDDI.json" && \ - ckan config-tool "${CKAN_INI}" "ckanext.password_policy.password_length = 12" && \ - ckan config-tool "${CKAN_INI}" "ckanext.password_policy.failed_logins = 3" && \ - ckan config-tool "${CKAN_INI}" "ckanext.password_policy.user_locked_time = 600" && \ + ckan config-tool "${CKAN_INI}" "ckanext.dathere_theme.column_count = 4" && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.type = custom" && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.custom.url = https://tile.openstreetmap.de/{z}/{x}/{y}.png" && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.attribution = OpenStreetMap contributors." && \ - ckan config-tool "${CKAN_INI}" "who.timeout = 1800" && \ - ckan config-tool "${CKAN_INI}" "ckan.auth.public_user_details = False" && \ + ckan config-tool "${CKAN_INI}" "ckanext.security.redis.host = 127.0.0.1" && \ + ckan config-tool "${CKAN_INI}" "ckanext.security.redis.port = 6379" && \ + ckan config-tool "${CKAN_INI}" "ckanext.security.redis.db = 1" && \ + ckan config-tool "${CKAN_INI}" "ckanext.security.lock_timeout = 900" && \ + ckan config-tool "${CKAN_INI}" "ckanext.security.login_max_count = 3" && \ + ckan config-tool "${CKAN_INI}" "ckanext.security.brute_force_key = user_name" && \ + ckan config-tool "${CKAN_INI}" "ckanext.security.disable_password_reset_override = true" && \ + ckan config-tool "${CKAN_INI}" "ckanext.security.enable_totp = false" && \ + ckan config-tool "${CKAN_INI}" "ckan.fortify.enable_password_policy = True" && \ + ckan config-tool "${CKAN_INI}" "ckan.fortify.password_policy.min_length = 12" && \ + ckan config-tool "${CKAN_INI}" "PERMANENT_SESSION_LIFETIME = 600" && \ echo "${TZ}" > /etc/timezone && \ mkdir -p ${CKAN_STORAGE_PATH} && \ chown -R ckan:ckan ${APP_DIR} ${CKAN_STORAGE_PATH} && \ # Remove wheels rm -rf ${APP_DIR}/ext_wheels -USER ckan +WORKDIR ${APP_DIR} + +ENV UWSGI_HARAKIRI=50 + +# Create local storage folder +RUN mkdir -p ${CKAN_STORAGE_PATH} && \ + chown -R ckan:ckan ${CKAN_STORAGE_PATH} + +# Copy local scripts +COPY setup/prerun.py ${APP_DIR} +COPY setup/start_ckan.sh ${APP_DIR} +ADD https://raw.githubusercontent.com/ckan/ckan/${GIT_BRANCH}/wsgi.py ${APP_DIR} +RUN chmod 644 ${APP_DIR}/wsgi.py + +# Create entrypoint directory for children image scripts +ONBUILD RUN mkdir /docker-entrypoint.d + +EXPOSE 5000 + +HEALTHCHECK --interval=60s --timeout=5s --retries=5 CMD curl --fail http://localhost:5000/api/3/action/status_show || exit CMD ["/srv/app/start_ckan.sh"] + +CMD ["/srv/app/start_ckan.sh"] diff --git a/sddi-base/setup/prerun.py b/sddi-base/setup/prerun.py new file mode 100644 index 0000000..bc10087 --- /dev/null +++ b/sddi-base/setup/prerun.py @@ -0,0 +1,222 @@ +import os +import subprocess +import sys + +import psycopg2 + +try: + from urllib.error import URLError + from urllib.request import urlopen +except ImportError: + from urllib2 import urlopen + from urllib2 import URLError + +import json +import re +import time + +ckan_ini = os.environ.get("CKAN_INI", "/srv/app/production.ini") + +RETRY = 5 + + +def update_plugins(): + + plugins = os.environ.get("CKAN__PLUGINS", "") + print(("[prerun] Setting the following plugins in {}:".format(ckan_ini))) + print(plugins) + cmd = ["ckan", "config-tool", ckan_ini, "ckan.plugins = {}".format(plugins)] + subprocess.check_output(cmd, stderr=subprocess.STDOUT) + print("[prerun] Plugins set.") + + +def check_main_db_connection(retry=None): + + conn_str = os.environ.get("CKAN_SQLALCHEMY_URL") + if not conn_str: + print("[prerun] CKAN_SQLALCHEMY_URL not defined, not checking db") + return + return check_db_connection(conn_str, retry) + + +def check_datastore_db_connection(retry=None): + + conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") + if not conn_str: + print("[prerun] CKAN_DATASTORE_WRITE_URL not defined, not checking db") + return + return check_db_connection(conn_str, retry) + + +def check_db_connection(conn_str, retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print("[prerun] Giving up after 5 tries...") + sys.exit(1) + + try: + connection = psycopg2.connect(conn_str) + + except psycopg2.Error as e: + print(str(e)) + print("[prerun] Unable to connect to the database, waiting...") + time.sleep(10) + check_db_connection(conn_str, retry=retry - 1) + else: + connection.close() + + +def check_solr_connection(retry=None): + + if retry is None: + retry = RETRY + elif retry == 0: + print("[prerun] Giving up after 5 tries...") + sys.exit(1) + + url = os.environ.get("CKAN_SOLR_URL", "") + search_url = '{url}/schema/name?wt=json'.format(url=url) + + try: + connection = urlopen(search_url) + except URLError as e: + print(str(e)) + print("[prerun] Unable to connect to solr, waiting...") + time.sleep(10) + check_solr_connection(retry=retry - 1) + else: + import re + conn_info = connection.read() + schema_name = json.loads(conn_info) + if 'ckan' in schema_name['name']: + print('[prerun] Succesfully connected to solr and CKAN schema loaded') + else: + print('[prerun] Succesfully connected to solr, but CKAN schema not found') + + +def init_db(): + + db_command = ["ckan", "-c", ckan_ini, "db", "init"] + print("[prerun] Initializing or upgrading db - start") + try: + subprocess.check_output(db_command, stderr=subprocess.STDOUT) + print("[prerun] Initializing or upgrading db - end") + except subprocess.CalledProcessError as e: + if "OperationalError" in e.output: + print(e.output) + print("[prerun] Database not ready, waiting a bit before exit...") + time.sleep(5) + sys.exit(1) + else: + print(e.output) + raise e + + +def init_datastore_db(): + + conn_str = os.environ.get("CKAN_DATASTORE_WRITE_URL") + if not conn_str: + print("[prerun] Skipping datastore initialization") + return + + datastore_perms_command = ["ckan", "-c", ckan_ini, "datastore", "set-permissions"] + + connection = psycopg2.connect(conn_str) + cursor = connection.cursor() + + print("[prerun] Initializing datastore db - start") + try: + datastore_perms = subprocess.Popen( + datastore_perms_command, stdout=subprocess.PIPE + ) + + perms_sql = datastore_perms.stdout.read() + # Remove internal pg command as psycopg2 does not like it + perms_sql = re.sub(b'\\\\connect "(.*)"', b"", perms_sql) + cursor.execute(perms_sql) + for notice in connection.notices: + print(notice) + + connection.commit() + + print("[prerun] Initializing datastore db - end") + print(datastore_perms.stdout.read()) + except psycopg2.Error as e: + print("[prerun] Could not initialize datastore") + print(str(e)) + + except subprocess.CalledProcessError as e: + if "OperationalError" in e.output: + print(e.output) + print("[prerun] Database not ready, waiting a bit before exit...") + time.sleep(5) + sys.exit(1) + else: + print(e.output) + raise e + finally: + cursor.close() + connection.close() + + +def create_sysadmin(): + + name = os.environ.get("CKAN_SYSADMIN_NAME") + password = os.environ.get("CKAN_SYSADMIN_PASSWORD") + email = os.environ.get("CKAN_SYSADMIN_EMAIL") + + if name and password and email: + + # Check if user exists + command = ["ckan", "-c", ckan_ini, "user", "show", name] + + out = subprocess.check_output(command) + if b"User:None" not in re.sub(b"\s", b"", out): + print("[prerun] Sysadmin user exists, skipping creation") + return + + # Create user + command = [ + "ckan", + "-c", + ckan_ini, + "user", + "add", + name, + "password=" + password, + "email=" + email, + ] + + subprocess.call(command) + print("[prerun] Created user {0}".format(name)) + + # Make it sysadmin + command = ["ckan", "-c", ckan_ini, "sysadmin", "add", name] + + subprocess.call(command) + print("[prerun] Made user {0} a sysadmin".format(name)) + + # cleanup permissions + # We're running as root before pivoting to uwsgi and dropping privs + data_dir = "%s/storage" % os.environ['CKAN_STORAGE_PATH'] + + command = ["chown", "-R", "ckan:ckan", data_dir] + subprocess.call(command) + print("[prerun] Ensured storage directory is owned by ckan") + +if __name__ == "__main__": + + maintenance = os.environ.get("MAINTENANCE_MODE", "").lower() == "true" + + if maintenance: + print("[prerun] Maintenance mode, skipping setup...") + else: + check_main_db_connection() + init_db() + update_plugins() + check_datastore_db_connection() + init_datastore_db() + check_solr_connection() + create_sysadmin() \ No newline at end of file diff --git a/sddi-base/setup/start_ckan.sh b/sddi-base/setup/start_ckan.sh new file mode 100644 index 0000000..6cfb703 --- /dev/null +++ b/sddi-base/setup/start_ckan.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +if [[ $CKAN__PLUGINS == *"datapusher"* ]]; then + # Add ckan.datapusher.api_token to the CKAN config file (updated with corrected value later) + echo "Setting a temporary value for ckan.datapusher.api_token" + ckan config-tool $CKAN_INI ckan.datapusher.api_token=xxx +fi + +# Set up the Secret key used by Beaker and Flask +# This can be overriden using a CKAN___BEAKER__SESSION__SECRET env var +if grep -qE "beaker.session.secret ?= ?$" production.ini +then + echo "Setting beaker.session.secret in ini file" + ckan config-tool $CKAN_INI "beaker.session.secret=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + ckan config-tool $CKAN_INI "WTF_CSRF_SECRET_KEY=$(python3 -c 'import secrets; print(secrets.token_urlsafe())')" + JWT_SECRET=$(python3 -c 'import secrets; print("string:" + secrets.token_urlsafe())') + ckan config-tool $CKAN_INI "api_token.jwt.encode.secret=${JWT_SECRET}" + ckan config-tool $CKAN_INI "api_token.jwt.decode.secret=${JWT_SECRET}" +fi + +# Run the prerun script to init CKAN and create the default admin user +python3 prerun.py + +# Run any startup scripts provided by images extending this one +if [[ -d "/docker-entrypoint.d" ]] +then + for f in /docker-entrypoint.d/*; do + case "$f" in + *.sh) echo "$0: Running init file $f"; . "$f" ;; + *.py) echo "$0: Running init file $f"; python3 "$f"; echo ;; + *) echo "$0: Ignoring $f (not an sh or py file)" ;; + esac + done +fi + +# Set the common uwsgi options +UWSGI_OPTS="--plugins http,python \ + --socket /tmp/uwsgi.sock \ + --wsgi-file /srv/app/wsgi.py \ + --module wsgi:application \ + --uid 92 --gid 92 \ + --http 0.0.0.0:5000 \ + --master --enable-threads \ + --lazy-apps \ + -p 2 -L -b 32768 --vacuum \ + --harakiri $UWSGI_HARAKIRI" + +if [ $? -eq 0 ] +then + # Start supervisord + supervisord --configuration /etc/supervisord.conf & + # Start uwsgi + uwsgi $UWSGI_OPTS +else + echo "[prerun] failed...not starting CKAN." +fi \ No newline at end of file diff --git a/sddi-base/setup/supervisord.conf b/sddi-base/setup/supervisord.conf new file mode 100644 index 0000000..052dbc5 --- /dev/null +++ b/sddi-base/setup/supervisord.conf @@ -0,0 +1,23 @@ +[unix_http_server] +file = /tmp/supervisor.sock +chmod = 0777 +chown = nobody:nogroup + +[supervisord] +logfile = /tmp/supervisord.log +logfile_maxbytes = 50MB +logfile_backups=10 +loglevel = info +pidfile = /tmp/supervisord.pid +nodaemon = true +umask = 022 +identifier = supervisor + +[supervisorctl] +serverurl = unix:///tmp/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[include] +files = /etc/supervisord.d/*.conf \ No newline at end of file diff --git a/sddi-base/setup/uwsgi.conf b/sddi-base/setup/uwsgi.conf new file mode 100644 index 0000000..ee45c80 --- /dev/null +++ b/sddi-base/setup/uwsgi.conf @@ -0,0 +1,2 @@ +[uwsgi] +route = ^(?!/api).*$ basicauth:Restricted,/srv/app/.htpasswd \ No newline at end of file From 6d335b9a4d9f954fc3eadc8ef7a772814683e2a6 Mon Sep 17 00:00:00 2001 From: Aleksandra Lazoroska Date: Thu, 31 Oct 2024 15:54:45 +0100 Subject: [PATCH 02/12] Execute permissions on start_ckan.sh --- sddi-base/setup/start_ckan.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 sddi-base/setup/start_ckan.sh diff --git a/sddi-base/setup/start_ckan.sh b/sddi-base/setup/start_ckan.sh old mode 100644 new mode 100755 From e75dcb07c59c1b704c0a36c20bde0962930c041d Mon Sep 17 00:00:00 2001 From: Ilche Bedelovski Date: Thu, 31 Oct 2024 18:22:36 +0100 Subject: [PATCH 03/12] Install specific version of sqlalchemy --- sddi-base/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index 9800d40..d3b0111 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -85,7 +85,7 @@ RUN apt-get update && apt-get install -y \ libxml2-dev \ libxslt-dev -RUN pip install -U markupsafe==2.0.1 +RUN pip install -U markupsafe==2.0.1 sqlalchemy==1.4.41 # Create a constraint file that limits the Cython version to a compatible one, see https://github.com/yaml/pyyaml/issues/736 RUN echo 'Cython < 3.0' > /tmp/constraint.txt @@ -314,7 +314,7 @@ RUN apt-get update && apt-get install -y \ # Cleanup to reduce image size RUN apt-get clean && rm -rf /var/lib/apt/lists/* -RUN pip install markupsafe==2.0.1 setuptools wheel +RUN pip install markupsafe==2.0.1 setuptools wheel sqlalchemy==1.4.41 # Create a constraint file that limits the Cython version to a compatible one, see https://github.com/yaml/pyyaml/issues/736 RUN echo 'Cython < 3.0' > /tmp/constraint.txt From 214459e1d753f05d49c0777781a599df2e79633e Mon Sep 17 00:00:00 2001 From: Ilche Bedelovski Date: Fri, 1 Nov 2024 14:09:24 +0100 Subject: [PATCH 04/12] Envvars and dcat upgrade --- sddi-base/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index d3b0111..5444038 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -106,8 +106,8 @@ RUN set -ex && \ ls -lah /wheels # ckanext-envvars -ENV ENVVARS_GIT_URL=https://github.com/okfn/ckanext-envvars -ENV ENVVARS_GIT_BRANCH=0.0.2 +ENV ENVVARS_GIT_URL=https://github.com/ckan/ckanext-envvars/ +ENV ENVVARS_GIT_BRANCH=v0.0.6 RUN set -ex && \ pip3 wheel --wheel-dir=/wheels git+${ENVVARS_GIT_URL}@${ENVVARS_GIT_BRANCH}#egg=ckanext-envvars @@ -241,7 +241,7 @@ RUN set -ex && \ git+${CKANEXT_FORTIFY_GITHUB_URL}.git@${CKANEXT_FORTIFY_VERSION}#egg=ckanext-fortify # ckanext-dcat ########################################################## -ARG CKANEXT_DCAT_VERSION="v2.0.0" +ARG CKANEXT_DCAT_VERSION="v2.1.0" ENV CKANEXT_DCAT_VERSION=${CKANEXT_DCAT_VERSION} ENV CKANEXT_DCAT_GITHUB_URL="https://github.com/ckan/ckanext-dcat" @@ -398,13 +398,14 @@ RUN set -ex && \ # ckanext-dcat ########################################################## RUN set -ex && \ + pip install --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-dcat-requirements.txt && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-dcat # ckanext-security ###################################################### RUN set -ex && \ + pip install --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-security-requirements.txt && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-security - ENV CKAN__PLUGINS "envvars image_view text_view recline_view webpage_view datastore \ harvest ckan_harvester \ hierarchy_display hierarchy_form \ From 612df9363699883aa0e5b0f69166c15a3c642eb0 Mon Sep 17 00:00:00 2001 From: Ilche Bedelovski Date: Fri, 1 Nov 2024 23:09:13 +0100 Subject: [PATCH 05/12] Fix dcat and harvest extensions --- sddi-base/Dockerfile | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index 5444038..b5417e6 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -144,13 +144,13 @@ RUN set -ex && \ git+${CKANEXT_DATESEARCH_VERSION_GITHUB_URL}.git@${CKANEXT_DATESEARCH_VERSION}#egg=ckanext-datesearch # ckanext-harvest ########################################################### -ARG CKANEXT_HARVEST_VERSION="v1.5.6" +ARG CKANEXT_HARVEST_VERSION="master" ENV CKANEXT_HARVEST_VERSION=${CKANEXT_HARVEST_VERSION} ENV CKANEXT_HARVEST_GITHUB_URL="https://github.com/ckan/ckanext-harvest.git" RUN set -ex && \ mkdir -p /wheels && \ - pip install -r https://raw.githubusercontent.com/ckan/ckanext-harvest/${CKANEXT_HARVEST_VERSION}/dev-requirements.txt + pip install -r https://raw.githubusercontent.com/ckan/ckanext-harvest/${CKANEXT_HARVEST_VERSION}/requirements.txt RUN set -ex && \ pip wheel --wheel-dir=/wheels -r https://raw.githubusercontent.com/ckan/ckanext-harvest/${CKANEXT_HARVEST_VERSION}/requirements.txt && \ @@ -232,7 +232,7 @@ RUN set -ex && \ git+${CKANEXT_CLAMAV_GITHUB_URL}.git@${CKANEXT_CLAMAV_VERSION}#egg=ckanext-clamav # ckanext-fortify ############################## -ARG CKANEXT_FORTIFY_VERSION="bda5d12" +ARG CKANEXT_FORTIFY_VERSION="3.0.0" ENV CKANEXT_FORTIFY_VERSION=${CKANEXT_FORTIFY_VERSION} ENV CKANEXT_FORTIFY_GITHUB_URL="https://github.com/salsadigitalauorg/ckanext-fortify" @@ -241,7 +241,7 @@ RUN set -ex && \ git+${CKANEXT_FORTIFY_GITHUB_URL}.git@${CKANEXT_FORTIFY_VERSION}#egg=ckanext-fortify # ckanext-dcat ########################################################## -ARG CKANEXT_DCAT_VERSION="v2.1.0" +ARG CKANEXT_DCAT_VERSION="v1.5.1" ENV CKANEXT_DCAT_VERSION=${CKANEXT_DCAT_VERSION} ENV CKANEXT_DCAT_GITHUB_URL="https://github.com/ckan/ckanext-dcat" @@ -406,11 +406,9 @@ RUN set -ex && \ pip install --find-links=${APP_DIR}/ext_wheels -r ${APP_DIR}/ext_wheels/ckanext-security-requirements.txt && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-security -ENV CKAN__PLUGINS "envvars image_view text_view recline_view webpage_view datastore \ +ENV CKAN__PLUGINS "envvars image_view text_view webpage_view datastore \ harvest ckan_harvester \ hierarchy_display hierarchy_form \ - # datapusher Token required \ - # relation ImportError: cannot import name 'Mapping' from 'collections' \ spatial_metadata spatial_query \ scheming_datasets \ datesearch \ @@ -419,10 +417,9 @@ ENV CKAN__PLUGINS "envvars image_view text_view recline_view webpage_view datast theme_sddi \ heroslideradmin \ clamav \ - fortify \ - dcat \ + # fortify \ + dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface \ security" - # password_policy No module named 'ckan.lib.repoze_plugins' \ RUN set -ex && \ ckan generate config ${APP_DIR}/production.ini @@ -449,6 +446,11 @@ RUN set -ex && \ ckan config-tool "${CKAN_INI}" "ckanext.security.enable_totp = false" && \ ckan config-tool "${CKAN_INI}" "ckan.fortify.enable_password_policy = True" && \ ckan config-tool "${CKAN_INI}" "ckan.fortify.password_policy.min_length = 12" && \ + ckan config-tool "${CKAN_INI}" "ckan.fortify.check_parent_org_allowed = True" && \ + ckan config-tool "${CKAN_INI}" "ckanext.dcat.enable_content_negotiation = True" && \ + ckan config-tool "${CKAN_INI}" "ckan.harvest.log_scope = 0" && \ + ckan config-tool "${CKAN_INI}" "ckan.harvest.log_level = debug" && \ + ckan config-tool "${CKAN_INI}" "ckan.harvest.log_timeframe = 10" && \ ckan config-tool "${CKAN_INI}" "PERMANENT_SESSION_LIFETIME = 600" && \ echo "${TZ}" > /etc/timezone && \ mkdir -p ${CKAN_STORAGE_PATH} && \ From 6743178aafc3f12957a16233aefa230c72749ac7 Mon Sep 17 00:00:00 2001 From: Ilche Bedelovski Date: Mon, 4 Nov 2024 12:22:10 +0100 Subject: [PATCH 06/12] Harvest plugin to version 1.6.0 --- sddi-base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index b5417e6..5d88687 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -144,7 +144,7 @@ RUN set -ex && \ git+${CKANEXT_DATESEARCH_VERSION_GITHUB_URL}.git@${CKANEXT_DATESEARCH_VERSION}#egg=ckanext-datesearch # ckanext-harvest ########################################################### -ARG CKANEXT_HARVEST_VERSION="master" +ARG CKANEXT_HARVEST_VERSION="v1.6.0" ENV CKANEXT_HARVEST_VERSION=${CKANEXT_HARVEST_VERSION} ENV CKANEXT_HARVEST_GITHUB_URL="https://github.com/ckan/ckanext-harvest.git" From da6348e01dca16f3bd164a34d32ab29727aab0a6 Mon Sep 17 00:00:00 2001 From: Ilche Bedelovski Date: Mon, 4 Nov 2024 21:23:48 +0100 Subject: [PATCH 07/12] Skip wheels installation for fortify and heroslider --- sddi-base/Dockerfile | 29 ++++------------------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index 5d88687..aaf98c4 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -207,18 +207,6 @@ RUN set -ex && \ pip wheel --wheel-dir=/wheels \ git+${CKANEXT_THEME_SDDI_GITHUB_URL}.git@${CKANEXT_THEME_SDDI_VERSION}#egg=ckanext-theme-sddi -# ckanext-heroslideradmin ############################################################# -ARG CKANEXT_HEROSLIDERADMIN_VERSION="4b60e00" -ENV CKANEXT_HEROSLIDERADMIN_VERSION=${CKANEXT_HEROSLIDERADMIN_VERSION} -ENV CKANEXT_HEROSLIDERADMIN_GITHUB_URL="https://github.com/dathere/ckanext-heroslideradmin" - -RUN set -ex && \ - curl -o /wheels/ckanext-heroslideradmin-requirements.txt \ - ${CKANEXT_HEROSLIDERADMIN_GITHUB_URL}/raw/${CKANEXT_HEROSLIDERADMIN_VERSION}/requirements.txt && \ - pip install -r /wheels/ckanext-heroslideradmin-requirements.txt && \ - pip wheel --wheel-dir=/wheels \ - git+${CKANEXT_HEROSLIDERADMIN_GITHUB_URL}.git@${CKANEXT_HEROSLIDERADMIN_VERSION}#egg=ckanext-heroslideradmin - # ckanext-clamav ############################################################# ARG CKANEXT_CLAMAV_VERSION="a1d23ac" ENV CKANEXT_CLAMAV_VERSION=${CKANEXT_CLAMAV_VERSION} @@ -231,15 +219,6 @@ RUN set -ex && \ pip wheel --wheel-dir=/wheels \ git+${CKANEXT_CLAMAV_GITHUB_URL}.git@${CKANEXT_CLAMAV_VERSION}#egg=ckanext-clamav -# ckanext-fortify ############################## -ARG CKANEXT_FORTIFY_VERSION="3.0.0" -ENV CKANEXT_FORTIFY_VERSION=${CKANEXT_FORTIFY_VERSION} -ENV CKANEXT_FORTIFY_GITHUB_URL="https://github.com/salsadigitalauorg/ckanext-fortify" - -RUN set -ex && \ - pip wheel --wheel-dir=/wheels \ - git+${CKANEXT_FORTIFY_GITHUB_URL}.git@${CKANEXT_FORTIFY_VERSION}#egg=ckanext-fortify - # ckanext-dcat ########################################################## ARG CKANEXT_DCAT_VERSION="v1.5.1" ENV CKANEXT_DCAT_VERSION=${CKANEXT_DCAT_VERSION} @@ -386,15 +365,15 @@ RUN set -ex && \ # ckanext-heroslideradmin ############################################################# RUN set -ex && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-heroslideradmin - + pip install -e "git+https://github.com/dathere/ckanext-heroslideradmin.git@4b60e00#egg=ckanext-heroslideradmin" + # ckanext-clamav ############################################################# RUN set -ex && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-clamav # ckanext-fortify ############################## RUN set -ex && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-fortify + pip install -e "git+https://github.com/salsadigitalauorg/ckanext-fortify#egg=ckanext-fortify" # ckanext-dcat ########################################################## RUN set -ex && \ @@ -417,7 +396,7 @@ ENV CKAN__PLUGINS "envvars image_view text_view webpage_view datastore \ theme_sddi \ heroslideradmin \ clamav \ - # fortify \ + fortify \ dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface \ security" From 249b89a70c19dc8501d76e4dda78ff923e3989a4 Mon Sep 17 00:00:00 2001 From: Aleksandra Lazoroska Date: Tue, 5 Nov 2024 18:33:55 +0100 Subject: [PATCH 08/12] Add uwsgi-plugin-python3 package --- sddi-base/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index aaf98c4..8603464 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -288,6 +288,7 @@ RUN apt-get update && apt-get install -y \ libproj-dev \ proj-data \ python3-cffi \ + uwsgi-plugin-python3 \ supervisor # Cleanup to reduce image size From c56dbd7161f4caba4b5e55582db9ad755250dc9f Mon Sep 17 00:00:00 2001 From: Aleksandra Lazoroska Date: Thu, 7 Nov 2024 11:57:57 +0100 Subject: [PATCH 09/12] Removed redis configuration for CKANext security --- sddi-base/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index 8603464..c18e001 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -416,9 +416,6 @@ RUN set -ex && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.type = custom" && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.custom.url = https://tile.openstreetmap.de/{z}/{x}/{y}.png" && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.attribution = OpenStreetMap contributors." && \ - ckan config-tool "${CKAN_INI}" "ckanext.security.redis.host = 127.0.0.1" && \ - ckan config-tool "${CKAN_INI}" "ckanext.security.redis.port = 6379" && \ - ckan config-tool "${CKAN_INI}" "ckanext.security.redis.db = 1" && \ ckan config-tool "${CKAN_INI}" "ckanext.security.lock_timeout = 900" && \ ckan config-tool "${CKAN_INI}" "ckanext.security.login_max_count = 3" && \ ckan config-tool "${CKAN_INI}" "ckanext.security.brute_force_key = user_name" && \ From d62fa86bc5450a98fa36cb89172c42b5071a6523 Mon Sep 17 00:00:00 2001 From: Aleksandra Lazoroska Date: Thu, 7 Nov 2024 15:01:06 +0100 Subject: [PATCH 10/12] Add relation plugin and fix the plugin order --- sddi-base/Dockerfile | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index c18e001..a9a7cad 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -339,6 +339,10 @@ RUN set -ex && \ RUN set -ex && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-envvars +# ckanext-relation ############################################################ +RUN set -ex && \ + pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-relation + # ckanext-scheming ############################################################ RUN set -ex && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-scheming @@ -389,17 +393,18 @@ RUN set -ex && \ ENV CKAN__PLUGINS "envvars image_view text_view webpage_view datastore \ harvest ckan_harvester \ hierarchy_display hierarchy_form \ + relation \ spatial_metadata spatial_query \ - scheming_datasets \ datesearch \ - geo_view geojson_view wmts_view shp_view \ scheme_sddi \ theme_sddi \ - heroslideradmin \ - clamav \ + scheming_datasets \ + geo_view geojson_view wmts_view shp_view \ fortify \ + security \ + heroslideradmin \ dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface \ - security" + clamav" RUN set -ex && \ ckan generate config ${APP_DIR}/production.ini @@ -409,7 +414,6 @@ RUN set -ex && \ ckan config-tool "${CKAN_INI}" "ckan.spatial.srid = 4326" && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.search_backend = solr-bbox" && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.use_postgis_sorting = true" && \ - ckan config-tool "${CKAN_INI}" "scheming.dataset_schemas = ckanext.scheming:ckan_dataset.yaml" && \ ckan config-tool "${CKAN_INI}" "scheming.presets = ckanext.scheming:presets.json ckanext.scheme_sddi:sddi_presets.json" && \ ckan config-tool "${CKAN_INI}" "scheming.dataset_fallback = false" && \ ckan config-tool "${CKAN_INI}" "ckanext.dathere_theme.column_count = 4" && \ From ed34729223d53d40f6d830d635c2ad08e7a7c5a2 Mon Sep 17 00:00:00 2001 From: Aleksandra Lazoroska Date: Thu, 7 Nov 2024 18:00:52 +0100 Subject: [PATCH 11/12] Change sddi-scheme installation step --- sddi-base/Dockerfile | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index a9a7cad..cb79b00 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -180,21 +180,6 @@ RUN set -ex && \ pip wheel --wheel-dir=/wheels \ git+${CKANEXT_GEOVIEW_GITHUB_URL}.git@${CKANEXT_GEOVIEW_VERSION}#egg=ckanext-geoview -# ckanext-scheme-sddi ############################################################# -ARG CKANEXT_SCHEME_SDDI_VERSION="0.0.1" -ENV CKANEXT_SCHEME_SDDI_VERSION=${CKANEXT_SCHEME_SDDI_VERSION} -ENV CKANEXT_SCHEME_SDDI_GITHUB_URL="https://github.com/MarijaKnezevic/ckanext-scheme-sddi" - -RUN set -ex && \ - mkdir -p /wheels && \ - pip install -r ${CKANEXT_SCHEME_SDDI_GITHUB_URL}/raw/${CKANEXT_SCHEME_SDDI_VERSION}/dev-requirements.txt - -RUN set -ex && \ - pip wheel --wheel-dir=/wheels -r ${CKANEXT_SCHEME_SDDI_GITHUB_URL}/raw/${CKANEXT_SCHEME_SDDI_VERSION}/requirements.txt && \ - pip wheel --wheel-dir=/wheels git+${CKANEXT_SCHEME_SDDI_GITHUB_URL}.git@${CKANEXT_SCHEME_SDDI_VERSION}#egg=ckanext-scheme-sddi && \ - curl -o /wheels/ckanext-scheme-sddi.txt ${CKANEXT_SCHEME_SDDI_GITHUB_URL}/raw/${CKANEXT_SCHEME_SDDI_VERSION}/requirements.txt && \ - ls -lah /wheels - # ckanext-theme-sddi ############################################################# ARG CKANEXT_THEME_SDDI_VERSION="0.0.1" ENV CKANEXT_THEME_SDDI_VERSION=${CKANEXT_THEME_SDDI_VERSION} @@ -362,7 +347,7 @@ RUN set -ex && \ # ckanext-scheme-sddi ############################################################# RUN set -ex && \ - pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-scheme-sddi + pip install -e "git+https://github.com/MarijaKnezevic/ckanext-scheme-sddi@0.0.1#egg=ckanext-scheme-sddi" # ckanext-theme-sddi ############################################################# RUN set -ex && \ @@ -414,6 +399,7 @@ RUN set -ex && \ ckan config-tool "${CKAN_INI}" "ckan.spatial.srid = 4326" && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.search_backend = solr-bbox" && \ ckan config-tool "${CKAN_INI}" "ckanext.spatial.use_postgis_sorting = true" && \ + ckan config-tool "${CKAN_INI}" "scheming.dataset_schemas = ckanext.scheme_sddi:sddi_dataset.yaml" && \ ckan config-tool "${CKAN_INI}" "scheming.presets = ckanext.scheming:presets.json ckanext.scheme_sddi:sddi_presets.json" && \ ckan config-tool "${CKAN_INI}" "scheming.dataset_fallback = false" && \ ckan config-tool "${CKAN_INI}" "ckanext.dathere_theme.column_count = 4" && \ From 2f14036b2f66da060c5a527aed166f3f0f734cf6 Mon Sep 17 00:00:00 2001 From: Aleksandra Lazoroska Date: Fri, 8 Nov 2024 14:31:02 +0100 Subject: [PATCH 12/12] Trigger new pipeline build --- sddi-base/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sddi-base/Dockerfile b/sddi-base/Dockerfile index cb79b00..9c9c14e 100644 --- a/sddi-base/Dockerfile +++ b/sddi-base/Dockerfile @@ -345,7 +345,7 @@ RUN set -ex && \ RUN set -ex && \ pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-geoview -# ckanext-scheme-sddi ############################################################# +# ckanext-scheme-sddi ############################################################ RUN set -ex && \ pip install -e "git+https://github.com/MarijaKnezevic/ckanext-scheme-sddi@0.0.1#egg=ckanext-scheme-sddi"