From 8ed04525be99b23e01a1046a94461b7329673493 Mon Sep 17 00:00:00 2001 From: Chlod Alejandro Date: Fri, 16 Aug 2024 03:04:07 +0800 Subject: [PATCH] Refactor Docker setup Re-add ability to use Docker for development post-switch to Cloud VPS. Support for Trove has been added, along with a few tweaks (like moving execution to a non-root user, as required by Symfony). Also moved off of Toolforge images and onto global `php` images, since CopyPatrol isn't expected to run on Toolforge anymore. --- .github/workflows/ci.yml | 127 ++++++++++++++----------------- README.md | 76 ++++++++++++++++++- docker-compose.yml | 13 +++- docker/Dockerfile | 145 +++++++++++++++++++++--------------- docker/docker-entrypoint.sh | 35 +++++---- 5 files changed, 245 insertions(+), 151 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6aa0c883..a15dd428 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: name: Build and test runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Setup PHP @@ -29,74 +29,59 @@ jobs: ./bin/console lint:yaml ./config ./vendor/bin/minus-x check . ./bin/phpunit --exclude-group=integration + build_image: + name: Build Docker image + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v3 + + - name: Build image + id: docker_build + uses: docker/build-push-action@v6 + with: + context: . + file: docker/Dockerfile + target: production + tags: wikimedia/copypatrol:latest + outputs: type=docker,dest=/tmp/copypatrol-production.image.tar + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} + + - name: Upload Docker image to artifacts + uses: actions/upload-artifact@v4 + with: + name: image-production + path: /tmp/copypatrol-production.image.tar + analysis: + name: Analyze Docker images + runs-on: ubuntu-latest + needs: build_image + + steps: + - name: Download Docker image from artifacts + uses: actions/download-artifact@v4 + with: + name: image-production + path: /tmp -# build_image: -# name: Build Docker image -# runs-on: ubuntu-latest -# needs: build -# strategy: -# matrix: -# targets: -# - name: production -# tag: wikimedia/copypatrol -# - name: development -# tag: wikimedia/copypatrol-development -# steps: -# - name: Checkout code -# uses: actions/checkout@v2 -# -# - name: Set up QEMU -# uses: docker/setup-qemu-action@v2 -# -# - name: Set up Docker Buildx -# id: buildx -# uses: docker/setup-buildx-action@v2 -# -# - name: Build image -# id: docker_build -# uses: docker/build-push-action@v4 -# with: -# context: . -# file: docker/Dockerfile -# target: ${{ matrix.targets.name }} -# tags: ${{ matrix.targets.tag }}:latest -# outputs: type=docker,dest=/tmp/copypatrol-${{ matrix.targets.name }}.image.tar -# cache-from: type=gha -# cache-to: type=gha,mode=max -# -# - name: Image digest -# run: echo ${{ steps.docker_build.outputs.digest }} -# -# - name: Upload Docker image to artifacts -# uses: actions/upload-artifact@v2 -# with: -# name: image-${{ matrix.targets.name }} -# path: /tmp/copypatrol-${{ matrix.targets.name }}.image.tar -# analysis: -# name: Analyze Docker images -# runs-on: ubuntu-latest -# needs: build_image -# strategy: -# matrix: -# targets: -# - name: production -# tag: wikimedia/copypatrol -# - name: development -# tag: wikimedia/copypatrol-development -# -# steps: -# - name: Download Docker image from artifacts -# uses: actions/download-artifact@v2 -# with: -# name: image-${{ matrix.targets.name }} -# path: /tmp -# -# - name: Load image -# run: | -# docker load --input /tmp/copypatrol-${{ matrix.targets.name }}.image.tar -# docker image ls -a -# - name: Dive -# uses: yuichielectric/dive-action@0.0.4 -# with: -# image: ${{ matrix.targets.tag }}:latest -# github-token: ${{ secrets.GITHUB_TOKEN }} + - name: Load image + run: | + docker load --input /tmp/copypatrol-production.image.tar + docker image ls -a + - name: Dive + uses: MaxymVlasov/dive-action@v1.0.1 + with: + image: wikimedia/copypatrol:latest + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index d5432f2d..28694e62 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,80 @@ Assets are compiled using Webpack Encore. The compiled assets **must** be commit ## Installing using Docker -_A new Docker image needs to be created following the move to Wikimedia VPS._ -_You can use the manual installation instructions above in the meantime._ +Development through Docker is suggested if you have a different version of PHP locally +installed, or if you wish to keep an isolated installation of PHP 8.2 for CopyPatrol. + +1. Copy [.env](.env) to [.env.local](.env.local) and fill in the appropriate details. + 1. Set `REPLICAS_HOST_*` and `TROVE_HOST` to `127.0.0.1` + 2. Use the credentials in your `replica.my.cnf` file in the home directory of your + Toolforge account for `REPLICAS_USERNAME` and `REPLICAS_PASSWORD`. + 3. Set the rest of the `TROVE_*` variables to that of the installation of the CopyPatrol + database (`COPYPATROL_DB_NAME`). + 4. If you need to test OAuth, obtain tokens by registering a new consumer on Meta at + [Special:OAuthConsumerRegistration](https://meta.wikimedia.org/wiki/Special:OAuthConsumerRegistration). + Alternatively, you can set `LOGGED_IN_USER` to any value to simulate being that user + after clicking on 'Login'. + 5. If you need to test the "iThenticate report" functionality, set `TCA_DOMAIN` and `TCA_KEY`. + Reports older than `AppController::ITHENTICATE_V2_TIMESTAMP` need to connect to the older + iThenticate API, using the credentials set by `ITHENTICATE_USERNAME` and `ITHENTICATE_PASSWORD`. +2. Build the development image once and install Composer dependencies with the following + ```bash + docker compose build + # On Windows, use `%CD%` instead of `$(pwd)`. + docker run --rm -ti -v $(pwd):/app wikimedia/copypatrol:development composer install + ``` + Run the second command again every time you change `composer.json`, or when `composer.json` + is changed in an upstream commit. This can take a while on Windows. +3. (*Windows only*) Set the `HOME` environment variable to your user profile directory. + ```cmd + setx HOME %UserProfile% + set HOME=%UserProfile% + ``` + The first command sets `HOME` for future shells. The second command sets `HOME` for the current shell. +4. Open a new terminal and start the development container with + ```bash + docker compose up + ``` + Starting the local development server will be delayed until the next + step is finished. +5. Open up an SSH tunnel to access the databases on Toolforge. + ```bash + # Your SSH config at $HOME/.ssh will be mounted into the container. + # Your passphrase will be requested if your private key is protected. + # If your Toolforge shell name is different from the default, append + # your shell name after "ssh". (e.g. `... start ssh exampleuser`) + docker compose exec copypatrol start ssh + ``` + This terminal will stay open as long as SSH is connected. No successful + connection message is shown, but Symfony will start immediately once the + ports are open. This extra step is required for you to be able to enter + your SSH key password through a TTY. + +Changes to this folder will automatically be applied to the running Docker container. This includes +changes to `src` files, `.env.local`, etc. XDebug is set up to connect to the host machine +(the computer running the Docker container) on port 9003 upon request ([more info](https://xdebug.org/docs/step_debug)). + +If the Trove host changes, you must set the `TROVE_HOST` environment variable to the correct host. +Review [OpenStack browser](https://openstack-browser.toolforge.org/project/copypatrol/database/copypatrol-dev-db-01) for +the latest host. + +An XDebug configuration is provided by default. To customize this config, mount a +`xdebug.ini` file to `/usr/local/etc/php/conf.d/xdebug.ini` in the container. + +
+Production image + +A production image can be built with the following command: +```bash +docker build -t wikimedia/copypatrol:latest -f docker/Dockerfile . +``` +This image does not contain XDebug or OpenSSH, and does not have an SSH tunnel to Toolforge. +You can test it out by running the following command: +```bash +# On Windows, use `%CD%` instead of `$(pwd)`. +docker run -ti --rm -p 8000:8000 wikimedia/copypatrol:latest +``` +
## Adding new languages diff --git a/docker-compose.yml b/docker-compose.yml index 351ad446..8110c002 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.8' - # ========================================================= # # This file allows anyone to start the CopyPatrol web interface @@ -26,12 +24,17 @@ services: command: serve stdin_open: true tty: true + environment: + - TROVE_HOST extra_hosts: - host.docker.internal:host-gateway ports: - "8000:8000" volumes: # This binds your SSH configuration into the container. + # If you don't want to do this, comment this entry out. + # TODO: Disable `copypatrol` user access to /ssh, when that becomes possible. + # https://github.com/docker/roadmap/issues/398 - type: bind source: "$HOME/.ssh" target: "/ssh" @@ -40,4 +43,8 @@ services: - type: bind source: "." target: "/app" - stop_signal: SIGINT \ No newline at end of file + read_only: true + - type: bind + source: "./var" + target: "/app/var" + stop_signal: SIGINT diff --git a/docker/Dockerfile b/docker/Dockerfile index d1ef5da7..dc7aa492 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,105 +1,130 @@ -FROM docker-registry.tools.wmflabs.org/toolforge-php74-sssd-web:latest AS vendor +ARG PHP_VERSION="8.2" + +FROM php:${PHP_VERSION} AS base # =============================================== -# COMPOSER INSTALL -# Post-install scripts are run in a later stage. +# BASE IMAGE +# Used to set up dependencies and other things required in both the vendor and main stages. # =============================================== +ARG PHP_EXTENSIONS="apcu intl pdo_mysql" ENV COPYPATROL_ROOT=/app +ENV DEBIAN_FRONTEND=noninteractive WORKDIR ${COPYPATROL_ROOT} +# Add mlocati/php-extension-installer +ADD --chmod=0755 https://github.com/mlocati/docker-php-extension-installer/releases/latest/download/install-php-extensions /usr/local/bin/ + +# Install required PHP extensions +RUN install-php-extensions $PHP_EXTENSIONS + +# Add Composer +RUN curl https://getcomposer.org/installer | php && \ + mv composer.phar /usr/local/bin/composer + +# Add copypatrol user for user-level command execution +RUN useradd -mrs /bin/bash copypatrol + +# Set permission on app folder +RUN chown copypatrol:copypatrol ${COPYPATROL_ROOT} + +FROM base AS vendor +# =============================================== +# COMPOSER INSTALL +# Post-install scripts are run in a later stage. +# =============================================== + # Install unzip for safety -RUN apt update && apt install -y unzip +RUN apt-get clean && \ + apt-get update && \ + DEBIAN_FRONTEND=noninteractive && \ + apt-get install --yes unzip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* # Copy composer lock file, Symfony config, and bin/ folder COPY composer.* ${COPYPATROL_ROOT} -RUN composer install --no-scripts +# Set permissions for app directory +RUN chown -R copypatrol:copypatrol ${COPYPATROL_ROOT} && \ + chmod -R 755 ${COPYPATROL_ROOT} -# :~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~: +USER copypatrol +RUN composer install --no-scripts -FROM docker-registry.tools.wmflabs.org/toolforge-php74-sssd-web:latest as production +FROM base AS development # =============================================== -# PRODUCTION IMAGE +# DEVELOPMENT IMAGE # =============================================== ENV COPYPATROL_ROOT=/app WORKDIR ${COPYPATROL_ROOT} -# Disable file error logging for Lighttpd (enables error logging to stderr) -RUN sed -i 's!server.errorlog!# server.errorlog!g' /etc/lighttpd/lighttpd.conf +RUN install-php-extensions xdebug -# Enable required Lighttpd modules (rewrite, php) -RUN lighty-enable-mod fastcgi-php -RUN lighty-enable-mod rewrite +# Add XDebug configuration +RUN echo -e "error_reporting=E_ALL\\n\ +\\n\ +[xdebug]\\n\ +xdebug.mode=develop,coverage,debug,profile\\n\ +xdebug.start_with_request=yes\\n\ +xdebug.log=/tmp/xdebug.log\\n\ +xdebug.log_level=0\\n\ +xdebug.client_host=host.docker.internal\\n" >> /usr/local/etc/php/conf.d/xdebug.ini -# Add rewrite rules -RUN echo 'url.rewrite-if-not-file += ( "^(/.*)" => "/index.php$0" )' >> /etc/lighttpd/conf-enabled/90-copypatrol.conf +# Install OpenSSH (client), sudo, Symfony CLI, and unzip +RUN curl -1sLf 'https://dl.cloudsmith.io/public/symfony/stable/setup.deb.sh' | bash && \ + apt-get update && \ + apt-get install --yes openssh-client sudo symfony-cli unzip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* -## Everything before this was to set up a Toolforge-like environment -## for local development. +# Add copypatrol-ssh user for SSH access +RUN useradd -mrs /bin/bash copypatrol-ssh -# Symlink CopyPatrol public to document root -RUN rm -rf /var/www/html -RUN ln -s ${COPYPATROL_ROOT}/public /var/www/html +# Copy files +COPY --chown=copypatrol:copypatrol --chmod=755 . ${COPYPATROL_ROOT} # Copy vendor files -COPY --from=vendor ${COPYPATROL_ROOT}/vendor ${COPYPATROL_ROOT}/vendor +COPY --from=vendor --chown=copypatrol:copypatrol --chmod=755 \ + ${COPYPATROL_ROOT}/vendor ${COPYPATROL_ROOT}/vendor/ -# Copy files -COPY . ${COPYPATROL_ROOT} +# Run post-install scripts (which we skipped in the vendor container) +RUN sudo -u copypatrol composer run-script post-install-cmd -# Run post-install scripts (which we skipped in the vendor stages) -RUN composer run-script post-install-cmd - -EXPOSE 80 -# Set start command (enable FastCGI and start lighttpd) -CMD [ "lighttpd", "-D", "-f", "/etc/lighttpd/lighttpd.conf" ] +# Copy the entrypoint file, convert from CRLF to LF (if not +# already LF), set permissions, and link +RUN tr -d '\015' /docker-entrypoint.sh && \ + chmod 700 /docker-entrypoint.sh && \ + ln -s /docker-entrypoint.sh /usr/local/bin/start -# :~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~:~: +# Set start command (development entrypoint) +ENTRYPOINT [ "/docker-entrypoint.sh" ] -FROM docker-registry.tools.wmflabs.org/toolforge-php74-sssd-base:latest as development +FROM base AS production # =============================================== -# DEVELOPMENT IMAGE +# PRODUCTION IMAGE # =============================================== ENV COPYPATROL_ROOT=/app WORKDIR ${COPYPATROL_ROOT} -# Add OpenSSH, XDebug, and Symfony CLI +# Install Symfony CLI RUN curl -1sLf 'https://dl.cloudsmith.io/public/symfony/stable/setup.deb.sh' | bash && \ - apt-get clean && \ apt-get update && \ - DEBIAN_FRONTEND=noninteractive && \ - apt-get install --yes openssh-client php7.4-xdebug symfony-cli && \ + apt-get install --yes symfony-cli && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* - -# Add Composer -RUN curl https://getcomposer.org/installer | php && \ - mv composer.phar /usr/local/bin/composer -RUN echo -e "error_reporting=E_ALL\\n\ -\\n\ -[xdebug]\\n\ -xdebug.remote_enable=1\\n\ -xdebug.mode=develop,coverage,debug,profile\\n\ -xdebug.start_with_request=yes\\n\ -xdebug.log=/tmp/xdebug.log\\n\ -xdebug.log_level=0\\n\ -xdebug.remote_host=host.docker.internal\\n\ -xdebug.client_host=host.docker.internal\\n" >> /etc/php/7.4/mods-available/xdebug.ini +# Copy files +COPY --chown=copypatrol:copypatrol --chmod=755 . ${COPYPATROL_ROOT} # Copy vendor files -COPY --from=vendor ${COPYPATROL_ROOT}/vendor ${COPYPATROL_ROOT}/vendor +COPY --from=vendor --chown=copypatrol:copypatrol --chmod=755 \ + ${COPYPATROL_ROOT}/vendor ${COPYPATROL_ROOT}/vendor/ -# Copy files -COPY . ${COPYPATROL_ROOT} +# Switch to `copypatrol` user +USER copypatrol # Run post-install scripts (which we skipped in the vendor stages) RUN composer run-script post-install-cmd -# Copy the entrypoint file, convert from CRLF to LF (if not -# already LF), set permissions, and link -RUN tr -d '\015' /docker-entrypoint.sh && \ - chmod 700 /docker-entrypoint.sh && \ - ln -s /docker-entrypoint.sh /usr/local/bin/start - # Set start command (symfony serve) -ENTRYPOINT [ "/docker-entrypoint.sh" ] \ No newline at end of file +ENTRYPOINT [ "symfony" ] +CMD [ "serve" ] diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 4632e105..cecdb5f9 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -1,28 +1,29 @@ #!/usr/bin/env bash -TOOLSDB_PORT=4720 -TOOLSDB_PORT_HEX=1270 +TROVE_HOST="${TROVE_HOST:-hxmnwriu2vm.svc.trove.eqiad1.wikimedia.cloud}" +TROVE_PORT=4721 +TROVE_PORT_HEX=1271 # Also includes the rem_address column field to ensure that we're # checking the local_address column -TOOLSDB_PORT_REGEX="0100007F:$TOOLSDB_PORT_HEX\s[0-9A-F]+:[0-9A-F]+" +TROVE_PORT_REGEX="0100007F:$TROVE_PORT_HEX\s[0-9A-F]+:[0-9A-F]+" -if [ $1 == "serve" ] || [ -z $1 ]; then - if [ -z $SKIP_PORT_CHECK ]; then +if [ "$1" == "serve" ] || [ -z "$1" ]; then + if [ -d "/ssh" ] && [ -z "$SKIP_PORT_CHECK" ]; then echo "Open a new terminal and start the SSH connections." echo "See README for more information." echo - echo "Waiting for ToolsDB SQL port ($TOOLSDB_PORT) to open..." + echo "Waiting for TROVE SQL port ($TROVE_PORT) to open..." echo "(Set the SKIP_PORT_CHECK environment variable to skip this.)" trap "exit" SIGINT SIGTERM - while [ ! "$(grep -Pc "$TOOLSDB_PORT_REGEX" /proc/net/tcp)" -ge 1 ]; do + while [ ! "$(grep -Pc "$TROVE_PORT_REGEX" /proc/net/tcp)" -ge 1 ]; do # Waiting... sleep 1 done trap SIGINT SIGTERM - echo "Port ($TOOLSDB_PORT) opened, serving..." + echo "Port ($TROVE_PORT) opened, serving..." fi - exec symfony serve -elif [ $1 == "ssh" ]; then + exec sudo -u copypatrol-ssh symfony serve +elif [ "$1" == "ssh" ]; then # Check for /ssh if [ ! -d "/ssh" ]; then echo "/ssh directory not found. Mount your $HOME/.ssh folder to /ssh." @@ -30,16 +31,20 @@ elif [ $1 == "ssh" ]; then fi # Copy /ssh - cp -r /ssh /root/.ssh + rm -rf /home/copypatrol-ssh/.ssh + cp -r /ssh /home/copypatrol-ssh/.ssh # Fix permissions - chmod 700 -R /root/.ssh + chmod 700 -R /home/copypatrol-ssh/.ssh + chown copypatrol-ssh:copypatrol-ssh -R /home/copypatrol-ssh/.ssh # Check for a username provided in the SSH config - username=$(ssh -G login.toolforge.org | grep "user " | sed 's/^user //' -) + username=$(su -l copypatrol-ssh -c "ssh -G login.toolforge.org" | grep "user " | sed 's/^user //' -) # Start SSH - symfony console toolforge:ssh --toolsdb -b 127.0.0.1 $username ${@:2} + # Intentional echo to allow for word splitting + # shellcheck disable=SC2116 + exec sudo -u copypatrol-ssh symfony console toolforge:ssh --trove="$TROVE_HOST" -b 127.0.0.1 "$username" $(echo ${@:2}) else exec $@ -fi \ No newline at end of file +fi