From 9d0e60f9ea05a92cc8d1bde9cad021a1503810ac Mon Sep 17 00:00:00 2001 From: firm1 Date: Sun, 29 Nov 2020 22:08:08 +0100 Subject: [PATCH] =?UTF-8?q?Migration=20de=20la=20CI=20de=20travis=20=C3=A0?= =?UTF-8?q?=20Github=20Actions=20(#5994)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove .travis.yml * fix coverage * @Amaury remarks * remplace le push de la doc * Améliore la forme du fichier de workflow - Ajoute plein de lignes vide pour pouvoir plus facilement scanner le fichier. - Renomme toute les sections pour mieux scanner le log de GitHub Actions. - Explicite les versions utilisées dans ce log pour ne pas se faire surprendre. - Ajoute des commentaires expliquant en une phrase ce que fait chaque job. * GitHub n'aime pas les commentaires, semble-t-il. * Revert "GitHub n'aime pas les commentaires […]" This reverts commit 08b65c1265ec96f87ba2c48ab987e37dd8b251ed. * Petite erreur YAML passée inaperçue * Rassemble toutes les versions dans l'environnement. - Toutes les versions des divers services utilisés par la CI sont précisées tout en haut du fichier, en tant que variables d'environnement. * Plus de cohérence de nommage avec GitHub * La tâche push_doc ne tourne pas du tout sur une PR - Modifie la CI avec un test au niveau de la tâche directement, afin qu'elle ne soit pas du tout démarrée si on est sur une PR. * Petite erreur de syntaxe * Pas d'environnement dans les services - GitHub Actions ne permet pas d'utiliser les variables d'environnement hors des noms, des tests (`if`) et des variables passées à des actions (`with`). Donc, pas dans les définitions de services. Par conséquent, nous sommes forcés de renseigner les versions en dur. Pour référence, j'ai gardé les versions en haut du fichier avec un commentaire précisant clairement qu'il faut mettre à jour les deux. * Final rename, I guess * Renomme test.yml en ci.yml Le nom est plus représentatif de tout ce que ça fait (beaucoup plus que juste des tests). * On n'est pas à un renomamge près * Retire une installation explicite inutile Remarque de Situphen Co-authored-by: Amaury Carrade --- .github/workflows/ci.yml | 285 ++++++++++++++++++++++++++++ .travis.yml | 101 ---------- requirements-ci.txt | 4 + scripts/ci_mysql_setup.sh | 21 -- scripts/ci_turbo.sh | 31 --- scripts/deploy.sh | 104 ---------- scripts/install_zds.sh | 9 +- scripts/push_doc.sh | 12 -- scripts/travis_header.sh | 136 ------------- scripts/travis_run.sh | 59 ------ scripts/travis_script.sh | 119 ------------ scripts/update_and_deploy.sh | 31 --- update.md | 10 - zds/member/tests/tests_front.py | 12 +- zds/settings/ci_test.py | 32 +++- zds/tutorialv2/tests/tests_front.py | 18 +- 16 files changed, 336 insertions(+), 648 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .travis.yml create mode 100644 requirements-ci.txt delete mode 100755 scripts/ci_mysql_setup.sh delete mode 100755 scripts/ci_turbo.sh delete mode 100755 scripts/deploy.sh delete mode 100755 scripts/push_doc.sh delete mode 100755 scripts/travis_header.sh delete mode 100755 scripts/travis_run.sh delete mode 100755 scripts/travis_script.sh delete mode 100755 scripts/update_and_deploy.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..da441c808b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,285 @@ +name: CI + +on: ["push", "pull_request"] + +env: + NODE_VERSION: "12" + PYTHON_VERSION: "3.7" + MARIADB_VERSION: "10.4.10" + + # As GitHub Action does not allow environment variables + # to be used in services definitions, these are only for + # reference. If you update these versions, you HAVE TO + # update the versions in the services definitions of the + # test job. + ELASTICSEARCH_VERSION: "5.5.2" + MEMCACHED_VERSION: "1.6" + +jobs: + # Lint the Python back-end with flake8. + lint-back: + name: Lint back-end + runs-on: ubuntu-18.04 + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v2 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Install python dependencies + run: pip install flake8 + + - name: Lint back-end + run: make lint-back + + # Build the documentation and upload it as an artifact. + build-doc: + name: Build Sphinx documentation + runs-on: ubuntu-18.04 + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v2 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Upgrade pip + run: pip install --upgrade pip + + - name: Retrieve pip cache directory + id: pip-cache + run: echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache pip packages + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install python dependencies + run: pip install -r requirements-dev.txt + + - name: Build documentation + run: make generate-doc + + - name: Upload documentation as an artifact + uses: actions/upload-artifact@v2 + with: + name: doc + path: doc/build/html + retention-days: 1 + + # Build the website front-end and upload built assets as an artifact. + build-front: + name: Lint and build front-end + runs-on: ubuntu-18.04 + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up NodeJS ${{ env.NODE_VERSION }} + uses: actions/setup-node@v1 + with: + node-version: "${{ env.NODE_VERSION }}" + + - name: Retrieve yarn cache directory + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + + - name: Cache Node modules + uses: actions/cache@v2 + id: yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Install front-end + run: make install-front + + - name: Lint front-end + run: make lint-front + + - name: Build front-end + run: make build-front + + - name: Upload font-end assets for subsequent tests + uses: actions/upload-artifact@v2 + with: + name: assets + path: dist + retention-days: 1 + + # Test the zds-site project. + # Install the project, using assets created during the previous job, + # and install elasticsearch & memcache as a service. Then, run the tests + # in a matrix build to parallelize multiple components. + test: + name: Install and test zds-site + needs: build-front + runs-on: ubuntu-18.04 + + strategy: + matrix: + module: + [ + "zds.tutorialv2", + "zds.member zds.gallery zds.searchv2 zds.middlewares zds.pages", + "zds.forum zds.featured zds.mp zds.notification zds.utils", + ] + + services: + elasticsearch: + image: "elasticsearch:5.5.2" + ports: + - "9200:9200" + env: + "http.host": "0.0.0.0" + "transport.host": "127.0.0.1" + "xpack.security.enabled": false + "ES_JAVA_OPTS": "-Xms512m -Xmx512m" + options: >- + -e="discovery.type=single-node" + --health-cmd="curl http://localhost:9200/_cluster/health" + --health-interval=10s + --health-timeout=5s + --health-retries=10 + + memcached: + image: "memcached:1.6" + ports: + - "11211:11211" + + steps: + - name: Shutdown Ubuntu MySQL + run: sudo service mysql stop + + - name: Set up MariaDB ${{ env.MARIADB_VERSION }} + uses: getong/mariadb-action@v1.1 + with: + character set server: "utf8mb4" + collation server: "utf8mb4_unicode_ci" + mariadb version: "${{ env.MARIADB_VERSION }}" + mysql database: "ci_db_name" + mysql root password: "ci_root_password" + + - name: Checkout + uses: actions/checkout@v2 + + - name: Download previously built assets + uses: actions/download-artifact@v2 + with: + name: assets + path: dist + + - name: Upgrade pip + run: | + pip install --upgrade pip + + - name: Retrieve pip cache directory + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache pip packages + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Cache Node modules + uses: actions/cache@v2 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('zmd/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v2 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Set up NodeJS ${{ env.NODE_VERSION }} + uses: actions/setup-node@v1 + with: + node-version: "${{ env.NODE_VERSION }}" + + - name: Install Python dependencies + run: pip install -r requirements-ci.txt + + - name: Build and start zmarkdown + run: | + make zmd-install + make zmd-start + + - name: Run tests for ${{ matrix.module }} + run: | + export PATH="$PATH:$GECKOWEBDRIVER" + coverage run --source='.' manage.py test -v=2 --keepdb --settings zds.settings.ci_test ${{ matrix.module }} + + - name: Analyze coverage + shell: bash -l {0} + run: | + echo $COVERALLS_FLAG_NAME + python -m pip install coveralls + coveralls + env: + GITHUB_TOKEN: ${{ secrets.github_token }} + COVERALLS_PARALLEL: true + COVERALLS_FLAG_NAME: "${{ matrix.module }}" + + # Push coverage data to Coveralls. + coverage: + name: Push coverage to Coveralls + needs: test + runs-on: ubuntu-18.04 + + steps: + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v2 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Upload coverage data + run: | + python -m pip install coveralls + coveralls --finish + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # If we're on the dev branch (i.e. for merged pull requests), push the built + # documentation to the gh-page branch. + push_doc: + name: Push documentation to GitHub Pages + needs: ["build-doc", "test"] + runs-on: ubuntu-18.04 + if: "github.ref == 'refs/heads/dev'" + + steps: + - name: Download previously built documentation + uses: actions/download-artifact@v2 + with: + name: doc + path: doc/build/html + + - name: Deploy to GitHub Pages + uses: JamesIves/github-pages-deploy-action@3.7.1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: gh-pages + FOLDER: doc/build/html + CLEAN: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1ec9e0807c..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,101 +0,0 @@ -os: linux -dist: bionic -language: python - - -python: - - 3.7 - - -services: - - memcached - - -addons: - firefox: "latest" - mariadb: 10.4 - apt: - packages: - - language-pack-fr - - -git: - depth: 1 - - -env: - global: - - COVERALLS_PARALLEL: true - - secure: "azmDZZQZzf88zpbkYpLpxI66vpEVyv+kniW0QdWAt4qlys8P5OcO3VJBR5YT85vlvnjN9b6raWQAL1ymee0WmVdTmzXed8XjZv7t9QXVw7pfezxMKlEftVp/4Cu4wtvbew0ViZXNWV2hNXHzEqlhgnoIOq94i0UzZ7grMrI0xm0=" - - -notifications: - webhooks: https://coveralls.io/webhook - - -jobs: - fast_finish: true - include: - - name: "Job 1 : tuto (zds.tutorialv2)" - env: ZDS_TEST_JOB="zds.tutorialv2" - - name: "Job 2 : backend test1 (zds.member zds.utils zds.forum zds.middlewares)" - env: ZDS_TEST_JOB="zds.member zds.utils zds.forum zds.middlewares" - - name: "Job 3 : backend test2 (zds.mp zds.gallery zds.pages zds.featured zds.notification zds.searchv2)" - env: ZDS_TEST_JOB="zds.mp zds.gallery zds.pages zds.featured zds.notification zds.searchv2" - - name: "Job 4 : frontend test" - env: ZDS_TEST_JOB="front fixture selenium" - - name: "Job 5 : doc" - env: ZDS_TEST_JOB="doc" - - -cache: - apt: true - pip: true - yarn: true - npm: true - directories: - - $HOME/.local/share/fonts - - $HOME/.texlive - - $HOME/.cache/pip - - $HOME/virtualenv/python$TRAVIS_PYTHON_VERSION/bin - - $HOME/build/zestedesavoir/zds-site/zdsenv - - $HOME/.nvm - - $HOME/node_modules - - $HOME/build/zestedesavoir/zds-site/node_modules - - $HOME/build/zestedesavoir/zds-site/zmd/node_modules - - -before_install: - - source ./scripts/travis_header.sh # Need to be sourced - - -install: - - ./scripts/install_zds.sh --answer-yes --detect-os-version --travis-output $zds_install_argument - - -script: - - source ./scripts/travis_run.sh - - - ./scripts/travis_overview/travis_overview.sh --travis-output - - -after_success: - - du -sh $HOME/.texlive 2>/dev/null | true - - - | - # upload coverage - if [[ "$ZDS_TEST_JOB" != "none" && "$ZDS_TEST_JOB" != *"ci_turbo"* ]]; then - coveralls - fi - - - | - # upload built documentation to GitHub Pages - if [[ "$ZDS_TEST_JOB" == *"doc"* ]] && [[ "$TRAVIS_BRANCH" == "dev" ]]; then - # Adding GitHub OAuth token to login - echo -e "machine github.com login $BOT_LOGIN\n password $BOT_PASSWORD" > $HOME/.netrc \ - && git config --global url."https://".insteadOf git:// \ - && git config --global user.name "Build bot" \ - && git config --global user.email "zestedesavoir@gmail.com" \ - && git config --global push.default simple \ - && ./scripts/push_doc.sh - fi diff --git a/requirements-ci.txt b/requirements-ci.txt new file mode 100644 index 0000000000..85486ead05 --- /dev/null +++ b/requirements-ci.txt @@ -0,0 +1,4 @@ +-r requirements-dev.txt +-r requirements-prod.txt + +coverage==5.3 diff --git a/scripts/ci_mysql_setup.sh b/scripts/ci_mysql_setup.sh deleted file mode 100755 index fcfc6c7feb..0000000000 --- a/scripts/ci_mysql_setup.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -sudo sed -i'' 's/\[mysqld\]/\[mysqld\]\ninnodb_file_per_table=on\ninnodb_file_format=barracuda\ninnodb_large_prefix=on\ncharacter-set-client-handshake=false\ncharacter-set-server=utf8mb4\ncollation-server=utf8mb4_unicode_ci/' /etc/mysql/mysql.conf.d/mysqld.cnf -sudo systemctl restart mysql - -# Travis should fail as soon as possible -sudo mysql -u root -e "SET GLOBAL sql_mode = 'NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES';" - -# Avoid "mysql has gone away" errors -sudo mysql -u root -e "SET GLOBAL wait_timeout = 36000;" -sudo mysql -u root -e "SET GLOBAL max_allowed_packet = 134209536;" - -# Ensures correct charset and collation -sudo mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4';" -sudo mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci';" - -# Ensures the root user is able to connect without password -sudo mysql -u root -e "SET Password=PASSWORD('')" - -# Create database with the correct charset and collation -sudo mysql -u root -e "CREATE DATABASE zds_test CHARACTER SET = 'utf8mb4' COLLATE = 'utf8mb4_unicode_ci';" diff --git a/scripts/ci_turbo.sh b/scripts/ci_turbo.sh deleted file mode 100755 index 5fb201d702..0000000000 --- a/scripts/ci_turbo.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# This script is sourced from the .travis.yml file - -default_branch=dev - -git remote set-branches origin $default_branch -git fetch --unshallow origin $default_branch - -if [[ ! -z "$TRAVIS_TAG" ]] -then - # Remember, we are sourced, we must not exit the main shell - return -fi - -changed_files=$(git --no-pager diff --name-only $TRAVIS_COMMIT $(git merge-base $TRAVIS_COMMIT origin/$default_branch)) -echo "changed files:" -echo $changed_files - -if ! echo "$changed_files" | egrep -v "^assets" -then - # Don't test the backend if only the `/assets/` directory changed - if [[ "$ZDS_TEST_JOB" == *"front"* ]] - then - export ZDS_TEST_JOB="front ci_turbo" - else - export ZDS_TEST_JOB="none" - fi - - echo "skipping backend tests" -fi diff --git a/scripts/deploy.sh b/scripts/deploy.sh deleted file mode 100755 index 827f362e74..0000000000 --- a/scripts/deploy.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash -# -# Zeste de Savoir deployment script -# -# Deploys specified version of Zeste de Savoir -# -# DON'T RUN THIS SCRIPT DIRECTLY -# Running update_and_deploy.sh instead will first get the -# appropriate version of this script and run it instead. - -set -euo pipefail -IFS=$'\n\t' - -ENV_PATH="/opt/zds/zdsenv" -REPO_PATH="/opt/zds/zds-site" - -if [ "$1" != "ok" ]; then - echo "This script shouldn't be run directly. Run ./update_and_deploy.sh instead." >&2 - exit 1 -fi - -# Shift the first arg "ok" -shift - -if [ "$(whoami)" != "zds" ]; then - echo "This script must be run by zds user" >&2 - exit 1 -fi - -# Check if the git working directory is clean (excluding scripts/ folder) -if ! git diff-index --quiet HEAD -- . ':!scripts/'; then - echo "Git repo has uncommited changes. Make sure it's clean before trying again" >&2 - exit 1 -fi - -read -p "Did you run specific tasks for this version as described in update.md? [y/N] " -r -echo # move to a new line - -if [[ ! $REPLY =~ ^[Yy]$ ]] then - echo "Do it, now!" - exit 1 -fi - -cd $REPO_PATH - -# Enable maintenance mode -sudo ln -sf errors/maintenance.html $ENV_PATH/webroot/ - -# Delete old branch if exists -git checkout refs/heads/prod - -if git rev-parse --verify "refs/heads/$1" > /dev/null; then - git branch -D $1 -fi - -# Removes dist/ folder to avoid conflicts -rm -rf ./dist/ -# Switch to new tag -git fetch --tags -# Server has git < 1.9, git fetch --tags doesn't retrieve commits... -git fetch - -if git rev-parse $1 >/dev/null 2>&1 -then - echo "Tag $1 found!" -else - echo "Tag $1 doesn't exist." - exit 1; -fi - -# Checkout the tag -git checkout refs/heads/$1-build -# Create a branch with the same name - required to have version data in footer -git checkout -b $1 - -# Update application data -source $ENV_PATH/bin/activate -pip install --upgrade -r requirements-prod.txt -python manage.py migrate -python manage.py compilemessages - -## Collect all staticfiles from dist/ and python packages to static/ -python manage.py collectstatic --noinput --clear -## Exit venv -deactivate - -# Restart zds -sudo systemctl restart zds.{service,socket} - -# Clean the cache by restarting it -sudo service memcached restart - -# Disable maintenance mode -sudo rm $ENV_PATH/webroot/maintenance.html - -# update latex -mkdir -p $HOME/texmf/tex/latex/ -wget "https://raw.githubusercontent.com/zestedesavoir/latex-template/${1-build}/zmdocument.cls" -mv zmdocument.cls $HOME/texmf/tex/latex/ -sudo texhash -# Display current branch and commit -git status -echo "Deployed commit: `git rev-parse HEAD`" -echo "Dont forget to run specific tasks for this version as described in update.md" diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 4f1cb54bdc..4baca484fe 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -47,14 +47,6 @@ LOCAL_DIR="$(cd "$(dirname "$0")" && pwd)" source $LOCAL_DIR/define_variable.sh source $LOCAL_DIR/define_function.sh - -# enable travis fold -ZDS_SHOW_TRAVIS_FOLD=0 -if $(_in "--travis-output" $@); then - ZDS_SHOW_TRAVIS_FOLD=1 - export DJANGO_SETTINGS_MODULE="zds.settings.travis_fixture" -fi - zds_fold_category "install" @@ -156,6 +148,7 @@ fi # virtualenv + if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then zds_fold_start "virtualenv" "* Create virtualenv" diff --git a/scripts/push_doc.sh b/scripts/push_doc.sh deleted file mode 100755 index 592a440450..0000000000 --- a/scripts/push_doc.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# -# Push documentation on the gh-pages branch - -echo "Move the generated doc in a new branch" -touch doc/build/html/.nojekyll -echo "docs.zestedesavoir.com" > doc/build/html/CNAME -git add doc/build/html -f -git commit -m "Build documentation" -git subtree split --branch build_doc --prefix doc/build/html -echo "Push the documentation" -git push origin build_doc:gh-pages -f diff --git a/scripts/travis_header.sh b/scripts/travis_header.sh deleted file mode 100755 index b2107bdad0..0000000000 --- a/scripts/travis_header.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/bash - - -zds_install_argument="" -function zds_register_for_install { - if [[ "$zds_install_argument" == "" ]]; then - zds_install_argument="$1" - else - zds_install_argument="$zds_install_argument $1" - fi -} - - -function zds_register_module_for_installation { - zds_register_for_install "+base +prod" - - # install elastic-local - if [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then - print_info "* Register elasticsearch with jdk. It needed to test : zds.searchv2." - zds_register_for_install "+jdk-local +elastic-local" - fi - - # install backend dependencies - if ! ( [[ "$ZDS_TEST_JOB" == *"zds."* ]] || \ - [[ "$ZDS_TEST_JOB" == *"selenium"* ]] || \ - [[ "$ZDS_TEST_JOB" == *"doc"* ]]); then - print_info "* Don't register back because zds.* tasks, doc and selenium are not registered." - zds_register_for_install "-back" - else - print_info "* Back is registered because zds.* tasks, doc or selenium are registered." - - if ! ( [[ "$ZDS_TEST_JOB" == *"zds."* ]] || \ - [[ "$ZDS_TEST_JOB" == *"selenium"* ]]); then - print_info "* Don't migrate-db, if only doc are registered." - zds_register_for_install "-back-migrate-db" - fi - fi - - - # install frontend dependencies - if ! ( [[ "$ZDS_TEST_JOB" == *"front"* ]] || \ - [[ "$ZDS_TEST_JOB" == *"selenium"* ]]); then - print_info "* Don't register front because front task and selenium are not registered." - zds_register_for_install "-front" - else - print_info "* Front is registered because front task or selenium are registered." - fi - - - # Run fixture only when it is asked - if [[ "$ZDS_TEST_JOB" != *"fixture"* ]]; then - print_info "* Don't register fixture." - zds_register_for_install "-data" - else - print_info "* Fixture is registered." - fi - - print_info "* Argument for installation : $zds_install_argument" -} - - -function install_geckodriver { - wget "https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz" - mkdir "geckodriver" - tar -xzf "geckodriver-v0.23.0-linux64.tar.gz" -C "geckodriver" -} - - -source ./scripts/define_variable.sh -source ./scripts/define_function.sh --travis-output - - -zds_fold_category "before_install" - - -zds_fold_start "packages" "* [packages] apt-get update : make sure our source list is up-to-date (for newest dependencies version)" - sudo apt-get update -qq -zds_fold_end - - -zds_fold_start "coveralls" "* [coveralls] Install with pip" - pip install -q coveralls -zds_fold_end - - -zds_fold_start "ci_turbo" "* [ci_turbo] Skip task depending on directory changes (task will run only if needed)" - source ./scripts/ci_turbo.sh # This script exports environment variables, it must be sourced -zds_fold_end - - -if [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - #display print_info - forwho="" - if [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - forwho="'zds.*' tasks (-> needed for tests)" - - if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - forwho="$forwho and selenium" - fi - else - forwho="selenium" - fi - - zds_fold_start "mysql" "* [mysql] Install mysql for $forwho." - ./scripts/ci_mysql_setup.sh - zds_fold_end -fi - - -if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - zds_fold_start "webdriver" "* [webdriver] Install webdriver for selenium" - install_geckodriver - zds_fold_end -fi - -if [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then - # install latex - zds_fold_start "latex" "* [latex] Install latex & Run texhash (install: texlive + latex-template)" - # this script is faster than zds_install.sh +tex-local +latex-template - git clone $ZDS_LATEX_REPO - TEMPLATEDIR=$HOME/.texlive/texmf-local/tex/latex/ - ./latex-template/scripts/install_font.sh - ./latex-template/scripts/install_texlive.sh - export PATH=$HOME/.texlive/bin/x86_64-linux:$PATH - rm -rf $TEMPLATEDIR/latex-template - mkdir -p $TEMPLATEDIR - cp -r ./latex-template $TEMPLATEDIR - texhash - zds_fold_end -fi - -if [[ "$ZDS_TEST_JOB" != "none" ]]; then - zds_fold_start "register_module" "* [packages] Register module for installation" - zds_register_module_for_installation - zds_fold_end -fi diff --git a/scripts/travis_run.sh b/scripts/travis_run.sh deleted file mode 100755 index 693203cf7d..0000000000 --- a/scripts/travis_run.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - - -if [[ "$ZDS_TEST_JOB" == "none" ]]; then - exit 0 -fi - - -function print_info { - echo -en "\033[0;36m" - echo "$1" - echo -en "\033[00m" -} - - -function error_handler { - if [[ $exVal != 0 ]]; then - print_error $1 - exit 1 - fi -} - - -function run_script { - ./scripts/travis_script.sh $1; exVal=$? - error_handler "!! Some error on the last task ($1)." -} - - -function activate_env { - print_info "source $1/bin/activate" - source $1/bin/activate; exVal=$? - error_handler "!! Error: environnement not load.\n - Value = $1" -} - - -activate_env "./$ZDS_VENV" - # Add geckodriver in the PATH of zdsenv - export PATH="$PATH:/home/travis/build/zestedesavoir/zds-site/geckodriver" - export DISPLAY=":99.0" - # Use hack for virtualenv (fix some task with "command not found") - export PATH="$PATH:/home/travis/virtualenv/python${TRAVIS_PYTHON_VERSION}/bin" - - run_script "start_elasticsearch" - - run_script "make_migrations" - - run_script "lint_frontend" - - run_script "print_zmarkdown_log" - - run_script "selenium_test" - - run_script "coverage_backend" - - run_script "lint_backend" - - run_script "build_documentation" - diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh deleted file mode 100755 index e8fd25d5fa..0000000000 --- a/scripts/travis_script.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash - -source ./scripts/define_function.sh --travis-output - -zds_fold_category "script" - -exVal=0 - - -# start elastic -if [[ "$1" == "start_elasticsearch" ]] && [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then - zds_fold_start "elasticsearch" "* Start elasticsearch as service" - sudo service elasticsearch start; exVal=$? - - gateway "!! elasticsearch didn't start" $exVal - zds_fold_end -fi - - -# start latex -if [[ "$1" == "start_latex" ]] && [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then - zds_fold_start "latex" "* Start texhash -> latex" - texhash; exVal=$? - - gateway "!! Texhash failed" $exVal - zds_fold_end -fi - - -# lint backend -if [[ "$1" == "lint_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds.gallery"* ]]; then - zds_fold_start "lint_backend" "* Run lint for backend" - ./scripts/no_import_zds_settings.sh \ - && flake8 \ - && flake8 --config=zds/settings/.flake8 zds/settings; exVal=$? - - gateway "!! Test failed" $exVal - zds_fold_end -fi - -# make migrations -if [[ "$1" == "make_migrations" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - zds_fold_start "make_migrations" "* Make migrations for backend tasks" - python manage.py makemigrations --dry-run --check; exVal=$? - - gateway "!! Test failed" $exVal - zds_fold_end -fi - - -# lint frontend -if [[ "$1" == "lint_frontend" ]] && [[ "$ZDS_TEST_JOB" == *"front"* ]]; then - zds_fold_start "lint_frontend" "* Run lint for frontend" - npm run lint; exVal=$? - - gateway "!! Test failed" $exVal - zds_fold_end -fi - - -# coverage backend -if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - zds_fold_start "coverage_backend" "* Run coverage for backend" - - zds_start_zmd - - coverage run --source='.' manage.py \ - test -v=2\ - --keepdb \ - --settings zds.settings.ci_test \ - --exclude-tag=front \ - ${ZDS_TEST_JOB/front/}; exVal=$? - - gateway "!! Test failed" $exVal - - zds_stop_zmd - - zds_fold_end -fi - - -# print zmarkdown log -if [[ "$1" == "print_zmarkdown_log" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - zds_fold_start "zmarkdown_log" "* Print zmarkdown log" - node ./zmd/node_modules/pm2/bin/pm2 logs --nostream --raw --lines 1000 - zds_fold_end -fi - - -# selenium test -if [[ "$1" == "selenium_test" ]] && [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - zds_fold_start "selenium_test" "* Run selenium test for frontend" - - zds_start_zmd - - xvfb-run --server-args="-screen 0 1280x720x8" coverage run --source='.' manage.py \ - test -v=2\ - --settings zds.settings.ci_test \ - --tag=front \ - --keepdb; exVal=$? - - gateway "!! Test failed" $exVal - - zds_stop_zmd - - zds_fold_end -fi - - -# build documentation -if [[ "$1" == "build_documentation" ]] && [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then - zds_fold_start "doc" "* Run SphinxBuild to build documentation" - print_info "* Build documentation" - if [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then - make generate-doc; exVal=$? - gateway "!! Build documentation failed" $exVal - fi - zds_fold_end -fi diff --git a/scripts/update_and_deploy.sh b/scripts/update_and_deploy.sh deleted file mode 100755 index 642ff12f8d..0000000000 --- a/scripts/update_and_deploy.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# -# Zeste de Savoir deployment script updater -# -# Updates the ZdS deployment script and launch it -# -# Usage: -# - This script must be run by zds user -# - This script has exactly 1 parameter: the tag name to deploy -# - -if [ "$(whoami)" != "zds" ]; then - echo "This script must be run by zds user" >&2 - exit 1 -fi - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -echo "Backing up the previous deployment script" -mv -f deploy.sh deploy.sh.bak - -echo "Fetching the appropriate version of the deployment script" -wget https://raw.githubusercontent.com/zestedesavoir/zds-site/$1/scripts/deploy.sh -chmod +x deploy.sh - -echo "Runing the deployment script…" -./deploy.sh ok $@ - diff --git a/update.md b/update.md index 734cb67218..a68e805c81 100644 --- a/update.md +++ b/update.md @@ -1093,16 +1093,6 @@ Smileys Clem (#4408) + Éditer `/etc/nginx/sites-available/zestedesavoir` et ajouter `include snippets/clem_smileys.conf;` dans le bloc `location ~* ^/(static|media|errors)/ {` + Tester la configuration : `nginx -t` -Script de déploiement ---------------------- - -Le script de mise à jour du script de déploiement ayant changé, il faut d'abord récupérer la nouvelle version en faisant dans /opt/zds/zds-site - -```sh -git fetch origin -git checkout origin/dev scripts/update_and_deploy.sh -``` - Lancer le script de déploiement ------------------------------- diff --git a/zds/member/tests/tests_front.py b/zds/member/tests/tests_front.py index 937e4e191b..15388a7a4d 100644 --- a/zds/member/tests/tests_front.py +++ b/zds/member/tests/tests_front.py @@ -1,17 +1,19 @@ from django.contrib.staticfiles.testing import StaticLiveServerTestCase -from django.urls import reverse from django.test import tag -from selenium.webdriver.firefox.webdriver import WebDriver +from django.urls import reverse +from selenium.webdriver import Firefox +from selenium.webdriver.firefox.options import Options -# NOTE In Django 1.11.4 there is a --selenium option for python manage.py test @tag('front') class MemberFrontTests(StaticLiveServerTestCase): @classmethod def setUpClass(cls): super(MemberFrontTests, cls).setUpClass() - cls.selenium = WebDriver() - cls.selenium.implicitly_wait(10) + options = Options() + options.headless = True + cls.selenium = Firefox(options=options) + cls.selenium.implicitly_wait(30) @classmethod def tearDownClass(cls): diff --git a/zds/settings/ci_test.py b/zds/settings/ci_test.py index 6d67173279..a27a687a29 100644 --- a/zds/settings/ci_test.py +++ b/zds/settings/ci_test.py @@ -1,14 +1,16 @@ +from colorlog import ColoredFormatter + from .abstract_base import * from .abstract_test import * DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'zds_test', + 'NAME': 'ci_db_name', 'USER': 'root', - 'PASSWORD': '', + 'PASSWORD': 'ci_root_password', 'HOST': '127.0.0.1', - 'PORT': '', + 'PORT': '3306', 'CONN_MAX_AGE': 600, 'OPTIONS': { 'charset': 'utf8mb4', @@ -19,3 +21,27 @@ for logger in LOGGING['loggers'].values(): logger['level'] = 'ERROR' + logger['formatters'] = { + 'verbose': { + '()': ColoredFormatter, + 'format': '%(log_color)s %(levelname)s %(reset)s %(bold_black)s%(name)s%(reset)s %(message)s', + 'log_colors': { + 'DEBUG': 'fg_white,bg_black', + 'INFO': 'fg_black,bg_bold_white', + 'WARNING': 'fg_black,bg_bold_yellow', + 'ERROR': 'fg_bold_white,bg_bold_red', + 'CRITICAL': 'fg_bold_white,bg_bold_red', + }, + }, + + 'django.server': { + '()': ColoredFormatter, + 'format': '%(log_color)s%(message)s', + 'log_colors': { + 'INFO': 'bold_black', + 'WARNING': 'bold_yellow', + 'ERROR': 'bold_red', + 'CRITICAL': 'bold_red', + }, + }, + } diff --git a/zds/tutorialv2/tests/tests_front.py b/zds/tutorialv2/tests/tests_front.py index 16b02e7289..5645af01cd 100644 --- a/zds/tutorialv2/tests/tests_front.py +++ b/zds/tutorialv2/tests/tests_front.py @@ -1,14 +1,17 @@ +from copy import deepcopy +from django.conf import settings from django.contrib.staticfiles.testing import StaticLiveServerTestCase from django.test import override_settings +from django.test import tag from django.urls import reverse from selenium.common.exceptions import WebDriverException +from selenium.webdriver import Firefox +from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.common.by import By -from selenium.webdriver.firefox.webdriver import WebDriver -from django.test import tag +from selenium.webdriver.firefox.options import Options from selenium.webdriver.support import expected_conditions -from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as ec -from selenium.webdriver.common.action_chains import ActionChains +from selenium.webdriver.support.ui import WebDriverWait from zds.member.factories import StaffProfileFactory, ProfileFactory from zds.tutorialv2.factories import ( @@ -20,9 +23,6 @@ ) from zds.tutorialv2.models.database import PublishedContent, PublishableContent from zds.tutorialv2.tests import TutorialTestMixin, TutorialFrontMixin -from copy import deepcopy -from django.conf import settings - from zds.utils.factories import CategoryFactory overridden_zds_app = deepcopy(settings.ZDS_APP) @@ -38,7 +38,9 @@ class PublicationFronttest(StaticLiveServerTestCase, TutorialTestMixin, Tutorial @classmethod def setUpClass(cls): super(PublicationFronttest, cls).setUpClass() - cls.selenium = WebDriver() + options = Options() + options.headless = True + cls.selenium = Firefox(options=options) cls.selenium.implicitly_wait(10) @classmethod