diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000000..da441c808b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,285 @@ +name: CI + +on: ["push", "pull_request"] + +env: + NODE_VERSION: "12" + PYTHON_VERSION: "3.7" + MARIADB_VERSION: "10.4.10" + + # As GitHub Action does not allow environment variables + # to be used in services definitions, these are only for + # reference. If you update these versions, you HAVE TO + # update the versions in the services definitions of the + # test job. + ELASTICSEARCH_VERSION: "5.5.2" + MEMCACHED_VERSION: "1.6" + +jobs: + # Lint the Python back-end with flake8. + lint-back: + name: Lint back-end + runs-on: ubuntu-18.04 + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v2 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Install python dependencies + run: pip install flake8 + + - name: Lint back-end + run: make lint-back + + # Build the documentation and upload it as an artifact. + build-doc: + name: Build Sphinx documentation + runs-on: ubuntu-18.04 + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v2 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Upgrade pip + run: pip install --upgrade pip + + - name: Retrieve pip cache directory + id: pip-cache + run: echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache pip packages + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements-dev.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Install python dependencies + run: pip install -r requirements-dev.txt + + - name: Build documentation + run: make generate-doc + + - name: Upload documentation as an artifact + uses: actions/upload-artifact@v2 + with: + name: doc + path: doc/build/html + retention-days: 1 + + # Build the website front-end and upload built assets as an artifact. + build-front: + name: Lint and build front-end + runs-on: ubuntu-18.04 + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up NodeJS ${{ env.NODE_VERSION }} + uses: actions/setup-node@v1 + with: + node-version: "${{ env.NODE_VERSION }}" + + - name: Retrieve yarn cache directory + id: yarn-cache-dir-path + run: echo "::set-output name=dir::$(yarn cache dir)" + + - name: Cache Node modules + uses: actions/cache@v2 + id: yarn-cache + with: + path: ${{ steps.yarn-cache-dir-path.outputs.dir }} + key: ${{ runner.os }}-yarn-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-yarn- + + - name: Install front-end + run: make install-front + + - name: Lint front-end + run: make lint-front + + - name: Build front-end + run: make build-front + + - name: Upload font-end assets for subsequent tests + uses: actions/upload-artifact@v2 + with: + name: assets + path: dist + retention-days: 1 + + # Test the zds-site project. + # Install the project, using assets created during the previous job, + # and install elasticsearch & memcache as a service. Then, run the tests + # in a matrix build to parallelize multiple components. + test: + name: Install and test zds-site + needs: build-front + runs-on: ubuntu-18.04 + + strategy: + matrix: + module: + [ + "zds.tutorialv2", + "zds.member zds.gallery zds.searchv2 zds.middlewares zds.pages", + "zds.forum zds.featured zds.mp zds.notification zds.utils", + ] + + services: + elasticsearch: + image: "elasticsearch:5.5.2" + ports: + - "9200:9200" + env: + "http.host": "0.0.0.0" + "transport.host": "127.0.0.1" + "xpack.security.enabled": false + "ES_JAVA_OPTS": "-Xms512m -Xmx512m" + options: >- + -e="discovery.type=single-node" + --health-cmd="curl http://localhost:9200/_cluster/health" + --health-interval=10s + --health-timeout=5s + --health-retries=10 + + memcached: + image: "memcached:1.6" + ports: + - "11211:11211" + + steps: + - name: Shutdown Ubuntu MySQL + run: sudo service mysql stop + + - name: Set up MariaDB ${{ env.MARIADB_VERSION }} + uses: getong/mariadb-action@v1.1 + with: + character set server: "utf8mb4" + collation server: "utf8mb4_unicode_ci" + mariadb version: "${{ env.MARIADB_VERSION }}" + mysql database: "ci_db_name" + mysql root password: "ci_root_password" + + - name: Checkout + uses: actions/checkout@v2 + + - name: Download previously built assets + uses: actions/download-artifact@v2 + with: + name: assets + path: dist + + - name: Upgrade pip + run: | + pip install --upgrade pip + + - name: Retrieve pip cache directory + id: pip-cache + run: | + echo "::set-output name=dir::$(pip cache dir)" + + - name: Cache pip packages + uses: actions/cache@v2 + with: + path: ${{ steps.pip-cache.outputs.dir }} + key: ${{ runner.os }}-pip-${{ hashFiles('requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-pip- + + - name: Cache Node modules + uses: actions/cache@v2 + with: + path: ~/.npm + key: ${{ runner.os }}-node-${{ hashFiles('zmd/package-lock.json') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v2 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Set up NodeJS ${{ env.NODE_VERSION }} + uses: actions/setup-node@v1 + with: + node-version: "${{ env.NODE_VERSION }}" + + - name: Install Python dependencies + run: pip install -r requirements-ci.txt + + - name: Build and start zmarkdown + run: | + make zmd-install + make zmd-start + + - name: Run tests for ${{ matrix.module }} + run: | + export PATH="$PATH:$GECKOWEBDRIVER" + coverage run --source='.' manage.py test -v=2 --keepdb --settings zds.settings.ci_test ${{ matrix.module }} + + - name: Analyze coverage + shell: bash -l {0} + run: | + echo $COVERALLS_FLAG_NAME + python -m pip install coveralls + coveralls + env: + GITHUB_TOKEN: ${{ secrets.github_token }} + COVERALLS_PARALLEL: true + COVERALLS_FLAG_NAME: "${{ matrix.module }}" + + # Push coverage data to Coveralls. + coverage: + name: Push coverage to Coveralls + needs: test + runs-on: ubuntu-18.04 + + steps: + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v2 + with: + python-version: "${{ env.PYTHON_VERSION }}" + + - name: Upload coverage data + run: | + python -m pip install coveralls + coveralls --finish + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + # If we're on the dev branch (i.e. for merged pull requests), push the built + # documentation to the gh-page branch. + push_doc: + name: Push documentation to GitHub Pages + needs: ["build-doc", "test"] + runs-on: ubuntu-18.04 + if: "github.ref == 'refs/heads/dev'" + + steps: + - name: Download previously built documentation + uses: actions/download-artifact@v2 + with: + name: doc + path: doc/build/html + + - name: Deploy to GitHub Pages + uses: JamesIves/github-pages-deploy-action@3.7.1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: gh-pages + FOLDER: doc/build/html + CLEAN: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 1ec9e0807c..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,101 +0,0 @@ -os: linux -dist: bionic -language: python - - -python: - - 3.7 - - -services: - - memcached - - -addons: - firefox: "latest" - mariadb: 10.4 - apt: - packages: - - language-pack-fr - - -git: - depth: 1 - - -env: - global: - - COVERALLS_PARALLEL: true - - secure: "azmDZZQZzf88zpbkYpLpxI66vpEVyv+kniW0QdWAt4qlys8P5OcO3VJBR5YT85vlvnjN9b6raWQAL1ymee0WmVdTmzXed8XjZv7t9QXVw7pfezxMKlEftVp/4Cu4wtvbew0ViZXNWV2hNXHzEqlhgnoIOq94i0UzZ7grMrI0xm0=" - - -notifications: - webhooks: https://coveralls.io/webhook - - -jobs: - fast_finish: true - include: - - name: "Job 1 : tuto (zds.tutorialv2)" - env: ZDS_TEST_JOB="zds.tutorialv2" - - name: "Job 2 : backend test1 (zds.member zds.utils zds.forum zds.middlewares)" - env: ZDS_TEST_JOB="zds.member zds.utils zds.forum zds.middlewares" - - name: "Job 3 : backend test2 (zds.mp zds.gallery zds.pages zds.featured zds.notification zds.searchv2)" - env: ZDS_TEST_JOB="zds.mp zds.gallery zds.pages zds.featured zds.notification zds.searchv2" - - name: "Job 4 : frontend test" - env: ZDS_TEST_JOB="front fixture selenium" - - name: "Job 5 : doc" - env: ZDS_TEST_JOB="doc" - - -cache: - apt: true - pip: true - yarn: true - npm: true - directories: - - $HOME/.local/share/fonts - - $HOME/.texlive - - $HOME/.cache/pip - - $HOME/virtualenv/python$TRAVIS_PYTHON_VERSION/bin - - $HOME/build/zestedesavoir/zds-site/zdsenv - - $HOME/.nvm - - $HOME/node_modules - - $HOME/build/zestedesavoir/zds-site/node_modules - - $HOME/build/zestedesavoir/zds-site/zmd/node_modules - - -before_install: - - source ./scripts/travis_header.sh # Need to be sourced - - -install: - - ./scripts/install_zds.sh --answer-yes --detect-os-version --travis-output $zds_install_argument - - -script: - - source ./scripts/travis_run.sh - - - ./scripts/travis_overview/travis_overview.sh --travis-output - - -after_success: - - du -sh $HOME/.texlive 2>/dev/null | true - - - | - # upload coverage - if [[ "$ZDS_TEST_JOB" != "none" && "$ZDS_TEST_JOB" != *"ci_turbo"* ]]; then - coveralls - fi - - - | - # upload built documentation to GitHub Pages - if [[ "$ZDS_TEST_JOB" == *"doc"* ]] && [[ "$TRAVIS_BRANCH" == "dev" ]]; then - # Adding GitHub OAuth token to login - echo -e "machine github.com login $BOT_LOGIN\n password $BOT_PASSWORD" > $HOME/.netrc \ - && git config --global url."https://".insteadOf git:// \ - && git config --global user.name "Build bot" \ - && git config --global user.email "zestedesavoir@gmail.com" \ - && git config --global push.default simple \ - && ./scripts/push_doc.sh - fi diff --git a/requirements-ci.txt b/requirements-ci.txt new file mode 100644 index 0000000000..85486ead05 --- /dev/null +++ b/requirements-ci.txt @@ -0,0 +1,4 @@ +-r requirements-dev.txt +-r requirements-prod.txt + +coverage==5.3 diff --git a/scripts/ci_mysql_setup.sh b/scripts/ci_mysql_setup.sh deleted file mode 100755 index fcfc6c7feb..0000000000 --- a/scripts/ci_mysql_setup.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -sudo sed -i'' 's/\[mysqld\]/\[mysqld\]\ninnodb_file_per_table=on\ninnodb_file_format=barracuda\ninnodb_large_prefix=on\ncharacter-set-client-handshake=false\ncharacter-set-server=utf8mb4\ncollation-server=utf8mb4_unicode_ci/' /etc/mysql/mysql.conf.d/mysqld.cnf -sudo systemctl restart mysql - -# Travis should fail as soon as possible -sudo mysql -u root -e "SET GLOBAL sql_mode = 'NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES';" - -# Avoid "mysql has gone away" errors -sudo mysql -u root -e "SET GLOBAL wait_timeout = 36000;" -sudo mysql -u root -e "SET GLOBAL max_allowed_packet = 134209536;" - -# Ensures correct charset and collation -sudo mysql -u root -e "SET GLOBAL character_set_server = 'utf8mb4';" -sudo mysql -u root -e "SET GLOBAL collation_server = 'utf8mb4_unicode_ci';" - -# Ensures the root user is able to connect without password -sudo mysql -u root -e "SET Password=PASSWORD('')" - -# Create database with the correct charset and collation -sudo mysql -u root -e "CREATE DATABASE zds_test CHARACTER SET = 'utf8mb4' COLLATE = 'utf8mb4_unicode_ci';" diff --git a/scripts/ci_turbo.sh b/scripts/ci_turbo.sh deleted file mode 100755 index 5fb201d702..0000000000 --- a/scripts/ci_turbo.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -# This script is sourced from the .travis.yml file - -default_branch=dev - -git remote set-branches origin $default_branch -git fetch --unshallow origin $default_branch - -if [[ ! -z "$TRAVIS_TAG" ]] -then - # Remember, we are sourced, we must not exit the main shell - return -fi - -changed_files=$(git --no-pager diff --name-only $TRAVIS_COMMIT $(git merge-base $TRAVIS_COMMIT origin/$default_branch)) -echo "changed files:" -echo $changed_files - -if ! echo "$changed_files" | egrep -v "^assets" -then - # Don't test the backend if only the `/assets/` directory changed - if [[ "$ZDS_TEST_JOB" == *"front"* ]] - then - export ZDS_TEST_JOB="front ci_turbo" - else - export ZDS_TEST_JOB="none" - fi - - echo "skipping backend tests" -fi diff --git a/scripts/deploy.sh b/scripts/deploy.sh deleted file mode 100755 index 827f362e74..0000000000 --- a/scripts/deploy.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash -# -# Zeste de Savoir deployment script -# -# Deploys specified version of Zeste de Savoir -# -# DON'T RUN THIS SCRIPT DIRECTLY -# Running update_and_deploy.sh instead will first get the -# appropriate version of this script and run it instead. - -set -euo pipefail -IFS=$'\n\t' - -ENV_PATH="/opt/zds/zdsenv" -REPO_PATH="/opt/zds/zds-site" - -if [ "$1" != "ok" ]; then - echo "This script shouldn't be run directly. Run ./update_and_deploy.sh instead." >&2 - exit 1 -fi - -# Shift the first arg "ok" -shift - -if [ "$(whoami)" != "zds" ]; then - echo "This script must be run by zds user" >&2 - exit 1 -fi - -# Check if the git working directory is clean (excluding scripts/ folder) -if ! git diff-index --quiet HEAD -- . ':!scripts/'; then - echo "Git repo has uncommited changes. Make sure it's clean before trying again" >&2 - exit 1 -fi - -read -p "Did you run specific tasks for this version as described in update.md? [y/N] " -r -echo # move to a new line - -if [[ ! $REPLY =~ ^[Yy]$ ]] then - echo "Do it, now!" - exit 1 -fi - -cd $REPO_PATH - -# Enable maintenance mode -sudo ln -sf errors/maintenance.html $ENV_PATH/webroot/ - -# Delete old branch if exists -git checkout refs/heads/prod - -if git rev-parse --verify "refs/heads/$1" > /dev/null; then - git branch -D $1 -fi - -# Removes dist/ folder to avoid conflicts -rm -rf ./dist/ -# Switch to new tag -git fetch --tags -# Server has git < 1.9, git fetch --tags doesn't retrieve commits... -git fetch - -if git rev-parse $1 >/dev/null 2>&1 -then - echo "Tag $1 found!" -else - echo "Tag $1 doesn't exist." - exit 1; -fi - -# Checkout the tag -git checkout refs/heads/$1-build -# Create a branch with the same name - required to have version data in footer -git checkout -b $1 - -# Update application data -source $ENV_PATH/bin/activate -pip install --upgrade -r requirements-prod.txt -python manage.py migrate -python manage.py compilemessages - -## Collect all staticfiles from dist/ and python packages to static/ -python manage.py collectstatic --noinput --clear -## Exit venv -deactivate - -# Restart zds -sudo systemctl restart zds.{service,socket} - -# Clean the cache by restarting it -sudo service memcached restart - -# Disable maintenance mode -sudo rm $ENV_PATH/webroot/maintenance.html - -# update latex -mkdir -p $HOME/texmf/tex/latex/ -wget "https://raw.githubusercontent.com/zestedesavoir/latex-template/${1-build}/zmdocument.cls" -mv zmdocument.cls $HOME/texmf/tex/latex/ -sudo texhash -# Display current branch and commit -git status -echo "Deployed commit: `git rev-parse HEAD`" -echo "Dont forget to run specific tasks for this version as described in update.md" diff --git a/scripts/install_zds.sh b/scripts/install_zds.sh index 4f1cb54bdc..4baca484fe 100755 --- a/scripts/install_zds.sh +++ b/scripts/install_zds.sh @@ -47,14 +47,6 @@ LOCAL_DIR="$(cd "$(dirname "$0")" && pwd)" source $LOCAL_DIR/define_variable.sh source $LOCAL_DIR/define_function.sh - -# enable travis fold -ZDS_SHOW_TRAVIS_FOLD=0 -if $(_in "--travis-output" $@); then - ZDS_SHOW_TRAVIS_FOLD=1 - export DJANGO_SETTINGS_MODULE="zds.settings.travis_fixture" -fi - zds_fold_category "install" @@ -156,6 +148,7 @@ fi # virtualenv + if ! $(_in "-virtualenv" $@) && ( $(_in "+virtualenv" $@) || $(_in "+base" $@) || $(_in "+full" $@) ); then zds_fold_start "virtualenv" "* Create virtualenv" diff --git a/scripts/push_doc.sh b/scripts/push_doc.sh deleted file mode 100755 index 592a440450..0000000000 --- a/scripts/push_doc.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# -# Push documentation on the gh-pages branch - -echo "Move the generated doc in a new branch" -touch doc/build/html/.nojekyll -echo "docs.zestedesavoir.com" > doc/build/html/CNAME -git add doc/build/html -f -git commit -m "Build documentation" -git subtree split --branch build_doc --prefix doc/build/html -echo "Push the documentation" -git push origin build_doc:gh-pages -f diff --git a/scripts/travis_header.sh b/scripts/travis_header.sh deleted file mode 100755 index b2107bdad0..0000000000 --- a/scripts/travis_header.sh +++ /dev/null @@ -1,136 +0,0 @@ -#!/bin/bash - - -zds_install_argument="" -function zds_register_for_install { - if [[ "$zds_install_argument" == "" ]]; then - zds_install_argument="$1" - else - zds_install_argument="$zds_install_argument $1" - fi -} - - -function zds_register_module_for_installation { - zds_register_for_install "+base +prod" - - # install elastic-local - if [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then - print_info "* Register elasticsearch with jdk. It needed to test : zds.searchv2." - zds_register_for_install "+jdk-local +elastic-local" - fi - - # install backend dependencies - if ! ( [[ "$ZDS_TEST_JOB" == *"zds."* ]] || \ - [[ "$ZDS_TEST_JOB" == *"selenium"* ]] || \ - [[ "$ZDS_TEST_JOB" == *"doc"* ]]); then - print_info "* Don't register back because zds.* tasks, doc and selenium are not registered." - zds_register_for_install "-back" - else - print_info "* Back is registered because zds.* tasks, doc or selenium are registered." - - if ! ( [[ "$ZDS_TEST_JOB" == *"zds."* ]] || \ - [[ "$ZDS_TEST_JOB" == *"selenium"* ]]); then - print_info "* Don't migrate-db, if only doc are registered." - zds_register_for_install "-back-migrate-db" - fi - fi - - - # install frontend dependencies - if ! ( [[ "$ZDS_TEST_JOB" == *"front"* ]] || \ - [[ "$ZDS_TEST_JOB" == *"selenium"* ]]); then - print_info "* Don't register front because front task and selenium are not registered." - zds_register_for_install "-front" - else - print_info "* Front is registered because front task or selenium are registered." - fi - - - # Run fixture only when it is asked - if [[ "$ZDS_TEST_JOB" != *"fixture"* ]]; then - print_info "* Don't register fixture." - zds_register_for_install "-data" - else - print_info "* Fixture is registered." - fi - - print_info "* Argument for installation : $zds_install_argument" -} - - -function install_geckodriver { - wget "https://github.com/mozilla/geckodriver/releases/download/v0.23.0/geckodriver-v0.23.0-linux64.tar.gz" - mkdir "geckodriver" - tar -xzf "geckodriver-v0.23.0-linux64.tar.gz" -C "geckodriver" -} - - -source ./scripts/define_variable.sh -source ./scripts/define_function.sh --travis-output - - -zds_fold_category "before_install" - - -zds_fold_start "packages" "* [packages] apt-get update : make sure our source list is up-to-date (for newest dependencies version)" - sudo apt-get update -qq -zds_fold_end - - -zds_fold_start "coveralls" "* [coveralls] Install with pip" - pip install -q coveralls -zds_fold_end - - -zds_fold_start "ci_turbo" "* [ci_turbo] Skip task depending on directory changes (task will run only if needed)" - source ./scripts/ci_turbo.sh # This script exports environment variables, it must be sourced -zds_fold_end - - -if [[ "$ZDS_TEST_JOB" == *"zds."* ]] || [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - #display print_info - forwho="" - if [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - forwho="'zds.*' tasks (-> needed for tests)" - - if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - forwho="$forwho and selenium" - fi - else - forwho="selenium" - fi - - zds_fold_start "mysql" "* [mysql] Install mysql for $forwho." - ./scripts/ci_mysql_setup.sh - zds_fold_end -fi - - -if [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - zds_fold_start "webdriver" "* [webdriver] Install webdriver for selenium" - install_geckodriver - zds_fold_end -fi - -if [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then - # install latex - zds_fold_start "latex" "* [latex] Install latex & Run texhash (install: texlive + latex-template)" - # this script is faster than zds_install.sh +tex-local +latex-template - git clone $ZDS_LATEX_REPO - TEMPLATEDIR=$HOME/.texlive/texmf-local/tex/latex/ - ./latex-template/scripts/install_font.sh - ./latex-template/scripts/install_texlive.sh - export PATH=$HOME/.texlive/bin/x86_64-linux:$PATH - rm -rf $TEMPLATEDIR/latex-template - mkdir -p $TEMPLATEDIR - cp -r ./latex-template $TEMPLATEDIR - texhash - zds_fold_end -fi - -if [[ "$ZDS_TEST_JOB" != "none" ]]; then - zds_fold_start "register_module" "* [packages] Register module for installation" - zds_register_module_for_installation - zds_fold_end -fi diff --git a/scripts/travis_run.sh b/scripts/travis_run.sh deleted file mode 100755 index 693203cf7d..0000000000 --- a/scripts/travis_run.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - - -if [[ "$ZDS_TEST_JOB" == "none" ]]; then - exit 0 -fi - - -function print_info { - echo -en "\033[0;36m" - echo "$1" - echo -en "\033[00m" -} - - -function error_handler { - if [[ $exVal != 0 ]]; then - print_error $1 - exit 1 - fi -} - - -function run_script { - ./scripts/travis_script.sh $1; exVal=$? - error_handler "!! Some error on the last task ($1)." -} - - -function activate_env { - print_info "source $1/bin/activate" - source $1/bin/activate; exVal=$? - error_handler "!! Error: environnement not load.\n - Value = $1" -} - - -activate_env "./$ZDS_VENV" - # Add geckodriver in the PATH of zdsenv - export PATH="$PATH:/home/travis/build/zestedesavoir/zds-site/geckodriver" - export DISPLAY=":99.0" - # Use hack for virtualenv (fix some task with "command not found") - export PATH="$PATH:/home/travis/virtualenv/python${TRAVIS_PYTHON_VERSION}/bin" - - run_script "start_elasticsearch" - - run_script "make_migrations" - - run_script "lint_frontend" - - run_script "print_zmarkdown_log" - - run_script "selenium_test" - - run_script "coverage_backend" - - run_script "lint_backend" - - run_script "build_documentation" - diff --git a/scripts/travis_script.sh b/scripts/travis_script.sh deleted file mode 100755 index e8fd25d5fa..0000000000 --- a/scripts/travis_script.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash - -source ./scripts/define_function.sh --travis-output - -zds_fold_category "script" - -exVal=0 - - -# start elastic -if [[ "$1" == "start_elasticsearch" ]] && [[ "$ZDS_TEST_JOB" == *"zds.searchv2"* ]]; then - zds_fold_start "elasticsearch" "* Start elasticsearch as service" - sudo service elasticsearch start; exVal=$? - - gateway "!! elasticsearch didn't start" $exVal - zds_fold_end -fi - - -# start latex -if [[ "$1" == "start_latex" ]] && [[ "$ZDS_TEST_JOB" == *"zds.tutorialv2"* ]]; then - zds_fold_start "latex" "* Start texhash -> latex" - texhash; exVal=$? - - gateway "!! Texhash failed" $exVal - zds_fold_end -fi - - -# lint backend -if [[ "$1" == "lint_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds.gallery"* ]]; then - zds_fold_start "lint_backend" "* Run lint for backend" - ./scripts/no_import_zds_settings.sh \ - && flake8 \ - && flake8 --config=zds/settings/.flake8 zds/settings; exVal=$? - - gateway "!! Test failed" $exVal - zds_fold_end -fi - -# make migrations -if [[ "$1" == "make_migrations" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - zds_fold_start "make_migrations" "* Make migrations for backend tasks" - python manage.py makemigrations --dry-run --check; exVal=$? - - gateway "!! Test failed" $exVal - zds_fold_end -fi - - -# lint frontend -if [[ "$1" == "lint_frontend" ]] && [[ "$ZDS_TEST_JOB" == *"front"* ]]; then - zds_fold_start "lint_frontend" "* Run lint for frontend" - npm run lint; exVal=$? - - gateway "!! Test failed" $exVal - zds_fold_end -fi - - -# coverage backend -if [[ "$1" == "coverage_backend" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - zds_fold_start "coverage_backend" "* Run coverage for backend" - - zds_start_zmd - - coverage run --source='.' manage.py \ - test -v=2\ - --keepdb \ - --settings zds.settings.ci_test \ - --exclude-tag=front \ - ${ZDS_TEST_JOB/front/}; exVal=$? - - gateway "!! Test failed" $exVal - - zds_stop_zmd - - zds_fold_end -fi - - -# print zmarkdown log -if [[ "$1" == "print_zmarkdown_log" ]] && [[ "$ZDS_TEST_JOB" == *"zds."* ]]; then - zds_fold_start "zmarkdown_log" "* Print zmarkdown log" - node ./zmd/node_modules/pm2/bin/pm2 logs --nostream --raw --lines 1000 - zds_fold_end -fi - - -# selenium test -if [[ "$1" == "selenium_test" ]] && [[ "$ZDS_TEST_JOB" == *"selenium"* ]]; then - zds_fold_start "selenium_test" "* Run selenium test for frontend" - - zds_start_zmd - - xvfb-run --server-args="-screen 0 1280x720x8" coverage run --source='.' manage.py \ - test -v=2\ - --settings zds.settings.ci_test \ - --tag=front \ - --keepdb; exVal=$? - - gateway "!! Test failed" $exVal - - zds_stop_zmd - - zds_fold_end -fi - - -# build documentation -if [[ "$1" == "build_documentation" ]] && [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then - zds_fold_start "doc" "* Run SphinxBuild to build documentation" - print_info "* Build documentation" - if [[ "$ZDS_TEST_JOB" == *"doc"* ]]; then - make generate-doc; exVal=$? - gateway "!! Build documentation failed" $exVal - fi - zds_fold_end -fi diff --git a/scripts/update_and_deploy.sh b/scripts/update_and_deploy.sh deleted file mode 100755 index 642ff12f8d..0000000000 --- a/scripts/update_and_deploy.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -# -# Zeste de Savoir deployment script updater -# -# Updates the ZdS deployment script and launch it -# -# Usage: -# - This script must be run by zds user -# - This script has exactly 1 parameter: the tag name to deploy -# - -if [ "$(whoami)" != "zds" ]; then - echo "This script must be run by zds user" >&2 - exit 1 -fi - -if [ "$#" -ne 1 ]; then - echo "Usage: $0 " >&2 - exit 1 -fi - -echo "Backing up the previous deployment script" -mv -f deploy.sh deploy.sh.bak - -echo "Fetching the appropriate version of the deployment script" -wget https://raw.githubusercontent.com/zestedesavoir/zds-site/$1/scripts/deploy.sh -chmod +x deploy.sh - -echo "Runing the deployment script…" -./deploy.sh ok $@ - diff --git a/update.md b/update.md index 734cb67218..a68e805c81 100644 --- a/update.md +++ b/update.md @@ -1093,16 +1093,6 @@ Smileys Clem (#4408) + Éditer `/etc/nginx/sites-available/zestedesavoir` et ajouter `include snippets/clem_smileys.conf;` dans le bloc `location ~* ^/(static|media|errors)/ {` + Tester la configuration : `nginx -t` -Script de déploiement ---------------------- - -Le script de mise à jour du script de déploiement ayant changé, il faut d'abord récupérer la nouvelle version en faisant dans /opt/zds/zds-site - -```sh -git fetch origin -git checkout origin/dev scripts/update_and_deploy.sh -``` - Lancer le script de déploiement ------------------------------- diff --git a/zds/member/tests/tests_front.py b/zds/member/tests/tests_front.py index 937e4e191b..15388a7a4d 100644 --- a/zds/member/tests/tests_front.py +++ b/zds/member/tests/tests_front.py @@ -1,17 +1,19 @@ from django.contrib.staticfiles.testing import StaticLiveServerTestCase -from django.urls import reverse from django.test import tag -from selenium.webdriver.firefox.webdriver import WebDriver +from django.urls import reverse +from selenium.webdriver import Firefox +from selenium.webdriver.firefox.options import Options -# NOTE In Django 1.11.4 there is a --selenium option for python manage.py test @tag('front') class MemberFrontTests(StaticLiveServerTestCase): @classmethod def setUpClass(cls): super(MemberFrontTests, cls).setUpClass() - cls.selenium = WebDriver() - cls.selenium.implicitly_wait(10) + options = Options() + options.headless = True + cls.selenium = Firefox(options=options) + cls.selenium.implicitly_wait(30) @classmethod def tearDownClass(cls): diff --git a/zds/settings/ci_test.py b/zds/settings/ci_test.py index 6d67173279..a27a687a29 100644 --- a/zds/settings/ci_test.py +++ b/zds/settings/ci_test.py @@ -1,14 +1,16 @@ +from colorlog import ColoredFormatter + from .abstract_base import * from .abstract_test import * DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'zds_test', + 'NAME': 'ci_db_name', 'USER': 'root', - 'PASSWORD': '', + 'PASSWORD': 'ci_root_password', 'HOST': '127.0.0.1', - 'PORT': '', + 'PORT': '3306', 'CONN_MAX_AGE': 600, 'OPTIONS': { 'charset': 'utf8mb4', @@ -19,3 +21,27 @@ for logger in LOGGING['loggers'].values(): logger['level'] = 'ERROR' + logger['formatters'] = { + 'verbose': { + '()': ColoredFormatter, + 'format': '%(log_color)s %(levelname)s %(reset)s %(bold_black)s%(name)s%(reset)s %(message)s', + 'log_colors': { + 'DEBUG': 'fg_white,bg_black', + 'INFO': 'fg_black,bg_bold_white', + 'WARNING': 'fg_black,bg_bold_yellow', + 'ERROR': 'fg_bold_white,bg_bold_red', + 'CRITICAL': 'fg_bold_white,bg_bold_red', + }, + }, + + 'django.server': { + '()': ColoredFormatter, + 'format': '%(log_color)s%(message)s', + 'log_colors': { + 'INFO': 'bold_black', + 'WARNING': 'bold_yellow', + 'ERROR': 'bold_red', + 'CRITICAL': 'bold_red', + }, + }, + } diff --git a/zds/tutorialv2/tests/tests_front.py b/zds/tutorialv2/tests/tests_front.py index 16b02e7289..5645af01cd 100644 --- a/zds/tutorialv2/tests/tests_front.py +++ b/zds/tutorialv2/tests/tests_front.py @@ -1,14 +1,17 @@ +from copy import deepcopy +from django.conf import settings from django.contrib.staticfiles.testing import StaticLiveServerTestCase from django.test import override_settings +from django.test import tag from django.urls import reverse from selenium.common.exceptions import WebDriverException +from selenium.webdriver import Firefox +from selenium.webdriver.common.action_chains import ActionChains from selenium.webdriver.common.by import By -from selenium.webdriver.firefox.webdriver import WebDriver -from django.test import tag +from selenium.webdriver.firefox.options import Options from selenium.webdriver.support import expected_conditions -from selenium.webdriver.support.ui import WebDriverWait from selenium.webdriver.support import expected_conditions as ec -from selenium.webdriver.common.action_chains import ActionChains +from selenium.webdriver.support.ui import WebDriverWait from zds.member.factories import StaffProfileFactory, ProfileFactory from zds.tutorialv2.factories import ( @@ -20,9 +23,6 @@ ) from zds.tutorialv2.models.database import PublishedContent, PublishableContent from zds.tutorialv2.tests import TutorialTestMixin, TutorialFrontMixin -from copy import deepcopy -from django.conf import settings - from zds.utils.factories import CategoryFactory overridden_zds_app = deepcopy(settings.ZDS_APP) @@ -38,7 +38,9 @@ class PublicationFronttest(StaticLiveServerTestCase, TutorialTestMixin, Tutorial @classmethod def setUpClass(cls): super(PublicationFronttest, cls).setUpClass() - cls.selenium = WebDriver() + options = Options() + options.headless = True + cls.selenium = Firefox(options=options) cls.selenium.implicitly_wait(10) @classmethod