From dcf069d27b3966ee551b4c24575749363d1bcd06 Mon Sep 17 00:00:00 2001 From: Tristiisch Date: Tue, 27 Aug 2024 00:46:56 +0200 Subject: [PATCH 1/2] cicd: reorganize (#19) --- .coveragerc | 2 + .github/workflows/python.yml | 510 ++++++++---------- Dockerfile | 13 +- Makefile | 15 +- src/__main__.py | 1 - src/cli.py | 10 +- src/pyramid/connector/discord/bot_cmd.py | 6 +- .../data/functional/application_info.py | 85 ++- src/pyramid/data/functional/git_info.py | 102 ---- src/pyramid/data/functional/main.py | 15 +- src/pyramid/git.py | 9 - 11 files changed, 279 insertions(+), 489 deletions(-) create mode 100644 .coveragerc delete mode 100644 src/pyramid/data/functional/git_info.py delete mode 100644 src/pyramid/git.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..b0f11ed --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[run] +data_file = ./cover/result diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index ffb6037..24d5634 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -4,6 +4,8 @@ on: push: branches: - "*" + tags: + - '[0-9]+.[0-9]+.[0-9]+' paths: - "src/**/*.py" - "requirements.txt" @@ -12,8 +14,6 @@ on: - "Dockerfile" - "docker-compose*.yml" - ".github/workflows/python.yml" - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' pull_request: types: [opened, synchronize] branches: @@ -38,271 +38,123 @@ env: jobs: - compile: - name: "Compile Python 3.11" - runs-on: ubuntu-latest - outputs: - json: ${{ steps.version.outputs.json }} - version: ${{ steps.version.outputs.version }} - commit_id: ${{ steps.version.outputs.commit_id }} - branch: ${{ steps.version.outputs.branch }} - last_author: ${{ steps.version.outputs.last_author }} - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Cache dependencies - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - - ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} - - ${{ runner.os }}-pip - - - name: Install dependencies - run: | - pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - - name: Test compilation - run: | - python -m compileall ${{ env.SRC }} - - - name: Save version - run: | - FULL_JSON=$(python ${{ env.SRC }} --version) - echo "json=$(echo $FULL_JSON | jq -c)" >> $GITHUB_OUTPUT - echo "version=$(echo $FULL_JSON | jq -r '.version')" >> $GITHUB_OUTPUT - echo "commit_id=$(echo $FULL_JSON | jq -r '.git_info.commit_id')" >> $GITHUB_OUTPUT - echo "branch=$(echo $FULL_JSON | jq -r '.git_info.branch')" >> $GITHUB_OUTPUT - echo "last_author=$(echo $FULL_JSON | jq -r '.git_info.last_author')" >> $GITHUB_OUTPUT - echo "Output $GITHUB_OUTPUT" - cat $GITHUB_OUTPUT - id: version - - unit_test: - name: "Unit tests" - needs: compile - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - clean: false - - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Cache dependencies - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-python_test-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - - ${{ runner.os }}-python_test-${{ hashFiles('**/requirements.txt') }} - - ${{ runner.os }}-python_test-${{ hashFiles('**/requirements.txt') }} - - ${{ runner.os }}-python - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - - name: Install project for tests - run: | - pip install pytest-cov - pip install -e . - - - name: Units tests - env: - DEEZER__ARL: ${{ secrets.CONFIG_DEEZER_ARL }} - SPOTIFY__CLIENT_ID: ${{ secrets.CONFIG_SPOTIFY_CLIENT_ID }} - SPOTIFY__CLIENT_SECRET: ${{ secrets.CONFIG_SPOTIFY_CLIENT_SECRET }} - run: | - pytest --cov=${{ env.MODULE_NAME }} ${{ env.TEST_DIR }} - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4.5.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - slug: tristiisch/PyRamid - - unit_test_compatibility: - name: "Envs unit tests" - needs: unit_test - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.12" - platform: - - linux/amd64 - - linux/arm64/v8 - continue-on-error: true - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Cache dependencies - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-python_test_${{ matrix.python-version }}_${{ matrix.platform }}-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - - ${{ runner.os }}-python_test_${{ matrix.python-version }}_${{ matrix.platform }} - - ${{ runner.os }}-python_test_${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - - name: Install project for tests - run: | - pip install pytest-cov - pip install -e . - - - name: Units tests - env: - DEEZER__ARL: ${{ secrets.CONFIG_DEEZER_ARL }} - SPOTIFY__CLIENT_ID: ${{ secrets.CONFIG_SPOTIFY_CLIENT_ID }} - SPOTIFY__CLIENT_SECRET: ${{ secrets.CONFIG_SPOTIFY_CLIENT_SECRET }} - run: | - pytest --cov=${{ env.MODULE_NAME }} ${{ env.TEST_DIR }} - - version_compatibility: - name: "Envs compatibility" - needs: compile - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: - - "3.8" - - "3.9" - - "3.10" - - "3.12" - platform: - - linux/amd64 - - linux/arm64/v8 - continue-on-error: true - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: "${{ matrix.python-version }}" - - - name: Cache dependencies - uses: actions/cache@v4 - with: - path: ~/.cache/pip - key: ${{ runner.os }}-python_${{ matrix.python-version }}_${{ matrix.platform }}-${{ hashFiles('**/requirements.txt') }} - restore-keys: | - - ${{ runner.os }}-python_${{ matrix.python-version }}_${{ matrix.platform }} - - ${{ runner.os }}-python_${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - - name: Test compilation - run: | - python -m compileall ${{ env.SRC }} - info: - name: "Build information" - needs: ["compile"] + name: "Informations" runs-on: ubuntu-latest outputs: - environment: ${{ steps.environment.outputs.environment }} - version_tag: ${{ steps.environment.outputs.tag }} - version_number: ${{ steps.environment.outputs.version }} - last_release_ref: ${{ steps.last_release.outputs.last_release_ref }} - commit_messages: ${{ steps.commit_messages.outputs.commit_messages }} - if: github.event_name == 'push' + project_version: ${{ steps.environment.outputs.result.project_version }} + environment: ${{ steps.environment.outputs.result.git_environment }} + docker_tag: ${{ steps.environment.outputs.result.docker_tag }} + commit_id: ${{ steps.environment.outputs.result.commit_id }} + last_release_ref: ${{ steps.last_release.outputs.result.last_release_ref }} + changelog: ${{ steps.changelog.outputs.result.changelog }} steps: - name: Checkout repository uses: actions/checkout@v4 - with: - fetch-depth: 100 - - - name: Get tags - run: git fetch --tags origin - - name: Get environnement names + - name: Define build variables id: environment - run: | - if [ ${{ github.event_name }} == 'create' && ${{ github.ref_type }} == 'tag' ]; then - echo "tag=latest" >> $GITHUB_OUTPUT - echo "environment=production" >> $GITHUB_OUTPUT - echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT - elif [ ${{ github.ref }} = 'refs/heads/main' ]; then - echo "tag=pre-prod" >> $GITHUB_OUTPUT - echo "environment=pre-production" >> $GITHUB_OUTPUT - echo "version=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT - else - echo "tag=dev" >> $GITHUB_OUTPUT - echo "environment=developement" >> $GITHUB_OUTPUT - echo "version=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT - fi - echo "Output $GITHUB_OUTPUT" - cat $GITHUB_OUTPUT - - - name: Get Github last release tag + uses: actions/github-script@v7 + with: + script: | + const refType = context.ref_type; + const ref = context.ref; + const refName = context.ref_name; + const sha = context.sha; + const shortSha = sha.slice(0, 7); + + let dockerTag; + let gitEnvironment; + let projectVersion; + + if (refType === 'tag') { + dockerTag = 'latest'; + gitEnvironment = 'production'; + projectVersion = refName; + } else if (ref === 'refs/heads/main') { + dockerTag = 'pre-prod'; + gitEnvironment = 'pre-production'; + projectVersion = `${refName}-${shortSha}`; + } else { + dockerTag = 'dev'; + gitEnvironment = 'development'; + projectVersion = `${refName}-${shortSha}`; + } + + const reset = "\x1b[0m"; + const textColor = "\x1b[36m"; // Cyan for static text + const varColor = "\x1b[35m"; // Magenta for variables + console.log(`${textColor}${projectVersion} in environment ${varColor}${gitEnvironment}${textColor} with tag ${varColor}${dockerTag}${reset}.`); + + return { docker_tag: dockerTag, git_environment: gitEnvironment, commit_id: shortSha, project_version: projectVersion }; + + - name: Get Github last release name id: last_release - run: | - RESPONSE=$(curl -s "https://api.github.com/repos/${{ github.repository }}/releases/latest") - if [[ $(echo "$RESPONSE" | jq -r .message) == "Not Found" ]]; then - LAST_RELEASE_REF=$(git rev-list --max-parents=0 HEAD) - else - LAST_RELEASE_REF=$(echo "$RESPONSE" | jq -r .tag_name) - fi - echo "last_release_ref=${LAST_RELEASE_REF}" >> $GITHUB_OUTPUT - echo "Output $GITHUB_OUTPUT" - cat $GITHUB_OUTPUT - - - name: Get Github commit messages - id: commit_messages - run: | - COMMIT_MESSAGES=$(git log ${{ steps.last_release.outputs.last_release_ref }}..${{ github.sha }} --oneline --no-merges | sed 's/^/* /' | sed "s/\n/\\\\n/g") - EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) - echo "commit_messages<<$EOF" >> $GITHUB_OUTPUT - echo "$COMMIT_MESSAGES" >> $GITHUB_OUTPUT - echo "$EOF" >> $GITHUB_OUTPUT - echo "Output $GITHUB_OUTPUT" - cat $GITHUB_OUTPUT + uses: actions/github-script@v7 + with: + script: | + const latestRelease = await github.rest.repos.getLatestRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + }); + + const reset = "\x1b[0m"; + const textColor = "\x1b[36m"; // Cyan for static text + const varColor = "\x1b[35m"; // Magenta for variables + console.log(`${textColor}The last release is ${varColor}${latestRelease.data.tag_name}${textColor}.${reset}`); + + return { last_release_ref: latestRelease.data.tag_name }; + + - name: Generate Changelog + id: changelog + uses: actions/github-script@v7 + with: + script: | + const lastRelease = '${{ steps.last_release.outputs.result.last_release_ref }}'; + const currentSha = context.sha; + + const pulls = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'closed', + per_page: 100, + }); + + const mergedPulls = pulls.data.filter(pr => + pr.merged_at && + pr.merge_commit_sha >= lastRelease && + pr.merge_commit_sha <= currentSha + ); + + let changes = "## What's Changed\n"; + let contributorsNames = new Set(); + + mergedPulls.forEach(pr => { + changes += `* ${pr.title} ${pr.html_url}\n`; + contributorsNames.add(pr.user.login); + }); + + let contributors = `### Contributors + Thanks to @${Array.from(contributorsNames).join(', ')}.\n`; + + const fullChangelog = `**Full Changelog**: https://github.com/${context.repo.owner}/${context.repo.repo}/compare/${lastRelease}...${currentSha}`; + const changelog = changes + '\n' + (contributorsNames.size ? contributors + '\n' : '') + fullChangelog; + + const reset = "\x1b[0m"; + const textColor = "\x1b[36m"; // Cyan for static text + const varColor = "\x1b[35m"; // Magenta for variables + console.log(`${textColor}Changelog:${reset}\n${changelog}`); + + return { changelog: changelog }; + + - name: Output Changelog + run: echo "${{ steps.changelog.outputs.result.changelog }}" docker_image_build: - name: "Build Docker Images" - needs: ["compile", "info"] + name: "Build" + needs: ["info"] runs-on: ubuntu-latest - if: github.event_name == 'push' strategy: fail-fast: false matrix: @@ -319,10 +171,6 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Set git info - run: | - echo '${{ needs.compile.outputs.json }}' | jq -r '.git_info' > git_info.json - - name: Docker meta id: meta uses: docker/metadata-action@v5 @@ -345,18 +193,16 @@ jobs: id: build uses: docker/build-push-action@v6 with: - context: . + file: ./Dockerfile target: executable + context: . platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max provenance: mode=max build-args: | - VERSION=${{ needs.info.outputs.version_number }} - GIT_COMMIT_ID=${{ needs.compile.outputs.commit_id }} - GIT_BRANCH=${{ needs.compile.outputs.branch }} - GIT_LAST_AUTHOR=${{ needs.compile.outputs.last_author }} + VERSION=${{ needs.info.outputs.project_version }} outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - name: Export digest @@ -374,9 +220,10 @@ jobs: retention-days: 1 docker_image_push: - name: "Push Docker Images" + name: "Push" runs-on: ubuntu-latest - needs: ["compile", "unit_test", "info", "docker_image_build"] + needs: ["info", "docker_image_build"] + if: github.event_name == 'push' steps: - name: Download digests @@ -386,31 +233,32 @@ jobs: pattern: digests-* merge-multiple: true - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to DockerHub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Docker meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY_IMAGE }} tags: | - type=raw,value=${{ needs.info.outputs.version_number }},enable=${{ needs.info.outputs.version_tag == 'latest' }} - type=raw,value=${{ needs.info.outputs.version_tag }} + type=raw,value=${{ needs.info.outputs.docker_tag }} + type=raw,value=${{ needs.info.outputs.project_version }} - name: Create manifest list and push working-directory: /tmp/digests run: docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) docker_image_push_private: - name: "Push Privates Docker Images" + name: "Push Privates" runs-on: ubuntu-latest - needs: ["compile", "unit_test", "info", "docker_image_build"] + needs: ["info", "docker_image_build"] + if: github.event_name == 'push' steps: - name: Download digests @@ -420,9 +268,6 @@ jobs: pattern: digests-* merge-multiple: true - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Login to Github Container Registry uses: docker/login-action@v3 with: @@ -430,30 +275,33 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Docker meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY_IMAGE_PRIVATE }} tags: | - type=ref,event=branch - type=raw,value=${{ needs.compile.outputs.version }}-${{ needs.compile.outputs.commit_id }} + type=raw,value=${{ needs.info.outputs.docker_tag }} + type=raw,value=${{ needs.info.outputs.project_version }} - name: Create manifest list and push working-directory: /tmp/digests run: docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) docker_swarm_deploy: - name: "Deploy Docker Swarm" + name: "Deploy" needs: ["info", "docker_image_push"] runs-on: ubuntu-latest environment: ${{ needs.info.outputs.environment }} - if: (github.event_name == 'tag' && needs.info.outputs.version_tag == 'latest') || (github.event_name == 'push' && needs.info.outputs.version_tag != 'latest') + if: github.event_name == 'push' && (github.ref_type == 'tag' && needs.info.outputs.docker_tag == 'latest') || (github.ref_type == 'branch' && needs.info.outputs.docker_tag != 'latest') steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Docker Swarm Update + - name: Deploy v${{ needs.info.outputs.project_version }} to ${{ needs.info.outputs.environment }} uses: tristiisch/docker-stack-deployment@master with: deployment_mode: docker-swarm @@ -466,21 +314,105 @@ jobs: secrets: ${{ vars.DOCKER_COMPOSE_SERVICE}} ${{ vars.DOCKER_STACK_NAME }} DISCORD__TOKEN ${{ secrets.CONFIG_DISCORD_TOKEN }} DEEZER__ARL ${{ secrets.CONFIG_DEEZER_ARL }} SPOTIFY__CLIENT_ID ${{ secrets.CONFIG_SPOTIFY_CLIENT_ID }} SPOTIFY__CLIENT_SECRET ${{ secrets.CONFIG_SPOTIFY_CLIENT_SECRET }} release_publish: - name: "Publish release" - needs: ["compile", "info", "docker_image_push"] + name: "Release" + needs: ["info", "docker_image_push"] runs-on: ubuntu-latest - if: github.event_name == 'tag' && needs.info.outputs.version_tag == 'latest' + if: github.ref_type == 'tag' && github.event_name == 'push' && needs.info.outputs.docker_tag == 'latest' steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Update Release + - name: Update Release v${{ needs.info.outputs.project_version }} run: | - gh release edit "${GITHUB_REF#refs/tags/}" \ - --title "Release v${{ needs.compile.outputs.version }}" \ - --notes "This has been deployed on the Discord bot `PyRamid#6882`.\nTo use the latest version the bot, please refer to the instructions outlined at https://github.com/tristiisch/PyRamid/#usage.\n\n## Changes\n${{ needs.info.outputs.commit_messages }}\n\n## Docker\nThis version is now accessible through various Docker images. Each image creation corresponds to a unique snapshot of this version, while updating the image corresponds to using an updated Docker image tag.\n\n### Images creation\n* ${{ env.REGISTRY_IMAGE_PRIVATE }}:${{ needs.compile.outputs.version }}-${{ needs.compile.outputs.commit_id }}\n\n### Images update\n* ${{ env.REGISTRY_IMAGE }}:${{ needs.info.outputs.version_tag }}\n* ${{ env.REGISTRY_IMAGE }}:${{ needs.compile.outputs.version }}" - # --draft false \ - # --prerelease false + set -eu + cat << $EOF | gh release edit "${{ github.ref_name }}" \ + --title "Release v${{ needs.info.outputs.project_version }}" \ + --draft=false \ + --prerelease=false \ + --note-files - + The latest version of the Discord bot PyRamid#6882 has been successfully deployed. + To start using this updated version, please follow the instructions provided at [PyRamid Usage Guide](https://github.com/tristiisch/PyRamid/#usage). + + ${{ needs.info.outputs.changelog }} + + ## Docker + This version is now accessible through various Docker images. Each image creation corresponds to a unique snapshot of this version, while updating the image corresponds to using an updated Docker image tag. + + ### Images availables + * `${{ env.REGISTRY_IMAGE }}:${{ needs.info.outputs.docker_tag }}` + * `${{ env.REGISTRY_IMAGE }}:${{ needs.info.outputs.project_version }}` + $EOF env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + + docker_image_test_build: + name: "Build Tests" + needs: ["info", "docker_image_build"] + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker image Build + id: build + uses: docker/build-push-action@v6 + with: + file: ./Dockerfile + target: tests + context: . + tags: pyramid:tests + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + VERSION=${{ needs.info.outputs.project_version }}-tests + push: false + outputs: type=docker + + - name: Export digest + run: docker save pyramid:tests -o "./pyramid-tests.tar" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-tests + path: ./pyramid-tests.tar + if-no-files-found: error + retention-days: 1 + + tests_in_docker: + name: "Tests" + needs: ["docker_image_test_build"] + runs-on: ubuntu-latest + + steps: + - name: Download test digests + uses: actions/download-artifact@v4 + with: + name: digests-tests + + - name: Load Docker image + run: docker load -i "./pyramid-tests.tar" + + - name: Run unit tests + run: | + mkdir -p ./coverage && chmod 777 ./coverage + docker run --rm -v ./coverage:/app/coverage -e SPOTIFY__CLIENT_ID=${{ secrets.CONFIG_SPOTIFY_CLIENT_ID }} -e SPOTIFY__CLIENT_SECRET=${{ secrets.CONFIG_SPOTIFY_CLIENT_SECRET }} pyramid:tests + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.5.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: tristiisch/PyRamid + files: ./cover/result diff --git a/Dockerfile b/Dockerfile index 919f572..ee96123 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,6 @@ # Define the Python version and other build arguments ARG PYTHON_VERSION=3.12 ARG VERSION=0.0.0 -ARG GIT_COMMIT_ID=0000000 -ARG GIT_BRANCH=unknown -ARG GIT_LAST_AUTHOR=unknown ARG APP_USER=app-usr ARG APP_GROUP=app-grp @@ -49,12 +46,13 @@ RUN apk update && \ apk upgrade && \ apk add --no-cache ffmpeg opus-dev binutils && \ # Clean up apk cache - rm -rf /var/cache/apk/* /etc/apk/cache/* /root/.cache/* + ls -lah /var/cache/apk/ && \ + rm -rf /var/cache/apk/* WORKDIR /app # Create a user and group for running the application -RUN addgroup -S $APP_GROUP && adduser -S $APP_USER -G $APP_GROUP +RUN addgroup -g 1000 -S $APP_GROUP && adduser -u 1000 -S $APP_USER -G $APP_GROUP # Create and set permissions for directories RUN mkdir -p ./songs && chmod 770 ./songs && chown root:$APP_GROUP ./songs && \ @@ -64,13 +62,12 @@ RUN mkdir -p ./songs && chmod 770 ./songs && chown root:$APP_GROUP ./songs && \ FROM base AS executable ARG VERSION -ARG GIT_COMMIT_ID ARG APP_USER ARG APP_GROUP LABEL org.opencontainers.image.source="https://github.com/tristiisch/PyRamid" \ org.opencontainers.image.authors="tristiisch" \ - version="$VERSION-$GIT_COMMIT_ID" + version="$VERSION" # Copy the virtual environment from the builder stage COPY --chown=root:$APP_GROUP --chmod=550 --from=builder /opt/venv /opt/venv @@ -133,4 +130,4 @@ RUN pip install -e . USER $APP_USER # Run tests -CMD ["pytest", "--cov=pyramid tests/"] +CMD ["pytest", "--cov=pyramid tests/", "--cov-config=.coveragerc"] diff --git a/Makefile b/Makefile index 25c6cf8..0600ebc 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,13 @@ DOCKER_CONTEXT_PREPROD := cookie-pulsheberg all: up-b logs -start: +build: + @docker compose build --pull + +build-c: + @docker compose build --pull + +up: @docker compose up -d --remove-orphans up-f: @@ -45,9 +51,10 @@ exec-pp: dev: @docker compose -f $(DOCKER_COMPOSE_FILE_DEV) up -d --remove-orphans --pull always --force-recreate -test: +tests: @docker build -f Dockerfile --target tests -t pyramid:tests . - @docker run --rm -t pyramid:tests + @mkdir -p ./cover && chmod 777 ./cover + @docker run --rm -v ./cover:/app/cover pyramid:tests img-b: @python scripts/environnement.py --build @@ -60,3 +67,5 @@ img-c: clean: @python scripts/environnement.py --clean + +.PHONY: build tests \ No newline at end of file diff --git a/src/__main__.py b/src/__main__.py index 58d9b3d..c6be667 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -5,7 +5,6 @@ def startup(): main.args() main.logs() - main.git_info() main.config() main.clean_data() diff --git a/src/cli.py b/src/cli.py index 8a18de8..1fff624 100644 --- a/src/cli.py +++ b/src/cli.py @@ -12,8 +12,7 @@ parser = argparse.ArgumentParser(description="Readme at https://github.com/tristiisch/PyRamid") parser.add_argument("--version", action="store_true", help="Print version", required=False) -parser.add_argument("--git", action="store_true", help="Print git informations", required=False) -# parser.add_argument("--health", action="store_true", help="Print health", required=False) +parser.add_argument("--health", action="store_true", help="Print health", required=False) health_subparser = parser.add_subparsers(dest="health") health_parser = health_subparser.add_parser("health", help="Print health") @@ -27,12 +26,7 @@ args = parser.parse_args() if args.version: - info.load_git_info() - print(info.to_json()) - -elif args.git: - info.load_git_info() - print(info.git_info.to_json()) + print(info.get_version()) elif args.health: sc = SocketClient(args.host, args.port) diff --git a/src/pyramid/connector/discord/bot_cmd.py b/src/pyramid/connector/discord/bot_cmd.py index 8f5c58e..1bbce09 100644 --- a/src/pyramid/connector/discord/bot_cmd.py +++ b/src/pyramid/connector/discord/bot_cmd.py @@ -51,7 +51,7 @@ async def cmd_about(ctx: Interaction): self.__logger.warning("Unable to get self user instance") info = self.__info - embed = Embed(title=info.name.capitalize(), color=Color.gold()) + embed = Embed(title=info.__name.capitalize(), color=Color.gold()) if bot_user is not None and bot_user.avatar is not None: embed.set_thumbnail(url=bot_user.avatar.url) @@ -80,8 +80,8 @@ async def cmd_about(ctx: Interaction): icon_url=owner.avatar.url if owner.avatar is not None else None, ) - embed.add_field(name="Version", value=info.get_full_version(), inline=True) - embed.add_field(name="OS", value=info.os, inline=True) + embed.add_field(name="Version", value=info.get_version(), inline=True) + embed.add_field(name="OS", value=info.get_os(), inline=True) embed.add_field( name="Environment", value=self.__environment.name.capitalize(), diff --git a/src/pyramid/data/functional/application_info.py b/src/pyramid/data/functional/application_info.py index 5a7ade7..4323a85 100644 --- a/src/pyramid/data/functional/application_info.py +++ b/src/pyramid/data/functional/application_info.py @@ -1,64 +1,45 @@ import json +import os import platform import subprocess -from pyramid.data.functional.git_info import GitInfo - class ApplicationInfo: def __init__(self): - self.name = "pyramid" - self.os = get_os().lower() - self.version = "0.6.3" - self.git_info = GitInfo() - - def load_git_info(self): - git_info = GitInfo.read() - if git_info is not None: - self.git_info = git_info - else: - self.git_info.get() + self.__name = "pyramid" + self.__os = self.__detect_os().lower() + self.__version = os.getenv("VERSION") def get_version(self): - return f"v{self.version}" - - def get_full_version(self): - return f"v{self.version}-{self.git_info.commit_id}" - - def __str__(self): - return f"{self.name.capitalize()} {self.get_full_version()} on {self.os} by {self.git_info.last_author}" - - def to_json(self): - data = vars(self) - data["git_info"] = vars(self.git_info) - return json.dumps(data, indent=4) - - -def get_os() -> str: - os_name = platform.system() - if os_name == "Linux": - return __get_linux_distro() - elif os_name == "Windows": - return f"{os_name}_{platform.version()}" - elif os_name == "Darwin": - return f"{os_name}_{platform.mac_ver()[0]}" - else: - return os_name - + return f"v{self.__version}" + + def get_os(self): + return self.__os + + def __detect_os(self) -> str: + os_name = platform.system() + if os_name == "Linux": + return self.__detect_linux_distro() + elif os_name == "Windows": + return f"{os_name}_{platform.version()}" + elif os_name == "Darwin": + return f"{os_name}_{platform.mac_ver()[0]}" + else: + return os_name -def __get_linux_distro() -> str: - try: - dist_name = subprocess.check_output(["lsb_release", "-i", "-s"]).strip().decode("utf-8") - dist_version = subprocess.check_output(["lsb_release", "-r", "-s"]).strip().decode("utf-8") - return f"{dist_name}_{dist_version}" - except FileNotFoundError: + def __detect_linux_distro(self) -> str: try: - with open("/etc/os-release", "r") as f: - lines = f.readlines() - for line in lines: - if line.startswith("PRETTY_NAME"): - dist_info = line.split("=")[1].strip().strip('"') - return dist_info + dist_name = subprocess.check_output(["lsb_release", "-i", "-s"]).strip().decode("utf-8") + dist_version = subprocess.check_output(["lsb_release", "-r", "-s"]).strip().decode("utf-8") + return f"{dist_name}_{dist_version}" except FileNotFoundError: - pass - return "Linux distribution information not available." + try: + with open("/etc/os-release", "r") as f: + lines = f.readlines() + for line in lines: + if line.startswith("PRETTY_NAME"): + dist_info = line.split("=")[1].strip().strip('"') + return dist_info + except FileNotFoundError: + pass + return "Linux distribution information not available." diff --git a/src/pyramid/data/functional/git_info.py b/src/pyramid/data/functional/git_info.py deleted file mode 100644 index 23c2740..0000000 --- a/src/pyramid/data/functional/git_info.py +++ /dev/null @@ -1,102 +0,0 @@ -import json -import logging -import os -import pathlib - - -class GitInfo: - def __init__(self): - self.commit_id: str | None = None - self.branch: str | None = None - self.last_author: str | None = None - - def get(self, base_path=None, max_length=7) -> bool: - if not base_path: - directory = pathlib.Path() - else: - directory = pathlib.Path(base_path) - - git_dir = directory / ".git" - if not git_dir.exists(): - return False - - git_head = git_dir / "HEAD" - if not git_head.exists(): - return False - - ref = None - commit_hash = None - with (git_head).open("r") as f: - head_file = f.read().strip() - - prefix = "ref: " - if head_file.startswith(prefix): - head_file = head_file[len(prefix) :] - ref = head_file - prefix = "refs/heads/" - if head_file.startswith(prefix): - head_file = head_file[len(prefix) :] - self.branch = head_file - - git_ref = git_dir / ref - if git_ref.exists(): - with (git_ref).open("r") as git_hash: - commit_id = git_hash.readline().strip() - self.commit_id = commit_id[:max_length] - - # Repo is in detached HEAD - else: - commit_hash = head_file - self.commit_id = commit_hash[:max_length] - - heads_path = git_dir / "refs" / "heads" - for root, dirs, files in os.walk(heads_path): - for branch_name in files: - branch_path = os.path.join(root, branch_name) - with open(branch_path, "r") as branch_file: - if branch_file.read().strip() == commit_hash: - self.branch = os.path.relpath(branch_path, heads_path).replace( - "\\", "/" - ) - break - if self.branch is not None: - break - - git_log_head = git_dir / "logs" / "HEAD" - if git_log_head.exists(): - log_lines = git_log_head.read_text().strip().split("\n") - self.last_author = log_lines[-1].split(" ")[2] - - return True - - def to_json(self): - data = vars(self) - return json.dumps(data, indent=4) - - def save(self, file_name="git_info.json"): - data = vars(self) - with open(file_name, "w") as f: - json.dump(data, f, indent=4) - - @classmethod - def read(cls, file_name="git_info.json", max_length=8): - if not os.path.exists(file_name): - return None - try: - with open(file_name, "r") as f: - data = json.load(f) - git_info = cls() - - git_info.commit_id = data["commit_id"][:max_length] - git_info.branch = data["branch"] - git_info.last_author = data["last_author"] - return git_info - - except ( - FileNotFoundError, - json.JSONDecodeError, - UnicodeDecodeError, - TypeError, - ) as e: - logging.warning("Error occurred while read %s due to %s", file_name, e) - return None diff --git a/src/pyramid/data/functional/main.py b/src/pyramid/data/functional/main.py index 0d62c1b..e825ad1 100644 --- a/src/pyramid/data/functional/main.py +++ b/src/pyramid/data/functional/main.py @@ -26,18 +26,10 @@ def __init__(self): def args(self): parser = argparse.ArgumentParser(description="Music Bot Discord using Deezer.") parser.add_argument("--version", action="store_true", help="Print version", required=False) - parser.add_argument( - "--git", action="store_true", help="Print git informations", required=False - ) args = parser.parse_args() if args.version: - self._info.load_git_info() - print(f"{self._info.to_json()}") - sys.exit(0) - elif args.git: - self._info.load_git_info() - print(f"{self._info.git_info.to_json()}") + print(f"{self._info.get_version()}") sys.exit(0) # Logs management @@ -53,11 +45,6 @@ def logs(self): # Deletion of log files over 10 tools.keep_latest_files(log_dir, 10, "error") - # Logs management - def git_info(self): - self._info.load_git_info() - logging.info(self._info) - def config(self): # Config load self._config = Configuration(self.logger) diff --git a/src/pyramid/git.py b/src/pyramid/git.py deleted file mode 100644 index 319703d..0000000 --- a/src/pyramid/git.py +++ /dev/null @@ -1,9 +0,0 @@ -from pyramid.data.functional.git_info import GitInfo - - -git_info = GitInfo.read() -if git_info is None: - git_info = GitInfo() - git_info.get() - -print(git_info.to_json()) From 0b327061b57e1bc441fed0311bc2fdca3e7fc5e0 Mon Sep 17 00:00:00 2001 From: Tristiisch Date: Tue, 27 Aug 2024 01:32:20 +0200 Subject: [PATCH 2/2] Revert "cicd: reorganize (#19)" (#22) This reverts commit dcf069d27b3966ee551b4c24575749363d1bcd06. --- .coveragerc | 2 - .github/workflows/python.yml | 510 ++++++++++-------- Dockerfile | 13 +- Makefile | 15 +- src/__main__.py | 1 + src/cli.py | 10 +- src/pyramid/connector/discord/bot_cmd.py | 6 +- .../data/functional/application_info.py | 85 +-- src/pyramid/data/functional/git_info.py | 102 ++++ src/pyramid/data/functional/main.py | 15 +- src/pyramid/git.py | 9 + 11 files changed, 489 insertions(+), 279 deletions(-) delete mode 100644 .coveragerc create mode 100644 src/pyramid/data/functional/git_info.py create mode 100644 src/pyramid/git.py diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index b0f11ed..0000000 --- a/.coveragerc +++ /dev/null @@ -1,2 +0,0 @@ -[run] -data_file = ./cover/result diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 24d5634..ffb6037 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -4,8 +4,6 @@ on: push: branches: - "*" - tags: - - '[0-9]+.[0-9]+.[0-9]+' paths: - "src/**/*.py" - "requirements.txt" @@ -14,6 +12,8 @@ on: - "Dockerfile" - "docker-compose*.yml" - ".github/workflows/python.yml" + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' pull_request: types: [opened, synchronize] branches: @@ -38,123 +38,271 @@ env: jobs: - info: - name: "Informations" + compile: + name: "Compile Python 3.11" runs-on: ubuntu-latest outputs: - project_version: ${{ steps.environment.outputs.result.project_version }} - environment: ${{ steps.environment.outputs.result.git_environment }} - docker_tag: ${{ steps.environment.outputs.result.docker_tag }} - commit_id: ${{ steps.environment.outputs.result.commit_id }} - last_release_ref: ${{ steps.last_release.outputs.result.last_release_ref }} - changelog: ${{ steps.changelog.outputs.result.changelog }} + json: ${{ steps.version.outputs.json }} + version: ${{ steps.version.outputs.version }} + commit_id: ${{ steps.version.outputs.commit_id }} + branch: ${{ steps.version.outputs.branch }} + last_author: ${{ steps.version.outputs.last_author }} steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Define build variables - id: environment - uses: actions/github-script@v7 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 with: - script: | - const refType = context.ref_type; - const ref = context.ref; - const refName = context.ref_name; - const sha = context.sha; - const shortSha = sha.slice(0, 7); - - let dockerTag; - let gitEnvironment; - let projectVersion; - - if (refType === 'tag') { - dockerTag = 'latest'; - gitEnvironment = 'production'; - projectVersion = refName; - } else if (ref === 'refs/heads/main') { - dockerTag = 'pre-prod'; - gitEnvironment = 'pre-production'; - projectVersion = `${refName}-${shortSha}`; - } else { - dockerTag = 'dev'; - gitEnvironment = 'development'; - projectVersion = `${refName}-${shortSha}`; - } - - const reset = "\x1b[0m"; - const textColor = "\x1b[36m"; // Cyan for static text - const varColor = "\x1b[35m"; // Magenta for variables - console.log(`${textColor}${projectVersion} in environment ${varColor}${gitEnvironment}${textColor} with tag ${varColor}${dockerTag}${reset}.`); - - return { docker_tag: dockerTag, git_environment: gitEnvironment, commit_id: shortSha, project_version: projectVersion }; - - - name: Get Github last release name - id: last_release - uses: actions/github-script@v7 + python-version: "3.11" + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + - ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} + - ${{ runner.os }}-pip + + - name: Install dependencies + run: | + pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Test compilation + run: | + python -m compileall ${{ env.SRC }} + + - name: Save version + run: | + FULL_JSON=$(python ${{ env.SRC }} --version) + echo "json=$(echo $FULL_JSON | jq -c)" >> $GITHUB_OUTPUT + echo "version=$(echo $FULL_JSON | jq -r '.version')" >> $GITHUB_OUTPUT + echo "commit_id=$(echo $FULL_JSON | jq -r '.git_info.commit_id')" >> $GITHUB_OUTPUT + echo "branch=$(echo $FULL_JSON | jq -r '.git_info.branch')" >> $GITHUB_OUTPUT + echo "last_author=$(echo $FULL_JSON | jq -r '.git_info.last_author')" >> $GITHUB_OUTPUT + echo "Output $GITHUB_OUTPUT" + cat $GITHUB_OUTPUT + id: version + + unit_test: + name: "Unit tests" + needs: compile + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + clean: false + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-python_test-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + - ${{ runner.os }}-python_test-${{ hashFiles('**/requirements.txt') }} + - ${{ runner.os }}-python_test-${{ hashFiles('**/requirements.txt') }} + - ${{ runner.os }}-python + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Install project for tests + run: | + pip install pytest-cov + pip install -e . + + - name: Units tests + env: + DEEZER__ARL: ${{ secrets.CONFIG_DEEZER_ARL }} + SPOTIFY__CLIENT_ID: ${{ secrets.CONFIG_SPOTIFY_CLIENT_ID }} + SPOTIFY__CLIENT_SECRET: ${{ secrets.CONFIG_SPOTIFY_CLIENT_SECRET }} + run: | + pytest --cov=${{ env.MODULE_NAME }} ${{ env.TEST_DIR }} + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4.5.0 + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: tristiisch/PyRamid + + unit_test_compatibility: + name: "Envs unit tests" + needs: unit_test + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.12" + platform: + - linux/amd64 + - linux/arm64/v8 + continue-on-error: true + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Cache dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-python_test_${{ matrix.python-version }}_${{ matrix.platform }}-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + - ${{ runner.os }}-python_test_${{ matrix.python-version }}_${{ matrix.platform }} + - ${{ runner.os }}-python_test_${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Install project for tests + run: | + pip install pytest-cov + pip install -e . + + - name: Units tests + env: + DEEZER__ARL: ${{ secrets.CONFIG_DEEZER_ARL }} + SPOTIFY__CLIENT_ID: ${{ secrets.CONFIG_SPOTIFY_CLIENT_ID }} + SPOTIFY__CLIENT_SECRET: ${{ secrets.CONFIG_SPOTIFY_CLIENT_SECRET }} + run: | + pytest --cov=${{ env.MODULE_NAME }} ${{ env.TEST_DIR }} + + version_compatibility: + name: "Envs compatibility" + needs: compile + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: + - "3.8" + - "3.9" + - "3.10" + - "3.12" + platform: + - linux/amd64 + - linux/arm64/v8 + continue-on-error: true + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 with: - script: | - const latestRelease = await github.rest.repos.getLatestRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - }); - - const reset = "\x1b[0m"; - const textColor = "\x1b[36m"; // Cyan for static text - const varColor = "\x1b[35m"; // Magenta for variables - console.log(`${textColor}The last release is ${varColor}${latestRelease.data.tag_name}${textColor}.${reset}`); - - return { last_release_ref: latestRelease.data.tag_name }; - - - name: Generate Changelog - id: changelog - uses: actions/github-script@v7 + python-version: "${{ matrix.python-version }}" + + - name: Cache dependencies + uses: actions/cache@v4 with: - script: | - const lastRelease = '${{ steps.last_release.outputs.result.last_release_ref }}'; - const currentSha = context.sha; - - const pulls = await github.rest.pulls.list({ - owner: context.repo.owner, - repo: context.repo.repo, - state: 'closed', - per_page: 100, - }); - - const mergedPulls = pulls.data.filter(pr => - pr.merged_at && - pr.merge_commit_sha >= lastRelease && - pr.merge_commit_sha <= currentSha - ); - - let changes = "## What's Changed\n"; - let contributorsNames = new Set(); - - mergedPulls.forEach(pr => { - changes += `* ${pr.title} ${pr.html_url}\n`; - contributorsNames.add(pr.user.login); - }); - - let contributors = `### Contributors - Thanks to @${Array.from(contributorsNames).join(', ')}.\n`; - - const fullChangelog = `**Full Changelog**: https://github.com/${context.repo.owner}/${context.repo.repo}/compare/${lastRelease}...${currentSha}`; - const changelog = changes + '\n' + (contributorsNames.size ? contributors + '\n' : '') + fullChangelog; - - const reset = "\x1b[0m"; - const textColor = "\x1b[36m"; // Cyan for static text - const varColor = "\x1b[35m"; // Magenta for variables - console.log(`${textColor}Changelog:${reset}\n${changelog}`); - - return { changelog: changelog }; - - - name: Output Changelog - run: echo "${{ steps.changelog.outputs.result.changelog }}" + path: ~/.cache/pip + key: ${{ runner.os }}-python_${{ matrix.python-version }}_${{ matrix.platform }}-${{ hashFiles('**/requirements.txt') }} + restore-keys: | + - ${{ runner.os }}-python_${{ matrix.python-version }}_${{ matrix.platform }} + - ${{ runner.os }}-python_${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + + - name: Test compilation + run: | + python -m compileall ${{ env.SRC }} + + info: + name: "Build information" + needs: ["compile"] + runs-on: ubuntu-latest + outputs: + environment: ${{ steps.environment.outputs.environment }} + version_tag: ${{ steps.environment.outputs.tag }} + version_number: ${{ steps.environment.outputs.version }} + last_release_ref: ${{ steps.last_release.outputs.last_release_ref }} + commit_messages: ${{ steps.commit_messages.outputs.commit_messages }} + if: github.event_name == 'push' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 100 + + - name: Get tags + run: git fetch --tags origin + + - name: Get environnement names + id: environment + run: | + if [ ${{ github.event_name }} == 'create' && ${{ github.ref_type }} == 'tag' ]; then + echo "tag=latest" >> $GITHUB_OUTPUT + echo "environment=production" >> $GITHUB_OUTPUT + echo "version=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + elif [ ${{ github.ref }} = 'refs/heads/main' ]; then + echo "tag=pre-prod" >> $GITHUB_OUTPUT + echo "environment=pre-production" >> $GITHUB_OUTPUT + echo "version=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + else + echo "tag=dev" >> $GITHUB_OUTPUT + echo "environment=developement" >> $GITHUB_OUTPUT + echo "version=${GITHUB_SHA::7}" >> $GITHUB_OUTPUT + fi + echo "Output $GITHUB_OUTPUT" + cat $GITHUB_OUTPUT + + - name: Get Github last release tag + id: last_release + run: | + RESPONSE=$(curl -s "https://api.github.com/repos/${{ github.repository }}/releases/latest") + if [[ $(echo "$RESPONSE" | jq -r .message) == "Not Found" ]]; then + LAST_RELEASE_REF=$(git rev-list --max-parents=0 HEAD) + else + LAST_RELEASE_REF=$(echo "$RESPONSE" | jq -r .tag_name) + fi + echo "last_release_ref=${LAST_RELEASE_REF}" >> $GITHUB_OUTPUT + echo "Output $GITHUB_OUTPUT" + cat $GITHUB_OUTPUT + + - name: Get Github commit messages + id: commit_messages + run: | + COMMIT_MESSAGES=$(git log ${{ steps.last_release.outputs.last_release_ref }}..${{ github.sha }} --oneline --no-merges | sed 's/^/* /' | sed "s/\n/\\\\n/g") + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "commit_messages<<$EOF" >> $GITHUB_OUTPUT + echo "$COMMIT_MESSAGES" >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT + echo "Output $GITHUB_OUTPUT" + cat $GITHUB_OUTPUT docker_image_build: - name: "Build" - needs: ["info"] + name: "Build Docker Images" + needs: ["compile", "info"] runs-on: ubuntu-latest + if: github.event_name == 'push' strategy: fail-fast: false matrix: @@ -171,6 +319,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Set git info + run: | + echo '${{ needs.compile.outputs.json }}' | jq -r '.git_info' > git_info.json + - name: Docker meta id: meta uses: docker/metadata-action@v5 @@ -193,16 +345,18 @@ jobs: id: build uses: docker/build-push-action@v6 with: - file: ./Dockerfile - target: executable context: . + target: executable platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max provenance: mode=max build-args: | - VERSION=${{ needs.info.outputs.project_version }} + VERSION=${{ needs.info.outputs.version_number }} + GIT_COMMIT_ID=${{ needs.compile.outputs.commit_id }} + GIT_BRANCH=${{ needs.compile.outputs.branch }} + GIT_LAST_AUTHOR=${{ needs.compile.outputs.last_author }} outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true - name: Export digest @@ -220,10 +374,9 @@ jobs: retention-days: 1 docker_image_push: - name: "Push" + name: "Push Docker Images" runs-on: ubuntu-latest - needs: ["info", "docker_image_build"] - if: github.event_name == 'push' + needs: ["compile", "unit_test", "info", "docker_image_build"] steps: - name: Download digests @@ -233,32 +386,31 @@ jobs: pattern: digests-* merge-multiple: true + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to DockerHub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY_IMAGE }} tags: | - type=raw,value=${{ needs.info.outputs.docker_tag }} - type=raw,value=${{ needs.info.outputs.project_version }} + type=raw,value=${{ needs.info.outputs.version_number }},enable=${{ needs.info.outputs.version_tag == 'latest' }} + type=raw,value=${{ needs.info.outputs.version_tag }} - name: Create manifest list and push working-directory: /tmp/digests run: docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) docker_image_push_private: - name: "Push Privates" + name: "Push Privates Docker Images" runs-on: ubuntu-latest - needs: ["info", "docker_image_build"] - if: github.event_name == 'push' + needs: ["compile", "unit_test", "info", "docker_image_build"] steps: - name: Download digests @@ -268,6 +420,9 @@ jobs: pattern: digests-* merge-multiple: true + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Github Container Registry uses: docker/login-action@v3 with: @@ -275,33 +430,30 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Docker meta uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY_IMAGE_PRIVATE }} tags: | - type=raw,value=${{ needs.info.outputs.docker_tag }} - type=raw,value=${{ needs.info.outputs.project_version }} + type=ref,event=branch + type=raw,value=${{ needs.compile.outputs.version }}-${{ needs.compile.outputs.commit_id }} - name: Create manifest list and push working-directory: /tmp/digests run: docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) docker_swarm_deploy: - name: "Deploy" + name: "Deploy Docker Swarm" needs: ["info", "docker_image_push"] runs-on: ubuntu-latest environment: ${{ needs.info.outputs.environment }} - if: github.event_name == 'push' && (github.ref_type == 'tag' && needs.info.outputs.docker_tag == 'latest') || (github.ref_type == 'branch' && needs.info.outputs.docker_tag != 'latest') + if: (github.event_name == 'tag' && needs.info.outputs.version_tag == 'latest') || (github.event_name == 'push' && needs.info.outputs.version_tag != 'latest') steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Deploy v${{ needs.info.outputs.project_version }} to ${{ needs.info.outputs.environment }} + - name: Docker Swarm Update uses: tristiisch/docker-stack-deployment@master with: deployment_mode: docker-swarm @@ -314,105 +466,21 @@ jobs: secrets: ${{ vars.DOCKER_COMPOSE_SERVICE}} ${{ vars.DOCKER_STACK_NAME }} DISCORD__TOKEN ${{ secrets.CONFIG_DISCORD_TOKEN }} DEEZER__ARL ${{ secrets.CONFIG_DEEZER_ARL }} SPOTIFY__CLIENT_ID ${{ secrets.CONFIG_SPOTIFY_CLIENT_ID }} SPOTIFY__CLIENT_SECRET ${{ secrets.CONFIG_SPOTIFY_CLIENT_SECRET }} release_publish: - name: "Release" - needs: ["info", "docker_image_push"] + name: "Publish release" + needs: ["compile", "info", "docker_image_push"] runs-on: ubuntu-latest - if: github.ref_type == 'tag' && github.event_name == 'push' && needs.info.outputs.docker_tag == 'latest' + if: github.event_name == 'tag' && needs.info.outputs.version_tag == 'latest' steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Update Release v${{ needs.info.outputs.project_version }} + - name: Update Release run: | - set -eu - cat << $EOF | gh release edit "${{ github.ref_name }}" \ - --title "Release v${{ needs.info.outputs.project_version }}" \ - --draft=false \ - --prerelease=false \ - --note-files - - The latest version of the Discord bot PyRamid#6882 has been successfully deployed. - To start using this updated version, please follow the instructions provided at [PyRamid Usage Guide](https://github.com/tristiisch/PyRamid/#usage). - - ${{ needs.info.outputs.changelog }} - - ## Docker - This version is now accessible through various Docker images. Each image creation corresponds to a unique snapshot of this version, while updating the image corresponds to using an updated Docker image tag. - - ### Images availables - * `${{ env.REGISTRY_IMAGE }}:${{ needs.info.outputs.docker_tag }}` - * `${{ env.REGISTRY_IMAGE }}:${{ needs.info.outputs.project_version }}` - $EOF + gh release edit "${GITHUB_REF#refs/tags/}" \ + --title "Release v${{ needs.compile.outputs.version }}" \ + --notes "This has been deployed on the Discord bot `PyRamid#6882`.\nTo use the latest version the bot, please refer to the instructions outlined at https://github.com/tristiisch/PyRamid/#usage.\n\n## Changes\n${{ needs.info.outputs.commit_messages }}\n\n## Docker\nThis version is now accessible through various Docker images. Each image creation corresponds to a unique snapshot of this version, while updating the image corresponds to using an updated Docker image tag.\n\n### Images creation\n* ${{ env.REGISTRY_IMAGE_PRIVATE }}:${{ needs.compile.outputs.version }}-${{ needs.compile.outputs.commit_id }}\n\n### Images update\n* ${{ env.REGISTRY_IMAGE }}:${{ needs.info.outputs.version_tag }}\n* ${{ env.REGISTRY_IMAGE }}:${{ needs.compile.outputs.version }}" + # --draft false \ + # --prerelease false env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - docker_image_test_build: - name: "Build Tests" - needs: ["info", "docker_image_build"] - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ vars.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Docker image Build - id: build - uses: docker/build-push-action@v6 - with: - file: ./Dockerfile - target: tests - context: . - tags: pyramid:tests - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: | - VERSION=${{ needs.info.outputs.project_version }}-tests - push: false - outputs: type=docker - - - name: Export digest - run: docker save pyramid:tests -o "./pyramid-tests.tar" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digests-tests - path: ./pyramid-tests.tar - if-no-files-found: error - retention-days: 1 - - tests_in_docker: - name: "Tests" - needs: ["docker_image_test_build"] - runs-on: ubuntu-latest - - steps: - - name: Download test digests - uses: actions/download-artifact@v4 - with: - name: digests-tests - - - name: Load Docker image - run: docker load -i "./pyramid-tests.tar" - - - name: Run unit tests - run: | - mkdir -p ./coverage && chmod 777 ./coverage - docker run --rm -v ./coverage:/app/coverage -e SPOTIFY__CLIENT_ID=${{ secrets.CONFIG_SPOTIFY_CLIENT_ID }} -e SPOTIFY__CLIENT_SECRET=${{ secrets.CONFIG_SPOTIFY_CLIENT_SECRET }} pyramid:tests - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4.5.0 - with: - token: ${{ secrets.CODECOV_TOKEN }} - slug: tristiisch/PyRamid - files: ./cover/result diff --git a/Dockerfile b/Dockerfile index ee96123..919f572 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,9 @@ # Define the Python version and other build arguments ARG PYTHON_VERSION=3.12 ARG VERSION=0.0.0 +ARG GIT_COMMIT_ID=0000000 +ARG GIT_BRANCH=unknown +ARG GIT_LAST_AUTHOR=unknown ARG APP_USER=app-usr ARG APP_GROUP=app-grp @@ -46,13 +49,12 @@ RUN apk update && \ apk upgrade && \ apk add --no-cache ffmpeg opus-dev binutils && \ # Clean up apk cache - ls -lah /var/cache/apk/ && \ - rm -rf /var/cache/apk/* + rm -rf /var/cache/apk/* /etc/apk/cache/* /root/.cache/* WORKDIR /app # Create a user and group for running the application -RUN addgroup -g 1000 -S $APP_GROUP && adduser -u 1000 -S $APP_USER -G $APP_GROUP +RUN addgroup -S $APP_GROUP && adduser -S $APP_USER -G $APP_GROUP # Create and set permissions for directories RUN mkdir -p ./songs && chmod 770 ./songs && chown root:$APP_GROUP ./songs && \ @@ -62,12 +64,13 @@ RUN mkdir -p ./songs && chmod 770 ./songs && chown root:$APP_GROUP ./songs && \ FROM base AS executable ARG VERSION +ARG GIT_COMMIT_ID ARG APP_USER ARG APP_GROUP LABEL org.opencontainers.image.source="https://github.com/tristiisch/PyRamid" \ org.opencontainers.image.authors="tristiisch" \ - version="$VERSION" + version="$VERSION-$GIT_COMMIT_ID" # Copy the virtual environment from the builder stage COPY --chown=root:$APP_GROUP --chmod=550 --from=builder /opt/venv /opt/venv @@ -130,4 +133,4 @@ RUN pip install -e . USER $APP_USER # Run tests -CMD ["pytest", "--cov=pyramid tests/", "--cov-config=.coveragerc"] +CMD ["pytest", "--cov=pyramid tests/"] diff --git a/Makefile b/Makefile index 0600ebc..25c6cf8 100644 --- a/Makefile +++ b/Makefile @@ -9,13 +9,7 @@ DOCKER_CONTEXT_PREPROD := cookie-pulsheberg all: up-b logs -build: - @docker compose build --pull - -build-c: - @docker compose build --pull - -up: +start: @docker compose up -d --remove-orphans up-f: @@ -51,10 +45,9 @@ exec-pp: dev: @docker compose -f $(DOCKER_COMPOSE_FILE_DEV) up -d --remove-orphans --pull always --force-recreate -tests: +test: @docker build -f Dockerfile --target tests -t pyramid:tests . - @mkdir -p ./cover && chmod 777 ./cover - @docker run --rm -v ./cover:/app/cover pyramid:tests + @docker run --rm -t pyramid:tests img-b: @python scripts/environnement.py --build @@ -67,5 +60,3 @@ img-c: clean: @python scripts/environnement.py --clean - -.PHONY: build tests \ No newline at end of file diff --git a/src/__main__.py b/src/__main__.py index c6be667..58d9b3d 100644 --- a/src/__main__.py +++ b/src/__main__.py @@ -5,6 +5,7 @@ def startup(): main.args() main.logs() + main.git_info() main.config() main.clean_data() diff --git a/src/cli.py b/src/cli.py index 1fff624..8a18de8 100644 --- a/src/cli.py +++ b/src/cli.py @@ -12,7 +12,8 @@ parser = argparse.ArgumentParser(description="Readme at https://github.com/tristiisch/PyRamid") parser.add_argument("--version", action="store_true", help="Print version", required=False) -parser.add_argument("--health", action="store_true", help="Print health", required=False) +parser.add_argument("--git", action="store_true", help="Print git informations", required=False) +# parser.add_argument("--health", action="store_true", help="Print health", required=False) health_subparser = parser.add_subparsers(dest="health") health_parser = health_subparser.add_parser("health", help="Print health") @@ -26,7 +27,12 @@ args = parser.parse_args() if args.version: - print(info.get_version()) + info.load_git_info() + print(info.to_json()) + +elif args.git: + info.load_git_info() + print(info.git_info.to_json()) elif args.health: sc = SocketClient(args.host, args.port) diff --git a/src/pyramid/connector/discord/bot_cmd.py b/src/pyramid/connector/discord/bot_cmd.py index 1bbce09..8f5c58e 100644 --- a/src/pyramid/connector/discord/bot_cmd.py +++ b/src/pyramid/connector/discord/bot_cmd.py @@ -51,7 +51,7 @@ async def cmd_about(ctx: Interaction): self.__logger.warning("Unable to get self user instance") info = self.__info - embed = Embed(title=info.__name.capitalize(), color=Color.gold()) + embed = Embed(title=info.name.capitalize(), color=Color.gold()) if bot_user is not None and bot_user.avatar is not None: embed.set_thumbnail(url=bot_user.avatar.url) @@ -80,8 +80,8 @@ async def cmd_about(ctx: Interaction): icon_url=owner.avatar.url if owner.avatar is not None else None, ) - embed.add_field(name="Version", value=info.get_version(), inline=True) - embed.add_field(name="OS", value=info.get_os(), inline=True) + embed.add_field(name="Version", value=info.get_full_version(), inline=True) + embed.add_field(name="OS", value=info.os, inline=True) embed.add_field( name="Environment", value=self.__environment.name.capitalize(), diff --git a/src/pyramid/data/functional/application_info.py b/src/pyramid/data/functional/application_info.py index 4323a85..5a7ade7 100644 --- a/src/pyramid/data/functional/application_info.py +++ b/src/pyramid/data/functional/application_info.py @@ -1,45 +1,64 @@ import json -import os import platform import subprocess +from pyramid.data.functional.git_info import GitInfo + class ApplicationInfo: def __init__(self): - self.__name = "pyramid" - self.__os = self.__detect_os().lower() - self.__version = os.getenv("VERSION") + self.name = "pyramid" + self.os = get_os().lower() + self.version = "0.6.3" + self.git_info = GitInfo() - def get_version(self): - return f"v{self.__version}" - - def get_os(self): - return self.__os - - def __detect_os(self) -> str: - os_name = platform.system() - if os_name == "Linux": - return self.__detect_linux_distro() - elif os_name == "Windows": - return f"{os_name}_{platform.version()}" - elif os_name == "Darwin": - return f"{os_name}_{platform.mac_ver()[0]}" + def load_git_info(self): + git_info = GitInfo.read() + if git_info is not None: + self.git_info = git_info else: - return os_name + self.git_info.get() + + def get_version(self): + return f"v{self.version}" + + def get_full_version(self): + return f"v{self.version}-{self.git_info.commit_id}" + + def __str__(self): + return f"{self.name.capitalize()} {self.get_full_version()} on {self.os} by {self.git_info.last_author}" + + def to_json(self): + data = vars(self) + data["git_info"] = vars(self.git_info) + return json.dumps(data, indent=4) + + +def get_os() -> str: + os_name = platform.system() + if os_name == "Linux": + return __get_linux_distro() + elif os_name == "Windows": + return f"{os_name}_{platform.version()}" + elif os_name == "Darwin": + return f"{os_name}_{platform.mac_ver()[0]}" + else: + return os_name + - def __detect_linux_distro(self) -> str: +def __get_linux_distro() -> str: + try: + dist_name = subprocess.check_output(["lsb_release", "-i", "-s"]).strip().decode("utf-8") + dist_version = subprocess.check_output(["lsb_release", "-r", "-s"]).strip().decode("utf-8") + return f"{dist_name}_{dist_version}" + except FileNotFoundError: try: - dist_name = subprocess.check_output(["lsb_release", "-i", "-s"]).strip().decode("utf-8") - dist_version = subprocess.check_output(["lsb_release", "-r", "-s"]).strip().decode("utf-8") - return f"{dist_name}_{dist_version}" + with open("/etc/os-release", "r") as f: + lines = f.readlines() + for line in lines: + if line.startswith("PRETTY_NAME"): + dist_info = line.split("=")[1].strip().strip('"') + return dist_info except FileNotFoundError: - try: - with open("/etc/os-release", "r") as f: - lines = f.readlines() - for line in lines: - if line.startswith("PRETTY_NAME"): - dist_info = line.split("=")[1].strip().strip('"') - return dist_info - except FileNotFoundError: - pass - return "Linux distribution information not available." + pass + return "Linux distribution information not available." diff --git a/src/pyramid/data/functional/git_info.py b/src/pyramid/data/functional/git_info.py new file mode 100644 index 0000000..23c2740 --- /dev/null +++ b/src/pyramid/data/functional/git_info.py @@ -0,0 +1,102 @@ +import json +import logging +import os +import pathlib + + +class GitInfo: + def __init__(self): + self.commit_id: str | None = None + self.branch: str | None = None + self.last_author: str | None = None + + def get(self, base_path=None, max_length=7) -> bool: + if not base_path: + directory = pathlib.Path() + else: + directory = pathlib.Path(base_path) + + git_dir = directory / ".git" + if not git_dir.exists(): + return False + + git_head = git_dir / "HEAD" + if not git_head.exists(): + return False + + ref = None + commit_hash = None + with (git_head).open("r") as f: + head_file = f.read().strip() + + prefix = "ref: " + if head_file.startswith(prefix): + head_file = head_file[len(prefix) :] + ref = head_file + prefix = "refs/heads/" + if head_file.startswith(prefix): + head_file = head_file[len(prefix) :] + self.branch = head_file + + git_ref = git_dir / ref + if git_ref.exists(): + with (git_ref).open("r") as git_hash: + commit_id = git_hash.readline().strip() + self.commit_id = commit_id[:max_length] + + # Repo is in detached HEAD + else: + commit_hash = head_file + self.commit_id = commit_hash[:max_length] + + heads_path = git_dir / "refs" / "heads" + for root, dirs, files in os.walk(heads_path): + for branch_name in files: + branch_path = os.path.join(root, branch_name) + with open(branch_path, "r") as branch_file: + if branch_file.read().strip() == commit_hash: + self.branch = os.path.relpath(branch_path, heads_path).replace( + "\\", "/" + ) + break + if self.branch is not None: + break + + git_log_head = git_dir / "logs" / "HEAD" + if git_log_head.exists(): + log_lines = git_log_head.read_text().strip().split("\n") + self.last_author = log_lines[-1].split(" ")[2] + + return True + + def to_json(self): + data = vars(self) + return json.dumps(data, indent=4) + + def save(self, file_name="git_info.json"): + data = vars(self) + with open(file_name, "w") as f: + json.dump(data, f, indent=4) + + @classmethod + def read(cls, file_name="git_info.json", max_length=8): + if not os.path.exists(file_name): + return None + try: + with open(file_name, "r") as f: + data = json.load(f) + git_info = cls() + + git_info.commit_id = data["commit_id"][:max_length] + git_info.branch = data["branch"] + git_info.last_author = data["last_author"] + return git_info + + except ( + FileNotFoundError, + json.JSONDecodeError, + UnicodeDecodeError, + TypeError, + ) as e: + logging.warning("Error occurred while read %s due to %s", file_name, e) + return None diff --git a/src/pyramid/data/functional/main.py b/src/pyramid/data/functional/main.py index e825ad1..0d62c1b 100644 --- a/src/pyramid/data/functional/main.py +++ b/src/pyramid/data/functional/main.py @@ -26,10 +26,18 @@ def __init__(self): def args(self): parser = argparse.ArgumentParser(description="Music Bot Discord using Deezer.") parser.add_argument("--version", action="store_true", help="Print version", required=False) + parser.add_argument( + "--git", action="store_true", help="Print git informations", required=False + ) args = parser.parse_args() if args.version: - print(f"{self._info.get_version()}") + self._info.load_git_info() + print(f"{self._info.to_json()}") + sys.exit(0) + elif args.git: + self._info.load_git_info() + print(f"{self._info.git_info.to_json()}") sys.exit(0) # Logs management @@ -45,6 +53,11 @@ def logs(self): # Deletion of log files over 10 tools.keep_latest_files(log_dir, 10, "error") + # Logs management + def git_info(self): + self._info.load_git_info() + logging.info(self._info) + def config(self): # Config load self._config = Configuration(self.logger) diff --git a/src/pyramid/git.py b/src/pyramid/git.py new file mode 100644 index 0000000..319703d --- /dev/null +++ b/src/pyramid/git.py @@ -0,0 +1,9 @@ +from pyramid.data.functional.git_info import GitInfo + + +git_info = GitInfo.read() +if git_info is None: + git_info = GitInfo() + git_info.get() + +print(git_info.to_json())