From 660f7cb13dd6373bac0630e441e45021090e8c73 Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Mon, 15 Apr 2024 11:55:34 +0200 Subject: [PATCH 01/12] Test deployment locally with tmt Using a Vagrant libvirt based virtual machine and OpenShift Local. --- .ansible-lint | 1 + .fmf/version | 1 + Makefile | 38 ++++++++ containers/Vagrantfile | 94 +++++++++++++++++++ docs/deployment/testing-changes.md | 35 +++++++ openshift/redis.yml.j2 | 2 +- plans/main.fmf | 13 +++ playbooks/deploy.yml | 14 +-- playbooks/oc-cluster-run.yml | 19 ++++ playbooks/oc-cluster-setup.yml | 52 ++++++++++ playbooks/oc-cluster-tests-setup.yml | 45 +++++++++ .../generate_secrets/tasks/generate-cert.yml | 8 ++ playbooks/test_deploy_setup.yml | 80 ++++++++++++++++ tests/deployment/main.fmf | 12 +++ 14 files changed, 407 insertions(+), 7 deletions(-) create mode 100644 .fmf/version create mode 100644 containers/Vagrantfile create mode 100644 plans/main.fmf create mode 100644 playbooks/oc-cluster-run.yml create mode 100644 playbooks/oc-cluster-setup.yml create mode 100644 playbooks/oc-cluster-tests-setup.yml create mode 100644 playbooks/test_deploy_setup.yml create mode 100644 tests/deployment/main.fmf diff --git a/.ansible-lint b/.ansible-lint index 5b10e84..c26ee86 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -14,6 +14,7 @@ mock_modules: # Ansible 2.9.27 in F35 still contains the k8s module so we can ignore the error until F36, # where we can switch to kubernetes.core.k8s as ansible-5.x in F36 contains it. - k8s + - kubernetes.core.k8s # Ignore until F36, where these are in community.crypto collection (part of ansible-5.x rpm). - openssh_keypair - openssl_certificate diff --git a/.fmf/version b/.fmf/version new file mode 100644 index 0000000..d00491f --- /dev/null +++ b/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/Makefile b/Makefile index ed0b3b0..f6d19d1 100755 --- a/Makefile +++ b/Makefile @@ -6,12 +6,22 @@ AP := ansible-playbook -vv -c local -i localhost, -e ansible_python_interpreter= # "By default, Ansible runs as if --tags all had been specified." # https://docs.ansible.com/ansible/latest/user_guide/playbooks_tags.html#special-tags TAGS ?= all +VAGRANT_SSH_PORT = "$(shell cd containers && vagrant ssh-config | awk '/Port/{print $$2}')" +VAGRANT_SSH_USER = "$(shell cd containers && vagrant ssh-config | awk '/User/{print $$2}')" +VAGRANT_SSH_GUEST = "$(shell cd containers && vagrant ssh-config | awk '/HostName/{print $$2}')" +VAGRANT_SSH_IDENTITY_FILE = "$(shell cd containers && vagrant ssh-config | awk '/IdentityFile/{print $$2}')" +VAGRANT_SSH_CONFIG = $(shell cd containers && vagrant ssh-config | awk 'NR>1 {print " -o "$$1"="$$2}') +VAGRANT_SHARED_DIR = "/vagrant" + +CENTOS_VAGRANT_BOX = CentOS-Stream-Vagrant-8-latest.x86_64.vagrant-libvirt.box +CENTOS_VAGRANT_URL = https://cloud.centos.org/centos/8-stream/x86_64/images/$(CENTOS_VAGRANT_BOX) ifneq "$(shell whoami)" "root" ASK_PASS ?= --ask-become-pass endif # Only for Packit team members with access to Bitwarden vault +# if not working prepend OPENSSL_CONF=/dev/null to script invocation download-secrets: ./scripts/download_secrets.sh @@ -50,3 +60,31 @@ check: move-stable: [[ -d move_stable_repositories ]] || scripts/move_stable.py init scripts/move_stable.py move-all + +oc-cluster-create: +# vagrant pointer is broken... + [[ -f $(CENTOS_VAGRANT_BOX) ]] || wget $(CENTOS_VAGRANT_URL) + cd containers && vagrant up + +oc-cluster-destroy: + cd containers && vagrant destroy + +oc-cluster-up: + cd containers && vagrant up + cd containers && vagrant ssh -c "cd $(VAGRANT_SHARED_DIR) && $(AP) playbooks/oc-cluster-run.yml" + +oc-cluster-down: + cd containers && vagrant halt + +oc-cluster-ssh: oc-cluster-up + ssh $(VAGRANT_SSH_CONFIG) localhost + +test-deploy: +# to be run inside VM where the oc cluster is running! Call make tmt-tests instead. + DEPLOYMENT=dev $(AP) playbooks/generate-local-secrets.yml + DEPLOYMENT=dev $(AP) -e '{"src_dir": $(VAGRANT_SHARED_DIR)}' playbooks/test_deploy_setup.yml + DEPLOYMENT=dev $(AP) -e '{"container_engine": "podman", "registry": "default-route-openshift-image-registry.apps-crc.testing", "registry_user": "kubeadmin", "src_dir": $(VAGRANT_SHARED_DIR)}' playbooks/deploy.yml + DEPLOYMENT=dev $(AP) -e '{"container_engine": "podman", "registry": "default-route-openshift-image-registry.apps-crc.testing", "registry_user": "kubeadmin", "src_dir": $(VAGRANT_SHARED_DIR)}' playbooks/check.yml + +tmt-tests: + tmt run --all provision --how connect --user vagrant --guest $(VAGRANT_SSH_GUEST) --port $(VAGRANT_SSH_PORT) --key $(VAGRANT_SSH_IDENTITY_FILE) diff --git a/containers/Vagrantfile b/containers/Vagrantfile new file mode 100644 index 0000000..067d63d --- /dev/null +++ b/containers/Vagrantfile @@ -0,0 +1,94 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +VAGRANTFILE_API_VERSION = "2" +# export VAGRANT_EXPERIMENTAL="1" +# export VAGRANT_EXPERIMENTAL="disks" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + config.vm.box = "centos/stream8" + #config.vm.box_url = "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-Vagrant-8-latest.x86_64.vagrant-libvirt.box" gives 404 + config.vm.box_url = "file:///$VagrantProjectHome/../CentOS-Stream-Vagrant-8-latest.x86_64.vagrant-libvirt.box" + + + # Forward traffic on the host to the development server on the guest + config.vm.network "forwarded_port", guest: 5000, host: 5000 + # Forward traffic on the host to Redis on the guest + config.vm.network "forwarded_port", guest: 6379, host: 6379 + # Forward traffic on the host to the SSE server on the guest + config.vm.network "forwarded_port", guest: 8080, host: 8080 + + + if Vagrant.has_plugin?("vagrant-hostmanager") + config.hostmanager.enabled = true + config.hostmanager.manage_host = true + end + + # Vagrant can share the source directory using rsync, NFS, or SSHFS (with the vagrant-sshfs + # plugin). By default it rsyncs the current working directory to /vagrant. + # + # If you would prefer to use NFS to share the directory uncomment this and configure NFS + # config.vm.synced_folder ".", "/vagrant", type: "nfs", nfs_version: 4, nfs_udp: false + config.vm.synced_folder "..", "/vagrant" + # config.vm.synced_folder ".", "/vagrant", disabled: true + # config.vm.synced_folder ".", "/srv/pagure", + # ssh_opts_append: "-o IdentitiesOnly=yes", + # type: "sshfs" + + # To cache update packages (which is helpful if frequently doing `vagrant destroy && vagrant up`) + # you can create a local directory and share it to the guest's DNF cache. The directory needs to + # exist, so create it before you uncomment the line below. + #Dir.mkdir('.dnf-cache') unless File.exists?('.dnf-cache') + #config.vm.synced_folder ".dnf-cache", "/var/cache/dnf", + # type: "sshfs", + # sshfs_opts_append: "-o nonempty" + + # Comment this line if you would like to disable the automatic update during provisioning + config.vm.provision "shell", inline: "sudo dnf -y --disablerepo '*' --enablerepo=extras swap centos-linux-repos centos-stream-repos" + + # !!!!!!! resize disk image !!!!!!!!! + config.vm.provision "shell", inline: "sudo dnf install -y cloud-utils-growpart" + config.vm.provision "shell", inline: "sudo growpart /dev/vda 1" + config.vm.provision "shell", inline: "sudo xfs_growfs /dev/vda1" + + # bootstrap and run with ansible + config.vm.provision "ansible" do |ansible| + # ansible.verbose = "-vvv" + ansible.verbose = true + ansible.playbook = "../playbooks/oc-cluster-setup.yml" + end + config.vm.provision "ansible" do |ansible| + # ansible.verbose = "-vvv" + ansible.verbose = true + ansible.playbook = "../playbooks/oc-cluster-run.yml" + ansible.extra_vars = "../secrets/openshift-local-pull-secret.yml" + end + config.vm.provision "ansible" do |ansible| + # ansible.verbose = "-vvv" + ansible.verbose = true + ansible.playbook = "../playbooks/oc-cluster-tests-setup.yml" + ansible.extra_vars = {"src_dir": "/vagrant"} + end + + # Create the box + config.vm.define "packit-oc-cluster" do |oc| + oc.vm.host_name = "packit-oc-cluster.example.com" + + oc.vm.provider :libvirt do |domain| + # Season to taste + domain.cpus = 6 + domain.graphics_type = "spice" + domain.memory = 14336 + domain.video_type = "qxl" + domain.machine_virtual_size = 100 + + # Uncomment the following line if you would like to enable libvirt's unsafe cache + # mode. It is called unsafe for a reason, as it causes the virtual host to ignore all + # fsync() calls from the guest. Only do this if you are comfortable with the possibility of + # your development guest becoming corrupted (in which case you should only need to do a + # vagrant destroy and vagrant up to get a new one). + # + # domain.volume_cache = "unsafe" + end + end +end diff --git a/docs/deployment/testing-changes.md b/docs/deployment/testing-changes.md index 7917b63..b12b3c7 100644 --- a/docs/deployment/testing-changes.md +++ b/docs/deployment/testing-changes.md @@ -68,3 +68,38 @@ This repository provides helpful playbook to do this with one command: Zuul provides a public key for every project. The ansible playbook downloads Zuul repository and pass the project tenant and name as parameters to encryption script. This script then encrypts files with public key of the project. For more information please refer to [official docs](https://ansible.softwarefactory-project.io/docs/user/zuul_user.html#create-a-secret-to-be-used-in-jobs). + +### Test Deployment locally with OpenShift Local + +For using OpenShift Local you need a _pull secret_, download it here: https://console.redhat.com/openshift/create/local. Save it in a file called `secrets\openshift-local-pull-secret.yml` following this format: + +``` +--- +pull_secret: <<< DOWNLOADED PULL SECRET CONTENT >>> +``` + +Populate the `secrets` dir with all the other secrets. +You _should use_ your own secrets but if you have access to `stg` secrets +you can also do: + +``` +DEPLOYMENT=stg make download-secrets +``` + +Now you can create and start the OpenShift Local cluster (it take as long as an hour) in a Vagrant Virtual Machine with: + +``` +make oc-cluster-create +``` + +And once it is up and running you can test the `packit-service` deployment with the command: + +``` +make tmt-tests +``` + +This command will sshed the virtual machine and run the tests there (`make test-deploy`), +you can run the tests as many time you want as long as the virtual machine is up and running and the `crc cluster` is started (`make oc-cluster-up` after every `make oc-cluster-down`). +You can skip the `tmt` environment and run the test directly inside the VM using `make oc-cluster-ssh` and `cd /vagrant && make test-deploy`. + +You can destroy the `libvirt` machine with `make oc-cluster-destroy` and re-create it again with `make oc-cluster-create`. diff --git a/openshift/redis.yml.j2 b/openshift/redis.yml.j2 index a2fe295..d459fd9 100644 --- a/openshift/redis.yml.j2 +++ b/openshift/redis.yml.j2 @@ -20,7 +20,7 @@ spec: spec: containers: - name: redis - image: quay.io/sclorg/redis-7-c9s + image: quay.io/sclorg/redis-7-c9s:c9s ports: - containerPort: 6379 volumeMounts: diff --git a/plans/main.fmf b/plans/main.fmf new file mode 100644 index 0000000..f0d5726 --- /dev/null +++ b/plans/main.fmf @@ -0,0 +1,13 @@ +summary: Packit deployment checks + +discover: + how: fmf + +execute: + how: tmt + +/deployment: + summary: Run packit-service deployment test + discover+: + test: + - deployment diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index fc625a2..5cb2767 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -65,6 +65,8 @@ memory: "128Mi" cpu: "50m" appcode: PCKT-002 + registry: 172.30.1.1:5000 + registry_user: developer tasks: - name: Include tasks/project-dir.yml ansible.builtin.include_tasks: tasks/project-dir.yml @@ -124,25 +126,25 @@ tls_verify_false: "{{ '--tls-verify=false' if 'podman' in container_engine else '' }}" changed_when: false - name: Login to local cluster - ansible.builtin.shell: "{{ container_engine }} login -u developer -p $(oc whoami -t) 172.30.1.1:5000 {{ tls_verify_false }}" + ansible.builtin.shell: "{{ container_engine }} login -u {{ registry_user }} -p $(oc whoami -t) {{ registry }} {{ tls_verify_false }}" changed_when: false - name: Inspect service image ansible.builtin.command: "{{ container_engine }} inspect {{ image }}" changed_when: false - name: Tag the image with :dev - ansible.builtin.command: "{{ container_engine }} tag {{ image }} 172.30.1.1:5000/myproject/packit-service:dev" + ansible.builtin.command: "{{ container_engine }} tag {{ image }} {{ registry }}/myproject/packit-service:dev" changed_when: true - name: Push the image - ansible.builtin.command: "{{ container_engine }} push 172.30.1.1:5000/myproject/packit-service:dev {{ tls_verify_false }}" + ansible.builtin.command: "{{ container_engine }} push {{ registry }}/myproject/packit-service:dev {{ tls_verify_false }}" changed_when: true - name: Inspect worker image ansible.builtin.command: "{{ container_engine }} inspect {{ image_worker }}" changed_when: false - name: Tag the image with :dev - ansible.builtin.command: "{{ container_engine }} tag {{ image_worker }} 172.30.1.1:5000/myproject/packit-worker:dev" + ansible.builtin.command: "{{ container_engine }} tag {{ image_worker }} {{ registry }}/myproject/packit-worker:dev" changed_when: true - name: Push the image - ansible.builtin.command: "{{ container_engine }} push 172.30.1.1:5000/myproject/packit-worker:dev {{ tls_verify_false }}" + ansible.builtin.command: "{{ container_engine }} push {{ registry }}/myproject/packit-worker:dev {{ tls_verify_false }}" changed_when: true - name: Deploy secrets @@ -440,7 +442,7 @@ - name: Wait for deploymentconfig rollouts to complete # timeout 10min to not wait indefinitely in case of a problem - ansible.builtin.command: timeout 10m oc rollout status -w deploy/{{ item }} + ansible.builtin.command: timeout 15m oc rollout status -w deploy/{{ item }} register: oc_rollout_status changed_when: false failed_when: '"successfully rolled out" not in oc_rollout_status.stdout' diff --git a/playbooks/oc-cluster-run.yml b/playbooks/oc-cluster-run.yml new file mode 100644 index 0000000..846c1f0 --- /dev/null +++ b/playbooks/oc-cluster-run.yml @@ -0,0 +1,19 @@ +--- +- name: Start Openshift Local cluster + hosts: all + gather_facts: False + vars: + pull_secret: "!!!TAKEN FROM VAGRANT FILE!!!" + dest_pull_secret_file: /tmp/openshift-local-pull-secret.txt + tasks: + - name: Show pull secret + ansible.builtin.debug: + msg: "{{ pull_secret }}" + - name: Create secret file from pull request var + ansible.builtin.copy: + content: "{{ pull_secret }}" + dest: "{{ dest_pull_secret_file }}" + mode: "0640" + - name: Start cluster + ansible.builtin.command: crc start -p {{ dest_pull_secret_file }} + changed_when: False diff --git a/playbooks/oc-cluster-setup.yml b/playbooks/oc-cluster-setup.yml new file mode 100644 index 0000000..5487425 --- /dev/null +++ b/playbooks/oc-cluster-setup.yml @@ -0,0 +1,52 @@ +--- +- name: Install an Openshift Local cluster + hosts: all + vars: + crc_download_link: https://developers.redhat.com/content-gateway/rest/mirror/pub/openshift-v4/clients/crc/latest/crc-linux-amd64.tar.xz + tasks: + - name: Add vagrant user to sudoers + become: True + ansible.builtin.lineinfile: + path: /etc/sudoers + state: present + line: "vagrant ALL=(ALL) NOPASSWD: ALL" + validate: /usr/sbin/visudo -cf %s + - name: Download CRC + ansible.builtin.get_url: + url: "{{ crc_download_link }}" + dest: /tmp + mode: "0664" + register: download + - name: Unarchive tarball + ansible.builtin.unarchive: + src: "{{ download.dest }}" + dest: /tmp + list_files: True + remote_src: True + register: unarchive + - name: Show crc executable path + ansible.builtin.debug: + msg: "/tmp/{{ unarchive.files[0] }}crc" + - name: Create ~/.local/bin if it does not exist + ansible.builtin.file: + path: ~/.local/bin + state: directory + mode: "0755" + - name: Create symbolic link to crc + ansible.builtin.file: + src: "/tmp/{{ unarchive.files[0] }}/crc" + dest: ~/.local/bin/crc + state: link + force: True + - name: Setup consent-telemetry no + ansible.builtin.command: crc config set consent-telemetry no + changed_when: False + - name: Setup minimum packit-service memory requirements + ansible.builtin.command: crc config set memory 13312 MiB + changed_when: False + - name: Setup minimum packit-service disk space requirements + ansible.builtin.command: crc config set disk-size 60 Gi + changed_when: False + - name: Setup cluster + ansible.builtin.command: crc setup + changed_when: False diff --git a/playbooks/oc-cluster-tests-setup.yml b/playbooks/oc-cluster-tests-setup.yml new file mode 100644 index 0000000..37e511b --- /dev/null +++ b/playbooks/oc-cluster-tests-setup.yml @@ -0,0 +1,45 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +--- +- name: Create vars/packit/dev.yml and deploy + hosts: all + vars: + oc_download_link: https://mirror.openshift.com/pub/openshift-v4/amd64/clients/ocp/stable/openshift-client-linux.tar.gz + test_project_name: myproject + crc_url: https://api.crc.testing:6443 + image: quay.io/packit/packit-service:stg + image_worker: quay.io/packit/packit-worker:stg + image_fedmsg: quay.io/packit/packit-service-fedmsg:stg + image_dashboard: quay.io/packit/dashboard:stg + image_tokman: quay.io/packit/tokman:stg + tasks: + - name: Enable EPEL repositories + ansible.builtin.package: + name: + - epel-release + become: true + - name: Install packages for deployment + ansible.builtin.package: + name: + - ansible + - python3-openshift + - python3-pip + - python3-passlib # for using htpasswd ansible module + - make + - podman + become: true + + - name: Download OC + ansible.builtin.get_url: + url: "{{ oc_download_link }}" + dest: /tmp + mode: "0644" + register: download + - name: Unarchive tarball in /usr/bin + become: True + ansible.builtin.unarchive: + src: "{{ download.dest }}" + dest: /usr/bin + list_files: True + remote_src: True diff --git a/playbooks/roles/generate_secrets/tasks/generate-cert.yml b/playbooks/roles/generate_secrets/tasks/generate-cert.yml index 640362c..7004506 100644 --- a/playbooks/roles/generate_secrets/tasks/generate-cert.yml +++ b/playbooks/roles/generate_secrets/tasks/generate-cert.yml @@ -7,6 +7,14 @@ name: - python-cryptography become: true + when: (ansible_facts['distribution'] == "CentOS" and ansible_facts['distribution_major_version'] == "7") + +- name: Install Python cryptography module + ansible.builtin.package: + name: + - python3-cryptography + become: true + when: (ansible_facts['distribution'] == "CentOS" and ansible_facts['distribution_major_version'] == "8") - name: Generate OpenSSL private keys openssl_privatekey: diff --git a/playbooks/test_deploy_setup.yml b/playbooks/test_deploy_setup.yml new file mode 100644 index 0000000..2159640 --- /dev/null +++ b/playbooks/test_deploy_setup.yml @@ -0,0 +1,80 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +--- +- name: Ensure crc is started, create openshift project and vars/packit/dev.yml + hosts: all + vars: + test_project_name: myproject + crc_url: https://api.crc.testing:6443 + image: quay.io/packit/packit-service:stg + image_worker: quay.io/packit/packit-worker:stg + image_fedmsg: quay.io/packit/packit-service-fedmsg:stg + image_dashboard: quay.io/packit/dashboard:stg + image_tokman: quay.io/packit/tokman:stg + tasks: + - name: Ensure crc is started + ansible.builtin.command: crc start + changed_when: False + + - name: Get token + ansible.builtin.command: oc whoami -t + register: kubeconfig_token + changed_when: false + + - name: Delete test project namespace if already exist + kubernetes.core.k8s: + name: "{{ test_project_name }}" + api_version: v1 + kind: Namespace + state: absent + + - name: Create test project namespace + kubernetes.core.k8s: + name: "{{ test_project_name }}" + api_version: v1 + kind: Namespace + state: present + + - name: | + Get packit stage images to be used as dev images, + if you want to use local dev images change var values + and point to them + ansible.builtin.command: podman pull "{{ item }}" + loop: + - "{{ image }}" + - "{{ image_worker }}" + - "{{ image_fedmsg }}" + - "{{ image_dashboard }}" + - "{{ image_tokman }}" + loop_control: + pause: 1 # otherwise getting 500 Internal Server Error from registry + changed_when: False + + - name: Create packit/dev.yml + ansible.builtin.copy: + content: | + # See https://github.com/packit/deployment/blob/main/vars/packit/dev_template.yml + project: {{ test_project_name }} + host: {{ crc_url }} + api_key: {{ kubeconfig_token.stdout }} + validate_certs: false + check_up_to_date: false + # Whether to deploy and check that pod + # Let's preserve some resources + with_tokman: true + with_beat: true + with_fedmsg: true + with_dashboard: true + with_flower: false + with_fluentd_sidecar: false + with_pushgateway: false + # Use stg images + push_dev_images: true + image: {{ image }} + image_worker: {{ image_worker }} + image_fedmsg: {{ image_fedmsg }} + image_dashboard: {{ image_dashboard }} + image_tokman: {{ image_tokman }} + dest: "{{ src_dir }}/vars/packit/dev.yml" + mode: 0644 diff --git a/tests/deployment/main.fmf b/tests/deployment/main.fmf new file mode 100644 index 0000000..d8ab015 --- /dev/null +++ b/tests/deployment/main.fmf @@ -0,0 +1,12 @@ +test: + cd /vagrant && make test-deploy + +duration: 10m + +/deployment: + summary: Run packit-service deployment test + tags: [deployment] + duration: 60m + tier: 1 + environment+: + DEPLOYMENT: dev From 37346b7382a127df7849464fcfef57cd355f0553 Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Thu, 6 Jun 2024 11:02:54 +0200 Subject: [PATCH 02/12] Run packit-service deployment test inside tmt This commit breaks vagrant test! To be able to run OpenShift Local inside a local provisione tmt virtual machine you need to change tmt sources: change DEFAULT_CPU_COUNT in tmt/steps/provision/testcloud.py to 6 2 cpus are not enough for Openshift Local and is is not custumizable (will open an issue in tmt but our goal is to provision the machine in the cloud so this is not strictly necessary, for local usage is probably better to make vagrant work again) --- Makefile | 23 +- plans/deployment.fmf | 63 ++++ plans/main.fmf | 13 - playbooks/oc-cluster-run.yml | 29 +- playbooks/oc-cluster-setup.yml | 57 ++-- playbooks/oc-cluster-tests-setup.yml | 36 +- playbooks/oc-cluster-user.yml | 19 ++ playbooks/roles/deploy/defaults/main.yml | 62 ++++ playbooks/roles/deploy/handlers/main.yml | 21 ++ playbooks/roles/deploy/tasks/main.yml | 410 +++++++++++++++++++++++ playbooks/test_deploy.yml | 22 ++ playbooks/test_deploy_setup.yml | 32 +- tests/deployment/main.fmf | 25 +- 13 files changed, 735 insertions(+), 77 deletions(-) create mode 100644 plans/deployment.fmf delete mode 100644 plans/main.fmf create mode 100644 playbooks/oc-cluster-user.yml create mode 100644 playbooks/roles/deploy/defaults/main.yml create mode 100644 playbooks/roles/deploy/handlers/main.yml create mode 100644 playbooks/roles/deploy/tasks/main.yml create mode 100644 playbooks/test_deploy.yml diff --git a/Makefile b/Makefile index f6d19d1..59dd939 100755 --- a/Makefile +++ b/Makefile @@ -11,7 +11,8 @@ VAGRANT_SSH_USER = "$(shell cd containers && vagrant ssh-config | awk '/User/{pr VAGRANT_SSH_GUEST = "$(shell cd containers && vagrant ssh-config | awk '/HostName/{print $$2}')" VAGRANT_SSH_IDENTITY_FILE = "$(shell cd containers && vagrant ssh-config | awk '/IdentityFile/{print $$2}')" VAGRANT_SSH_CONFIG = $(shell cd containers && vagrant ssh-config | awk 'NR>1 {print " -o "$$1"="$$2}') -VAGRANT_SHARED_DIR = "/vagrant" +#VAGRANT_SHARED_DIR = "/vagrant" +VAGRANT_SHARED_DIR = "/home/tmt/deployment" CENTOS_VAGRANT_BOX = CentOS-Stream-Vagrant-8-latest.x86_64.vagrant-libvirt.box CENTOS_VAGRANT_URL = https://cloud.centos.org/centos/8-stream/x86_64/images/$(CENTOS_VAGRANT_BOX) @@ -80,11 +81,23 @@ oc-cluster-ssh: oc-cluster-up ssh $(VAGRANT_SSH_CONFIG) localhost test-deploy: -# to be run inside VM where the oc cluster is running! Call make tmt-tests instead. +# to be run inside VM where the oc cluster is running! Call make tmt-vagrant-tests instead from outside the vagrant machine. DEPLOYMENT=dev $(AP) playbooks/generate-local-secrets.yml DEPLOYMENT=dev $(AP) -e '{"src_dir": $(VAGRANT_SHARED_DIR)}' playbooks/test_deploy_setup.yml - DEPLOYMENT=dev $(AP) -e '{"container_engine": "podman", "registry": "default-route-openshift-image-registry.apps-crc.testing", "registry_user": "kubeadmin", "src_dir": $(VAGRANT_SHARED_DIR)}' playbooks/deploy.yml - DEPLOYMENT=dev $(AP) -e '{"container_engine": "podman", "registry": "default-route-openshift-image-registry.apps-crc.testing", "registry_user": "kubeadmin", "src_dir": $(VAGRANT_SHARED_DIR)}' playbooks/check.yml + cd $(VAGRANT_SHARED_DIR); DEPLOYMENT=dev $(AP) -e '{"container_engine": "podman", "registry": "default-route-openshift-image-registry.apps-crc.testing", "registry_user": "kubeadmin", "src_dir": $(VAGRANT_SHARED_DIR)}' playbooks/test_deploy.yml -tmt-tests: +tmt-vagrant-test: tmt run --all provision --how connect --user vagrant --guest $(VAGRANT_SSH_GUEST) --port $(VAGRANT_SSH_PORT) --key $(VAGRANT_SSH_IDENTITY_FILE) + +tf-deploy: + testing-farm request --compose Fedora-Rawhide --git-url https://github.com/majamassarini/deployment --git-ref tf-openshift-tests --plan deployment + +# tmt run --id packit-service-deployment --until execute +# tmt run --id packit-service-deployment prepare --force +# tmt run --id packit-service-deployment login --step prepare:start +# tmt run --id packit-service-deployment execute --force +# tmt run --id packit-service-deployment login --step execute:start +# tmt run --id packit-service-deployment finish +# tmt run --id packit-service-deployment clean + +# virsh list --all diff --git a/plans/deployment.fmf b/plans/deployment.fmf new file mode 100644 index 0000000..39ba7bb --- /dev/null +++ b/plans/deployment.fmf @@ -0,0 +1,63 @@ +tier: 1 + +execute: + how: tmt + +environment: + DEPLOYMENT: dev + #ANSIBLE_DEBUG: 1 + +report: + how: display + +discover: + how: fmf + test: + - deployment + +/local: + summary: Run packit-service deployment test on a local machine managed by tmt + discover: + how: fmf + test: + - deployment + provision: + how: virtual + image: centos-stream + disk: 100 + memory: 14336 + # cpu.cores: 6 # I can not customize cpu cores in virtual testcloud prepare plugin + # but OpenShift Cluster will not start without 4 cores at least + # change DEFAULT_CPU_COUNT in tmt/steps/provision/testcloud.py to 6 + prepare: + - how: ansible + playbook: playbooks/oc-cluster-user.yml + extra-args: '-vvv' + - how: ansible + playbook: playbooks/oc-cluster-setup.yml + extra-args: '-vvv' + - how: ansible + playbook: playbooks/oc-cluster-run.yml + extra-args: '-vvv --extra-vars @./secrets/openshift-local-pull-secret.yml' + - how: ansible + playbook: playbooks/oc-cluster-tests-setup.yml + extra-args: '-vvv' + #- how: ansible + # playbook: playbooks/generate-local-secrets.yml + # extra-args: '-vvv --extra-vars generate_secrets_project_dir="./"' + #- how: ansible + # playbook: playbooks/test_deploy_setup.yml + # extra-args: '-vvv' + +#/vagrant: +# summary: Run packit-service deployment test on a vagrant machine already up and running +# +# description: +# guest, port and key details are taken from the tmt-vagrant-test Makefile target. +# This provision is meant to be used just locally with the Vagrant machine already up +# and running. To start the Vagrant machine use the oc-cluster-up Makefile target. +# To access the Vagrant machine use the oc-cluster-ssh Makefile target. +# +# provision: +# how: connect +# user: vagrant diff --git a/plans/main.fmf b/plans/main.fmf deleted file mode 100644 index f0d5726..0000000 --- a/plans/main.fmf +++ /dev/null @@ -1,13 +0,0 @@ -summary: Packit deployment checks - -discover: - how: fmf - -execute: - how: tmt - -/deployment: - summary: Run packit-service deployment test - discover+: - test: - - deployment diff --git a/playbooks/oc-cluster-run.yml b/playbooks/oc-cluster-run.yml index 846c1f0..f9bba11 100644 --- a/playbooks/oc-cluster-run.yml +++ b/playbooks/oc-cluster-run.yml @@ -1,10 +1,13 @@ --- - name: Start Openshift Local cluster hosts: all + become: true + become_user: tmt gather_facts: False vars: - pull_secret: "!!!TAKEN FROM VAGRANT FILE!!!" + pull_secret: "!!!TAKEN FROM ANSIBLE ENV!!!" dest_pull_secret_file: /tmp/openshift-local-pull-secret.txt + crc_path: /home/tmt/.local/bin tasks: - name: Show pull secret ansible.builtin.debug: @@ -14,6 +17,28 @@ content: "{{ pull_secret }}" dest: "{{ dest_pull_secret_file }}" mode: "0640" + owner: tmt + + - name: Find user uid + ansible.builtin.command: "id -u tmt" + register: user_uid + changed_when: false + + - name: Determine XDG_RUNTIME_DIR + ansible.builtin.set_fact: + xdg_runtime_dir: "/run/user/{{ user_uid.stdout }}" + changed_when: false + - name: Start cluster - ansible.builtin.command: crc start -p {{ dest_pull_secret_file }} + ansible.builtin.command: "{{ crc_path }}/crc start -c 6 -m 12000 -p {{ dest_pull_secret_file }}" + #ansible.builtin.command: "crc start -p {{ dest_pull_secret_file }}" changed_when: False + + - name: Create symbolic link to oc + ansible.builtin.file: + # src: "/tmp/{{ unarchive.files[0] }}/crc" + src: /home/tmt/.crc/bin/oc/oc + # dest: "{{ crc_path }}/crc" + dest: /home/tmt/.local/bin/oc + state: link + force: True diff --git a/playbooks/oc-cluster-setup.yml b/playbooks/oc-cluster-setup.yml index 5487425..76a6cd3 100644 --- a/playbooks/oc-cluster-setup.yml +++ b/playbooks/oc-cluster-setup.yml @@ -1,21 +1,18 @@ --- - name: Install an Openshift Local cluster + become: true + become_user: tmt hosts: all vars: crc_download_link: https://developers.redhat.com/content-gateway/rest/mirror/pub/openshift-v4/clients/crc/latest/crc-linux-amd64.tar.xz + crc_path: /home/tmt/.local/bin + user: tmt tasks: - - name: Add vagrant user to sudoers - become: True - ansible.builtin.lineinfile: - path: /etc/sudoers - state: present - line: "vagrant ALL=(ALL) NOPASSWD: ALL" - validate: /usr/sbin/visudo -cf %s - name: Download CRC ansible.builtin.get_url: url: "{{ crc_download_link }}" dest: /tmp - mode: "0664" + mode: "0777" register: download - name: Unarchive tarball ansible.builtin.unarchive: @@ -23,30 +20,50 @@ dest: /tmp list_files: True remote_src: True + owner: tmt register: unarchive - - name: Show crc executable path - ansible.builtin.debug: - msg: "/tmp/{{ unarchive.files[0] }}crc" - name: Create ~/.local/bin if it does not exist ansible.builtin.file: - path: ~/.local/bin + path: "{{ crc_path }}" state: directory mode: "0755" - - name: Create symbolic link to crc - ansible.builtin.file: + - name: Show crc executable path + ansible.builtin.debug: + msg: "/tmp/{{ unarchive.files[0] }}crc" + - name: Copy crc to tmt usr + ansible.builtin.copy: src: "/tmp/{{ unarchive.files[0] }}/crc" - dest: ~/.local/bin/crc - state: link + remote_src: true + dest: "{{ crc_path }}/crc" + mode: "0755" force: True + - name: Setup consent-telemetry no - ansible.builtin.command: crc config set consent-telemetry no + ansible.builtin.command: "{{ crc_path }}/crc config set consent-telemetry no" + changed_when: False + - name: Setup disable-update-check true + ansible.builtin.command: "{{ crc_path }}/crc config set disable-update-check true" + changed_when: False + - name: Setup monitoring no + ansible.builtin.command: "{{ crc_path }}/crc config set enable-cluster-monitoring false" changed_when: False - name: Setup minimum packit-service memory requirements - ansible.builtin.command: crc config set memory 13312 MiB + ansible.builtin.command: "{{ crc_path }}/crc config set memory 13312 MiB" changed_when: False - name: Setup minimum packit-service disk space requirements - ansible.builtin.command: crc config set disk-size 60 Gi + ansible.builtin.command: "{{ crc_path }}/crc config set disk-size 60 Gi" changed_when: False + + - name: Find user uid + ansible.builtin.command: "id -u tmt" + register: user_uid + changed_when: false + + - name: Determine XDG_RUNTIME_DIR + ansible.builtin.set_fact: + xdg_runtime_dir: "/run/user/{{ user_uid.stdout }}" + changed_when: false + - name: Setup cluster - ansible.builtin.command: crc setup + ansible.builtin.command: "{{ crc_path }}/crc setup" changed_when: False diff --git a/playbooks/oc-cluster-tests-setup.yml b/playbooks/oc-cluster-tests-setup.yml index 37e511b..b35536f 100644 --- a/playbooks/oc-cluster-tests-setup.yml +++ b/playbooks/oc-cluster-tests-setup.yml @@ -2,23 +2,15 @@ # SPDX-License-Identifier: MIT --- -- name: Create vars/packit/dev.yml and deploy +- name: Install packit-service dependencies hosts: all vars: oc_download_link: https://mirror.openshift.com/pub/openshift-v4/amd64/clients/ocp/stable/openshift-client-linux.tar.gz - test_project_name: myproject - crc_url: https://api.crc.testing:6443 - image: quay.io/packit/packit-service:stg - image_worker: quay.io/packit/packit-worker:stg - image_fedmsg: quay.io/packit/packit-service-fedmsg:stg - image_dashboard: quay.io/packit/dashboard:stg - image_tokman: quay.io/packit/tokman:stg tasks: - name: Enable EPEL repositories ansible.builtin.package: name: - epel-release - become: true - name: Install packages for deployment ansible.builtin.package: name: @@ -28,18 +20,16 @@ - python3-passlib # for using htpasswd ansible module - make - podman - become: true - - name: Download OC - ansible.builtin.get_url: - url: "{{ oc_download_link }}" - dest: /tmp - mode: "0644" - register: download - - name: Unarchive tarball in /usr/bin - become: True - ansible.builtin.unarchive: - src: "{{ download.dest }}" - dest: /usr/bin - list_files: True - remote_src: True + #- name: Download OC + # ansible.builtin.get_url: + # url: "{{ oc_download_link }}" + # dest: /tmp + # mode: "0644" + # register: download + #- name: Unarchive tarball in /usr/bin + # ansible.builtin.unarchive: + # src: "{{ download.dest }}" + # dest: /usr/bin + # list_files: True + # remote_src: True diff --git a/playbooks/oc-cluster-user.yml b/playbooks/oc-cluster-user.yml new file mode 100644 index 0000000..b9042b0 --- /dev/null +++ b/playbooks/oc-cluster-user.yml @@ -0,0 +1,19 @@ +--- +- name: Configure a user able to run a Openshift Local Cluster + hosts: all + tasks: + - name: Add tmt user + ansible.builtin.user: + name: tmt + append: true + groups: wheel + - name: Add tmt user to sudoers + ansible.builtin.lineinfile: + path: /etc/sudoers + state: present + #line: "{{ user }} ALL=(ALL) NOPASSWD: ALL" + line: "tmt ALL=(ALL) NOPASSWD: ALL" + validate: /usr/sbin/visudo -cf %s + - name: Enable lingering for tmt + ansible.builtin.command: loginctl enable-linger tmt + changed_when: False diff --git a/playbooks/roles/deploy/defaults/main.yml b/playbooks/roles/deploy/defaults/main.yml new file mode 100644 index 0000000..31a61d9 --- /dev/null +++ b/playbooks/roles/deploy/defaults/main.yml @@ -0,0 +1,62 @@ +--- +validate_certs: true # noqa: var-naming[no-role-prefix] +service: "{{ lookup('env', 'SERVICE') | default('packit', True) }}" # noqa: var-naming[no-role-prefix] +deployment: "{{ lookup('env', 'DEPLOYMENT') }}" # noqa: var-naming[no-role-prefix] +tenant: packit # noqa: var-naming[no-role-prefix] # MP+ tenant +with_tokman: true # noqa: var-naming[no-role-prefix] +with_fedmsg: true # noqa: var-naming[no-role-prefix] +with_redis: false # noqa: var-naming[no-role-prefix] +with_redict: true # noqa: var-naming[no-role-prefix] +with_redis_commander: false # noqa: var-naming[no-role-prefix] +with_flower: false # noqa: var-naming[no-role-prefix] +with_dashboard: true # noqa: var-naming[no-role-prefix] +with_beat: true # noqa: var-naming[no-role-prefix] +with_pushgateway: true # noqa: var-naming[no-role-prefix] +with_repository_cache: true # noqa: var-naming[no-role-prefix] +repository_cache_storage: 4Gi # noqa: var-naming[no-role-prefix] +push_dev_images: false # noqa: var-naming[no-role-prefix] +with_fluentd_sidecar: false # noqa: var-naming[no-role-prefix] +postgres_version: 13 # noqa: var-naming[no-role-prefix] +image: quay.io/packit/packit-service:{{ deployment }} # noqa: var-naming[no-role-prefix] +image_worker: quay.io/packit/packit-worker:{{ deployment }} # noqa: var-naming[no-role-prefix] +image_fedmsg: quay.io/packit/packit-service-fedmsg:{{ deployment }} # noqa: var-naming[no-role-prefix] +image_dashboard: quay.io/packit/dashboard:{{ deployment }} # noqa: var-naming[no-role-prefix] +image_tokman: quay.io/packit/tokman:{{ deployment }} # noqa: var-naming[no-role-prefix] +image_fluentd: quay.io/packit/fluentd-splunk-hec:latest # noqa: var-naming[no-role-prefix] +# project_dir is set in tasks/project-dir.yml +path_to_secrets: "{{ project_dir }}/secrets/{{ service }}/{{ deployment }}" # noqa: var-naming[no-role-prefix] +# to be used in Image streams as importPolicy:scheduled value +auto_import_images: true # noqa: var-naming[no-role-prefix] +# used in dev/zuul deployment to tag & push images to cluster +# https://github.com/packit/deployment/issues/112#issuecomment-673343049 +# container_engine: "{{ lookup('pipe', 'command -v podman 2> /dev/null || echo docker') }}" +container_engine: docker # noqa: var-naming[no-role-prefix] +celery_app: packit_service.worker.tasks # noqa: var-naming[no-role-prefix] +celery_retry_limit: 2 # noqa: var-naming[no-role-prefix] +celery_retry_backoff: 3 # noqa: var-naming[no-role-prefix] +workers_all_tasks: 1 # noqa: var-naming[no-role-prefix] +workers_short_running: 0 # noqa: var-naming[no-role-prefix] +workers_long_running: 0 # noqa: var-naming[no-role-prefix] +distgit_url: https://src.fedoraproject.org/ # noqa: var-naming[no-role-prefix] +distgit_namespace: rpms # noqa: var-naming[no-role-prefix] +sourcegit_namespace: "" # noqa: var-naming[no-role-prefix] # fedora-source-git only +pushgateway_address: http://pushgateway # noqa: var-naming[no-role-prefix] +# Check that the deployment repo is up-to-date +check_up_to_date: true # noqa: var-naming[no-role-prefix] +# Check that the current vars file is up-to-date with the template +check_vars_template_diff: true # noqa: var-naming[no-role-prefix] +deployment_repo_url: https://github.com/packit/deployment.git # noqa: var-naming[no-role-prefix] +# used by a few tasks below +k8s_apply: false # noqa: var-naming[no-role-prefix] +tokman: # noqa: var-naming[no-role-prefix] + workers: 1 + resources: + requests: + memory: "88Mi" + cpu: "5m" + limits: + memory: "128Mi" + cpu: "50m" +appcode: PCKT-002 # noqa: var-naming[no-role-prefix] +registry: 172.30.1.1:5000 # noqa: var-naming[no-role-prefix] +registry_user: developer # noqa: var-naming[no-role-prefix] diff --git a/playbooks/roles/deploy/handlers/main.yml b/playbooks/roles/deploy/handlers/main.yml new file mode 100644 index 0000000..73a2589 --- /dev/null +++ b/playbooks/roles/deploy/handlers/main.yml @@ -0,0 +1,21 @@ +--- +- name: Restart redis-commander deployment + ansible.builtin.command: oc rollout restart deploy/redis-commander + # Restart/rollout deployment as a reaction to config change + # when the deployment hasn't been changed itself. + changed_when: false + when: not redis_commander.changed + +- name: Restart tokman deployment + ansible.builtin.command: oc rollout restart deploy/tokman + # Restart/rollout deployment as a reaction to config change + # when the deployment hasn't been changed itself. + changed_when: false + when: not tokman.changed + +- name: Restart nginx deployment + ansible.builtin.command: oc rollout restart deploy/nginx + # Restart/rollout deployment as a reaction to config change + # when the deployment hasn't been changed itself. + changed_when: false + when: not nginx.changed diff --git a/playbooks/roles/deploy/tasks/main.yml b/playbooks/roles/deploy/tasks/main.yml new file mode 100644 index 0000000..7152d8e --- /dev/null +++ b/playbooks/roles/deploy/tasks/main.yml @@ -0,0 +1,410 @@ +--- +- name: Include tasks/project-dir.yml + ansible.builtin.include_tasks: tasks/project-dir.yml + tags: + - always + +- name: Include variables + ansible.builtin.include_vars: "{{ project_dir }}/vars/{{ service }}/{{ deployment }}.yml" + tags: + - always + +- name: Include tasks/check-up-to-date.yml + ansible.builtin.include_tasks: tasks/check-up-to-date.yml + tags: + - always + +- name: Include deployment facts + ansible.builtin.include_tasks: tasks/set-deployment-facts.yml + tags: + - always + +- name: Include tasks/set-facts.yml + ansible.builtin.include_tasks: tasks/set-facts.yml + tags: + - always + +- name: Include extra secret vars + ansible.builtin.include_vars: + file: "{{ path_to_secrets }}/extra-vars.yml" + name: vault + tags: + - always + +- name: Get k8s token and check it + tags: + - always + block: + - name: Get kubeconfig token + ansible.builtin.command: oc whoami -t + register: kubeconfig_token + changed_when: false + - name: Check if tokens match + ansible.builtin.assert: + that: + - kubeconfig_token.stdout == api_key + msg: "OpenShift API token defined in vars/ does not match token from your current environment." + +- name: Push dev images to local registry + when: push_dev_images + tags: + - packit-service + - packit-worker + - packit-service-beat + block: + - name: Set tls-verify to false if podman is used + ansible.builtin.set_fact: + tls_verify_false: "{{ '--tls-verify=false' if 'podman' in container_engine else '' }}" + changed_when: false + - name: Login to local cluster + ansible.builtin.shell: "{{ container_engine }} login -u {{ registry_user }} -p $(oc whoami -t) {{ registry }} {{ tls_verify_false }}" + changed_when: false + - name: Inspect service image + ansible.builtin.command: "{{ container_engine }} inspect {{ image }}" + changed_when: false + - name: Tag the image with :dev + ansible.builtin.command: "{{ container_engine }} tag {{ image }} {{ registry }}/myproject/packit-service:dev" + changed_when: true + - name: Push the image + ansible.builtin.command: "{{ container_engine }} push {{ registry }}/myproject/packit-service:dev {{ tls_verify_false }}" + changed_when: true + - name: Inspect worker image + ansible.builtin.command: "{{ container_engine }} inspect {{ image_worker }}" + changed_when: false + - name: Tag the image with :dev + ansible.builtin.command: "{{ container_engine }} tag {{ image_worker }} {{ registry }}/myproject/packit-worker:dev" + changed_when: true + - name: Push the image + ansible.builtin.command: "{{ container_engine }} push {{ registry }}/myproject/packit-worker:dev {{ tls_verify_false }}" + changed_when: true + - name: Inspect fedmsg image + ansible.builtin.command: "{{ container_engine }} inspect {{ image_fedmsg }}" + changed_when: false + - name: Tag the image with :dev + ansible.builtin.command: "{{ container_engine }} tag {{ image_fedmsg }} {{ registry }}/myproject/packit-service-fedmsg:dev" + changed_when: true + - name: Push the image + ansible.builtin.command: "{{ container_engine }} push {{ registry }}/myproject/packit-service-fedmsg:dev {{ tls_verify_false }}" + changed_when: true + - name: Inspect dashboard image + ansible.builtin.command: "{{ container_engine }} inspect {{ image_dashboard }}" + changed_when: false + - name: Tag the image with :dev + ansible.builtin.command: "{{ container_engine }} tag {{ image_dashboard }} {{ registry }}/myproject/packit-dashboard:dev" + changed_when: true + - name: Push the image + ansible.builtin.command: "{{ container_engine }} push {{ registry }}/myproject/packit-dashboard:dev {{ tls_verify_false }}" + changed_when: true + - name: Inspect tokman image + ansible.builtin.command: "{{ container_engine }} inspect {{ image_tokman }}" + changed_when: false + - name: Tag the image with :dev + ansible.builtin.command: "{{ container_engine }} tag {{ image_tokman }} {{ registry }}/myproject/tokman:dev" + changed_when: true + - name: Push the image + ansible.builtin.command: "{{ container_engine }} push {{ registry }}/myproject/tokman:dev {{ tls_verify_false }}" + changed_when: true + +- name: Deploy secrets + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/secret-packit-ssh.yml.j2') }}" + - "{{ lookup('template', '{{ project_dir }}/openshift/secret-packit-secrets.yml.j2') }}" + - "{{ lookup('template', '{{ project_dir }}/openshift/secret-packit-config.yml.j2') }}" + - "{{ lookup('template', '{{ project_dir }}/openshift/secret-sentry.yml.j2') }}" + - "{{ lookup('template', '{{ project_dir }}/openshift/secret-postgres.yml.j2') }}" + - "{{ lookup('template', '{{ project_dir }}/openshift/secret-aws.yml.j2') }}" + - "{{ lookup('template', '{{ project_dir }}/openshift/secret-splunk.yml.j2') }}" + tags: + - secrets + +- name: Set up sandbox namespace + when: with_sandbox + block: + - name: Create sandbox namespace + k8s: + resource_definition: "{{ lookup('template', '{{ project_dir }}/openshift/sandbox-namespace.yml.j2') }}" + host: "{{ host }}" + api_key: "{{ api_key }}" + validate_certs: "{{ validate_certs }}" + - name: Add edit role to service account in sandbox namespace + ansible.builtin.command: oc adm policy add-role-to-user edit system:serviceaccount:{{ project }}:default -n {{ sandbox_namespace }} + register: rolebinding + changed_when: "'added:' in rolebinding.stdout" + +- name: Deploy postgres + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/postgres.yml.j2') }}" + tags: + - postgres + +- name: Deploy redis + vars: + k8s_apply: true + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/redis.yml.j2') }}" + when: with_redis + tags: + - redis + +- name: Deploy redict + vars: + k8s_apply: true + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/redict.yml.j2') }}" + when: with_redict + tags: + - redict + +- name: Deploy fluentd image stream and config + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/fluentd.yml.j2') }}" + tags: + - packit-service + - packit-worker + when: with_fluentd_sidecar + +- name: Deploy packit-service + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/packit-service.yml.j2') }}" + tags: + - packit-service + +- name: Deploy repository cache PVCs for packit-workers that serves both queues + vars: + component: "packit-worker-{{ item }}" + k8s: + namespace: "{{ sandbox_namespace }}" + definition: "{{ lookup('template', '{{ project_dir }}/openshift/sandcastle-volumes-for-cache.yml.j2') }}" + host: "{{ host }}" + api_key: "{{ api_key }}" + validate_certs: "{{ validate_certs }}" + loop: "{{ range(0, workers_all_tasks) | list }}" + tags: + - packit-worker + when: workers_all_tasks > 0 and with_repository_cache + +- name: Deploy packit-worker to serve both queues + vars: + component: packit-worker + queues: "short-running,long-running" + worker_replicas: "{{ workers_all_tasks }}" + worker_requests_memory: "384Mi" + worker_requests_cpu: "100m" + worker_limits_memory: "1024Mi" + worker_limits_cpu: "400m" + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/packit-worker.yml.j2') }}" + tags: + - packit-worker + when: workers_all_tasks > 0 + +- name: Deploy packit-worker to serve short-running queue + vars: + component: packit-worker-short-running + queues: "short-running" + worker_replicas: "{{ workers_short_running }}" + # Short-running tasks are just interactions with different services. + # They should not require a lot of memory/cpu. + worker_requests_memory: "320Mi" + worker_requests_cpu: "80m" + worker_limits_memory: "640Mi" + worker_limits_cpu: "400m" + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/packit-worker.yml.j2') }}" + tags: + - packit-worker + when: workers_short_running > 0 + +- name: Deploy repository cache PVCs for packit-workers that serves long-running queue + vars: + component: "packit-worker-long-running-{{ item }}" + k8s: + namespace: "{{ sandbox_namespace }}" + definition: "{{ lookup('template', '{{ project_dir }}/openshift/sandcastle-volumes-for-cache.yml.j2') }}" + host: "{{ host }}" + api_key: "{{ api_key }}" + validate_certs: "{{ validate_certs }}" + loop: "{{ range(0, workers_long_running) | list }}" + tags: + - packit-worker + when: workers_long_running > 0 and with_repository_cache + +- name: Deploy packit-worker to serve long-running queue + vars: + component: packit-worker-long-running + queues: "long-running" + worker_replicas: "{{ workers_long_running }}" + # cloning repos is memory intensive: glibc needs 300M+, kernel 600M+ + # during cloning, we need to account for git and celery worker processes + worker_requests_memory: "384Mi" + worker_requests_cpu: "100m" + worker_limits_memory: "1024Mi" + worker_limits_cpu: "600m" + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/packit-worker.yml.j2') }}" + tags: + - packit-worker + when: workers_long_running > 0 + +- name: Deploy packit-service-beat + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/packit-service-beat.yml.j2') }}" + when: with_beat + tags: + - packit-service-beat + +- name: Deploy dashboard + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/dashboard.yml.j2') }}" + when: with_dashboard + tags: + - dashboard + +- name: Create redis-commander secrets + k8s: + namespace: "{{ project }}" + definition: "{{ lookup('template', '{{ project_dir }}/openshift/secret-redis-commander.yml.j2') }}" + host: "{{ host }}" + api_key: "{{ api_key }}" + validate_certs: "{{ validate_certs }}" + apply: true + tags: + - redis-commander + notify: + - Restart redis-commander deployment + when: with_redis_commander + +- name: Deploy redis-commander + vars: + k8s_apply: true + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/redis-commander.yml.j2') }}" + when: with_redis_commander + tags: + - redis-commander + register: redis_commander + +- name: Deploy flower + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/flower.yml.j2') }}" + when: with_flower + tags: + - flower + +- name: Deploy packit-service-fedmsg + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/packit-service-fedmsg.yml.j2') }}" + tags: + - fedmsg + when: with_fedmsg + +- name: Deploy GitHub App Private Key + k8s: + namespace: "{{ project }}" + resource_definition: "{{ lookup('template', '{{ project_dir }}/openshift/github-app-private-key.yml.j2') }}" + host: "{{ host }}" + api_key: "{{ api_key }}" + validate_certs: "{{ validate_certs }}" + tags: + - tokman + notify: + - Restart tokman deployment + when: with_tokman + +- name: Deploy tokman + k8s: + namespace: "{{ project }}" + definition: "{{ lookup('template', '{{ project_dir }}/openshift/tokman.yml.j2') }}" + host: "{{ host }}" + api_key: "{{ api_key }}" + validate_certs: "{{ validate_certs }}" + tags: + - tokman + register: tokman + when: with_tokman + +- name: Deploy aggregating pushgateway + ansible.builtin.include_tasks: tasks/k8s.yml + loop: + - "{{ lookup('template', '{{ project_dir }}/openshift/pushgateway.yml.j2') }}" + tags: + - pushgateway + when: with_pushgateway + +- name: Create htpasswd file and deploy it as a secret + tags: + - flower + when: with_flower + block: + - name: Create htpasswd file + htpasswd: + path: "{{ flower_htpasswd_path }}" + name: "flower-boss" + password: "{{ vault.flower.basic_auth | regex_replace('flower-boss:', '') }}" + mode: 0640 + - name: Deploy flower-htpasswd secret + # Don't use tasks/k8s.yml here because the loop item is always evaluated + k8s: + namespace: "{{ project }}" + resource_definition: "{{ lookup('template', '{{ project_dir }}/openshift/secret-flower-htpasswd.yml.j2') }}" + host: "{{ host }}" + api_key: "{{ api_key }}" + validate_certs: "{{ validate_certs }}" + notify: + - Restart nginx deployment + +- name: Deploy nginx to reverse proxy the pushgateway and flower + k8s: + namespace: "{{ project }}" + definition: "{{ lookup('template', '{{ project_dir }}/openshift/nginx.yml.j2') }}" + host: "{{ host }}" + api_key: "{{ api_key }}" + validate_certs: "{{ validate_certs }}" + tags: + - pushgateway + register: nginx + when: with_pushgateway and with_flower + +- name: Wait for worker-0 to be running + vars: + pod_name: packit-worker-0 + ansible.builtin.include_tasks: tasks/wait_for_pod.yml + when: workers_all_tasks > 0 + +- name: Wait for worker-short-running-0 to be running + vars: + pod_name: packit-worker-short-running-0 + ansible.builtin.include_tasks: tasks/wait_for_pod.yml + when: workers_short_running > 0 + +- name: Wait for worker-long-running-0 to be running + vars: + pod_name: packit-worker-long-running-0 + ansible.builtin.include_tasks: tasks/wait_for_pod.yml + when: workers_long_running > 0 + +- name: Select project to check status on command line + ansible.builtin.command: "oc project {{ project }}" + changed_when: false + +- name: Wait for deploymentconfig rollouts to complete + # timeout 10min to not wait indefinitely in case of a problem + ansible.builtin.command: timeout 15m oc rollout status -w deploy/{{ item }} + register: oc_rollout_status + changed_when: false + failed_when: '"successfully rolled out" not in oc_rollout_status.stdout' + loop: "{{ deploymentconfigs }}" diff --git a/playbooks/test_deploy.yml b/playbooks/test_deploy.yml new file mode 100644 index 0000000..ece2eb8 --- /dev/null +++ b/playbooks/test_deploy.yml @@ -0,0 +1,22 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +--- +- name: Deploy as tmt user + hosts: all + become: true + become_user: tmt + become_flags: -i + vars: + test_project_name: myproject + + roles: + - role: deploy + + post_tasks: + - name: Delete test project namespace if already exist + kubernetes.core.k8s: + name: "{{ test_project_name }}" + api_version: v1 + kind: Namespace + state: absent diff --git a/playbooks/test_deploy_setup.yml b/playbooks/test_deploy_setup.yml index 2159640..b9567ce 100644 --- a/playbooks/test_deploy_setup.yml +++ b/playbooks/test_deploy_setup.yml @@ -4,7 +4,11 @@ --- - name: Ensure crc is started, create openshift project and vars/packit/dev.yml hosts: all + become: true + become_user: tmt + become_flags: -i vars: + dest_pull_secret_file: /tmp/openshift-local-pull-secret.txt test_project_name: myproject crc_url: https://api.crc.testing:6443 image: quay.io/packit/packit-service:stg @@ -12,9 +16,24 @@ image_fedmsg: quay.io/packit/packit-service-fedmsg:stg image_dashboard: quay.io/packit/dashboard:stg image_tokman: quay.io/packit/tokman:stg + tmt_tree: "{{ lookup('env', 'TMT_TREE') }}" tasks: + - name: Find user uid + ansible.builtin.command: "id -u tmt" + register: user_uid + changed_when: false + + - name: Determine XDG_RUNTIME_DIR + ansible.builtin.set_fact: + xdg_runtime_dir: "/run/user/{{ user_uid.stdout }}" + changed_when: false + - name: Ensure crc is started - ansible.builtin.command: crc start + ansible.builtin.command: "crc start -c 6 -m 12000 -p {{ dest_pull_secret_file }}" + changed_when: False + + - name: Login into cluster as kubeadmin + ansible.builtin.command: oc login -u kubeadmin {{ crc_url }} changed_when: False - name: Get token @@ -47,10 +66,15 @@ - "{{ image_fedmsg }}" - "{{ image_dashboard }}" - "{{ image_tokman }}" - loop_control: - pause: 1 # otherwise getting 500 Internal Server Error from registry changed_when: False + - name: Copy tmt_tree in tmt home (otherwise I can't write on it...) # noqa risky-file-permissions # this is a recursive copy + ansible.builtin.copy: + src: "{{ tmt_tree }}/" + dest: /home/tmt/deployment + owner: tmt + directory_mode: "0755" + - name: Create packit/dev.yml ansible.builtin.copy: content: | @@ -76,5 +100,5 @@ image_fedmsg: {{ image_fedmsg }} image_dashboard: {{ image_dashboard }} image_tokman: {{ image_tokman }} - dest: "{{ src_dir }}/vars/packit/dev.yml" + dest: "/home/tmt/deployment/vars/packit/dev.yml" mode: 0644 diff --git a/tests/deployment/main.fmf b/tests/deployment/main.fmf index d8ab015..ea2becf 100644 --- a/tests/deployment/main.fmf +++ b/tests/deployment/main.fmf @@ -1,12 +1,17 @@ -test: - cd /vagrant && make test-deploy +duration: 60m -duration: 10m +environment: + DEPLOYMENT: dev -/deployment: - summary: Run packit-service deployment test - tags: [deployment] - duration: 60m - tier: 1 - environment+: - DEPLOYMENT: dev +require: + - type: file + pattern: + - /openshift + - /playbooks + - /scripts + - /secrets + - /vars + - /Makefile + +path: / +test: make test-deploy From 83f273e7155a2eb4bedcc3620a36cea52188ecd0 Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Mon, 17 Jun 2024 13:44:35 +0200 Subject: [PATCH 03/12] Add testing farm github action --- .github/workflows/tf-tests.yml | 25 +++++++++++++++++++++++++ plans/deployment.fmf | 16 ++++++++++++++++ playbooks/oc-cluster-run.yml | 5 ++++- playbooks/test_deploy_setup.yml | 7 +++++++ 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tf-tests.yml diff --git a/.github/workflows/tf-tests.yml b/.github/workflows/tf-tests.yml new file mode 100644 index 0000000..c2e5e0e --- /dev/null +++ b/.github/workflows/tf-tests.yml @@ -0,0 +1,25 @@ +name: Schedule test on Testing Farm +on: + pull_request: + +# The concurrency key is used to prevent multiple workflows from running at the same time +concurrency: + group: my-concurrency-group + cancel-in-progress: true + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Schedule test on Testing Farm + uses: sclorg/testing-farm-as-github-action@v2 + with: + compose: CentOS-Stream-9 + api_key: ${{ secrets.TF_API_KEY }} + git_url: "https://github.com/packit/deployment" + git_ref: "tf-openshift-tests" + tmt_plan_regex: "deployment/remote" + tmt_hardware: '{"memory": ">= 13 GiB", "disk": [{"size": ">= 100 GB"}], "cpu": {"cores": ">= 6"}, "virtualization": {"is-supported": true}}' + pull_request_status_name: "Deployment" + timeout: 3600 + secrets: CRC_PULL_SECRET=${{ secrets.CRC_PULL_SECRET }} diff --git a/plans/deployment.fmf b/plans/deployment.fmf index 39ba7bb..83d9726 100644 --- a/plans/deployment.fmf +++ b/plans/deployment.fmf @@ -15,6 +15,22 @@ discover: test: - deployment +/remote: + summary: Run packit-service deployment test on a remote testing-farm machine (details are defined in the GitHub action) + prepare: + - how: ansible + playbook: playbooks/oc-cluster-user.yml + extra-args: '-vvv' + - how: ansible + playbook: playbooks/oc-cluster-setup.yml + extra-args: '-vvv' + - how: ansible + playbook: playbooks/oc-cluster-run.yml + extra-args: '-vvv' + - how: ansible + playbook: playbooks/oc-cluster-tests-setup.yml + extra-args: '-vvv' + /local: summary: Run packit-service deployment test on a local machine managed by tmt discover: diff --git a/playbooks/oc-cluster-run.yml b/playbooks/oc-cluster-run.yml index f9bba11..ac2e5c2 100644 --- a/playbooks/oc-cluster-run.yml +++ b/playbooks/oc-cluster-run.yml @@ -5,10 +5,13 @@ become_user: tmt gather_facts: False vars: - pull_secret: "!!!TAKEN FROM ANSIBLE ENV!!!" dest_pull_secret_file: /tmp/openshift-local-pull-secret.txt crc_path: /home/tmt/.local/bin tasks: + - name: Look for pull_secret in env vars if it is not already defined + ansible.builtin.set_fact: + pull_secret: "{{ lookup('env', 'CRC_PULL_SECRET') }}" + when: pull_secret is undefined - name: Show pull secret ansible.builtin.debug: msg: "{{ pull_secret }}" diff --git a/playbooks/test_deploy_setup.yml b/playbooks/test_deploy_setup.yml index b9567ce..8fe35f6 100644 --- a/playbooks/test_deploy_setup.yml +++ b/playbooks/test_deploy_setup.yml @@ -75,6 +75,13 @@ owner: tmt directory_mode: "0755" + - name: Copy secrets that are already been generated in tmt home # noqa risky-file-permissions # this is a recursive copy + ansible.builtin.copy: + src: "../secrets" + dest: /home/tmt/deployment + owner: tmt + directory_mode: "0755" + - name: Create packit/dev.yml ansible.builtin.copy: content: | From 1a9e5bbd0252cf719f9509f5a55528f10d84686c Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Fri, 5 Jul 2024 14:58:07 +0200 Subject: [PATCH 04/12] Make test work in vagrant, local tmt vm and remote testing farm vm --- Makefile | 65 +++++++++++++++++++--------- containers/Vagrantfile | 19 ++++---- plans/deployment.fmf | 53 +++++++++++------------ playbooks/oc-cluster-run.yml | 12 ++--- playbooks/oc-cluster-setup.yml | 11 +++-- playbooks/oc-cluster-tests-setup.yml | 13 ------ playbooks/oc-cluster-user.yml | 11 +++-- playbooks/test_deploy.yml | 4 +- playbooks/test_deploy_setup.yml | 8 ++-- tests/deployment/main.fmf | 2 +- 10 files changed, 104 insertions(+), 94 deletions(-) diff --git a/Makefile b/Makefile index 59dd939..49cadec 100755 --- a/Makefile +++ b/Makefile @@ -11,12 +11,13 @@ VAGRANT_SSH_USER = "$(shell cd containers && vagrant ssh-config | awk '/User/{pr VAGRANT_SSH_GUEST = "$(shell cd containers && vagrant ssh-config | awk '/HostName/{print $$2}')" VAGRANT_SSH_IDENTITY_FILE = "$(shell cd containers && vagrant ssh-config | awk '/IdentityFile/{print $$2}')" VAGRANT_SSH_CONFIG = $(shell cd containers && vagrant ssh-config | awk 'NR>1 {print " -o "$$1"="$$2}') -#VAGRANT_SHARED_DIR = "/vagrant" -VAGRANT_SHARED_DIR = "/home/tmt/deployment" +VAGRANT_SHARED_DIR = "/vagrant" CENTOS_VAGRANT_BOX = CentOS-Stream-Vagrant-8-latest.x86_64.vagrant-libvirt.box CENTOS_VAGRANT_URL = https://cloud.centos.org/centos/8-stream/x86_64/images/$(CENTOS_VAGRANT_BOX) +CRC_PULL_SECRET ?= "$(shell cat secrets/openshift-local-pull-secret.yml)" + ifneq "$(shell whoami)" "root" ASK_PASS ?= --ask-become-pass endif @@ -72,7 +73,7 @@ oc-cluster-destroy: oc-cluster-up: cd containers && vagrant up - cd containers && vagrant ssh -c "cd $(VAGRANT_SHARED_DIR) && $(AP) playbooks/oc-cluster-run.yml" + cd containers && vagrant ssh -c "cd $(VAGRANT_SHARED_DIR) && $(AP) --extra-vars user=vagrant playbooks/oc-cluster-run.yml" oc-cluster-down: cd containers && vagrant halt @@ -81,23 +82,45 @@ oc-cluster-ssh: oc-cluster-up ssh $(VAGRANT_SSH_CONFIG) localhost test-deploy: -# to be run inside VM where the oc cluster is running! Call make tmt-vagrant-tests instead from outside the vagrant machine. +# to be run inside VM where the oc cluster && tmt is running! Call make tmt-vagrant-tests instead from outside the vagrant machine. +# SHARED_DIR could be /vagrant or /home/tmt/deployment, it depends on the VM where tmt is being run +# look inside deployment.fmf to find out the value of SHARED_DIR DEPLOYMENT=dev $(AP) playbooks/generate-local-secrets.yml - DEPLOYMENT=dev $(AP) -e '{"src_dir": $(VAGRANT_SHARED_DIR)}' playbooks/test_deploy_setup.yml - cd $(VAGRANT_SHARED_DIR); DEPLOYMENT=dev $(AP) -e '{"container_engine": "podman", "registry": "default-route-openshift-image-registry.apps-crc.testing", "registry_user": "kubeadmin", "src_dir": $(VAGRANT_SHARED_DIR)}' playbooks/test_deploy.yml - -tmt-vagrant-test: - tmt run --all provision --how connect --user vagrant --guest $(VAGRANT_SSH_GUEST) --port $(VAGRANT_SSH_PORT) --key $(VAGRANT_SSH_IDENTITY_FILE) - -tf-deploy: - testing-farm request --compose Fedora-Rawhide --git-url https://github.com/majamassarini/deployment --git-ref tf-openshift-tests --plan deployment - -# tmt run --id packit-service-deployment --until execute -# tmt run --id packit-service-deployment prepare --force -# tmt run --id packit-service-deployment login --step prepare:start -# tmt run --id packit-service-deployment execute --force -# tmt run --id packit-service-deployment login --step execute:start -# tmt run --id packit-service-deployment finish -# tmt run --id packit-service-deployment clean - + DEPLOYMENT=dev $(AP) -e '{"user": $(USER), "src_dir": $(SHARED_DIR)}' playbooks/test_deploy_setup.yml + cd $(SHARED_DIR); DEPLOYMENT=dev $(AP) -e '{"container_engine": "podman", "registry": "default-route-openshift-image-registry.apps-crc.testing", "registry_user": "kubeadmin", "user": $(USER), "src_dir": $(SHARED_DIR)}' playbooks/test_deploy.yml + +# Openshift Local pull_secret must exist locally +# or you can also define the CRC_PULL_SECRET var +check-pull-secret: + if [ ! -f secrets/openshift-local-pull-secret.yml ] && [ ! -n "$(CRC_PULL_SECRET)" ]; then echo "no pull secret available create secrets/openshift-local-pull-secret.yml file or set CRC_PULL_SECRET variable"; exit 1; else echo "pull secret found"; fi + +# Execute tmt deployment test on a vagrant virtual machine +# The virtual machine has to be already up and running, +# using the target oc-cluster-up +tmt-vagrant-test: check-pull-secret + tmt run --all provision --how connect --user vagrant --guest $(VAGRANT_SSH_GUEST) --port $(VAGRANT_SSH_PORT) --key $(VAGRANT_SSH_IDENTITY_FILE) plan --name deployment/vagrant + +# Execute tmt deployment test on a local virtual machine provisioned by tmt +# +# tmt local provisioned virtual machine have by default 2 cpu cores +# you need to change tmt defaults to be able to run this test locally +# change DEFAULT_CPU_COUNT in tmt/steps/provision/testcloud.py to 6 +# +# For running this same test remotely, using testing farm, we need the +# github action, there are no other ways (at the moment) to deal with +# the secrets (in our case the pull_request Openshift Local secret). +# For this reason the deployment/remote plan is not called by this file +# but is called from the testing farm github action configured in this PR +# +# Useful tmt/virsh commands to debug this test are listed below +# tmt run --id deployment --until execute +# tmt run --id deployment prepare --force +# tmt run --id deployment login --step prepare:start +# tmt run --id deployment execute --force +# tmt run --id deployment login --step execute:start +# tmt run --id deployment finish +# tmt clean runs +# tmt clean guests # virsh list --all +tmt-local-test: check-pull-secret + tmt run --id deployment plans --name deployment/local diff --git a/containers/Vagrantfile b/containers/Vagrantfile index 067d63d..4a50130 100644 --- a/containers/Vagrantfile +++ b/containers/Vagrantfile @@ -2,13 +2,11 @@ # vi: set ft=ruby : VAGRANTFILE_API_VERSION = "2" -# export VAGRANT_EXPERIMENTAL="1" -# export VAGRANT_EXPERIMENTAL="disks" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - config.vm.box = "centos/stream8" - #config.vm.box_url = "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-Vagrant-8-latest.x86_64.vagrant-libvirt.box" gives 404 - config.vm.box_url = "file:///$VagrantProjectHome/../CentOS-Stream-Vagrant-8-latest.x86_64.vagrant-libvirt.box" + config.vm.box = "centos/stream9" + config.vm.box_url = "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-Vagrant-9-latest.x86_64.vagrant-libvirt.box" + #config.vm.box_url = "file:///$VagrantProjectHome/../CentOS-Stream-Vagrant-8-latest.x86_64.vagrant-libvirt.box" # Forward traffic on the host to the development server on the guest @@ -44,30 +42,33 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # sshfs_opts_append: "-o nonempty" # Comment this line if you would like to disable the automatic update during provisioning - config.vm.provision "shell", inline: "sudo dnf -y --disablerepo '*' --enablerepo=extras swap centos-linux-repos centos-stream-repos" + # config.vm.provision "shell", inline: "sudo dnf -y --disablerepo '*' --enablerepo=extras swap centos-linux-repos centos-stream-repos" # !!!!!!! resize disk image !!!!!!!!! config.vm.provision "shell", inline: "sudo dnf install -y cloud-utils-growpart" config.vm.provision "shell", inline: "sudo growpart /dev/vda 1" - config.vm.provision "shell", inline: "sudo xfs_growfs /dev/vda1" + config.vm.provision "shell", inline: "sudo resize2fs /dev/vda1" + # config.vm.provision "shell", inline: "sudo xfs_growfs /dev/vda1" # this was for CentOS Stream 8 # bootstrap and run with ansible config.vm.provision "ansible" do |ansible| # ansible.verbose = "-vvv" ansible.verbose = true ansible.playbook = "../playbooks/oc-cluster-setup.yml" + ansible.extra_vars = {"user": "vagrant"} end config.vm.provision "ansible" do |ansible| # ansible.verbose = "-vvv" ansible.verbose = true ansible.playbook = "../playbooks/oc-cluster-run.yml" - ansible.extra_vars = "../secrets/openshift-local-pull-secret.yml" + ansible.raw_arguments = ['--extra-vars', 'user=vagrant', '--extra-vars', '@../secrets/openshift-local-pull-secret.yml'] end config.vm.provision "ansible" do |ansible| # ansible.verbose = "-vvv" + ansible.become = true + ansible.become_user = "root" ansible.verbose = true ansible.playbook = "../playbooks/oc-cluster-tests-setup.yml" - ansible.extra_vars = {"src_dir": "/vagrant"} end # Create the box diff --git a/plans/deployment.fmf b/plans/deployment.fmf index 83d9726..8be5d2a 100644 --- a/plans/deployment.fmf +++ b/plans/deployment.fmf @@ -5,6 +5,8 @@ execute: environment: DEPLOYMENT: dev + SHARED_DIR: /home/tmt/deployment + USER: tmt #ANSIBLE_DEBUG: 1 report: @@ -16,20 +18,20 @@ discover: - deployment /remote: - summary: Run packit-service deployment test on a remote testing-farm machine (details are defined in the GitHub action) + summary: Run packit-service deployment test on a remote testing-farm machine (provisioning details are defined in the GitHub action) prepare: - how: ansible playbook: playbooks/oc-cluster-user.yml - extra-args: '-vvv' + extra-args: "-vvv --extra-vars user=tmt" - how: ansible playbook: playbooks/oc-cluster-setup.yml - extra-args: '-vvv' + extra-args: "-vvv --extra-vars user=tmt" - how: ansible playbook: playbooks/oc-cluster-run.yml - extra-args: '-vvv' + extra-args: "-vvv --extra-vars user=tmt" - how: ansible playbook: playbooks/oc-cluster-tests-setup.yml - extra-args: '-vvv' + extra-args: "-vvv" /local: summary: Run packit-service deployment test on a local machine managed by tmt @@ -48,32 +50,29 @@ discover: prepare: - how: ansible playbook: playbooks/oc-cluster-user.yml - extra-args: '-vvv' + extra-args: "-vvv --extra-vars user=tmt" - how: ansible playbook: playbooks/oc-cluster-setup.yml - extra-args: '-vvv' + extra-args: "-vvv --extra-vars user=tmt" - how: ansible playbook: playbooks/oc-cluster-run.yml - extra-args: '-vvv --extra-vars @./secrets/openshift-local-pull-secret.yml' + extra-args: "-vvv --extra-vars @./secrets/openshift-local-pull-secret.yml --extra-vars user=tmt" - how: ansible playbook: playbooks/oc-cluster-tests-setup.yml - extra-args: '-vvv' - #- how: ansible - # playbook: playbooks/generate-local-secrets.yml - # extra-args: '-vvv --extra-vars generate_secrets_project_dir="./"' - #- how: ansible - # playbook: playbooks/test_deploy_setup.yml - # extra-args: '-vvv' + extra-args: "-vvv" -#/vagrant: -# summary: Run packit-service deployment test on a vagrant machine already up and running -# -# description: -# guest, port and key details are taken from the tmt-vagrant-test Makefile target. -# This provision is meant to be used just locally with the Vagrant machine already up -# and running. To start the Vagrant machine use the oc-cluster-up Makefile target. -# To access the Vagrant machine use the oc-cluster-ssh Makefile target. -# -# provision: -# how: connect -# user: vagrant +/vagrant: + summary: Run packit-service deployment test on a vagrant machine already up and running + environment+: + SHARED_DIR: /vagrant + USER: vagrant + + description: + guest, port and key details are taken from the tmt-vagrant-test Makefile target. + This provision is meant to be used just locally with the Vagrant machine already up + and running. To start the Vagrant machine use the oc-cluster-up Makefile target. + To access the Vagrant machine use the oc-cluster-ssh Makefile target. + + provision: + how: connect + user: vagrant diff --git a/playbooks/oc-cluster-run.yml b/playbooks/oc-cluster-run.yml index ac2e5c2..677960d 100644 --- a/playbooks/oc-cluster-run.yml +++ b/playbooks/oc-cluster-run.yml @@ -2,11 +2,11 @@ - name: Start Openshift Local cluster hosts: all become: true - become_user: tmt + become_user: "{{ user }}" gather_facts: False vars: dest_pull_secret_file: /tmp/openshift-local-pull-secret.txt - crc_path: /home/tmt/.local/bin + crc_path: "/home/{{ user }}/.local/bin" tasks: - name: Look for pull_secret in env vars if it is not already defined ansible.builtin.set_fact: @@ -20,10 +20,10 @@ content: "{{ pull_secret }}" dest: "{{ dest_pull_secret_file }}" mode: "0640" - owner: tmt + owner: "{{ user }}" - name: Find user uid - ansible.builtin.command: "id -u tmt" + ansible.builtin.command: "id -u {{ user }}" register: user_uid changed_when: false @@ -40,8 +40,8 @@ - name: Create symbolic link to oc ansible.builtin.file: # src: "/tmp/{{ unarchive.files[0] }}/crc" - src: /home/tmt/.crc/bin/oc/oc + src: "/home/{{ user }}/.crc/bin/oc/oc" # dest: "{{ crc_path }}/crc" - dest: /home/tmt/.local/bin/oc + dest: "/home/{{ user }}/.local/bin/oc" state: link force: True diff --git a/playbooks/oc-cluster-setup.yml b/playbooks/oc-cluster-setup.yml index 76a6cd3..46ff421 100644 --- a/playbooks/oc-cluster-setup.yml +++ b/playbooks/oc-cluster-setup.yml @@ -1,12 +1,11 @@ --- - name: Install an Openshift Local cluster become: true - become_user: tmt + become_user: "{{ user }}" hosts: all vars: crc_download_link: https://developers.redhat.com/content-gateway/rest/mirror/pub/openshift-v4/clients/crc/latest/crc-linux-amd64.tar.xz - crc_path: /home/tmt/.local/bin - user: tmt + crc_path: "/home/{{ user }}/.local/bin" tasks: - name: Download CRC ansible.builtin.get_url: @@ -20,7 +19,7 @@ dest: /tmp list_files: True remote_src: True - owner: tmt + owner: "{{ user }}" register: unarchive - name: Create ~/.local/bin if it does not exist ansible.builtin.file: @@ -30,7 +29,7 @@ - name: Show crc executable path ansible.builtin.debug: msg: "/tmp/{{ unarchive.files[0] }}crc" - - name: Copy crc to tmt usr + - name: Copy crc to user dir ansible.builtin.copy: src: "/tmp/{{ unarchive.files[0] }}/crc" remote_src: true @@ -55,7 +54,7 @@ changed_when: False - name: Find user uid - ansible.builtin.command: "id -u tmt" + ansible.builtin.command: "id -u {{ user }}" register: user_uid changed_when: false diff --git a/playbooks/oc-cluster-tests-setup.yml b/playbooks/oc-cluster-tests-setup.yml index b35536f..0048388 100644 --- a/playbooks/oc-cluster-tests-setup.yml +++ b/playbooks/oc-cluster-tests-setup.yml @@ -20,16 +20,3 @@ - python3-passlib # for using htpasswd ansible module - make - podman - - #- name: Download OC - # ansible.builtin.get_url: - # url: "{{ oc_download_link }}" - # dest: /tmp - # mode: "0644" - # register: download - #- name: Unarchive tarball in /usr/bin - # ansible.builtin.unarchive: - # src: "{{ download.dest }}" - # dest: /usr/bin - # list_files: True - # remote_src: True diff --git a/playbooks/oc-cluster-user.yml b/playbooks/oc-cluster-user.yml index b9042b0..a4045cf 100644 --- a/playbooks/oc-cluster-user.yml +++ b/playbooks/oc-cluster-user.yml @@ -2,18 +2,17 @@ - name: Configure a user able to run a Openshift Local Cluster hosts: all tasks: - - name: Add tmt user + - name: Add user ansible.builtin.user: - name: tmt + name: "{{ user }}" append: true groups: wheel - name: Add tmt user to sudoers ansible.builtin.lineinfile: path: /etc/sudoers state: present - #line: "{{ user }} ALL=(ALL) NOPASSWD: ALL" - line: "tmt ALL=(ALL) NOPASSWD: ALL" + line: "{{ user }} ALL=(ALL) NOPASSWD: ALL" validate: /usr/sbin/visudo -cf %s - - name: Enable lingering for tmt - ansible.builtin.command: loginctl enable-linger tmt + - name: Enable lingering for user + ansible.builtin.command: "loginctl enable-linger {{ user }}" changed_when: False diff --git a/playbooks/test_deploy.yml b/playbooks/test_deploy.yml index ece2eb8..d69aa6f 100644 --- a/playbooks/test_deploy.yml +++ b/playbooks/test_deploy.yml @@ -2,10 +2,10 @@ # SPDX-License-Identifier: MIT --- -- name: Deploy as tmt user +- name: Deploy as tmt/vagrant user hosts: all become: true - become_user: tmt + become_user: "{{ user }}" become_flags: -i vars: test_project_name: myproject diff --git a/playbooks/test_deploy_setup.yml b/playbooks/test_deploy_setup.yml index 8fe35f6..d4d2070 100644 --- a/playbooks/test_deploy_setup.yml +++ b/playbooks/test_deploy_setup.yml @@ -5,7 +5,7 @@ - name: Ensure crc is started, create openshift project and vars/packit/dev.yml hosts: all become: true - become_user: tmt + become_user: "{{ user }}" become_flags: -i vars: dest_pull_secret_file: /tmp/openshift-local-pull-secret.txt @@ -19,7 +19,7 @@ tmt_tree: "{{ lookup('env', 'TMT_TREE') }}" tasks: - name: Find user uid - ansible.builtin.command: "id -u tmt" + ansible.builtin.command: "id -u {{ user }}" register: user_uid changed_when: false @@ -74,6 +74,7 @@ dest: /home/tmt/deployment owner: tmt directory_mode: "0755" + when: src_dir == "/home/tmt/deployment" - name: Copy secrets that are already been generated in tmt home # noqa risky-file-permissions # this is a recursive copy ansible.builtin.copy: @@ -81,6 +82,7 @@ dest: /home/tmt/deployment owner: tmt directory_mode: "0755" + when: src_dir == "/home/tmt/deployment" - name: Create packit/dev.yml ansible.builtin.copy: @@ -107,5 +109,5 @@ image_fedmsg: {{ image_fedmsg }} image_dashboard: {{ image_dashboard }} image_tokman: {{ image_tokman }} - dest: "/home/tmt/deployment/vars/packit/dev.yml" + dest: "{{ src_dir }}/vars/packit/dev.yml" mode: 0644 diff --git a/tests/deployment/main.fmf b/tests/deployment/main.fmf index ea2becf..6912688 100644 --- a/tests/deployment/main.fmf +++ b/tests/deployment/main.fmf @@ -1,6 +1,6 @@ duration: 60m -environment: +environment+: DEPLOYMENT: dev require: From 006a156b5f36b96515c02f9d890fab4d9871f896 Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Wed, 11 Sep 2024 09:47:34 +0200 Subject: [PATCH 05/12] Manually repeat changes in deploy role and update deploy test after rebase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Before using the deploy role in the deploy playbook. Wait for the deploy tests to be reviewed, up and running. Every change in the deploy playbook has to be repeated manually back in the deploy role, in the meantime. Co-authored-by: Laura Barcziová <49026743+lbarcziova@users.noreply.github.com> --- playbooks/deploy.yml | 2 +- playbooks/roles/deploy/tasks/main.yml | 1 + .../files/centpkg-sig.conf.j2 | 32 +++++++++++++++++++ .../tasks/generate-configs.yml | 1 + playbooks/test_deploy_setup.yml | 2 +- 5 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 playbooks/roles/generate_secrets/files/centpkg-sig.conf.j2 diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index 5cb2767..dbfaf50 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -441,7 +441,7 @@ when: workers_long_running > 0 - name: Wait for deploymentconfig rollouts to complete - # timeout 10min to not wait indefinitely in case of a problem + # timeout 15min to not wait indefinitely in case of a problem ansible.builtin.command: timeout 15m oc rollout status -w deploy/{{ item }} register: oc_rollout_status changed_when: false diff --git a/playbooks/roles/deploy/tasks/main.yml b/playbooks/roles/deploy/tasks/main.yml index 7152d8e..200840e 100644 --- a/playbooks/roles/deploy/tasks/main.yml +++ b/playbooks/roles/deploy/tasks/main.yml @@ -115,6 +115,7 @@ - "{{ lookup('template', '{{ project_dir }}/openshift/secret-postgres.yml.j2') }}" - "{{ lookup('template', '{{ project_dir }}/openshift/secret-aws.yml.j2') }}" - "{{ lookup('template', '{{ project_dir }}/openshift/secret-splunk.yml.j2') }}" + - "{{ lookup('template', '{{ project_dir }}/openshift/secret-centpkg-sig.yml.j2') }}" tags: - secrets diff --git a/playbooks/roles/generate_secrets/files/centpkg-sig.conf.j2 b/playbooks/roles/generate_secrets/files/centpkg-sig.conf.j2 new file mode 100644 index 0000000..3ea5e50 --- /dev/null +++ b/playbooks/roles/generate_secrets/files/centpkg-sig.conf.j2 @@ -0,0 +1,32 @@ +[centpkg-sig] +lookaside = https://git.centos.org/sources +lookasidehash = sha512 +lookaside_cgi = https://git.centos.org/sources/upload_sig.cgi +# lookaside_cgi = https://git.centos.org/sources/upload.cgi +distgit_namespaced = True +distgit_namespaces = rpms +gitbaseurl = git+ssh://git@gitlab.com/CentOS/%(repo)s.git +anongiturl = https://gitlab.com/CentOS/%(repo)s.git +branchre = c\d+s?-sig-.+|master$ +kojiprofile = cbs +build_client = cbs +lookaside_namespaced = True +git_excludes = + i386/ + i686/ + x86_64/ + ppc/ + ppc64/ + ia64/ + mips/ + arm/ + noarch/ + /*.src.rpm + /build*.log + /.build-*.log + results_*/ + clog + +[centpkg-sig.distgit] +apibaseurl = https://gitlab.com +token = 111aaaAAA diff --git a/playbooks/roles/generate_secrets/tasks/generate-configs.yml b/playbooks/roles/generate_secrets/tasks/generate-configs.yml index e0fdabe..d7e06c8 100644 --- a/playbooks/roles/generate_secrets/tasks/generate-configs.yml +++ b/playbooks/roles/generate_secrets/tasks/generate-configs.yml @@ -13,3 +13,4 @@ - fedora.toml.j2 - copr - ssh_config + - centpkg-sig.conf.j2 diff --git a/playbooks/test_deploy_setup.yml b/playbooks/test_deploy_setup.yml index d4d2070..61b8d7c 100644 --- a/playbooks/test_deploy_setup.yml +++ b/playbooks/test_deploy_setup.yml @@ -95,7 +95,7 @@ check_up_to_date: false # Whether to deploy and check that pod # Let's preserve some resources - with_tokman: true + with_tokman: false with_beat: true with_fedmsg: true with_dashboard: true From 696883782c53edb303e26740806f62b672ac17a5 Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Wed, 11 Sep 2024 09:48:29 +0200 Subject: [PATCH 06/12] Create Makefile for Vagrant related targets Co-authored-by: Matej Focko --- Makefile | 40 ++++------------------------------------ containers/Makefile | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 36 deletions(-) create mode 100755 containers/Makefile diff --git a/Makefile b/Makefile index 49cadec..eecd6cc 100755 --- a/Makefile +++ b/Makefile @@ -6,15 +6,6 @@ AP := ansible-playbook -vv -c local -i localhost, -e ansible_python_interpreter= # "By default, Ansible runs as if --tags all had been specified." # https://docs.ansible.com/ansible/latest/user_guide/playbooks_tags.html#special-tags TAGS ?= all -VAGRANT_SSH_PORT = "$(shell cd containers && vagrant ssh-config | awk '/Port/{print $$2}')" -VAGRANT_SSH_USER = "$(shell cd containers && vagrant ssh-config | awk '/User/{print $$2}')" -VAGRANT_SSH_GUEST = "$(shell cd containers && vagrant ssh-config | awk '/HostName/{print $$2}')" -VAGRANT_SSH_IDENTITY_FILE = "$(shell cd containers && vagrant ssh-config | awk '/IdentityFile/{print $$2}')" -VAGRANT_SSH_CONFIG = $(shell cd containers && vagrant ssh-config | awk 'NR>1 {print " -o "$$1"="$$2}') -VAGRANT_SHARED_DIR = "/vagrant" - -CENTOS_VAGRANT_BOX = CentOS-Stream-Vagrant-8-latest.x86_64.vagrant-libvirt.box -CENTOS_VAGRANT_URL = https://cloud.centos.org/centos/8-stream/x86_64/images/$(CENTOS_VAGRANT_BOX) CRC_PULL_SECRET ?= "$(shell cat secrets/openshift-local-pull-secret.yml)" @@ -63,28 +54,11 @@ move-stable: [[ -d move_stable_repositories ]] || scripts/move_stable.py init scripts/move_stable.py move-all -oc-cluster-create: -# vagrant pointer is broken... - [[ -f $(CENTOS_VAGRANT_BOX) ]] || wget $(CENTOS_VAGRANT_URL) - cd containers && vagrant up - -oc-cluster-destroy: - cd containers && vagrant destroy - -oc-cluster-up: - cd containers && vagrant up - cd containers && vagrant ssh -c "cd $(VAGRANT_SHARED_DIR) && $(AP) --extra-vars user=vagrant playbooks/oc-cluster-run.yml" - -oc-cluster-down: - cd containers && vagrant halt - -oc-cluster-ssh: oc-cluster-up - ssh $(VAGRANT_SSH_CONFIG) localhost - -test-deploy: -# to be run inside VM where the oc cluster && tmt is running! Call make tmt-vagrant-tests instead from outside the vagrant machine. +# to be run inside VM where the oc cluster && tmt is running! +# cd containers; make tmt-vagrant-tests instead for testing in the vagrant machine. # SHARED_DIR could be /vagrant or /home/tmt/deployment, it depends on the VM where tmt is being run # look inside deployment.fmf to find out the value of SHARED_DIR +test-deploy: DEPLOYMENT=dev $(AP) playbooks/generate-local-secrets.yml DEPLOYMENT=dev $(AP) -e '{"user": $(USER), "src_dir": $(SHARED_DIR)}' playbooks/test_deploy_setup.yml cd $(SHARED_DIR); DEPLOYMENT=dev $(AP) -e '{"container_engine": "podman", "registry": "default-route-openshift-image-registry.apps-crc.testing", "registry_user": "kubeadmin", "user": $(USER), "src_dir": $(SHARED_DIR)}' playbooks/test_deploy.yml @@ -94,12 +68,6 @@ test-deploy: check-pull-secret: if [ ! -f secrets/openshift-local-pull-secret.yml ] && [ ! -n "$(CRC_PULL_SECRET)" ]; then echo "no pull secret available create secrets/openshift-local-pull-secret.yml file or set CRC_PULL_SECRET variable"; exit 1; else echo "pull secret found"; fi -# Execute tmt deployment test on a vagrant virtual machine -# The virtual machine has to be already up and running, -# using the target oc-cluster-up -tmt-vagrant-test: check-pull-secret - tmt run --all provision --how connect --user vagrant --guest $(VAGRANT_SSH_GUEST) --port $(VAGRANT_SSH_PORT) --key $(VAGRANT_SSH_IDENTITY_FILE) plan --name deployment/vagrant - # Execute tmt deployment test on a local virtual machine provisioned by tmt # # tmt local provisioned virtual machine have by default 2 cpu cores @@ -110,7 +78,7 @@ tmt-vagrant-test: check-pull-secret # github action, there are no other ways (at the moment) to deal with # the secrets (in our case the pull_request Openshift Local secret). # For this reason the deployment/remote plan is not called by this file -# but is called from the testing farm github action configured in this PR +# instead it is called from the testing farm github action # # Useful tmt/virsh commands to debug this test are listed below # tmt run --id deployment --until execute diff --git a/containers/Makefile b/containers/Makefile new file mode 100755 index 0000000..a063e3f --- /dev/null +++ b/containers/Makefile @@ -0,0 +1,45 @@ +ANSIBLE_PYTHON ?= $(shell command -v /usr/bin/python3 2> /dev/null || echo /usr/bin/python2) +AP := ansible-playbook -vv -c local -i localhost, -e ansible_python_interpreter=$(ANSIBLE_PYTHON) +VAGRANT_SSH_PORT = "$(shell vagrant ssh-config | awk '/Port/{print $$2}')" +VAGRANT_SSH_USER = "$(shell vagrant ssh-config | awk '/User/{print $$2}')" +VAGRANT_SSH_GUEST = "$(shell vagrant ssh-config | awk '/HostName/{print $$2}')" +VAGRANT_SSH_IDENTITY_FILE = "$(shell vagrant ssh-config | awk '/IdentityFile/{print $$2}')" +VAGRANT_SSH_CONFIG = $(shell vagrant ssh-config | awk 'NR>1 {print " -o "$$1"="$$2}') +VAGRANT_SHARED_DIR = "/vagrant" + +# to be used when the vagrant box link is broken, should be kept in sync with the Vagrant file +#CENTOS_VAGRANT_BOX = CentOS-Stream-Vagrant-8-latest.x86_64.vagrant-libvirt.box +#CENTOS_VAGRANT_URL = https://cloud.centos.org/centos/8-stream/x86_64/images/$(CENTOS_VAGRANT_BOX) + +CRC_PULL_SECRET ?= "$(shell cat secrets/openshift-local-pull-secret.yml)" + +# for this command to work, you may need to: +# sudo systemctl enable --now libvirtd +# sudo systemctl enable --now virtnetworkd +oc-cluster-create: + if [ ! -z "$(CENTOS_VAGRANT_BOX)" ] && [ -f $(CENTOS_VAGRANT_BOX) ]; then wget $(CENTOS_VAGRANT_URL); fi; + vagrant up + +oc-cluster-destroy: + vagrant destroy + +oc-cluster-up: + vagrant up + vagrant ssh -c "cd $(VAGRANT_SHARED_DIR) && $(AP) --extra-vars user=vagrant playbooks/oc-cluster-run.yml" + +oc-cluster-down: + vagrant halt + +oc-cluster-ssh: oc-cluster-up + ssh $(VAGRANT_SSH_CONFIG) localhost + +# Openshift Local pull_secret must exist locally +# or you can also define the CRC_PULL_SECRET var +check-pull-secret: + if [ ! -f ../secrets/openshift-local-pull-secret.yml ] && [ ! -n "$(CRC_PULL_SECRET)" ]; then echo "no pull secret available create secrets/openshift-local-pull-secret.yml file or set CRC_PULL_SECRET variable"; exit 1; else echo "pull secret found"; fi + +# Execute tmt deployment test on a vagrant virtual machine +# The virtual machine has to be already up and running, +# use the target oc-cluster-up +tmt-vagrant-test: check-pull-secret + tmt run --all provision --how connect --user vagrant --guest $(VAGRANT_SSH_GUEST) --port $(VAGRANT_SSH_PORT) --key $(VAGRANT_SSH_IDENTITY_FILE) plan --name deployment/vagrant From 8673ec13c37fa62616c4a692e7c6d9d6bf59494f Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Wed, 11 Sep 2024 10:44:50 +0200 Subject: [PATCH 07/12] Always install python3-cryptography Instead of python-cryptography. CentOS 8 Stream was not able to resolve python-cryptography. We don't need to support old CentOS versions anymore, so we can use the python3 package. --- playbooks/roles/generate_secrets/tasks/generate-cert.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/playbooks/roles/generate_secrets/tasks/generate-cert.yml b/playbooks/roles/generate_secrets/tasks/generate-cert.yml index 7004506..e82a959 100644 --- a/playbooks/roles/generate_secrets/tasks/generate-cert.yml +++ b/playbooks/roles/generate_secrets/tasks/generate-cert.yml @@ -2,19 +2,11 @@ # SPDX-License-Identifier: MIT --- -- name: Install Python cryptography module - ansible.builtin.package: - name: - - python-cryptography - become: true - when: (ansible_facts['distribution'] == "CentOS" and ansible_facts['distribution_major_version'] == "7") - - name: Install Python cryptography module ansible.builtin.package: name: - python3-cryptography become: true - when: (ansible_facts['distribution'] == "CentOS" and ansible_facts['distribution_major_version'] == "8") - name: Generate OpenSSL private keys openssl_privatekey: From b7b78045cce68e62ac27747969620344eba90894 Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Wed, 11 Sep 2024 11:42:49 +0200 Subject: [PATCH 08/12] Improve docs related with testing changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Laura Barcziová <49026743+lbarcziova@users.noreply.github.com> Co-authored-by: Matej Focko --- Makefile | 7 +-- docs/deployment/testing-changes.md | 74 ++++++++++++++++++++++++------ 2 files changed, 65 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index eecd6cc..18e0430 100755 --- a/Makefile +++ b/Makefile @@ -54,10 +54,11 @@ move-stable: [[ -d move_stable_repositories ]] || scripts/move_stable.py init scripts/move_stable.py move-all -# to be run inside VM where the oc cluster && tmt is running! -# cd containers; make tmt-vagrant-tests instead for testing in the vagrant machine. +# To be run inside VM where the oc cluster is running! +# `cd /vagrant; SHARED_DIR=/vagrant make test-deploy` for using it inside the vagrant VM. +# `SHARED_DIR=/home/tmt/deployment make test-deploy` for using it inside the tmt VM. # SHARED_DIR could be /vagrant or /home/tmt/deployment, it depends on the VM where tmt is being run -# look inside deployment.fmf to find out the value of SHARED_DIR +# look inside deployment.fmf to find out the value of SHARED_DIR set through tmt test-deploy: DEPLOYMENT=dev $(AP) playbooks/generate-local-secrets.yml DEPLOYMENT=dev $(AP) -e '{"user": $(USER), "src_dir": $(SHARED_DIR)}' playbooks/test_deploy_setup.yml diff --git a/docs/deployment/testing-changes.md b/docs/deployment/testing-changes.md index b12b3c7..7b1fe0a 100644 --- a/docs/deployment/testing-changes.md +++ b/docs/deployment/testing-changes.md @@ -71,35 +71,83 @@ For more information please refer to [official docs](https://ansible.softwarefac ### Test Deployment locally with OpenShift Local -For using OpenShift Local you need a _pull secret_, download it here: https://console.redhat.com/openshift/create/local. Save it in a file called `secrets\openshift-local-pull-secret.yml` following this format: +For using OpenShift Local you need a _pull secret_, download it here: https://console.redhat.com/openshift/create/local. Save it in a file called `secrets/openshift-local-pull-secret.yml` following this format: ``` --- pull_secret: <<< DOWNLOADED PULL SECRET CONTENT >>> ``` -Populate the `secrets` dir with all the other secrets. -You _should use_ your own secrets but if you have access to `stg` secrets -you can also do: +[Populate the `secrets` dir (`secrets/{SERVICE}/dev/`) with the other secrets.](secrets#running-a-servicebot-locally) + +You can choose if you want to use a Virtual Machine created by Vagrant or one created by tmt. + +Calling a test multiple times, modifyng and debugging it is simpler in a Vagrant VM. + +The tmt environment ensure a more reproducible test. + +#### Using Vagrant + +Create and start the OpenShift Local cluster in a Vagrant VM with (it takes as long as an hour in my X1 ThinkPad): ``` -DEPLOYMENT=stg make download-secrets +cd containers; make oc-cluster-create ``` -Now you can create and start the OpenShift Local cluster (it take as long as an hour) in a Vagrant Virtual Machine with: +Once OC is up and running you can test the `packit-service` deployment with the command: ``` -make oc-cluster-create +cd containers; make tmt-vagrant-test ``` -And once it is up and running you can test the `packit-service` deployment with the command: +This command will connect tmt to the Vagrant virtual machine and run the deploy test there (`make test-deploy`). +You can run the test as many times as you want as long as the virtual machine is up and running and the `crc cluster` is started (`make oc-cluster-up` after every `make oc-cluster-down`). +You can skip the `tmt` environment and run the test directly inside the VM: ``` -make tmt-tests +cd containers; +make oc-cluster-ssh ``` -This command will sshed the virtual machine and run the tests there (`make test-deploy`), -you can run the tests as many time you want as long as the virtual machine is up and running and the `crc cluster` is started (`make oc-cluster-up` after every `make oc-cluster-down`). -You can skip the `tmt` environment and run the test directly inside the VM using `make oc-cluster-ssh` and `cd /vagrant && make test-deploy`. +Inside the Vagrant VM as vagrant user you do: -You can destroy the `libvirt` machine with `make oc-cluster-destroy` and re-create it again with `make oc-cluster-create`. +``` +cd /vagrant +SHARED_DIR=/vagrant make test-deploy +``` + +You can directly work on the cluster: + +``` +oc login -u kubeadmin https://api.crc.testing:6443 +oc project myproject +oc describe node +oc describe pods +oc describe pod packit-worker-0 +... +``` + +You can destroy the `libvirt` machine with `cd containers; make oc-cluster-destroy` and re-create it again with `cd containers; make oc-cluster-create`. + +#### Using tmt + +You can test the packit-service deployment using a tmt created local VM with the command: + +``` +make tmt-local-test +``` + +It is quite hard to change a test inside a tmt created VM and debug it. +But, in case you need it this is a list of commands that can be handy: + +``` +tmt run --id deployment --until execute +tmt run --id deployment prepare --force +tmt run --id deployment login --step prepare:start +tmt run --id deployment execute --force +tmt run --id deployment login --step execute:start +tmt run --id deployment finish +tmt clean runs +tmt clean guests +virsh list --all +``` From e906d882a283d0e3d4e8fc19174434d9ef7d9a0d Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Thu, 12 Sep 2024 16:52:15 +0200 Subject: [PATCH 09/12] Deploy github-app-private-key for worker and service Tokman is no longer used. The github-app-private-key is now used directly by the service and the workers. --- playbooks/deploy.yml | 1 + playbooks/roles/deploy/tasks/main.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/playbooks/deploy.yml b/playbooks/deploy.yml index dbfaf50..e24142f 100644 --- a/playbooks/deploy.yml +++ b/playbooks/deploy.yml @@ -158,6 +158,7 @@ - "{{ lookup('template', '{{ project_dir }}/openshift/secret-aws.yml.j2') }}" - "{{ lookup('template', '{{ project_dir }}/openshift/secret-splunk.yml.j2') }}" - "{{ lookup('template', '{{ project_dir }}/openshift/secret-centpkg-sig.yml.j2') }}" + - "{{ lookup('template', '{{ project_dir }}/openshift/github-app-private-key.yml.j2') }}" tags: - secrets diff --git a/playbooks/roles/deploy/tasks/main.yml b/playbooks/roles/deploy/tasks/main.yml index 200840e..64f4c08 100644 --- a/playbooks/roles/deploy/tasks/main.yml +++ b/playbooks/roles/deploy/tasks/main.yml @@ -116,6 +116,7 @@ - "{{ lookup('template', '{{ project_dir }}/openshift/secret-aws.yml.j2') }}" - "{{ lookup('template', '{{ project_dir }}/openshift/secret-splunk.yml.j2') }}" - "{{ lookup('template', '{{ project_dir }}/openshift/secret-centpkg-sig.yml.j2') }}" + - "{{ lookup('template', '{{ project_dir }}/openshift/github-app-private-key.yml.j2') }}" tags: - secrets From 4dcdc99af2332c94deda2fc9bab6b9e4fc2da500 Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Fri, 13 Sep 2024 10:10:29 +0200 Subject: [PATCH 10/12] Make deploy test work when called through tmt inside vagrant VM --- playbooks/test_deploy_setup.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/test_deploy_setup.yml b/playbooks/test_deploy_setup.yml index 61b8d7c..d1653fa 100644 --- a/playbooks/test_deploy_setup.yml +++ b/playbooks/test_deploy_setup.yml @@ -71,18 +71,18 @@ - name: Copy tmt_tree in tmt home (otherwise I can't write on it...) # noqa risky-file-permissions # this is a recursive copy ansible.builtin.copy: src: "{{ tmt_tree }}/" - dest: /home/tmt/deployment + dest: "{{ src_dir }}" owner: tmt directory_mode: "0755" - when: src_dir == "/home/tmt/deployment" + when: user == "tmt" - name: Copy secrets that are already been generated in tmt home # noqa risky-file-permissions # this is a recursive copy ansible.builtin.copy: src: "../secrets" - dest: /home/tmt/deployment + dest: "{{ src_dir }}" owner: tmt directory_mode: "0755" - when: src_dir == "/home/tmt/deployment" + when: user == "tmt" - name: Create packit/dev.yml ansible.builtin.copy: From 49425a1b253574c91c7dc0a1eb7d7b5b0ad55ff2 Mon Sep 17 00:00:00 2001 From: Maja Massarini <2678400+majamassarini@users.noreply.github.com> Date: Fri, 13 Sep 2024 10:52:38 +0200 Subject: [PATCH 11/12] Disable by default plans designed to be used manually Co-authored-by: Matej Focko --- plans/deployment.fmf | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plans/deployment.fmf b/plans/deployment.fmf index 8be5d2a..44cb5f3 100644 --- a/plans/deployment.fmf +++ b/plans/deployment.fmf @@ -39,6 +39,10 @@ discover: how: fmf test: - deployment + adjust: + enabled: false + when: initiator != human + because: this plan is used to test the deployment locally via vagrant provision: how: virtual image: centos-stream @@ -66,6 +70,10 @@ discover: environment+: SHARED_DIR: /vagrant USER: vagrant + adjust: + enabled: false + when: initiator != human + because: this plan is used to set up vagrant locally description: guest, port and key details are taken from the tmt-vagrant-test Makefile target. From 9702af0166f1a3b68eba74f25bd67ba446d9de98 Mon Sep 17 00:00:00 2001 From: Maja Massarini Date: Fri, 13 Sep 2024 14:00:15 +0200 Subject: [PATCH 12/12] Do not call deploy test from zuul Now we are using testing farm for it. --- .github/workflows/tf-tests.yml | 4 ++-- .zuul.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tf-tests.yml b/.github/workflows/tf-tests.yml index c2e5e0e..1c21f89 100644 --- a/.github/workflows/tf-tests.yml +++ b/.github/workflows/tf-tests.yml @@ -1,4 +1,4 @@ -name: Schedule test on Testing Farm +name: Schedule tests on Testing Farm on: pull_request: @@ -11,7 +11,7 @@ jobs: tests: runs-on: ubuntu-latest steps: - - name: Schedule test on Testing Farm + - name: Schedule tests on Testing Farm uses: sclorg/testing-farm-as-github-action@v2 with: compose: CentOS-Stream-9 diff --git a/.zuul.yaml b/.zuul.yaml index 9846e1b..bc2324f 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -3,7 +3,7 @@ check: jobs: - pre-commit - - deployment-tests + # - deployment-tests gate: jobs: - pre-commit