diff --git a/README.md b/README.md index 91eb3a7..892cc67 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,12 @@ This is a collection of Ansible playbooks, Terraform configurations and scripts to deploy and operate Incus clusters. ## How to get the test setup run: -### Install incus and OpenTofu -Install incus stable or LTS on your system from the [zabbly/incus](https://github.com/zabbly/incus) release and initialize it on your local machine. +### Install Incus and OpenTofu +Install Incus stable or LTS on your system from the [zabbly/incus](https://github.com/zabbly/incus) release and initialize it on your local machine. Install [OpenTofu](https://opentofu.org/docs/intro/install/). -Install the required ceph packages for ansible on the controller, on Debian that's the `ceph-base` and `ceph-common` packages: +Install the required ceph packages for Ansible on the controller, on Debian that's the `ceph-base` and `ceph-common` packages: ``` apt install --no-install-recommends ceph-base ceph-common ``` @@ -24,9 +24,13 @@ Init the terraform project: tofu init ``` -Create the VMs for testing: +Create 5 VMs and associated networks and storage volumes for testing an Incus cluster: +If your Incus host needs different values from the default, you may need +to copy `terraform.tfvars.example` to `terraform.tfvars` and update the +variables. + ``` -tofu apply +tofu apply -target=module.baremetal ``` ### Run the Ansible Playbook @@ -35,22 +39,58 @@ Go to the ansible directory: cd ../ansible/ ``` +NOTE: If you need the same version of Ansible this was tested with: +``` +pyenv install 3.13.1 +pipenv --python "3.13.1" install +pipenv shell +ansible-galaxy install -r ansible_requirements.yml +``` + Copy the example inventory file: ``` cp hosts.yaml.example hosts.yaml ``` +NOTE: If you are connecting to a remote Incus host you will need to change the `ansible_incus_remote` variable to match the name of the Incus remote (see: `incus remote list` for a list of remote names to use). Run the Playbooks: ``` ansible-playbook deploy.yaml ``` -NOTE: When re-deploying the same cluster (e.g. following a `terraform -destroy`), you need to make sure to also clear any local state from the +NOTE: When re-deploying the same cluster (e.g. following a `terraform destroy`), +you need to make sure to also clear any local state from the `data` directory, failure to do so will cause Ceph/OVN to attempt connection to the previously deployed systems which will cause the deployment to get stuck. +``` +rm ansible/data/ceph/* +rm ansible/data/lvmcluster/* +rm ansible/data/ovn/* +``` + +### Test a VM and Contrainer on the new Incus cluster + +``` +# Open a shell on one of the Incus cluster nodes +incus exec server01 bash + +# List all instances +incus list + +# Launch a system container +incus launch images:ubuntu/22.04 ubuntu-container + +# Launch a virtual machine +incus launch images:ubuntu/22.04 ubuntu-vm --vm + +# Launch an application container +incus remote add oci-docker https://docker.io --protocol=oci +incus launch oci-docker:hello-world --ephemeral --console +incus launch oci-docker:nginx nginx-app-container +``` + ## Deploying against production systems ### Requirements (when using Incus with both Ceph and OVN) diff --git a/ansible/.gitignore b/ansible/.gitignore index 15205e5..620494d 100644 --- a/ansible/.gitignore +++ b/ansible/.gitignore @@ -1,2 +1,3 @@ data/* hosts.yaml +Pipfile.lock \ No newline at end of file diff --git a/ansible/Pipfile b/ansible/Pipfile new file mode 100644 index 0000000..26f7e06 --- /dev/null +++ b/ansible/Pipfile @@ -0,0 +1,14 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +jmespath = "*" +ansible-core = "==2.18.1" + +[dev-packages] + +[requires] +python_version = "3.13" +python_full_version = "3.13.1" \ No newline at end of file diff --git a/ansible/ansible_requirements.yml b/ansible/ansible_requirements.yml new file mode 100644 index 0000000..2319af2 --- /dev/null +++ b/ansible/ansible_requirements.yml @@ -0,0 +1,9 @@ +--- +# Install Collections and Roles with Ansible Galaxy +# ansible-galaxy install -r ansible_requirements.yml + +collections: + - name: community.crypto + - name: community.general + +roles: diff --git a/ansible/books/ceph.yaml b/ansible/books/ceph.yaml index 652f5b2..487a7ff 100644 --- a/ansible/books/ceph.yaml +++ b/ansible/books/ceph.yaml @@ -1,83 +1,4 @@ --- -- name: Ceph - Generate cluster keys and maps - hosts: all - order: shuffle - gather_facts: yes - gather_subset: - - "default_ipv4" - - "default_ipv6" - vars: - task_fsid: "{{ ceph_fsid | default('') }}" - task_bootstrap_osd_keyring: ../data/ceph/cluster.{{ task_fsid }}.bootstrap-osd.keyring - task_client_admin_keyring: ../data/ceph/cluster.{{ task_fsid }}.client.admin.keyring - task_mon_keyring: ../data/ceph/cluster.{{ task_fsid }}.mon.keyring - task_mon_map: ../data/ceph/cluster.{{ task_fsid }}.mon.map - task_release: "{{ ceph_release | default('squid') }}" - task_roles: "{{ ceph_roles | default([]) }}" - - task_release_majors: - luminous: 12 - mimic: 13 - nautilus: 14 - octopus: 15 - pacific: 16 - quincy: 17 - reef: 18 - squid: 19 - any_errors_fatal: true - tasks: - - name: Generate mon keyring - delegate_to: 127.0.0.1 - shell: - cmd: ceph-authtool --create-keyring {{ task_mon_keyring }} --gen-key -n mon. --cap mon 'allow *' - creates: '{{ task_mon_keyring }}' - throttle: 1 - when: 'task_fsid' - - - name: Generate client.admin keyring - delegate_to: 127.0.0.1 - shell: - cmd: ceph-authtool --create-keyring {{ task_client_admin_keyring }} --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' - creates: '{{ task_client_admin_keyring }}' - throttle: 1 - notify: Add key to client.admin keyring - when: 'task_fsid' - - - name: Generate bootstrap-osd keyring - delegate_to: 127.0.0.1 - shell: - cmd: ceph-authtool --create-keyring {{ task_bootstrap_osd_keyring }} --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r' - creates: '{{ task_bootstrap_osd_keyring }}' - throttle: 1 - notify: Add key to bootstrap-osd keyring - when: 'task_fsid' - - - name: Generate mon map - delegate_to: 127.0.0.1 - shell: - cmd: monmaptool --create{% if task_release_majors[task_release] | default(None) %} --set-min-mon-release={{ task_release_majors[task_release] }}{% endif %} --fsid {{ task_fsid }} {{ task_mon_map }} - creates: '{{ task_mon_map }}' - throttle: 1 - notify: Add nodes to mon map - when: 'task_fsid' - - handlers: - - name: Add key to client.admin keyring - delegate_to: 127.0.0.1 - shell: - cmd: ceph-authtool {{ task_mon_keyring }} --import-keyring {{ task_client_admin_keyring }} - - - name: Add key to bootstrap-osd keyring - delegate_to: 127.0.0.1 - shell: - cmd: ceph-authtool {{ task_mon_keyring }} --import-keyring {{ task_bootstrap_osd_keyring }} - - - name: Add nodes to mon map - delegate_to: 127.0.0.1 - shell: - cmd: monmaptool --add {{ item.name }} {{ item.ip }} {{ task_mon_map }} - loop: "{{ lookup('template', '../files/ceph/ceph.monitors.tpl') | from_yaml | default([]) }}" - - name: Ceph - Add package repository hosts: all order: shuffle @@ -191,6 +112,85 @@ state: present when: '"rgw" in task_roles' +- name: Ceph - Generate cluster keys and maps + hosts: all + order: shuffle + gather_facts: yes + gather_subset: + - "default_ipv4" + - "default_ipv6" + vars: + task_fsid: "{{ ceph_fsid | default('') }}" + task_bootstrap_osd_keyring: ../data/ceph/cluster.{{ task_fsid }}.bootstrap-osd.keyring + task_client_admin_keyring: ../data/ceph/cluster.{{ task_fsid }}.client.admin.keyring + task_mon_keyring: ../data/ceph/cluster.{{ task_fsid }}.mon.keyring + task_mon_map: ../data/ceph/cluster.{{ task_fsid }}.mon.map + task_release: "{{ ceph_release | default('squid') }}" + task_roles: "{{ ceph_roles | default([]) }}" + + task_release_majors: + luminous: 12 + mimic: 13 + nautilus: 14 + octopus: 15 + pacific: 16 + quincy: 17 + reef: 18 + squid: 19 + any_errors_fatal: true + tasks: + - name: Generate mon keyring + delegate_to: 127.0.0.1 + shell: + cmd: ceph-authtool --create-keyring {{ task_mon_keyring }} --gen-key -n mon. --cap mon 'allow *' + creates: '{{ task_mon_keyring }}' + throttle: 1 + when: 'task_fsid' + + - name: Generate client.admin keyring + delegate_to: 127.0.0.1 + shell: + cmd: ceph-authtool --create-keyring {{ task_client_admin_keyring }} --gen-key -n client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' + creates: '{{ task_client_admin_keyring }}' + throttle: 1 + notify: Add key to client.admin keyring + when: 'task_fsid' + + - name: Generate bootstrap-osd keyring + delegate_to: 127.0.0.1 + shell: + cmd: ceph-authtool --create-keyring {{ task_bootstrap_osd_keyring }} --gen-key -n client.bootstrap-osd --cap mon 'profile bootstrap-osd' --cap mgr 'allow r' + creates: '{{ task_bootstrap_osd_keyring }}' + throttle: 1 + notify: Add key to bootstrap-osd keyring + when: 'task_fsid' + + - name: Generate mon map + delegate_to: 127.0.0.1 + shell: + cmd: monmaptool --create{% if task_release_majors[task_release] | default(None) %} --set-min-mon-release={{ task_release_majors[task_release] }}{% endif %} --fsid {{ task_fsid }} {{ task_mon_map }} + creates: '{{ task_mon_map }}' + throttle: 1 + notify: Add nodes to mon map + when: 'task_fsid' + + handlers: + - name: Add key to client.admin keyring + delegate_to: 127.0.0.1 + shell: + cmd: ceph-authtool {{ task_mon_keyring }} --import-keyring {{ task_client_admin_keyring }} + + - name: Add key to bootstrap-osd keyring + delegate_to: 127.0.0.1 + shell: + cmd: ceph-authtool {{ task_mon_keyring }} --import-keyring {{ task_bootstrap_osd_keyring }} + + - name: Add nodes to mon map + delegate_to: 127.0.0.1 + shell: + cmd: monmaptool --add {{ item.name }} {{ item.ip }} {{ task_mon_map }} + loop: "{{ lookup('template', '../files/ceph/ceph.monitors.tpl') | from_yaml | default([]) }}" + - name: Ceph - Set up config and keyrings hosts: all order: shuffle diff --git a/ansible/files/ceph/ceph.monitors.tpl b/ansible/files/ceph/ceph.monitors.tpl index 1039ff6..d2ab862 100644 --- a/ansible/files/ceph/ceph.monitors.tpl +++ b/ansible/files/ceph/ceph.monitors.tpl @@ -1,4 +1,4 @@ -{% for host in vars['ansible_play_hosts'] %} +{% for host in groups['all'] %} {% if hostvars[host]['ceph_fsid'] == task_fsid and "mon" in hostvars[host]['ceph_roles'] %} - name: "{{ host }}" ip: "{{ hostvars[host]['ceph_ip_address'] | default(hostvars[host]['ansible_default_ipv6']['address'] | default(hostvars[host]['ansible_default_ipv4']['address'])) }}" diff --git a/terraform/.gitignore b/terraform/.gitignore index 90771d9..1851c44 100644 --- a/terraform/.gitignore +++ b/terraform/.gitignore @@ -2,3 +2,6 @@ .terraform.lock.hcl terraform.tfstate terraform.tfstate.backup + +*.tfvars +!*.auto.tfvars diff --git a/terraform/baremetal-incus/main.tf b/terraform/baremetal-incus/main.tf index f177938..44ee171 100644 --- a/terraform/baremetal-incus/main.tf +++ b/terraform/baremetal-incus/main.tf @@ -17,9 +17,9 @@ resource "incus_network" "this" { description = "Network used to test incus-deploy (OVN uplink)" config = { - "ipv4.address" = "172.31.254.1/24" + "ipv4.address" = var.ovn_uplink_ipv4_address "ipv4.nat" = "true" - "ipv6.address" = "fd00:1e4d:637d:1234::1/64" + "ipv6.address" = var.ovn_uplink_ipv6_address "ipv6.nat" = "true" } } @@ -49,7 +49,7 @@ resource "incus_profile" "this" { name = "eth0" properties = { - "network" = "incusbr0" + "network" = var.network "name" = "eth0" } } diff --git a/terraform/baremetal-incus/variables.tf b/terraform/baremetal-incus/variables.tf index 2cca72d..c01e26b 100644 --- a/terraform/baremetal-incus/variables.tf +++ b/terraform/baremetal-incus/variables.tf @@ -17,3 +17,17 @@ variable "memory" { variable "storage_pool" { type = string } + +variable "network" { + type = string +} + +variable "ovn_uplink_ipv4_address" { + type = string + default = "" +} + +variable "ovn_uplink_ipv6_address" { + type = string + default = "" +} diff --git a/terraform/main.tf b/terraform/main.tf index 3c5646f..59758d8 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -5,7 +5,12 @@ module "baremetal" { instance_names = ["server01", "server02", "server03", "server04", "server05"] image = "images:ubuntu/22.04" memory = "4GiB" - storage_pool = "default" + + storage_pool = var.incus_storage_pool + network = var.incus_network + + ovn_uplink_ipv4_address = var.ovn_uplink_ipv4_address + ovn_uplink_ipv6_address = var.ovn_uplink_ipv6_address } module "services" { @@ -14,5 +19,7 @@ module "services" { project_name = "dev-incus-deploy-services" instance_names = ["ceph-mds01", "ceph-mds02", "ceph-mds03", "ceph-mgr01", "ceph-mgr02", "ceph-mgr03", "ceph-rgw01", "ceph-rgw02", "ceph-rgw03"] image = "images:ubuntu/24.04" - storage_pool = "default" + + storage_pool = var.incus_storage_pool + network = var.incus_network } diff --git a/terraform/provider_incus.tf b/terraform/provider_incus.tf new file mode 100644 index 0000000..6867e14 --- /dev/null +++ b/terraform/provider_incus.tf @@ -0,0 +1,6 @@ +provider "incus" { + remote { + name = var.incus_remote + default = true + } +} diff --git a/terraform/services/main.tf b/terraform/services/main.tf index e28b178..b44719a 100644 --- a/terraform/services/main.tf +++ b/terraform/services/main.tf @@ -37,7 +37,7 @@ resource "incus_profile" "this" { name = "eth0" properties = { - "network" = "incusbr0" + "network" = var.network "name" = "eth0" } } diff --git a/terraform/services/variables.tf b/terraform/services/variables.tf index 22eb8e2..72a92fc 100644 --- a/terraform/services/variables.tf +++ b/terraform/services/variables.tf @@ -13,3 +13,8 @@ variable "image" { variable "storage_pool" { type = string } + +variable "network" { + type = string + default = "" +} diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example new file mode 100755 index 0000000..a290838 --- /dev/null +++ b/terraform/terraform.tfvars.example @@ -0,0 +1,18 @@ +# Example of terraform.tfvars +# +# You can copy the contents of this file to terraform.tfvars and modify +# values for your own setup to operate with values different from the defaults +# +# A terraform.tfvars will override any other defined variables such as the ones +# in *.auto.tfvars or defaults in variables.tf +# https://opentofu.org/docs/language/values/variables/#variable-definitions-tfvars-files +# https://opentofu.org/docs/language/values/variables/#variable-definition-precedence + +# Incus variables +incus_remote = "local" # Name of the Incus remote to deploy on (see `incus remote list`) +incus_storage_pool = "default" # Name of the storage pool to use for the VMs and volumes +incus_network = "incusbr0" # Name of the network to use for the VMs + +# OVN uplink configuration +ovn_uplink_ipv4_address = "172.31.254.1/24" +ovn_uplink_ipv6_address = "fd00:1e4d:637d:1234::1/64" diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100755 index 0000000..ee56bb8 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,24 @@ +variable "incus_remote" { + type = string + default = "local" +} + +variable "incus_storage_pool" { + type = string + default = "default" +} + +variable "incus_network" { + type = string + default = "incusbr0" +} + +variable "ovn_uplink_ipv4_address" { + type = string + default = "172.31.254.1/24" +} + +variable "ovn_uplink_ipv6_address" { + type = string + default = "fd00:1e4d:637d:1234::1/64" +} diff --git a/terraform/versions.tf b/terraform/versions.tf index d9d2883..0bff758 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -7,6 +7,3 @@ terraform { } } } - -provider "incus" { -}