🌱 E2E: Use kind instead of minikube

.gitignore

@@ -28,6 +28,7 @@ go.work.sum
 *~
 *.tmp
 .DS_Store
+.zed*
 
 # Tilt files.
 .tiltbuild

config/overlays/e2e-release-0.5/ironic.env

@@ -1,3 +1,3 @@
-DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel
-DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs
-IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/
+DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel
+DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs
+IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/

config/overlays/e2e-release-0.6/ironic.env

@@ -1,3 +1,3 @@
-DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel
-DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs
-IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/
+DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel
+DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs
+IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/

config/overlays/e2e/ironic.env

@@ -1,3 +1,3 @@
-DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel
-DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs
-IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/
+DEPLOY_KERNEL_URL=http://192.168.222.1:6180/images/ironic-python-agent.kernel
+DEPLOY_RAMDISK_URL=http://192.168.222.1:6180/images/ironic-python-agent.initramfs
+IRONIC_ENDPOINT=https://192.168.222.1:6385/v1/

hack/ci-e2e.sh

@@ -44,12 +44,10 @@ case "${GINKGO_FOCUS:-}" in
     export GINKGO_SKIP="${GINKGO_SKIP:-upgrade}"
     ;;
 esac
-export USE_EXISTING_CLUSTER="true"
 
 # Ensure requirements are installed
 "${REPO_ROOT}/hack/e2e/ensure_go.sh"
 export PATH="/usr/local/go/bin:${PATH}"
-"${REPO_ROOT}/hack/e2e/ensure_minikube.sh"
 "${REPO_ROOT}/hack/e2e/ensure_htpasswd.sh"
 # CAPI test framework uses kubectl in the background
 "${REPO_ROOT}/hack/e2e/ensure_kubectl.sh"
@@ -58,28 +56,8 @@ export PATH="/usr/local/go/bin:${PATH}"
 # Build the container image with e2e tag (used in tests)
 IMG=quay.io/metal3-io/baremetal-operator:e2e make docker
 
-# Set up minikube
-minikube start --driver=kvm2
-
 virsh -c qemu:///system net-define "${REPO_ROOT}/hack/e2e/net.xml"
 virsh -c qemu:///system net-start baremetal-e2e
-# Attach baremetal-e2e interface to minikube with specific mac.
-# This will give minikube a known reserved IP address that we can use for Ironic
-virsh -c qemu:///system attach-interface --domain minikube --mac="52:54:00:6c:3c:01" \
-  --model virtio --source baremetal-e2e --type network --config
-
-# Restart minikube to apply the changes
-minikube stop
-## Following loop is to avoid minikube restart issue
-## https://github.com/kubernetes/minikube/issues/14456
-while ! minikube start; do sleep 30; done
-
-# Load the BMO e2e image into it
-# minikube image load quay.io/metal3-io/baremetal-operator:e2e
-# Temporary workaround for https://github.com/kubernetes/minikube/issues/18021
-docker image save -o /tmp/bmo-e2e.tar quay.io/metal3-io/baremetal-operator:e2e
-minikube image load /tmp/bmo-e2e.tar
-rm /tmp/bmo-e2e.tar
 
 # This IP is defined by the network we created above.
 IP_ADDRESS="192.168.222.1"
@@ -100,7 +78,7 @@ if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then
     address=$(echo "${bmc}" | jq -r '.address')
     hostName=$(echo "${bmc}" | jq -r '.hostName')
     vbmc_port="${address##*:}"
-    "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${hostName}" "${vbmc_port}"
+    "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${hostName}" "${vbmc_port}" "${IP_ADDRESS}"
   done
 
 elif [[ "${BMO_E2E_EMULATOR}" == "sushy-tools" ]]; then

hack/clean-e2e.sh

@@ -3,7 +3,6 @@
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 cd "${REPO_ROOT}" || exit 1
 
-minikube delete
 docker rm -f vbmc
 docker rm -f image-server-e2e
 docker rm -f sushy-tools

hack/e2e/net.xml

@@ -9,8 +9,7 @@
   <ip address='192.168.222.1' netmask='255.255.255.0'>
     <dhcp>
       <range start='192.168.222.2' end='192.168.222.199'/>
-      <host mac='52:54:00:6c:3c:01' name='minikube' ip='192.168.222.199'/>
-      <bootp file='http://192.168.222.199:6180/boot.ipxe'/>
+      <bootp file='http://192.168.222.1:6180/boot.ipxe'/>
     </dhcp>
   </ip>
 </network>

ironic-deployment/components/tls/certificate.yaml

@@ -12,8 +12,6 @@ metadata:
 spec:
   commonName: ironic-ca
   isCA: true
-  ipAddresses:
-  - IRONIC_HOST_IP
   issuerRef:
     kind: Issuer
     name: selfsigned-issuer

ironic-deployment/overlays/e2e-release-24.0-with-inspector/ironic_bmo_configmap.env

@@ -1,6 +1,6 @@
 HTTP_PORT=6180
-PROVISIONING_IP=192.168.222.199
-CACHEURL=http://192.168.222.199/images
-IRONIC_FAST_TRACK=true
+PROVISIONING_INTERFACE=eth0
+CACHEURL=http://192.168.222.1/images
+IRONIC_EXTERNAL_CALLBACK_URL=https://192.168.222.1:6385
 IRONIC_KERNEL_PARAMS=console=ttyS0
 IRONIC_INSPECTOR_VLAN_INTERFACES=all

ironic-deployment/overlays/e2e-release-24.1/ironic_bmo_configmap.env

@@ -1,6 +1,7 @@
 HTTP_PORT=6180
-PROVISIONING_IP=192.168.222.199
-CACHEURL=http://192.168.222.199/images
+PROVISIONING_INTERFACE=eth0
+CACHEURL=http://192.168.222.1/images
+IRONIC_EXTERNAL_CALLBACK_URL=https://192.168.222.1:6385
 IRONIC_KERNEL_PARAMS=console=ttyS0
 IRONIC_INSPECTOR_VLAN_INTERFACES=all
 USE_IRONIC_INSPECTOR=false

ironic-deployment/overlays/e2e-release-24.1/kustomization.yaml

@@ -16,7 +16,24 @@ configMapGenerator:
   behavior: create
 
 patches:
-  - path: ironic-patch.yaml
+- path: ironic-patch.yaml
+# The TLS component adds certificates but it cannot know the exact IPs of our environment.
+# Here we patch the certificates to have the correct IPs.
+# - 192.168.222.1: management computer IP, forwarded to ironic inside kind
+- patch: |-
+    - op: replace
+      path: /spec/ipAddresses/0
+      value: 192.168.222.1
+  target:
+    kind: Certificate
+    name: ironic-cert
+- patch: |-
+    - op: replace
+      path: /spec/ipAddresses/0
+      value: 192.168.222.1
+  target:
+    kind: Certificate
+    name: ironic-cacert
 
 images:
 - name: quay.io/metal3-io/ironic
@@ -28,18 +45,3 @@ secretGenerator:
   behavior: create
   envs:
   - ironic-htpasswd
-
-replacements:
-  # Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap
-  - source:
-      kind: ConfigMap
-      name: ironic-bmo-configmap
-      fieldPath: .data.PROVISIONING_IP
-    targets:
-      - select:
-          version: v1
-          group: cert-manager.io
-          kind: Certificate
-          name:
-        fieldPaths:
-          - .spec.ipAddresses.0

ironic-deployment/overlays/e2e-with-inspector/ironic_bmo_configmap.env

@@ -1,7 +1,7 @@
 HTTP_PORT=6180
-PROVISIONING_IP=192.168.222.199
-CACHEURL=http://192.168.222.199/images
-IRONIC_FAST_TRACK=true
+PROVISIONING_INTERFACE=eth0
+CACHEURL=http://192.168.222.1/images
+IRONIC_EXTERNAL_CALLBACK_URL=https://192.168.222.1:6385
 IRONIC_KERNEL_PARAMS=console=ttyS0
 IRONIC_INSPECTOR_VLAN_INTERFACES=all
 USE_IRONIC_INSPECTOR=true

ironic-deployment/overlays/e2e-with-inspector/kustomization.yaml

@@ -17,6 +17,23 @@ configMapGenerator:
 
 patches:
 - path: ironic-patch.yaml
+# The TLS component adds certificates but it cannot know the exact IPs of our environment.
+# Here we patch the certificates to have the correct IPs.
+# - 192.168.222.1: management computer IP, forwarded to ironic inside kind
+- patch: |-
+    - op: replace
+      path: /spec/ipAddresses/0
+      value: 192.168.222.1
+  target:
+    kind: Certificate
+    name: ironic-cert
+- patch: |-
+    - op: replace
+      path: /spec/ipAddresses/0
+      value: 192.168.222.1
+  target:
+    kind: Certificate
+    name: ironic-cacert
 
 # NOTE: These credentials are generated automatically in hack/ci-e2e.sh
 secretGenerator:
@@ -34,18 +51,3 @@ secretGenerator:
 - name: ironic-inspector-auth-config
   files:
   - auth-config=ironic-inspector-auth-config
-
-replacements:
-# Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap
-- source:
-    kind: ConfigMap
-    name: ironic-bmo-configmap
-    fieldPath: .data.PROVISIONING_IP
-  targets:
-  - select:
-      version: v1
-      group: cert-manager.io
-      kind: Certificate
-      name:
-    fieldPaths:
-    - .spec.ipAddresses.0

ironic-deployment/overlays/e2e/ironic_bmo_configmap.env

@@ -1,6 +1,7 @@
 HTTP_PORT=6180
-PROVISIONING_IP=192.168.222.199
-CACHEURL=http://192.168.222.199/images
+PROVISIONING_INTERFACE=eth0
+CACHEURL=http://192.168.222.1/images
+IRONIC_EXTERNAL_CALLBACK_URL=https://192.168.222.1:6385
 IRONIC_KERNEL_PARAMS=console=ttyS0
 IRONIC_INSPECTOR_VLAN_INTERFACES=all
 USE_IRONIC_INSPECTOR=false

ironic-deployment/overlays/e2e/kustomization.yaml

@@ -16,7 +16,17 @@ configMapGenerator:
   behavior: create
 
 patches:
-  - path: ironic-patch.yaml
+- path: ironic-patch.yaml
+# The TLS component adds certificates but it cannot know the exact IPs of our environment.
+# Here we patch the certificates to have the correct IPs.
+# - 192.168.222.1: management computer IP, forwarded to ironic inside kind
+- patch: |-
+    - op: replace
+      path: /spec/ipAddresses/0
+      value: 192.168.222.1
+  target:
+    kind: Certificate
+    name: ironic-cert
 
 # NOTE: These credentials are generated automatically in hack/ci-e2e.sh
 secretGenerator:
@@ -25,18 +35,3 @@ secretGenerator:
   files:
   - htpasswd=ironic-htpasswd
   type: Opaque
-
-replacements:
-  # Replace IRONIC_HOST_IP in certificates with the PROVISIONING_IP from the configmap
-  - source:
-      kind: ConfigMap
-      name: ironic-bmo-configmap
-      fieldPath: .data.PROVISIONING_IP
-    targets:
-      - select:
-          version: v1
-          group: cert-manager.io
-          kind: Certificate
-          name:
-        fieldPaths:
-          - .spec.ipAddresses.0

test/e2e/config/fixture.yaml

@@ -21,6 +21,9 @@ variables:
 
   # This setting is for the separate cluster that are used for upgrade tests
   UPGRADE_DEPLOY_CERT_MANAGER: "true"
+  # We can create separate clusters for each upgrade test with fixture.
+  # Since there is no ironic involved, we do not risk having any port collisions.
+  UPGRADE_USE_EXISTING_CLUSTER: "false"
 
   IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
   ISO_IMAGE_URL: "http://192.168.222.1/cirros.iso"

test/e2e/config/ironic.yaml

@@ -2,8 +2,8 @@ images:
 # Use locally built e2e images
 - name: quay.io/metal3-io/baremetal-operator:e2e
   loadBehavior: tryLoad
-- name: quay.io/metal3-io/ironic:e2e
-  loadBehavior: tryLoad
+# - name: quay.io/metal3-io/ironic:local
+#   loadBehavior: tryLoad
 # Save some time and network by using cached images if available
 - name: quay.io/metal3-io/baremetal-operator:release-0.4
   loadBehavior: tryLoad
@@ -25,6 +25,11 @@ variables:
 
   # This setting is for the separate cluster that are used for upgrade tests
   UPGRADE_DEPLOY_CERT_MANAGER: "true"
+  # We run upgrade tests separately, so they can use the default kind cluster
+  # since there are no other tests running at the same time.
+  # Creating separate upgrade clusters would cause port collisions with the default kind cluster.
+  # We also configure these jobs to not deploy BMO/Ironic so that can be handled by the upgrade test.
+  UPGRADE_USE_EXISTING_CLUSTER: "true"
 
   IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
   ISO_IMAGE_URL: "http://192.168.222.1/minimal_linux_live-v2.iso"
@@ -53,31 +58,51 @@ intervals:
   default/wait-secret-deletion: ["1m", "1s"]
   default/wait-connect-ssh: ["2m", "10s"]
 
+kindExtraPortMappings:
+# Expose Ironic ports so they are reachable outside of kind
+- containerPort: 6180
+  hostPort: 6180
+  listenAddress: "0.0.0.0"
+  protocol: TCP
+- containerPort: 6385
+  hostPort: 6385
+  listenAddress: "0.0.0.0"
+  protocol: TCP
+- containerPort: 5050
+  hostPort: 5050
+  listenAddress: "0.0.0.0"
+  protocol: TCP
+
 bmoIronicUpgradeSpecs:
+# Upgrade BMO 0.4 -> latest
 - deployIronic: true
   deployBMO: true
   initBMOKustomization: "../../config/overlays/e2e-release-0.4"
   initIronicKustomization: "../../ironic-deployment/overlays/e2e-with-inspector"
   upgradeEntityKustomization: "../../config/overlays/e2e"
   upgradeEntityName: "bmo"
+# Upgrade BMO 0.5 -> latest
 - deployIronic: true
   deployBMO: true
   initBMOKustomization: "../../config/overlays/e2e-release-0.5"
   initIronicKustomization: "../../ironic-deployment/overlays/e2e"
   upgradeEntityKustomization: "../../config/overlays/e2e"
   upgradeEntityName: "bmo"
+# Upgrade BMO 0.6 -> latest
 - deployIronic: true
   deployBMO: true
   initBMOKustomization: "../../config/overlays/e2e-release-0.6"
   initIronicKustomization: "../../ironic-deployment/overlays/e2e"
   upgradeEntityKustomization: "../../config/overlays/e2e"
   upgradeEntityName: "bmo"
+# Upgrade Ironic 24.0-with-inspector -> latest | BMO 0.6
 - deployIronic: true
   deployBMO: true
   initBMOKustomization: "../../config/overlays/e2e-release-0.6"
   initIronicKustomization: "../../ironic-deployment/overlays/e2e-release-24.0-with-inspector"
   upgradeEntityKustomization: "../../ironic-deployment/overlays/e2e"
   upgradeEntityName: "ironic"
+# Upgrade Ironic 24.1 -> latest | BMO 0.6
 - deployIronic: true
   deployBMO: true
   initBMOKustomization: "../../config/overlays/e2e-release-0.6"