diff --git a/.gitignore b/.gitignore index 9f265bb9..15cc1211 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,15 @@ # IntelliJ IDEA files .idea/ + +# Terraform files and artifacts +**/.terraform +**/.terraform.lock.hcl +**/terraform.tfstate +**/terraform.tfstate.backup +**/terraform.tfvars +**/VMs.yaml + +# MKE 4 config files + +**/mke4.yaml +**/mke.yaml diff --git a/content/docs/getting-started/create-cluster.md b/content/docs/getting-started/create-cluster.md index 02ef2359..d9e9a608 100644 --- a/content/docs/getting-started/create-cluster.md +++ b/content/docs/getting-started/create-cluster.md @@ -3,7 +3,6 @@ title: Create a cluster weight: 3 --- - ## Install dependecies Verify that you have installed `mkectl` and other dependencies on your system @@ -68,12 +67,72 @@ process and ensures consistency in cluster deployments. license: refresh: true apiServer: - sans: ["mydomain.com"] + externalAddress: mke.example.com + sans: [] ingressController: - enabled: false + enabled: true + replicaCount: 2 + extraArgs: + httpPort: 80 + httpsPort: 443 + enableSslPassthrough: false + defaultSslCertificate: mke/auth-https.tls monitoring: enableGrafana: true enableOpscare: false + network: + kubeProxy: + disabled: false + mode: iptables + metricsbindaddress: 0.0.0.0:10249 + iptables: + masqueradebit: null + masqueradeall: false + localhostnodeports: null + syncperiod: + duration: 0s + minsyncperiod: + duration: 0s + ipvs: + syncperiod: + duration: 0s + minsyncperiod: + duration: 0s + scheduler: "" + excludecidrs: [] + strictarp: false + tcptimeout: + duration: 0s + tcpfintimeout: + duration: 0s + udptimeout: + duration: 0s + nodeportaddresses: [] + nllb: + disabled: true + cplb: + disabled: true + providers: + - provider: calico + enabled: true + CALICO_DISABLE_FILE_LOGGING: true + CALICO_STARTUP_LOGLEVEL: DEBUG + FELIX_LOGSEVERITYSCREEN: DEBUG + clusterCIDRIPv4: 192.168.0.0/16 + deployWithOperator: false + enableWireguard: false + ipAutodetectionMethod: null + mode: vxlan + overlay: Always + vxlanPort: 4789 + vxlanVNI: 10000 + windowsNodes: false + - provider: kuberouter + enabled: false + deployWithOperator: false + - provider: custom + enabled: false + deployWithOperator: false ``` {{< /details >}} @@ -84,16 +143,22 @@ process and ensures consistency in cluster deployments. mkectl init > mke.yaml ``` -2. In the configuration file, edit the `hosts` section to match your roster - of nodes. Provide the SSH information for each cluster node, as well as - the role of the node based on their functions within the cluster. The table - below provides the list of available node roles and their descriptions: +2. In the generated configuration file: + + - Edit the `hosts` section to match your roster of nodes. Provide the SSH + information for each cluster node, as well as the role of the node based + on their functions within the cluster. The table below provides the list + of available node roles and their descriptions: + + | Node Role | Description | + |-----------------------|-------------------------------------------------------------------------------------------------| + | **controller+worker** | A manager node that runs both control plane and data plane components. This role combines the responsibilities of managing cluster operations and executing workloads. | + | **worker** | A worker node that runs the data plane components. These nodes are dedicated to executing workloads and handling the operational tasks assigned by the control plane. | + | **single** | A special role used when the cluster consists of a single node. This node handles both control plane and data plane components, effectively managing and executing workloads within a standalone environment. | - | Node Role | Description | - |-----------------------|-------------------------------------------------------------------------------------------------| - | **controller+worker** | A manager node that runs both control plane and data plane components. This role combines the responsibilities of managing cluste operations and executing workloads. | - | **worker** | A worker node that runs the data plane components. These nodes are dedicated to executing workloads and handling the operational task assigned by the control plane. | - | **single** | A special role used when the cluster consists of a single node. This node handles both control plane and data plane components, effectivel managing and executing workloads within a standalone environment. | + - Specify the external address in the in `apiServer.externalAddress` field. + The external address is the domain name of the load balancer configured + as described in [System Requirements: Load balancer](../system-requirements#load-balancer-requirements). ## Create a cluster diff --git a/content/docs/getting-started/system-requirements.md b/content/docs/getting-started/system-requirements.md index ac4cd247..ca026788 100644 --- a/content/docs/getting-started/system-requirements.md +++ b/content/docs/getting-started/system-requirements.md @@ -19,3 +19,36 @@ documentation](https://docs.k0sproject.io/v1.29.4+k0s.0/system-requirements/). - Ubuntu 20.04 Linux - Architecture: `amd64` - CNI: Calico + +## Load balancer requirements + +The load balancer can be implemented in many different ways. You can use for example +HAProxy, NGINX, or the load balancer of your cloud provider. + +To ensure the MKE Dashboard functions properly, MKE requires a TCP load balancer. +This load balancer acts as a single point of contact to access the controllers. +With the default MKE configuration, the load balancer must allow and route traffic +to each controller through the following ports: + +| Listen port | Target port | Purpose | Configurable | +|-------------|-------------|---------------------|-----------------------| +| 6443 | 6443 | Kubernetes API | {{< icon "ban" >}} | +| 8132 | 8132 | Konnectivity | {{< icon "ban" >}} | +| 9443 | 9443 | Controller join API | {{< icon "ban" >}} | +| 443 | 33001 | Ingress Controller | {{< icon "check" >}} | + +You can configure the listen port of the Ingress Controller to be different from +the default port `443`. However, if you change the listen port, you must append +the new port number to the external address in the configuration file. For example, +if you set the listen port to be the same as the target port, `33001`, the configuration +should look as follows: + +```yaml +apiServer: + externalAddress: "mke.example.com:33001" +``` + +The target port must match the HTTPS node port of the Ingress Controller, +which is `33001` by default, but can be adjusted as needed. See the configuration +details for `nodePorts` in the +[Ingress Controller configuration](../../operations/ingress#configuration) diff --git a/content/docs/migrate-from-MKE-3.md b/content/docs/migrate-from-MKE-3.md index 67987518..866f11a9 100644 --- a/content/docs/migrate-from-MKE-3.md +++ b/content/docs/migrate-from-MKE-3.md @@ -94,20 +94,23 @@ are performed through the use of the `mkectl` tool: - Carry out post-upgrade cleanup, to remove MKE 3 components. - Output the new MKE 4 config file. -To upgrade an MKE 3 cluster, run the following command: +To upgrade an MKE 3 cluster, use the `mkectl upgrade` command: ```shell -mkectl upgrade --hosts-path --admin-username --admin-password +mkectl upgrade --hosts-path \ + --mke3-admin-username \ + --mke3-admin-password \ + --external-address \ + --config-out ``` -{{< callout type="info" >}} - -The MKE 4 config file prints to your console when the migration is complete. To -output the config file to a file for future use, run `mkectl apply`. -Alternatively, you can set the `--config-out` flag to the path where you want -to save the MKE 4 config file. +The external address is the domain name of the load balancer. For details, +see [System requirements: Load balancer requirements](../getting-started/system-requirements#load-balancer-requirements). -{{< /callout >}} +The `--config-out` flag allows you to specify a path where the MKE 4 configuration +file will be automatically created and saved during migration. If not specified, +the configuration file prints to your console on completion. In this case, save +the output to a file for future reference The upgrade process requires time to complete. Once the process is complete, run the following command to verify that the MKE 4 cluster is operating: diff --git a/content/docs/operations/ingress/_index.md b/content/docs/operations/ingress/_index.md index 83d0735d..4267f7af 100644 --- a/content/docs/operations/ingress/_index.md +++ b/content/docs/operations/ingress/_index.md @@ -15,36 +15,31 @@ currently supports. ## Configuration You can configure NGINX Ingress Controller through the `ingressController` -section of the MKE 4 configuration file. The function is disabled by default -and can be enabled by setting the `enabled` parameter to `true`. +section of the MKE 4 configuration file. The function is enabled by default +and must not be disabled for the cluster to function correctly. -```yaml -ingressController: - enabled: true -``` - -Other optional ingress controller parameters that you can configure are +Ingress controller parameters that you can configure are detailed in the following table. -| Field | Description | Default | -|---------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------| -| replicaCount | Sets the number of NGINX Ingress Controller deployment replicas. | 2 | -| enableLoadBalancer | Enables an external load balancer.

Valid values: `true`, `false`. | `false` | -| extraArgs | Additional command line arguments to pass to Ingress-Nginx Controller. | {} (empty) | | -| extraArgs.httpPort | Sets the container port for servicing HTTP traffic. | `80` | -| extraArgs.httpsPort | Sets the container port for servicing HTTPS traffic. | `443` | -| extraArgs.enableSslPassthrough | Enables SSL passthrough. | false | -| extraArgs.defaultSslCertificate | Sets the Secret that contains an SSL certificate to use as a default TLS certificate.

Valid value: ``/`` | "" | -| preserveClientIP | Enables preserving inbound traffic source IP.

Valid values: `true`, `false`. | `false` | -| externalIPs | Sets the list of external IPs for Ingress service. | [] (empty) | -| affinity | Sets node affinity. [Example](#affinity)

For more information, refer to the Kubernetes documentation [Affinity and anti-affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node#affinity-and-anti-affinity). | {} (empty) | -| tolerations | Sets node toleration. [Example](#tolerations)

Refer to the Kubernetes documentation [Assigning Pods to Nodes](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for more detail. | [] (empty) | | -| configMap | Adds custom configuration options to Nginx.

For a complete list of available options, refer to the [NGINX Ingress Controller ConfigMap](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#configuration-options). | {} (empty) | -| tcpServices | Sets TCP service key-value pairs; enables TCP services. [Example](./tcp_udp_services.md)

Refer to the NGINX Ingress documentation [Exposing TCP and UDP services](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md) for more information. for more information. | {} (empty) | -| udpServices | Sets UDP service key-value pairs; enables UDP services. [Example](./tcp_udp_services.md)

Refer to the NGINX Ingress documentation [Exposing TCP and UDP services](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md) for more information. | {} (empty) | -| nodePorts | Sets the node ports for the externalHTTP/HTTPS/TCP/UDP listener. | HTTP: 33000, HTTPS: 33001 | -| ports | Sets the port for the internalHTTP/HTTPS listener. | HTTP: 80, HTTPS: 443 | -| disableHttp | Disables the HTTP listener. | false | +| Field | Description | Default | +|---------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------| +| replicaCount | Sets the number of NGINX Ingress Controller deployment replicas. | 2 | +| enableLoadBalancer | Enables an external load balancer.

Valid values: `true`, `false`. | `true` if `apiServer.externalAddress` is set in the config file; `false` otherwise | +| extraArgs | Additional command line arguments to pass to Ingress-Nginx Controller. | {} (empty) | +| extraArgs.httpPort | Sets the container port for servicing HTTP traffic. | `80` | +| extraArgs.httpsPort | Sets the container port for servicing HTTPS traffic. | `443` | +| extraArgs.enableSslPassthrough | Enables SSL passthrough. | false | +| extraArgs.defaultSslCertificate | The only valid value is `mke/auth-https.tls`. Must NOT be changed. | `mke/auth-https.tls` | +| preserveClientIP | Enables preserving inbound traffic source IP.

Valid values: `true`, `false`. | `false` | +| externalIPs | Sets the list of external IPs for Ingress service. IP addresses of managers nodes are always added automatically. | [] | +| affinity | Sets node affinity. [Example](#affinity)
Affinity is always configured to schedule ingress controller pods on manager nodes. Additional rules may be added, but it's not recommended.

For more information, refer to the Kubernetes documentation [Affinity and anti-affinity](https://kubernetes.io/docs/concepts/configuration/assign-pod-node#affinity-and-anti-affinity). | {} (empty) | +| tolerations | Sets node toleration. [Example](#tolerations)
Tolerations are always configured to allow scheduling on manager nodes. Additional rules may be added, but it's not recommended.

Refer to the Kubernetes documentation [Assigning Pods to Nodes](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/) for more detail. | [] (empty) | +| configMap | Adds custom configuration options to Nginx.

For a complete list of available options, refer to the [NGINX Ingress Controller ConfigMap](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#configuration-options). | {} (empty) | +| tcpServices | Sets TCP service key-value pairs; enables TCP services. [Example](./tcp_udp_services.md)

Refer to the NGINX Ingress documentation [Exposing TCP and UDP services](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md) for more information. for more information. | {} (empty) | +| udpServices | Sets UDP service key-value pairs; enables UDP services. [Example](./tcp_udp_services.md)

Refer to the NGINX Ingress documentation [Exposing TCP and UDP services](https://github.com/kubernetes/ingress-nginx/blob/main/docs/user-guide/exposing-tcp-udp-services.md) for more information. | {} (empty) | +| nodePorts | Sets the node ports for the external HTTP/HTTPS/TCP/UDP listener. You should not change the HTTPS port, but if you do so, make sure to change the target port of the MKE Dashboard in your load balancer configuration. See [prerequisites docs](../../../getting-started/system-requirements.md#load-balancer-requirements) for details. | HTTP: 33000, HTTPS: 33001 | +| ports | Sets the port for the internalHTTP/HTTPS listener. | HTTP: 80, HTTPS: 443 | +| disableHttp | Disables the HTTP listener. | false | ### Affinity @@ -99,10 +94,9 @@ ingressController: value: "value1" effect: "NoExecute" extraArgs: - httpPort: 0 - httpsPort: 0 - enableSslPassthrough: true - defaultSslCertificate: "" + httpPort: 80 + httpsPort: 443 + enableSslPassthrough: false configMap: access-log-path: "/var/log/nginx/access.log" generate-request-id: "true" @@ -113,13 +107,13 @@ ingressController: udpServices: 5005: "default/udp-listener:5005" nodePorts: - http: 33003 - https: 33004 + http: 33000 + https: 33001 tcp: 9000: 33011 udp: 5005: 33012 - ports: + ports: http: 8080 https: 4443 affinity: diff --git a/content/docs/release-notes/known-issues.md b/content/docs/release-notes/known-issues.md index 3f3f4b9d..76772c84 100644 --- a/content/docs/release-notes/known-issues.md +++ b/content/docs/release-notes/known-issues.md @@ -19,30 +19,24 @@ following an upgrade performed with `mkectl`. 3. Apply the updated MKE 4 configuration. -## [BOP-686] kubectl fails during MKE 3.x to 4.x migration +## [BOP-879] External Address flag is ignored when upgrading from a single manager MKE cluster -For a cluster with multiple controller nodes, [k0s requires the presence of -a load balancer for the controller node](https://docs.k0sproject.io/head/high-availability/ ). -Without a load balancer, the controller nodes is unable to reach the kubelet on worker -nodes, and thus the user will be presented with `No agent available` errors. +When you upgrade a single-manager MKE 3 cluster, the `--external-address` flag of the `mkectl upgrade` +command is ignored. **Workaround:** -1. If an external load balancer is not already in place, create one that - targets all controllers and forwards the following ports: - - - `443` for controller - - `6443` for Kubernetes API - - `8132` for Konnectivity - -2. Use `k0sctl` to update the `k0s` config to set `externalAddress`: +1. Open for editing the MKE 4 configuration file that is generated by `mkectl upgrade` + at the end of upgrade. +2. Define the external address in the `apiServer.externalAddress` field as required: ```yaml - k0s: - config: - spec: - api: - externalAddress: - sans: - - + apiServer: + externalAddress: "mke.example.com" ``` + +3. Apply the configuration change: + + ```sh + mkectl apply + ``` \ No newline at end of file diff --git a/content/docs/tutorials/k0s-in-aws/terraform-scenario.md b/content/docs/tutorials/k0s-in-aws/terraform-scenario.md index bbc55833..9fa84ce4 100644 --- a/content/docs/tutorials/k0s-in-aws/terraform-scenario.md +++ b/content/docs/tutorials/k0s-in-aws/terraform-scenario.md @@ -50,13 +50,13 @@ by selecting the desired region from the dropdown menu in the top-right corner. ## Install MKE on k0s -1. Generate a sample `mke.yaml` file: +1. Generate a sample `mke4.yaml` file: ```shell - mkectl init > mke.yaml + mkectl init > mke4.yaml ``` -2. Edit the `hosts` section in `mke.yaml` using the values from the `VMs.yaml` +2. Edit the `hosts` section in `mke4.yaml` using the values from the `VMs.yaml` file. Example configuration of the `hosts` section: ```yaml @@ -75,11 +75,26 @@ by selecting the desired region from the dropdown menu in the top-right corner. user: ubuntu ``` -3. Create the MKE cluster: +3. Edit the `apiServer.externalAddress` in the configuration file + + ```sh + terraform output -raw lb_dns_name | { read lb; yq -i ".apiServer.externalAddress = \"$lb\"" mke4.yaml; } + ``` + + If you do not have the `yq` tool installed, edit the `mke4.yaml` file manually + setting `apiServer.externalAddress` to the output of the `terraform output -raw lb_dns_name` command. + +4. Create the MKE cluster: ```shell - mkectl apply -f mke.yaml + mkectl apply -f mke4.yaml ``` + {{< callout type="info" >}} + Upon successful completion of the MKE 4 installation, a username and password + will be automatically generated and displayed once for you to use. + + To explicitly set a password value, run `mkectl apply -f mke4.yaml --admin-password ` . + {{< /callout >}} ## Clean up infrastructure diff --git a/content/docs/tutorials/k0s-in-aws/terraform/controller.tf b/content/docs/tutorials/k0s-in-aws/terraform/controller.tf index d71f0252..ef72e08f 100644 --- a/content/docs/tutorials/k0s-in-aws/terraform/controller.tf +++ b/content/docs/tutorials/k0s-in-aws/terraform/controller.tf @@ -1,7 +1,21 @@ +locals { + lb_ports = { + 443: 33001, + 6443: 6443, + 8132: 8132, + 9443: 9443, + } +} + +resource "aws_default_subnet" "subnet" { + availability_zone = "${var.region}a" +} + resource "aws_instance" "cluster-controller" { count = var.controller_count ami = data.aws_ami.ubuntu.id instance_type = var.cluster_flavor + subnet_id = aws_default_subnet.subnet.id tags = { Name = "controller+worker" @@ -15,3 +29,25 @@ resource "aws_instance" "cluster-controller" { volume_size = 10 } } + +resource "aws_default_vpc" "vpc" { +} + +resource "aws_lb" "lb" { + name = "${var.cluster_name}-lb" + internal = false + load_balancer_type = "network" + subnets = [aws_default_subnet.subnet.id] +} + +module "lb_targets" { + source = "./lb_targets" + for_each = local.lb_ports + listen_port = each.key + target_port = each.value + arn = aws_lb.lb.arn + node_ids = aws_instance.cluster-controller[*].id + node_count = var.controller_count + cluster_name = var.cluster_name + vpc_id = aws_default_vpc.vpc.id +} diff --git a/content/docs/tutorials/k0s-in-aws/terraform/lb_targets/main.tf b/content/docs/tutorials/k0s-in-aws/terraform/lb_targets/main.tf new file mode 100644 index 00000000..0c53de4a --- /dev/null +++ b/content/docs/tutorials/k0s-in-aws/terraform/lb_targets/main.tf @@ -0,0 +1,29 @@ +resource "aws_lb_target_group" "api" { + name = "${var.cluster_name}-${var.listen_port}-${var.target_port}" + protocol = "TCP" + vpc_id = var.vpc_id + port = var.target_port + health_check { + unhealthy_threshold = 2 + healthy_threshold = 4 + interval = 10 + protocol = "TCP" + } +} + +resource "aws_lb_listener" "api" { + load_balancer_arn = var.arn + port = var.listen_port + protocol = "TCP" + default_action { + target_group_arn = aws_lb_target_group.api.arn + type = "forward" + } +} + +resource "aws_lb_target_group_attachment" "api" { + count = var.node_count + target_group_arn = aws_lb_target_group.api.arn + target_id = var.node_ids[count.index] + port = var.target_port +} diff --git a/content/docs/tutorials/k0s-in-aws/terraform/lb_targets/variables.tf b/content/docs/tutorials/k0s-in-aws/terraform/lb_targets/variables.tf new file mode 100644 index 00000000..46afc8cb --- /dev/null +++ b/content/docs/tutorials/k0s-in-aws/terraform/lb_targets/variables.tf @@ -0,0 +1,32 @@ +variable "listen_port" { + type = string + default = "443" + description = "Port for the listener." +} + +variable "target_port" { + type = string + default = "443" + description = "Port for the target group." +} + +variable "cluster_name" { + description = "Name of the cluster." +} + +variable "node_count" { + type = number + description = "Number of nodes in the cluster." +} + +variable "node_ids" { + description = "List of node instance IDs." +} + +variable "arn" { + description = "LB-specific ARN" +} + +variable "vpc_id" { + description = "VPC ID" +} diff --git a/content/docs/tutorials/k0s-in-aws/terraform/main.tf b/content/docs/tutorials/k0s-in-aws/terraform/main.tf index 06ac56b7..7ce5548a 100644 --- a/content/docs/tutorials/k0s-in-aws/terraform/main.tf +++ b/content/docs/tutorials/k0s-in-aws/terraform/main.tf @@ -105,3 +105,7 @@ locals { output "k0s_cluster" { value = yamlencode(local.k0s_tmpl) } + +output "lb_dns_name" { + value = aws_lb.lb.dns_name +}