From c43cad00d3e127d27383401100a903b2f51bc839 Mon Sep 17 00:00:00 2001 From: Orzelius <33936483+Orzelius@users.noreply.github.com> Date: Tue, 14 Jan 2025 00:35:36 +0900 Subject: [PATCH] tmp Signed-off-by: Orzelius <33936483+Orzelius@users.noreply.github.com> --- cmd/talosctl/cmd/mgmt/cluster/cluster.go | 35 +- cmd/talosctl/cmd/mgmt/cluster/create.go | 1632 ----------------- .../cmd/mgmt/cluster/create/common.go | 474 +++++ .../cmd/mgmt/cluster/create/create.go | 598 ++++++ .../cmd/mgmt/cluster/create/docker.go | 160 ++ cmd/talosctl/cmd/mgmt/cluster/create/qemu.go | 934 ++++++++++ .../qemu_darwin.go} | 12 +- .../cmd/mgmt/cluster/create/qemu_linux.go | 70 + cmd/talosctl/cmd/mgmt/cluster/create_linux.go | 19 - cmd/talosctl/cmd/mgmt/cluster/destroy.go | 4 +- cmd/talosctl/cmd/mgmt/cluster/show.go | 8 +- .../{qemu_launch_linux.go => qemu_launch.go} | 2 +- go.mod | 1 + go.sum | 2 + pkg/cluster/apply-config.go | 27 +- pkg/provision/providers/docker/crashdump.go | 2 +- pkg/provision/providers/docker/create.go | 12 +- pkg/provision/providers/docker/destroy.go | 2 +- pkg/provision/providers/docker/docker.go | 31 +- pkg/provision/providers/docker/image.go | 2 +- pkg/provision/providers/docker/network.go | 8 +- pkg/provision/providers/docker/node.go | 14 +- pkg/provision/providers/docker/reflect.go | 2 +- pkg/provision/providers/docker/request.go | 38 + pkg/provision/providers/factory.go | 21 +- .../providers/{qemu_linux.go => qemu.go} | 2 +- pkg/provision/providers/qemu/arch.go | 32 +- pkg/provision/providers/qemu/create.go | 31 +- pkg/provision/providers/qemu/create_darwin.go | 14 + pkg/provision/providers/qemu/create_linux.go | 15 + pkg/provision/providers/qemu/destroy.go | 2 +- pkg/provision/providers/qemu/launch.go | 283 +-- pkg/provision/providers/qemu/launch_darwin.go | 89 + pkg/provision/providers/qemu/launch_linux.go | 249 +++ pkg/provision/providers/qemu/node.go | 61 +- pkg/provision/providers/qemu/node_darwin.go | 29 + pkg/provision/providers/qemu/node_linux.go | 57 + pkg/provision/providers/qemu/pflash.go | 2 +- pkg/provision/providers/qemu/preflight.go | 137 +- .../providers/qemu/preflight_darwin.go | 17 + .../providers/qemu/preflight_linux.go | 134 ++ pkg/provision/providers/qemu/qemu.go | 26 +- pkg/provision/providers/qemu/tpm2.go | 6 +- pkg/provision/providers/qemu_other.go | 2 +- pkg/provision/providers/vm/dhcpd_darwin.go | 15 + .../providers/vm/{dhcpd.go => dhcpd_linux.go} | 4 +- pkg/provision/providers/vm/disk.go | 7 +- pkg/provision/providers/vm/json-logs.go | 2 +- pkg/provision/providers/vm/kms.go | 2 +- pkg/provision/providers/vm/loadbalancer.go | 7 +- .../providers/vm/loadbalancer_darwin.go | 7 + .../providers/vm/loadbalancer_linux.go | 7 + pkg/provision/providers/vm/network_darwin.go | 21 + .../vm/{network.go => network_linux.go} | 4 +- pkg/provision/providers/vm/request.go | 117 ++ pkg/provision/providers/vm/request_darwin.go | 14 + pkg/provision/providers/vm/request_linux.go | 48 + .../providers/vm/siderolink-agent.go | 4 +- pkg/provision/provision.go | 8 +- pkg/provision/request.go | 159 +- pkg/provision/result.go | 3 +- 61 files changed, 3405 insertions(+), 2322 deletions(-) delete mode 100644 cmd/talosctl/cmd/mgmt/cluster/create.go create mode 100644 cmd/talosctl/cmd/mgmt/cluster/create/common.go create mode 100644 cmd/talosctl/cmd/mgmt/cluster/create/create.go create mode 100644 cmd/talosctl/cmd/mgmt/cluster/create/docker.go create mode 100644 cmd/talosctl/cmd/mgmt/cluster/create/qemu.go rename cmd/talosctl/cmd/mgmt/cluster/{create_other.go => create/qemu_darwin.go} (63%) create mode 100644 cmd/talosctl/cmd/mgmt/cluster/create/qemu_linux.go delete mode 100644 cmd/talosctl/cmd/mgmt/cluster/create_linux.go rename cmd/talosctl/cmd/mgmt/{qemu_launch_linux.go => qemu_launch.go} (96%) create mode 100644 pkg/provision/providers/docker/request.go rename pkg/provision/providers/{qemu_linux.go => qemu.go} (94%) create mode 100644 pkg/provision/providers/qemu/create_darwin.go create mode 100644 pkg/provision/providers/qemu/create_linux.go create mode 100644 pkg/provision/providers/qemu/launch_darwin.go create mode 100644 pkg/provision/providers/qemu/launch_linux.go create mode 100644 pkg/provision/providers/qemu/node_darwin.go create mode 100644 pkg/provision/providers/qemu/node_linux.go create mode 100644 pkg/provision/providers/qemu/preflight_darwin.go create mode 100644 pkg/provision/providers/qemu/preflight_linux.go create mode 100644 pkg/provision/providers/vm/dhcpd_darwin.go rename pkg/provision/providers/vm/{dhcpd.go => dhcpd_linux.go} (98%) create mode 100644 pkg/provision/providers/vm/loadbalancer_darwin.go create mode 100644 pkg/provision/providers/vm/loadbalancer_linux.go create mode 100644 pkg/provision/providers/vm/network_darwin.go rename pkg/provision/providers/vm/{network.go => network_linux.go} (98%) create mode 100644 pkg/provision/providers/vm/request.go create mode 100644 pkg/provision/providers/vm/request_darwin.go create mode 100644 pkg/provision/providers/vm/request_linux.go diff --git a/cmd/talosctl/cmd/mgmt/cluster/cluster.go b/cmd/talosctl/cmd/mgmt/cluster/cluster.go index 4e81ed6666..8581f89c58 100644 --- a/cmd/talosctl/cmd/mgmt/cluster/cluster.go +++ b/cmd/talosctl/cmd/mgmt/cluster/cluster.go @@ -6,12 +6,12 @@ package cluster import ( - "errors" "path/filepath" "github.com/spf13/cobra" clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" + "github.com/siderolabs/talos/pkg/provision/providers" ) // Cmd represents the cluster command. @@ -19,32 +19,27 @@ var Cmd = &cobra.Command{ Use: "cluster", Short: "A collection of commands for managing local docker-based or QEMU-based clusters", Long: ``, - PersistentPreRunE: func(*cobra.Command, []string) error { - if provisionerName == docker && !bootloaderEnabled { - return errors.New("docker provisioner requires bootloader to be enabled") - } - - return nil - }, } -var ( - provisionerName string - stateDir string - clusterName string +type ClusterCmdOps struct { + ProvisionerName string + StateDir string + ClusterName string - defaultStateDir string - defaultCNIDir string -) + DefaultStateDir string + DefaultCNIDir string +} + +var Flags ClusterCmdOps func init() { talosDir, err := clientconfig.GetTalosDirectory() if err == nil { - defaultStateDir = filepath.Join(talosDir, "clusters") - defaultCNIDir = filepath.Join(talosDir, "cni") + Flags.DefaultStateDir = filepath.Join(talosDir, "clusters") + Flags.DefaultCNIDir = filepath.Join(talosDir, "cni") } - Cmd.PersistentFlags().StringVar(&provisionerName, "provisioner", docker, "Talos cluster provisioner to use") - Cmd.PersistentFlags().StringVar(&stateDir, "state", defaultStateDir, "directory path to store cluster state") - Cmd.PersistentFlags().StringVar(&clusterName, "name", "talos-default", "the name of the cluster") + Cmd.PersistentFlags().StringVar(&Flags.ProvisionerName, "provisioner", providers.DockerProviderName, "Talos cluster provisioner to use") + Cmd.PersistentFlags().StringVar(&Flags.StateDir, "state", Flags.DefaultStateDir, "directory path to store cluster state") + Cmd.PersistentFlags().StringVar(&Flags.ClusterName, "name", "talos-default", "the name of the cluster") } diff --git a/cmd/talosctl/cmd/mgmt/cluster/create.go b/cmd/talosctl/cmd/mgmt/cluster/create.go deleted file mode 100644 index 92f608331c..0000000000 --- a/cmd/talosctl/cmd/mgmt/cluster/create.go +++ /dev/null @@ -1,1632 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package cluster - -import ( - "bytes" - "context" - "encoding/base64" - "errors" - "fmt" - "math/big" - "net" - "net/netip" - "net/url" - "os" - "path/filepath" - stdruntime "runtime" - "slices" - "strconv" - "strings" - "time" - - "github.com/docker/cli/opts" - "github.com/dustin/go-humanize" - "github.com/google/uuid" - "github.com/hashicorp/go-getter/v2" - "github.com/klauspost/compress/zstd" - "github.com/siderolabs/crypto/x509" - "github.com/siderolabs/gen/maps" - "github.com/siderolabs/go-blockdevice/v2/encryption" - "github.com/siderolabs/go-kubeconfig" - "github.com/siderolabs/go-pointer" - "github.com/siderolabs/go-procfs/procfs" - sideronet "github.com/siderolabs/net" - "github.com/spf13/cobra" - "k8s.io/client-go/tools/clientcmd" - - "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster/internal/firewallpatch" - "github.com/siderolabs/talos/cmd/talosctl/pkg/mgmt/helpers" - "github.com/siderolabs/talos/pkg/cli" - "github.com/siderolabs/talos/pkg/cluster/check" - "github.com/siderolabs/talos/pkg/images" - clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" - "github.com/siderolabs/talos/pkg/machinery/config" - "github.com/siderolabs/talos/pkg/machinery/config/bundle" - "github.com/siderolabs/talos/pkg/machinery/config/configloader" - "github.com/siderolabs/talos/pkg/machinery/config/configpatcher" - "github.com/siderolabs/talos/pkg/machinery/config/container" - "github.com/siderolabs/talos/pkg/machinery/config/encoder" - "github.com/siderolabs/talos/pkg/machinery/config/generate" - "github.com/siderolabs/talos/pkg/machinery/config/machine" - "github.com/siderolabs/talos/pkg/machinery/config/types/security" - "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" - "github.com/siderolabs/talos/pkg/machinery/constants" - "github.com/siderolabs/talos/pkg/machinery/nethelpers" - "github.com/siderolabs/talos/pkg/machinery/version" - "github.com/siderolabs/talos/pkg/provision" - "github.com/siderolabs/talos/pkg/provision/access" - "github.com/siderolabs/talos/pkg/provision/providers" -) - -const ( - docker = "docker" - - // gatewayOffset is the offset from the network address of the IP address of the network gateway. - gatewayOffset = 1 - - // nodesOffset is the offset from the network address of the beginning of the IP addresses to be used for nodes. - nodesOffset = 2 - - // vipOffset is the offset from the network address of the CIDR to use for allocating the Virtual (shared) IP address, if enabled. - vipOffset = 50 - - inputDirFlag = "input-dir" - networkIPv4Flag = "ipv4" - networkIPv6Flag = "ipv6" - networkMTUFlag = "mtu" - networkCIDRFlag = "cidr" - networkNoMasqueradeCIDRsFlag = "no-masquerade-cidrs" - nameserversFlag = "nameservers" - clusterDiskPreallocateFlag = "disk-preallocate" - clusterDisksFlag = "user-disk" - clusterDiskSizeFlag = "disk" - useVIPFlag = "use-vip" - bootloaderEnabledFlag = "with-bootloader" - controlPlanePortFlag = "control-plane-port" - firewallFlag = "with-firewall" - tpm2EnabledFlag = "with-tpm2" - withDebugShellFlag = "with-debug-shell" - - // The following flags are the gen options - the options that are only used in machine configuration (i.e., not during the qemu/docker provisioning). - // They are not applicable when no machine configuration is generated, hence mutually exclusive with the --input-dir flag. - - nodeInstallImageFlag = "install-image" - configDebugFlag = "with-debug" - dnsDomainFlag = "dns-domain" - withClusterDiscoveryFlag = "with-cluster-discovery" - registryMirrorFlag = "registry-mirror" - registryInsecureFlag = "registry-insecure-skip-verify" - customCNIUrlFlag = "custom-cni-url" - talosVersionFlag = "talos-version" - encryptStatePartitionFlag = "encrypt-state" - encryptEphemeralPartitionFlag = "encrypt-ephemeral" - enableKubeSpanFlag = "with-kubespan" - forceEndpointFlag = "endpoint" - kubePrismFlag = "kubeprism-port" - diskEncryptionKeyTypesFlag = "disk-encryption-key-types" -) - -var ( - talosconfig string - nodeImage string - nodeInstallImage string - registryMirrors []string - registryInsecure []string - kubernetesVersion string - nodeVmlinuzPath string - nodeInitramfsPath string - nodeISOPath string - nodeDiskImagePath string - nodeIPXEBootScript string - applyConfigEnabled bool - bootloaderEnabled bool - uefiEnabled bool - tpm2Enabled bool - extraUEFISearchPaths []string - configDebug bool - networkCIDR string - networkNoMasqueradeCIDRs []string - networkMTU int - networkIPv4 bool - networkIPv6 bool - wireguardCIDR string - nameservers []string - dnsDomain string - workers int - controlplanes int - controlPlaneCpus string - workersCpus string - controlPlaneMemory int - workersMemory int - clusterDiskSize int - clusterDiskPreallocate bool - clusterDisks []string - extraDisks int - extraDiskSize int - extraDisksDrivers []string - targetArch string - clusterWait bool - clusterWaitTimeout time.Duration - forceInitNodeAsEndpoint bool - forceEndpoint string - inputDir string - cniBinPath []string - cniConfDir string - cniCacheDir string - cniBundleURL string - ports string - dockerHostIP string - withInitNode bool - customCNIUrl string - crashdumpOnFailure bool - skipKubeconfig bool - skipInjectingConfig bool - talosVersion string - encryptStatePartition bool - encryptEphemeralPartition bool - useVIP bool - enableKubeSpan bool - enableClusterDiscovery bool - configPatch []string - configPatchControlPlane []string - configPatchWorker []string - badRTC bool - extraBootKernelArgs string - dockerDisableIPv6 bool - controlPlanePort int - kubePrismPort int - dhcpSkipHostname bool - skipK8sNodeReadinessCheck bool - networkChaos bool - jitter time.Duration - latency time.Duration - packetLoss float64 - packetReorder float64 - packetCorrupt float64 - bandwidth int - diskEncryptionKeyTypes []string - withFirewall string - withUUIDHostnames bool - withSiderolinkAgent agentFlag - withJSONLogs bool - debugShellEnabled bool - configInjectionMethodFlag string - mountOpts opts.MountOpt -) - -// createCmd represents the cluster up command. -var createCmd = &cobra.Command{ - Use: "create", - Short: "Creates a local docker-based or QEMU-based kubernetes cluster", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - return cli.WithContext(context.Background(), create) - }, -} - -//nolint:gocyclo -func downloadBootAssets(ctx context.Context) error { - // download & cache images if provides as URLs - for _, downloadableImage := range []struct { - path *string - disableArchive bool - }{ - { - path: &nodeVmlinuzPath, - }, - { - path: &nodeInitramfsPath, - disableArchive: true, - }, - { - path: &nodeISOPath, - }, - { - path: &nodeDiskImagePath, - }, - } { - if *downloadableImage.path == "" { - continue - } - - u, err := url.Parse(*downloadableImage.path) - if err != nil || !(u.Scheme == "http" || u.Scheme == "https") { - // not a URL - continue - } - - defaultStateDir, err := clientconfig.GetTalosDirectory() - if err != nil { - return err - } - - cacheDir := filepath.Join(defaultStateDir, "cache") - - if os.MkdirAll(cacheDir, 0o755) != nil { - return err - } - - destPath := strings.ReplaceAll( - strings.ReplaceAll(u.String(), "/", "-"), - ":", "-") - - _, err = os.Stat(filepath.Join(cacheDir, destPath)) - if err == nil { - *downloadableImage.path = filepath.Join(cacheDir, destPath) - - // already cached - continue - } - - fmt.Fprintf(os.Stderr, "downloading asset from %q to %q\n", u.String(), filepath.Join(cacheDir, destPath)) - - client := getter.Client{ - Getters: []getter.Getter{ - &getter.HttpGetter{ - HeadFirstTimeout: 30 * time.Minute, - ReadTimeout: 30 * time.Minute, - }, - }, - } - - if downloadableImage.disableArchive { - q := u.Query() - - q.Set("archive", "false") - - u.RawQuery = q.Encode() - } - - _, err = client.Get(ctx, &getter.Request{ - Src: u.String(), - Dst: filepath.Join(cacheDir, destPath), - GetMode: getter.ModeFile, - }) - if err != nil { - // clean up the destination on failure - os.Remove(filepath.Join(cacheDir, destPath)) //nolint:errcheck - - return err - } - - *downloadableImage.path = filepath.Join(cacheDir, destPath) - } - - return nil -} - -//nolint:gocyclo,cyclop -func create(ctx context.Context) error { - if err := downloadBootAssets(ctx); err != nil { - return err - } - - if controlplanes < 1 { - return errors.New("number of controlplanes can't be less than 1") - } - - controlPlaneNanoCPUs, err := parseCPUShare(controlPlaneCpus) - if err != nil { - return fmt.Errorf("error parsing --cpus: %s", err) - } - - workerNanoCPUs, err := parseCPUShare(workersCpus) - if err != nil { - return fmt.Errorf("error parsing --cpus-workers: %s", err) - } - - controlPlaneMemory := int64(controlPlaneMemory) * 1024 * 1024 - workerMemory := int64(workersMemory) * 1024 * 1024 - - // Validate CIDR range and allocate IPs - fmt.Fprintln(os.Stderr, "validating CIDR and reserving IPs") - - cidr4, err := netip.ParsePrefix(networkCIDR) - if err != nil { - return fmt.Errorf("error validating cidr block: %w", err) - } - - if !cidr4.Addr().Is4() { - return errors.New("--cidr is expected to be IPV4 CIDR") - } - - // use ULA IPv6 network fd00::/8, add 'TAL' in hex to build /32 network, add IPv4 CIDR to build /64 unique network - cidr6, err := netip.ParsePrefix( - fmt.Sprintf( - "fd74:616c:%02x%02x:%02x%02x::/64", - cidr4.Addr().As4()[0], cidr4.Addr().As4()[1], cidr4.Addr().As4()[2], cidr4.Addr().As4()[3], - ), - ) - if err != nil { - return fmt.Errorf("error validating cidr IPv6 block: %w", err) - } - - var cidrs []netip.Prefix - - if networkIPv4 { - cidrs = append(cidrs, cidr4) - } - - if networkIPv6 { - cidrs = append(cidrs, cidr6) - } - - if len(cidrs) == 0 { - return errors.New("neither IPv4 nor IPv6 network was enabled") - } - - // Gateway addr at 1st IP in range, ex. 192.168.0.1 - gatewayIPs := make([]netip.Addr, len(cidrs)) - - for j := range gatewayIPs { - gatewayIPs[j], err = sideronet.NthIPInNetwork(cidrs[j], gatewayOffset) - if err != nil { - return err - } - } - - // Set starting ip at 2nd ip in range, ex: 192.168.0.2 - ips := make([][]netip.Addr, len(cidrs)) - - for j := range cidrs { - ips[j] = make([]netip.Addr, controlplanes+workers) - - for i := range ips[j] { - ips[j][i], err = sideronet.NthIPInNetwork(cidrs[j], nodesOffset+i) - if err != nil { - return err - } - } - } - - noMasqueradeCIDRs := make([]netip.Prefix, 0, len(networkNoMasqueradeCIDRs)) - - for _, cidr := range networkNoMasqueradeCIDRs { - var parsedCIDR netip.Prefix - - parsedCIDR, err = netip.ParsePrefix(cidr) - if err != nil { - return fmt.Errorf("error parsing non-masquerade CIDR %q: %w", cidr, err) - } - - noMasqueradeCIDRs = append(noMasqueradeCIDRs, parsedCIDR) - } - - // Parse nameservers - nameserverIPs := make([]netip.Addr, len(nameservers)) - - for i := range nameserverIPs { - nameserverIPs[i], err = netip.ParseAddr(nameservers[i]) - if err != nil { - return fmt.Errorf("failed parsing nameserver IP %q: %w", nameservers[i], err) - } - } - - // Virtual (shared) IP at the vipOffset IP in range, ex. 192.168.0.50 - var vip netip.Addr - - if useVIP { - vip, err = sideronet.NthIPInNetwork(cidrs[0], vipOffset) - if err != nil { - return err - } - } - - // Validate network chaos flags - if !networkChaos { - if jitter != 0 || latency != 0 || packetLoss != 0 || packetReorder != 0 || packetCorrupt != 0 || bandwidth != 0 { - return errors.New("network chaos flags can only be used with --with-network-chaos") - } - } - - provisioner, err := providers.Factory(ctx, provisionerName) - if err != nil { - return err - } - - defer provisioner.Close() //nolint:errcheck - - // Craft cluster and node requests - request := provision.ClusterRequest{ - Name: clusterName, - - Network: provision.NetworkRequest{ - Name: clusterName, - CIDRs: cidrs, - NoMasqueradeCIDRs: noMasqueradeCIDRs, - GatewayAddrs: gatewayIPs, - MTU: networkMTU, - Nameservers: nameserverIPs, - LoadBalancerPorts: []int{controlPlanePort}, - CNI: provision.CNIConfig{ - BinPath: cniBinPath, - ConfDir: cniConfDir, - CacheDir: cniCacheDir, - - BundleURL: cniBundleURL, - }, - DHCPSkipHostname: dhcpSkipHostname, - DockerDisableIPv6: dockerDisableIPv6, - NetworkChaos: networkChaos, - Jitter: jitter, - Latency: latency, - PacketLoss: packetLoss, - PacketReorder: packetReorder, - PacketCorrupt: packetCorrupt, - Bandwidth: bandwidth, - }, - - Image: nodeImage, - KernelPath: nodeVmlinuzPath, - InitramfsPath: nodeInitramfsPath, - ISOPath: nodeISOPath, - IPXEBootScript: nodeIPXEBootScript, - DiskImagePath: nodeDiskImagePath, - - SelfExecutable: os.Args[0], - StateDirectory: stateDir, - } - - provisionOptions := []provision.Option{ - provision.WithDockerPortsHostIP(dockerHostIP), - provision.WithBootlader(bootloaderEnabled), - provision.WithUEFI(uefiEnabled), - provision.WithTPM2(tpm2Enabled), - provision.WithDebugShell(debugShellEnabled), - provision.WithExtraUEFISearchPaths(extraUEFISearchPaths), - provision.WithTargetArch(targetArch), - provision.WithSiderolinkAgent(withSiderolinkAgent.IsEnabled()), - } - - var configBundleOpts []bundle.Option - - if debugShellEnabled { - if provisionerName != "qemu" { - return errors.New("debug shell only supported with qemu provisioner") - } - } - - if ports != "" { - if provisionerName != docker { - return errors.New("exposed-ports flag only supported with docker provisioner") - } - - portList := strings.Split(ports, ",") - provisionOptions = append(provisionOptions, provision.WithDockerPorts(portList)) - } - - disks, err := getDisks() - if err != nil { - return err - } - - if inputDir != "" { - configBundleOpts = append(configBundleOpts, bundle.WithExistingConfigs(inputDir)) - } else { - genOptions := []generate.Option{ - generate.WithInstallImage(nodeInstallImage), - generate.WithDebug(configDebug), - generate.WithDNSDomain(dnsDomain), - generate.WithClusterDiscovery(enableClusterDiscovery), - } - - for _, registryMirror := range registryMirrors { - components := strings.SplitN(registryMirror, "=", 2) - if len(components) != 2 { - return fmt.Errorf("invalid registry mirror spec: %q", registryMirror) - } - - genOptions = append(genOptions, generate.WithRegistryMirror(components[0], components[1])) - } - - for _, registryHost := range registryInsecure { - genOptions = append(genOptions, generate.WithRegistryInsecureSkipVerify(registryHost)) - } - - genOptions = append(genOptions, provisioner.GenOptions(request.Network)...) - - if customCNIUrl != "" { - genOptions = append(genOptions, generate.WithClusterCNIConfig(&v1alpha1.CNIConfig{ - CNIName: constants.CustomCNI, - CNIUrls: []string{customCNIUrl}, - })) - } - - if len(disks) > 1 { - // convert provision disks to machine disks - machineDisks := make([]*v1alpha1.MachineDisk, len(disks)-1) - for i, disk := range disks[1:] { - machineDisks[i] = &v1alpha1.MachineDisk{ - DeviceName: provisioner.UserDiskName(i + 1), - DiskPartitions: disk.Partitions, - } - } - - genOptions = append(genOptions, generate.WithUserDisks(machineDisks)) - } - - if talosVersion == "" { - if provisionerName == docker { - parts := strings.Split(nodeImage, ":") - - talosVersion = parts[len(parts)-1] - } else { - parts := strings.Split(nodeInstallImage, ":") - - talosVersion = parts[len(parts)-1] - } - } - - var versionContract *config.VersionContract - - if talosVersion != "latest" { - versionContract, err = config.ParseContractFromVersion(talosVersion) - if err != nil { - return fmt.Errorf("error parsing Talos version %q: %w", talosVersion, err) - } - - genOptions = append(genOptions, generate.WithVersionContract(versionContract)) - } - - if encryptStatePartition || encryptEphemeralPartition { - diskEncryptionConfig := &v1alpha1.SystemDiskEncryptionConfig{} - - var keys []*v1alpha1.EncryptionKey - - for i, key := range diskEncryptionKeyTypes { - switch key { - case "uuid": - keys = append(keys, &v1alpha1.EncryptionKey{ - KeyNodeID: &v1alpha1.EncryptionKeyNodeID{}, - KeySlot: i, - }) - case "kms": - var ip netip.Addr - - // get bridge IP - ip, err = sideronet.NthIPInNetwork(cidr4, 1) - if err != nil { - return err - } - - const port = 4050 - - keys = append(keys, &v1alpha1.EncryptionKey{ - KeyKMS: &v1alpha1.EncryptionKeyKMS{ - KMSEndpoint: "grpc://" + nethelpers.JoinHostPort(ip.String(), port), - }, - KeySlot: i, - }) - - provisionOptions = append(provisionOptions, provision.WithKMS(nethelpers.JoinHostPort("0.0.0.0", port))) - case "tpm": - keyTPM := &v1alpha1.EncryptionKeyTPM{} - - if versionContract.SecureBootEnrollEnforcementSupported() { - keyTPM.TPMCheckSecurebootStatusOnEnroll = pointer.To(true) - } - - keys = append(keys, &v1alpha1.EncryptionKey{ - KeyTPM: keyTPM, - KeySlot: i, - }) - default: - return fmt.Errorf("unknown key type %q", key) - } - } - - if len(keys) == 0 { - return errors.New("no disk encryption key types enabled") - } - - if encryptStatePartition { - diskEncryptionConfig.StatePartition = &v1alpha1.EncryptionConfig{ - EncryptionProvider: encryption.LUKS2, - EncryptionKeys: keys, - } - } - - if encryptEphemeralPartition { - diskEncryptionConfig.EphemeralPartition = &v1alpha1.EncryptionConfig{ - EncryptionProvider: encryption.LUKS2, - EncryptionKeys: keys, - } - } - - genOptions = append(genOptions, generate.WithSystemDiskEncryption(diskEncryptionConfig)) - } - - if useVIP { - genOptions = append(genOptions, - generate.WithNetworkOptions( - v1alpha1.WithNetworkInterfaceVirtualIP(provisioner.GetFirstInterface(), vip.String()), - ), - ) - } - - if enableKubeSpan { - genOptions = append(genOptions, - generate.WithNetworkOptions( - v1alpha1.WithKubeSpan(), - ), - ) - } - - if !bootloaderEnabled { - // disable kexec, as this would effectively use the bootloader - genOptions = append(genOptions, - generate.WithSysctls(map[string]string{ - "kernel.kexec_load_disabled": "1", - }), - ) - } - - if controlPlanePort != constants.DefaultControlPlanePort { - genOptions = append(genOptions, - generate.WithLocalAPIServerPort(controlPlanePort), - ) - } - - if kubePrismPort != constants.DefaultKubePrismPort { - genOptions = append(genOptions, - generate.WithKubePrismPort(kubePrismPort), - ) - } - - externalKubernetesEndpoint := provisioner.GetExternalKubernetesControlPlaneEndpoint(request.Network, controlPlanePort) - - if useVIP { - externalKubernetesEndpoint = "https://" + nethelpers.JoinHostPort(vip.String(), controlPlanePort) - } - - provisionOptions = append(provisionOptions, provision.WithKubernetesEndpoint(externalKubernetesEndpoint)) - - endpointList := provisioner.GetTalosAPIEndpoints(request.Network) - - switch { - case forceEndpoint != "": - // using non-default endpoints, provision additional cert SANs and fix endpoint list - endpointList = []string{forceEndpoint} - genOptions = append(genOptions, generate.WithAdditionalSubjectAltNames(endpointList)) - case forceInitNodeAsEndpoint: - endpointList = []string{ips[0][0].String()} - case len(endpointList) > 0: - for _, endpointHostPort := range endpointList { - endpointHost, _, err := net.SplitHostPort(endpointHostPort) - if err != nil { - endpointHost = endpointHostPort - } - - genOptions = append(genOptions, generate.WithAdditionalSubjectAltNames([]string{endpointHost})) - } - case endpointList == nil: - // use control plane nodes as endpoints, client-side load-balancing - for i := range controlplanes { - endpointList = append(endpointList, ips[0][i].String()) - } - } - - inClusterEndpoint := provisioner.GetInClusterKubernetesControlPlaneEndpoint(request.Network, controlPlanePort) - - if useVIP { - inClusterEndpoint = "https://" + nethelpers.JoinHostPort(vip.String(), controlPlanePort) - } - - genOptions = append(genOptions, generate.WithEndpointList(endpointList)) - configBundleOpts = append(configBundleOpts, - bundle.WithInputOptions( - &bundle.InputOptions{ - ClusterName: clusterName, - Endpoint: inClusterEndpoint, - KubeVersion: strings.TrimPrefix(kubernetesVersion, "v"), - GenOptions: genOptions, - }), - ) - } - - addConfigPatch := func(configPatches []string, configOpt func([]configpatcher.Patch) bundle.Option) error { - var patches []configpatcher.Patch - - patches, err = configpatcher.LoadPatches(configPatches) - if err != nil { - return fmt.Errorf("error parsing config JSON patch: %w", err) - } - - configBundleOpts = append(configBundleOpts, configOpt(patches)) - - return nil - } - - if err = addConfigPatch(configPatch, bundle.WithPatch); err != nil { - return err - } - - if err = addConfigPatch(configPatchControlPlane, bundle.WithPatchControlPlane); err != nil { - return err - } - - if err = addConfigPatch(configPatchWorker, bundle.WithPatchWorker); err != nil { - return err - } - - if withFirewall != "" { - var defaultAction nethelpers.DefaultAction - - defaultAction, err = nethelpers.DefaultActionString(withFirewall) - if err != nil { - return err - } - - var controlplaneIPs []netip.Addr - - for i := range ips { - controlplaneIPs = append(controlplaneIPs, ips[i][:controlplanes]...) - } - - configBundleOpts = append(configBundleOpts, - bundle.WithPatchControlPlane([]configpatcher.Patch{firewallpatch.ControlPlane(defaultAction, cidrs, gatewayIPs, controlplaneIPs)}), - bundle.WithPatchWorker([]configpatcher.Patch{firewallpatch.Worker(defaultAction, cidrs, gatewayIPs)}), - ) - } - - var slb *siderolinkBuilder - - if withSiderolinkAgent.IsEnabled() { - slb, err = newSiderolinkBuilder(gatewayIPs[0].String(), withSiderolinkAgent.IsTLS()) - if err != nil { - return err - } - } - - if trustedRootsConfig := slb.TrustedRootsConfig(); trustedRootsConfig != nil { - trustedRootsPatch, err := configloader.NewFromBytes(trustedRootsConfig) - if err != nil { - return fmt.Errorf("error loading trusted roots config: %w", err) - } - - configBundleOpts = append(configBundleOpts, bundle.WithPatch([]configpatcher.Patch{configpatcher.NewStrategicMergePatch(trustedRootsPatch)})) - } - - if withJSONLogs { - const port = 4003 - - provisionOptions = append(provisionOptions, provision.WithJSONLogs(nethelpers.JoinHostPort(gatewayIPs[0].String(), port))) - - cfg := container.NewV1Alpha1( - &v1alpha1.Config{ - ConfigVersion: "v1alpha1", - MachineConfig: &v1alpha1.MachineConfig{ - MachineLogging: &v1alpha1.LoggingConfig{ - LoggingDestinations: []v1alpha1.LoggingDestination{ - { - LoggingEndpoint: &v1alpha1.Endpoint{ - URL: &url.URL{ - Scheme: "tcp", - Host: nethelpers.JoinHostPort(gatewayIPs[0].String(), port), - }, - }, - LoggingFormat: "json_lines", - }, - }, - }, - }, - }) - configBundleOpts = append(configBundleOpts, bundle.WithPatch([]configpatcher.Patch{configpatcher.NewStrategicMergePatch(cfg)})) - } - - configBundle, err := bundle.NewBundle(configBundleOpts...) - if err != nil { - return err - } - - bundleTalosconfig := configBundle.TalosConfig() - if bundleTalosconfig == nil { - if clusterWait { - return errors.New("no talosconfig in the config bundle: cannot wait for cluster") - } - - if applyConfigEnabled { - return errors.New("no talosconfig in the config bundle: cannot apply config") - } - } - - if skipInjectingConfig { - types := []machine.Type{machine.TypeControlPlane, machine.TypeWorker} - - if withInitNode { - types = slices.Insert(types, 0, machine.TypeInit) - } - - if err = configBundle.Write(".", encoder.CommentsAll, types...); err != nil { - return err - } - } - - // Wireguard configuration. - var wireguardConfigBundle *helpers.WireguardConfigBundle - if wireguardCIDR != "" { - wireguardConfigBundle, err = helpers.NewWireguardConfigBundle(ips[0], wireguardCIDR, 51111, controlplanes) - if err != nil { - return err - } - } - - var extraKernelArgs *procfs.Cmdline - - if extraBootKernelArgs != "" || withSiderolinkAgent.IsEnabled() { - extraKernelArgs = procfs.NewCmdline(extraBootKernelArgs) - } - - err = slb.SetKernelArgs(extraKernelArgs, withSiderolinkAgent.IsTunnel()) - if err != nil { - return err - } - - // Add talosconfig to provision options, so we'll have it to parse there - provisionOptions = append(provisionOptions, provision.WithTalosConfig(configBundle.TalosConfig())) - - var configInjectionMethod provision.ConfigInjectionMethod - - switch configInjectionMethodFlag { - case "", "default", "http": - configInjectionMethod = provision.ConfigInjectionMethodHTTP - case "metal-iso": - configInjectionMethod = provision.ConfigInjectionMethodMetalISO - default: - return fmt.Errorf("unknown config injection method %q", configInjectionMethod) - } - - // Create the controlplane nodes. - for i := range controlplanes { - var cfg config.Provider - - nodeIPs := make([]netip.Addr, len(cidrs)) - for j := range nodeIPs { - nodeIPs[j] = ips[j][i] - } - - nodeUUID := uuid.New() - - err = slb.DefineIPv6ForUUID(nodeUUID) - if err != nil { - return err - } - - nodeReq := provision.NodeRequest{ - Name: nodeName(clusterName, "controlplane", i+1, nodeUUID), - Type: machine.TypeControlPlane, - IPs: nodeIPs, - Memory: controlPlaneMemory, - NanoCPUs: controlPlaneNanoCPUs, - Disks: disks, - Mounts: mountOpts.Value(), - SkipInjectingConfig: skipInjectingConfig, - ConfigInjectionMethod: configInjectionMethod, - BadRTC: badRTC, - ExtraKernelArgs: extraKernelArgs, - UUID: pointer.To(nodeUUID), - } - - if withInitNode && i == 0 { - cfg = configBundle.Init() - nodeReq.Type = machine.TypeInit - } else { - cfg = configBundle.ControlPlane() - } - - if wireguardConfigBundle != nil { - cfg, err = wireguardConfigBundle.PatchConfig(nodeIPs[0], cfg) - if err != nil { - return err - } - } - - nodeReq.Config = cfg - - request.Nodes = append(request.Nodes, nodeReq) - } - - // append extra disks - for i := range extraDisks { - driver := "ide" - - // ide driver is not supported on arm64 - if targetArch == "arm64" { - driver = "virtio" - } - - if i < len(extraDisksDrivers) { - driver = extraDisksDrivers[i] - } - - disks = append(disks, &provision.Disk{ - Size: uint64(extraDiskSize) * 1024 * 1024, - SkipPreallocate: !clusterDiskPreallocate, - Driver: driver, - }) - } - - for i := 1; i <= workers; i++ { - cfg := configBundle.Worker() - - nodeIPs := make([]netip.Addr, len(cidrs)) - for j := range nodeIPs { - nodeIPs[j] = ips[j][controlplanes+i-1] - } - - if wireguardConfigBundle != nil { - cfg, err = wireguardConfigBundle.PatchConfig(nodeIPs[0], cfg) - if err != nil { - return err - } - } - - nodeUUID := uuid.New() - - err = slb.DefineIPv6ForUUID(nodeUUID) - if err != nil { - return err - } - - request.Nodes = append(request.Nodes, - provision.NodeRequest{ - Name: nodeName(clusterName, "worker", i, nodeUUID), - Type: machine.TypeWorker, - IPs: nodeIPs, - Memory: workerMemory, - NanoCPUs: workerNanoCPUs, - Disks: disks, - Mounts: mountOpts.Value(), - Config: cfg, - ConfigInjectionMethod: configInjectionMethod, - SkipInjectingConfig: skipInjectingConfig, - BadRTC: badRTC, - ExtraKernelArgs: extraKernelArgs, - UUID: pointer.To(nodeUUID), - }) - } - - request.SiderolinkRequest = slb.SiderolinkRequest() - - cluster, err := provisioner.Create(ctx, request, provisionOptions...) - if err != nil { - return err - } - - if debugShellEnabled { - fmt.Println("You can now connect to debug shell on any node using these commands:") - - for _, node := range request.Nodes { - talosDir, err := clientconfig.GetTalosDirectory() - if err != nil { - return nil - } - - fmt.Printf("socat - UNIX-CONNECT:%s\n", filepath.Join(talosDir, "clusters", clusterName, node.Name+".serial")) - } - - return nil - } - - // No talosconfig in the bundle - skip the operations below - if bundleTalosconfig == nil { - return nil - } - - // Create and save the talosctl configuration file. - if err = saveConfig(bundleTalosconfig); err != nil { - return err - } - - clusterAccess := access.NewAdapter(cluster, provisionOptions...) - defer clusterAccess.Close() //nolint:errcheck - - if applyConfigEnabled { - err = clusterAccess.ApplyConfig(ctx, request.Nodes, request.SiderolinkRequest, os.Stdout) - if err != nil { - return err - } - } - - if err = postCreate(ctx, clusterAccess); err != nil { - if crashdumpOnFailure { - provisioner.CrashDump(ctx, cluster, os.Stderr) - } - - return err - } - - return showCluster(cluster) -} - -func nodeName(clusterName, role string, index int, uuid uuid.UUID) string { - if withUUIDHostnames { - return fmt.Sprintf("machine-%s", uuid) - } - - return fmt.Sprintf("%s-%s-%d", clusterName, role, index) -} - -func postCreate(ctx context.Context, clusterAccess *access.Adapter) error { - if !withInitNode { - if err := clusterAccess.Bootstrap(ctx, os.Stdout); err != nil { - return fmt.Errorf("bootstrap error: %w", err) - } - } - - if !clusterWait { - return nil - } - - // Run cluster readiness checks - checkCtx, checkCtxCancel := context.WithTimeout(ctx, clusterWaitTimeout) - defer checkCtxCancel() - - checks := check.DefaultClusterChecks() - - if skipK8sNodeReadinessCheck { - checks = slices.Concat(check.PreBootSequenceChecks(), check.K8sComponentsReadinessChecks()) - } - - checks = append(checks, check.ExtraClusterChecks()...) - - if err := check.Wait(checkCtx, clusterAccess, checks, check.StderrReporter()); err != nil { - return err - } - - if !skipKubeconfig { - if err := mergeKubeconfig(ctx, clusterAccess); err != nil { - return err - } - } - - return nil -} - -func saveConfig(talosConfigObj *clientconfig.Config) (err error) { - c, err := clientconfig.Open(talosconfig) - if err != nil { - return fmt.Errorf("error opening talos config: %w", err) - } - - renames := c.Merge(talosConfigObj) - for _, rename := range renames { - fmt.Fprintf(os.Stderr, "renamed talosconfig context %s\n", rename.String()) - } - - return c.Save(talosconfig) -} - -func mergeKubeconfig(ctx context.Context, clusterAccess *access.Adapter) error { - kubeconfigPath, err := kubeconfig.SinglePath() - if err != nil { - return err - } - - fmt.Fprintf(os.Stderr, "\nmerging kubeconfig into %q\n", kubeconfigPath) - - k8sconfig, err := clusterAccess.Kubeconfig(ctx) - if err != nil { - return fmt.Errorf("error fetching kubeconfig: %w", err) - } - - kubeConfig, err := clientcmd.Load(k8sconfig) - if err != nil { - return fmt.Errorf("error parsing kubeconfig: %w", err) - } - - if clusterAccess.ForceEndpoint != "" { - for name := range kubeConfig.Clusters { - kubeConfig.Clusters[name].Server = clusterAccess.ForceEndpoint - } - } - - _, err = os.Stat(kubeconfigPath) - if err != nil { - if !os.IsNotExist(err) { - return err - } - - return clientcmd.WriteToFile(*kubeConfig, kubeconfigPath) - } - - merger, err := kubeconfig.Load(kubeconfigPath) - if err != nil { - return fmt.Errorf("error loading existing kubeconfig: %w", err) - } - - err = merger.Merge(kubeConfig, kubeconfig.MergeOptions{ - ActivateContext: true, - OutputWriter: os.Stdout, - ConflictHandler: func(component kubeconfig.ConfigComponent, name string) (kubeconfig.ConflictDecision, error) { - return kubeconfig.RenameDecision, nil - }, - }) - if err != nil { - return fmt.Errorf("error merging kubeconfig: %w", err) - } - - return merger.Write(kubeconfigPath) -} - -func parseCPUShare(cpus string) (int64, error) { - cpu, ok := new(big.Rat).SetString(cpus) - if !ok { - return 0, fmt.Errorf("failed to parsing as a rational number: %s", cpus) - } - - nano := cpu.Mul(cpu, big.NewRat(1e9, 1)) - if !nano.IsInt() { - return 0, errors.New("value is too precise") - } - - return nano.Num().Int64(), nil -} - -func getDisks() ([]*provision.Disk, error) { - // should have at least a single primary disk - disks := []*provision.Disk{ - { - Size: uint64(clusterDiskSize) * 1024 * 1024, - SkipPreallocate: !clusterDiskPreallocate, - Driver: "virtio", - }, - } - - for _, disk := range clusterDisks { - var ( - partitions = strings.Split(disk, ":") - diskPartitions = make([]*v1alpha1.DiskPartition, len(partitions)/2) - diskSize uint64 - ) - - if len(partitions)%2 != 0 { - return nil, errors.New("failed to parse malformed partition definitions") - } - - partitionIndex := 0 - - for j := 0; j < len(partitions); j += 2 { - partitionPath := partitions[j] - - if !strings.HasPrefix(partitionPath, "/var") { - return nil, errors.New("user disk partitions can only be mounted into /var folder") - } - - value, e := strconv.ParseInt(partitions[j+1], 10, 0) - partitionSize := uint64(value) - - if e != nil { - partitionSize, e = humanize.ParseBytes(partitions[j+1]) - - if e != nil { - return nil, errors.New("failed to parse partition size") - } - } - - diskPartitions[partitionIndex] = &v1alpha1.DiskPartition{ - DiskSize: v1alpha1.DiskSize(partitionSize), - DiskMountPoint: partitionPath, - } - diskSize += partitionSize - partitionIndex++ - } - - disks = append(disks, &provision.Disk{ - // add 1 MB to make extra room for GPT and alignment - Size: diskSize + 2*1024*1024, - Partitions: diskPartitions, - SkipPreallocate: !clusterDiskPreallocate, - Driver: "ide", - }) - } - - return disks, nil -} - -func init() { - createCmd.Flags().StringVar( - &talosconfig, - "talosconfig", - "", - fmt.Sprintf("The path to the Talos configuration file. Defaults to '%s' env variable if set, otherwise '%s' and '%s' in order.", - constants.TalosConfigEnvVar, - filepath.Join("$HOME", constants.TalosDir, constants.TalosconfigFilename), - filepath.Join(constants.ServiceAccountMountPath, constants.TalosconfigFilename), - ), - ) - createCmd.Flags().StringVar(&nodeImage, "image", helpers.DefaultImage(images.DefaultTalosImageRepository), "the image to use") - createCmd.Flags().StringVar(&nodeInstallImage, nodeInstallImageFlag, helpers.DefaultImage(images.DefaultInstallerImageRepository), "the installer image to use") - createCmd.Flags().StringVar(&nodeVmlinuzPath, "vmlinuz-path", helpers.ArtifactPath(constants.KernelAssetWithArch), "the compressed kernel image to use") - createCmd.Flags().StringVar(&nodeISOPath, "iso-path", "", "the ISO path to use for the initial boot (VM only)") - createCmd.Flags().StringVar(&nodeInitramfsPath, "initrd-path", helpers.ArtifactPath(constants.InitramfsAssetWithArch), "initramfs image to use") - createCmd.Flags().StringVar(&nodeDiskImagePath, "disk-image-path", "", "disk image to use") - createCmd.Flags().StringVar(&nodeIPXEBootScript, "ipxe-boot-script", "", "iPXE boot script (URL) to use") - createCmd.Flags().BoolVar(&applyConfigEnabled, "with-apply-config", false, "enable apply config when the VM is starting in maintenance mode") - createCmd.Flags().BoolVar(&bootloaderEnabled, bootloaderEnabledFlag, true, "enable bootloader to load kernel and initramfs from disk image after install") - createCmd.Flags().BoolVar(&uefiEnabled, "with-uefi", true, "enable UEFI on x86_64 architecture") - createCmd.Flags().BoolVar(&tpm2Enabled, tpm2EnabledFlag, false, "enable TPM2 emulation support using swtpm") - createCmd.Flags().BoolVar(&debugShellEnabled, withDebugShellFlag, false, "drop talos into a maintenance shell on boot, this is for advanced debugging for developers only") - createCmd.Flags().MarkHidden("with-debug-shell") //nolint:errcheck - createCmd.Flags().StringSliceVar(&extraUEFISearchPaths, "extra-uefi-search-paths", []string{}, "additional search paths for UEFI firmware (only applies when UEFI is enabled)") - createCmd.Flags().StringSliceVar(®istryMirrors, registryMirrorFlag, []string{}, "list of registry mirrors to use in format: =") - createCmd.Flags().StringSliceVar(®istryInsecure, registryInsecureFlag, []string{}, "list of registry hostnames to skip TLS verification for") - createCmd.Flags().BoolVar(&configDebug, configDebugFlag, false, "enable debug in Talos config to send service logs to the console") - createCmd.Flags().IntVar(&networkMTU, networkMTUFlag, 1500, "MTU of the cluster network") - createCmd.Flags().StringVar(&networkCIDR, networkCIDRFlag, "10.5.0.0/24", "CIDR of the cluster network (IPv4, ULA network for IPv6 is derived in automated way)") - createCmd.Flags().StringSliceVar(&networkNoMasqueradeCIDRs, networkNoMasqueradeCIDRsFlag, []string{}, "list of CIDRs to exclude from NAT (QEMU provisioner only)") - createCmd.Flags().BoolVar(&networkIPv4, networkIPv4Flag, true, "enable IPv4 network in the cluster") - createCmd.Flags().BoolVar(&networkIPv6, networkIPv6Flag, false, "enable IPv6 network in the cluster (QEMU provisioner only)") - createCmd.Flags().StringVar(&wireguardCIDR, "wireguard-cidr", "", "CIDR of the wireguard network") - createCmd.Flags().StringSliceVar(&nameservers, nameserversFlag, []string{"8.8.8.8", "1.1.1.1", "2001:4860:4860::8888", "2606:4700:4700::1111"}, "list of nameservers to use") - createCmd.Flags().IntVar(&workers, "workers", 1, "the number of workers to create") - createCmd.Flags().IntVar(&controlplanes, "masters", 1, "the number of masters to create") - createCmd.Flags().MarkDeprecated("masters", "use --controlplanes instead") //nolint:errcheck - createCmd.Flags().IntVar(&controlplanes, "controlplanes", 1, "the number of controlplanes to create") - createCmd.Flags().StringVar(&controlPlaneCpus, "cpus", "2.0", "the share of CPUs as fraction (each control plane/VM)") - createCmd.Flags().StringVar(&workersCpus, "cpus-workers", "2.0", "the share of CPUs as fraction (each worker/VM)") - createCmd.Flags().IntVar(&controlPlaneMemory, "memory", 2048, "the limit on memory usage in MB (each control plane/VM)") - createCmd.Flags().IntVar(&workersMemory, "memory-workers", 2048, "the limit on memory usage in MB (each worker/VM)") - createCmd.Flags().IntVar(&clusterDiskSize, clusterDiskSizeFlag, 6*1024, "default limit on disk size in MB (each VM)") - createCmd.Flags().BoolVar(&clusterDiskPreallocate, clusterDiskPreallocateFlag, true, "whether disk space should be preallocated") - createCmd.Flags().StringSliceVar(&clusterDisks, clusterDisksFlag, []string{}, "list of disks to create for each VM in format: :::") - createCmd.Flags().IntVar(&extraDisks, "extra-disks", 0, "number of extra disks to create for each worker VM") - createCmd.Flags().StringSliceVar(&extraDisksDrivers, "extra-disks-drivers", nil, "driver for each extra disk (virtio, ide, ahci, scsi, nvme)") - createCmd.Flags().IntVar(&extraDiskSize, "extra-disks-size", 5*1024, "default limit on disk size in MB (each VM)") - createCmd.Flags().StringVar(&targetArch, "arch", stdruntime.GOARCH, "cluster architecture") - createCmd.Flags().BoolVar(&clusterWait, "wait", true, "wait for the cluster to be ready before returning") - createCmd.Flags().DurationVar(&clusterWaitTimeout, "wait-timeout", 20*time.Minute, "timeout to wait for the cluster to be ready") - createCmd.Flags().BoolVar(&forceInitNodeAsEndpoint, "init-node-as-endpoint", false, "use init node as endpoint instead of any load balancer endpoint") - createCmd.Flags().StringVar(&forceEndpoint, forceEndpointFlag, "", "use endpoint instead of provider defaults") - createCmd.Flags().StringVar(&kubernetesVersion, "kubernetes-version", constants.DefaultKubernetesVersion, "desired kubernetes version to run") - createCmd.Flags().StringVarP(&inputDir, inputDirFlag, "i", "", "location of pre-generated config files") - createCmd.Flags().StringSliceVar(&cniBinPath, "cni-bin-path", []string{filepath.Join(defaultCNIDir, "bin")}, "search path for CNI binaries (VM only)") - createCmd.Flags().StringVar(&cniConfDir, "cni-conf-dir", filepath.Join(defaultCNIDir, "conf.d"), "CNI config directory path (VM only)") - createCmd.Flags().StringVar(&cniCacheDir, "cni-cache-dir", filepath.Join(defaultCNIDir, "cache"), "CNI cache directory path (VM only)") - createCmd.Flags().StringVar(&cniBundleURL, "cni-bundle-url", fmt.Sprintf("https://github.com/%s/talos/releases/download/%s/talosctl-cni-bundle-%s.tar.gz", - images.Username, version.Trim(version.Tag), constants.ArchVariable), "URL to download CNI bundle from (VM only)") - createCmd.Flags().StringVarP(&ports, - "exposed-ports", - "p", - "", - "Comma-separated list of ports/protocols to expose on init node. Ex -p :/ (Docker provisioner only)", - ) - createCmd.Flags().StringVar(&dockerHostIP, "docker-host-ip", "0.0.0.0", "Host IP to forward exposed ports to (Docker provisioner only)") - createCmd.Flags().BoolVar(&withInitNode, "with-init-node", false, "create the cluster with an init node") - createCmd.Flags().StringVar(&customCNIUrl, customCNIUrlFlag, "", "install custom CNI from the URL (Talos cluster)") - createCmd.Flags().StringVar(&dnsDomain, dnsDomainFlag, "cluster.local", "the dns domain to use for cluster") - createCmd.Flags().BoolVar(&crashdumpOnFailure, "crashdump", false, "generate support zip when cluster startup fails") - createCmd.Flags().BoolVar(&skipKubeconfig, "skip-kubeconfig", false, "skip merging kubeconfig from the created cluster") - createCmd.Flags().BoolVar(&skipInjectingConfig, "skip-injecting-config", false, "skip injecting config from embedded metadata server, write config files to current directory") - createCmd.Flags().BoolVar(&encryptStatePartition, encryptStatePartitionFlag, false, "enable state partition encryption") - createCmd.Flags().BoolVar(&encryptEphemeralPartition, encryptEphemeralPartitionFlag, false, "enable ephemeral partition encryption") - createCmd.Flags().StringArrayVar(&diskEncryptionKeyTypes, diskEncryptionKeyTypesFlag, []string{"uuid"}, "encryption key types to use for disk encryption (uuid, kms)") - createCmd.Flags().StringVar(&talosVersion, talosVersionFlag, "", "the desired Talos version to generate config for (if not set, defaults to image version)") - createCmd.Flags().BoolVar(&useVIP, useVIPFlag, false, "use a virtual IP for the controlplane endpoint instead of the loadbalancer") - createCmd.Flags().BoolVar(&enableClusterDiscovery, withClusterDiscoveryFlag, true, "enable cluster discovery") - createCmd.Flags().BoolVar(&enableKubeSpan, enableKubeSpanFlag, false, "enable KubeSpan system") - createCmd.Flags().StringArrayVar(&configPatch, "config-patch", nil, "patch generated machineconfigs (applied to all node types), use @file to read a patch from file") - createCmd.Flags().StringArrayVar(&configPatchControlPlane, "config-patch-control-plane", nil, "patch generated machineconfigs (applied to 'init' and 'controlplane' types)") - createCmd.Flags().StringArrayVar(&configPatchWorker, "config-patch-worker", nil, "patch generated machineconfigs (applied to 'worker' type)") - createCmd.Flags().BoolVar(&badRTC, "bad-rtc", false, "launch VM with bad RTC state (QEMU only)") - createCmd.Flags().StringVar(&extraBootKernelArgs, "extra-boot-kernel-args", "", "add extra kernel args to the initial boot from vmlinuz and initramfs (QEMU only)") - createCmd.Flags().BoolVar(&dockerDisableIPv6, "docker-disable-ipv6", false, "skip enabling IPv6 in containers (Docker only)") - createCmd.Flags().IntVar(&controlPlanePort, controlPlanePortFlag, constants.DefaultControlPlanePort, "control plane port (load balancer and local API port, QEMU only)") - createCmd.Flags().IntVar(&kubePrismPort, kubePrismFlag, constants.DefaultKubePrismPort, "KubePrism port (set to 0 to disable)") - createCmd.Flags().BoolVar(&dhcpSkipHostname, "disable-dhcp-hostname", false, "skip announcing hostname via DHCP (QEMU only)") - createCmd.Flags().BoolVar(&skipK8sNodeReadinessCheck, "skip-k8s-node-readiness-check", false, "skip k8s node readiness checks") - createCmd.Flags().BoolVar(&networkChaos, "with-network-chaos", false, "enable to use network chaos parameters when creating a qemu cluster") - createCmd.Flags().DurationVar(&jitter, "with-network-jitter", 0, "specify jitter on the bridge interface when creating a qemu cluster") - createCmd.Flags().DurationVar(&latency, "with-network-latency", 0, "specify latency on the bridge interface when creating a qemu cluster") - createCmd.Flags().Float64Var(&packetLoss, "with-network-packet-loss", 0.0, "specify percent of packet loss on the bridge interface when creating a qemu cluster. e.g. 50% = 0.50 (default: 0.0)") - createCmd.Flags().Float64Var(&packetReorder, "with-network-packet-reorder", 0.0, - "specify percent of reordered packets on the bridge interface when creating a qemu cluster. e.g. 50% = 0.50 (default: 0.0)") - createCmd.Flags().Float64Var(&packetCorrupt, "with-network-packet-corrupt", 0.0, - "specify percent of corrupt packets on the bridge interface when creating a qemu cluster. e.g. 50% = 0.50 (default: 0.0)") - createCmd.Flags().IntVar(&bandwidth, "with-network-bandwidth", 0, "specify bandwidth restriction (in kbps) on the bridge interface when creating a qemu cluster") - createCmd.Flags().StringVar(&withFirewall, firewallFlag, "", "inject firewall rules into the cluster, value is default policy - accept/block (QEMU only)") - createCmd.Flags().BoolVar(&withUUIDHostnames, "with-uuid-hostnames", false, "use machine UUIDs as default hostnames (QEMU only)") - createCmd.Flags().Var(&withSiderolinkAgent, "with-siderolink", "enables the use of siderolink agent as configuration apply mechanism. `true` or `wireguard` enables the agent, `tunnel` enables the agent with grpc tunneling") //nolint:lll - createCmd.Flags().BoolVar(&withJSONLogs, "with-json-logs", false, "enable JSON logs receiver and configure Talos to send logs there") - createCmd.Flags().StringVar(&configInjectionMethodFlag, "config-injection-method", "", "a method to inject machine config: default is HTTP server, 'metal-iso' to mount an ISO (QEMU only)") - createCmd.Flags().Var(&mountOpts, "mount", "attach a mount to the container (Docker only)") - - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, nodeInstallImageFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, configDebugFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, dnsDomainFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, withClusterDiscoveryFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, registryMirrorFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, registryInsecureFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, customCNIUrlFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, talosVersionFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, encryptStatePartitionFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, encryptEphemeralPartitionFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, enableKubeSpanFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, forceEndpointFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, kubePrismFlag) - createCmd.MarkFlagsMutuallyExclusive(inputDirFlag, diskEncryptionKeyTypesFlag) - - Cmd.AddCommand(createCmd) -} - -func newSiderolinkBuilder(wgHost string, useTLS bool) (*siderolinkBuilder, error) { - prefix, err := networkPrefix("") - if err != nil { - return nil, err - } - - result := &siderolinkBuilder{ - wgHost: wgHost, - binds: map[uuid.UUID]netip.Addr{}, - prefix: prefix, - nodeIPv6Addr: prefix.Addr().Next().String(), - } - - if useTLS { - ca, err := x509.NewSelfSignedCertificateAuthority(x509.ECDSA(true), x509.IPAddresses([]net.IP{net.ParseIP(wgHost)})) - if err != nil { - return nil, err - } - - result.apiCert = ca.CrtPEM - result.apiKey = ca.KeyPEM - } - - var resultErr error - - for range 10 { - for _, d := range []struct { - field *int - net string - what string - }{ - {&result.wgPort, "udp", "WireGuard"}, - {&result.apiPort, "tcp", "gRPC API"}, - {&result.sinkPort, "tcp", "Event Sink"}, - {&result.logPort, "tcp", "Log Receiver"}, - } { - var err error - - *d.field, err = getDynamicPort(d.net) - if err != nil { - return nil, fmt.Errorf("failed to get dynamic port for %s: %w", d.what, err) - } - } - - resultErr = checkPortsDontOverlap(result.wgPort, result.apiPort, result.sinkPort, result.logPort) - if resultErr == nil { - break - } - } - - if resultErr != nil { - return nil, fmt.Errorf("failed to get non-overlapping dynamic ports in 10 attempts: %w", resultErr) - } - - return result, nil -} - -type siderolinkBuilder struct { - wgHost string - - binds map[uuid.UUID]netip.Addr - prefix netip.Prefix - nodeIPv6Addr string - wgPort int - apiPort int - sinkPort int - logPort int - - apiCert []byte - apiKey []byte -} - -// DefineIPv6ForUUID defines an IPv6 address for a given UUID. It is safe to call this method on a nil pointer. -func (slb *siderolinkBuilder) DefineIPv6ForUUID(id uuid.UUID) error { - if slb == nil { - return nil - } - - result, err := generateRandomNodeAddr(slb.prefix) - if err != nil { - return err - } - - slb.binds[id] = result.Addr() - - return nil -} - -// SiderolinkRequest returns a SiderolinkRequest based on the current state of the builder. -// It is safe to call this method on a nil pointer. -func (slb *siderolinkBuilder) SiderolinkRequest() provision.SiderolinkRequest { - if slb == nil { - return provision.SiderolinkRequest{} - } - - return provision.SiderolinkRequest{ - WireguardEndpoint: net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.wgPort)), - APIEndpoint: ":" + strconv.Itoa(slb.apiPort), - APICertificate: slb.apiCert, - APIKey: slb.apiKey, - SinkEndpoint: ":" + strconv.Itoa(slb.sinkPort), - LogEndpoint: ":" + strconv.Itoa(slb.logPort), - SiderolinkBind: maps.ToSlice(slb.binds, func(k uuid.UUID, v netip.Addr) provision.SiderolinkBind { - return provision.SiderolinkBind{ - UUID: k, - Addr: v, - } - }), - } -} - -// TrustedRootsConfig returns the trusted roots config for the current builder. -func (slb *siderolinkBuilder) TrustedRootsConfig() []byte { - if slb == nil || slb.apiCert == nil { - return nil - } - - trustedRootsConfig := security.NewTrustedRootsConfigV1Alpha1() - trustedRootsConfig.MetaName = "siderolink-ca" - trustedRootsConfig.Certificates = string(slb.apiCert) - - marshaled, err := encoder.NewEncoder(trustedRootsConfig, encoder.WithComments(encoder.CommentsDisabled)).Encode() - if err != nil { - panic(fmt.Sprintf("failed to marshal trusted roots config: %s", err)) - } - - return marshaled -} - -// SetKernelArgs sets the kernel arguments for the current builder. It is safe to call this method on a nil pointer. -func (slb *siderolinkBuilder) SetKernelArgs(extraKernelArgs *procfs.Cmdline, tunnel bool) error { - switch { - case slb == nil: - return nil - case extraKernelArgs.Get("siderolink.api") != nil, - extraKernelArgs.Get("talos.events.sink") != nil, - extraKernelArgs.Get("talos.logging.kernel") != nil: - return errors.New("siderolink kernel arguments are already set, cannot run with --with-siderolink") - default: - scheme := "grpc://" - - if slb.apiCert != nil { - scheme = "https://" - } - - apiLink := scheme + net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.apiPort)) + "?jointoken=foo" - - if tunnel { - apiLink += "&grpc_tunnel=true" - } - - extraKernelArgs.Append("siderolink.api", apiLink) - extraKernelArgs.Append("talos.events.sink", net.JoinHostPort(slb.nodeIPv6Addr, strconv.Itoa(slb.sinkPort))) - extraKernelArgs.Append("talos.logging.kernel", "tcp://"+net.JoinHostPort(slb.nodeIPv6Addr, strconv.Itoa(slb.logPort))) - - if trustedRootsConfig := slb.TrustedRootsConfig(); trustedRootsConfig != nil { - var buf bytes.Buffer - - zencoder, err := zstd.NewWriter(&buf) - if err != nil { - return fmt.Errorf("failed to create zstd encoder: %w", err) - } - - _, err = zencoder.Write(trustedRootsConfig) - if err != nil { - return fmt.Errorf("failed to write zstd data: %w", err) - } - - if err = zencoder.Close(); err != nil { - return fmt.Errorf("failed to close zstd encoder: %w", err) - } - - extraKernelArgs.Append(constants.KernelParamConfigInline, base64.StdEncoding.EncodeToString(buf.Bytes())) - } - - return nil - } -} - -func getDynamicPort(network string) (int, error) { - var ( - closeFn func() error - addrFn func() net.Addr - ) - - switch network { - case "tcp", "tcp4", "tcp6": - l, err := net.Listen(network, "127.0.0.1:0") - if err != nil { - return 0, err - } - - addrFn, closeFn = l.Addr, l.Close - case "udp", "udp4", "udp6": - l, err := net.ListenPacket(network, "127.0.0.1:0") - if err != nil { - return 0, err - } - - addrFn, closeFn = l.LocalAddr, l.Close - default: - return 0, fmt.Errorf("unsupported network: %s", network) - } - - _, portStr, err := net.SplitHostPort(addrFn().String()) - if err != nil { - return 0, handleCloseErr(err, closeFn()) - } - - port, err := strconv.Atoi(portStr) - if err != nil { - return 0, err - } - - return port, handleCloseErr(nil, closeFn()) -} - -func handleCloseErr(err error, closeErr error) error { - switch { - case err != nil && closeErr != nil: - return fmt.Errorf("error: %w, close error: %w", err, closeErr) - case err == nil && closeErr != nil: - return closeErr - case err != nil && closeErr == nil: - return err - default: - return nil - } -} - -func checkPortsDontOverlap(ports ...int) error { - slices.Sort(ports) - - if len(ports) != len(slices.Compact(ports)) { - return errors.New("generated ports overlap") - } - - return nil -} - -type agentFlag uint8 - -func (a *agentFlag) String() string { - switch *a { - case 1: - return "wireguard" - case 2: - return "grpc-tunnel" - case 3: - return "wireguard+tls" - case 4: - return "grpc-tunnel+tls" - default: - return "none" - } -} - -func (a *agentFlag) Set(s string) error { - switch s { - case "true", "wireguard": - *a = 1 - case "tunnel": - *a = 2 - case "wireguard+tls": - *a = 3 - case "grpc-tunnel+tls": - *a = 4 - default: - return fmt.Errorf("unknown type: %s, possible values: 'true', 'wireguard' for the usual WG; 'tunnel' for WG over GRPC, add '+tls' to enable TLS for API", s) - } - - return nil -} - -func (a *agentFlag) Type() string { return "agent" } -func (a *agentFlag) IsEnabled() bool { return *a != 0 } -func (a *agentFlag) IsTunnel() bool { return *a == 2 || *a == 4 } -func (a *agentFlag) IsTLS() bool { return *a == 3 || *a == 4 } diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/common.go b/cmd/talosctl/cmd/mgmt/cluster/create/common.go new file mode 100644 index 0000000000..4ddd68f8c4 --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/common.go @@ -0,0 +1,474 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package create + +import ( + "context" + "errors" + "fmt" + "math/big" + "net" + "net/netip" + "net/url" + "os" + "slices" + "strings" + + "github.com/siderolabs/go-kubeconfig" + sideronet "github.com/siderolabs/net" + "github.com/siderolabs/talos/pkg/cluster/check" + clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/bundle" + "github.com/siderolabs/talos/pkg/machinery/config/configpatcher" + "github.com/siderolabs/talos/pkg/machinery/config/container" + "github.com/siderolabs/talos/pkg/machinery/config/encoder" + "github.com/siderolabs/talos/pkg/machinery/config/generate" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/provision" + "github.com/siderolabs/talos/pkg/provision/access" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + // gatewayOffset is the offset from the network address of the IP address of the network gateway. + gatewayOffset = 1 + + // nodesOffset is the offset from the network address of the beginning of the IP addresses to be used for nodes. + nodesOffset = 2 + jsonLogsPort = 4003 +) + +// getNetworkRequestBase validates network related ops and creates a base network request +func getNetworkRequestBase(cOps CommonOps) (req provision.NetworkRequestBase, cidr4 netip.Prefix, err error) { + cidr4, err = netip.ParsePrefix(cOps.networkCIDR) + if err != nil { + return req, cidr4, fmt.Errorf("error validating cidr block: %w", err) + } + + if !cidr4.Addr().Is4() { + return req, cidr4, errors.New("--cidr is expected to be IPV4 CIDR") + } + var cidrs []netip.Prefix + if cOps.networkIPv4 { + cidrs = append(cidrs, cidr4) + } + + // Gateway addr at 1st IP in range, ex. 192.168.0.1 + gatewayIPs := make([]netip.Addr, len(cidrs)) + + for j := range gatewayIPs { + gatewayIPs[j], err = sideronet.NthIPInNetwork(cidrs[j], gatewayOffset) + if err != nil { + return req, cidr4, err + } + } + + return provision.NetworkRequestBase{ + Name: cOps.rootOps.ClusterName, + CIDRs: cidrs, + GatewayAddrs: gatewayIPs, + MTU: cOps.networkMTU, + }, cidr4, nil +} + +func getBaseNodeRequests(params CommonOps) (controls, workers provision.BaseNodeRequests, err error) { + if params.controlplanes < 1 { + return controls, workers, errors.New("number of controlplanes can't be less than 1") + } + + controlPlaneNanoCPUs, err := parseCPUShare(params.controlPlaneCpus) + if err != nil { + return controls, workers, fmt.Errorf("error parsing --cpus: %s", err) + } + + workerNanoCPUs, err := parseCPUShare(params.workersCpus) + if err != nil { + return controls, workers, fmt.Errorf("error parsing --cpus-workers: %s", err) + } + + controlPlaneMemory := int64(params.controlPlaneMemory) * 1024 * 1024 + workerMemory := int64(params.workersMemory) * 1024 * 1024 + + for i := range params.controlplanes { + controls = append(controls, provision.NodeRequestBase{ + Index: i, + Name: fmt.Sprintf("%s-%s-%d", params.rootOps.ClusterName, "controlplane", i+1), + Type: machine.TypeControlPlane, + Memory: controlPlaneMemory, + NanoCPUs: controlPlaneNanoCPUs, + SkipInjectingConfig: params.skipInjectingConfig, + }) + } + for i := range params.workers { + controls = append(controls, provision.NodeRequestBase{ + Index: params.controlplanes - 1 + i, + Name: fmt.Sprintf("%s-%s-%d", params.rootOps.ClusterName, "worker", i+1), + Type: machine.TypeWorker, + Memory: workerMemory, + NanoCPUs: workerNanoCPUs, + SkipInjectingConfig: params.skipInjectingConfig, + }) + } + + return controls, workers, nil +} + +func getBase(cOps CommonOps) (baseRequest provision.ClusterRequestBase, provisionOptions []provision.Option, cidr4 netip.Prefix, err error) { + networkRequestBase, _, err := getNetworkRequestBase(cOps) + if err != nil { + return + } + controlplanes, workers, err := getBaseNodeRequests(cOps) + if err != nil { + return + } + baseRequest = provision.ClusterRequestBase{ + Name: cOps.rootOps.ClusterName, + SelfExecutable: os.Args[0], + StateDirectory: cOps.rootOps.StateDir, + Workers: workers, + Controlplanes: controlplanes, + Network: networkRequestBase, + } + provisionOptions = getCommonProvisionOps(cOps, networkRequestBase.GatewayAddrs[0].String()) + return +} + +func postCreate(ctx context.Context, clusterAccess *access.Adapter, commonOps CommonOps) error { + if !commonOps.withInitNode { + if err := clusterAccess.Bootstrap(ctx, os.Stdout); err != nil { + return fmt.Errorf("bootstrap error: %w", err) + } + } + + if !commonOps.clusterWait { + return nil + } + + // Run cluster readiness checks + checkCtx, checkCtxCancel := context.WithTimeout(ctx, commonOps.clusterWaitTimeout) + defer checkCtxCancel() + + checks := check.DefaultClusterChecks() + + if commonOps.skipK8sNodeReadinessCheck { + checks = slices.Concat(check.PreBootSequenceChecks(), check.K8sComponentsReadinessChecks()) + } + + checks = append(checks, check.ExtraClusterChecks()...) + + if err := check.Wait(checkCtx, clusterAccess, checks, check.StderrReporter()); err != nil { + return err + } + + if !commonOps.skipKubeconfig { + if err := mergeKubeconfig(ctx, clusterAccess); err != nil { + return err + } + } + + return nil +} + +func saveConfig(talosConfigObj *clientconfig.Config, commonOps CommonOps) (err error) { + c, err := clientconfig.Open(commonOps.talosconfig) + if err != nil { + return fmt.Errorf("error opening talos config: %w", err) + } + + renames := c.Merge(talosConfigObj) + for _, rename := range renames { + fmt.Fprintf(os.Stderr, "renamed talosconfig context %s\n", rename.String()) + } + + return c.Save(commonOps.talosconfig) +} + +func mergeKubeconfig(ctx context.Context, clusterAccess *access.Adapter) error { + // TODO: the change below in different commit + kubeconfigPath, err := kubeconfig.DefaultPath() + if err != nil { + return err + } + + fmt.Fprintf(os.Stderr, "\nmerging kubeconfig into %q\n", kubeconfigPath) + + k8sconfig, err := clusterAccess.Kubeconfig(ctx) + if err != nil { + return fmt.Errorf("error fetching kubeconfig: %w", err) + } + + kubeConfig, err := clientcmd.Load(k8sconfig) + if err != nil { + return fmt.Errorf("error parsing kubeconfig: %w", err) + } + + if clusterAccess.ForceEndpoint != "" { + for name := range kubeConfig.Clusters { + kubeConfig.Clusters[name].Server = clusterAccess.ForceEndpoint + } + } + + _, err = os.Stat(kubeconfigPath) + if err != nil { + if !os.IsNotExist(err) { + return err + } + + return clientcmd.WriteToFile(*kubeConfig, kubeconfigPath) + } + + merger, err := kubeconfig.Load(kubeconfigPath) + if err != nil { + return fmt.Errorf("error loading existing kubeconfig: %w", err) + } + + err = merger.Merge(kubeConfig, kubeconfig.MergeOptions{ + ActivateContext: true, + OutputWriter: os.Stdout, + ConflictHandler: func(component kubeconfig.ConfigComponent, name string) (kubeconfig.ConflictDecision, error) { + return kubeconfig.RenameDecision, nil + }, + }) + if err != nil { + return fmt.Errorf("error merging kubeconfig: %w", err) + } + + return merger.Write(kubeconfigPath) +} + +func parseCPUShare(cpus string) (int64, error) { + cpu, ok := new(big.Rat).SetString(cpus) + if !ok { + return 0, fmt.Errorf("failed to parsing as a rational number: %s", cpus) + } + + nano := cpu.Mul(cpu, big.NewRat(1e9, 1)) + if !nano.IsInt() { + return 0, errors.New("value is too precise") + } + + return nano.Num().Int64(), nil +} + +// getIps calculates ips for nodes and the virtual ip +// preallocIps is false ips are nil and should not be preallocated +func getIps(cidrs []netip.Prefix, commonOps CommonOps) (ips [][]netip.Addr, err error) { + // Set starting ip at 2nd ip in range, ex: 192.168.0.2 + ips = make([][]netip.Addr, len(cidrs)) + + for j := range cidrs { + ips[j] = make([]netip.Addr, commonOps.controlplanes+commonOps.workers) + + for i := range ips[j] { + ips[j][i], err = sideronet.NthIPInNetwork(cidrs[j], nodesOffset+i) + if err != nil { + return ips, err + } + } + } + + return ips, err +} + +func getCommonGenOptions(cOps CommonOps) ([]generate.Option, *config.VersionContract, error) { + genOptions := []generate.Option{ + generate.WithDebug(cOps.configDebug), + generate.WithDNSDomain(cOps.dnsDomain), + generate.WithClusterDiscovery(cOps.enableClusterDiscovery), + } + + for _, registryMirror := range cOps.registryMirrors { + components := strings.SplitN(registryMirror, "=", 2) + if len(components) != 2 { + return genOptions, nil, fmt.Errorf("invalid registry mirror spec: %q", registryMirror) + } + + genOptions = append(genOptions, generate.WithRegistryMirror(components[0], components[1])) + } + + for _, registryHost := range cOps.registryInsecure { + genOptions = append(genOptions, generate.WithRegistryInsecureSkipVerify(registryHost)) + } + + if cOps.customCNIUrl != "" { + genOptions = append(genOptions, generate.WithClusterCNIConfig(&v1alpha1.CNIConfig{ + CNIName: constants.CustomCNI, + CNIUrls: []string{cOps.customCNIUrl}, + })) + } + + var versionContract *config.VersionContract + if cOps.talosVersion != "latest" { + versionContract, err := config.ParseContractFromVersion(cOps.talosVersion) + if err != nil { + return genOptions, nil, fmt.Errorf("error parsing Talos version %q: %w", cOps.talosVersion, err) + } + + genOptions = append(genOptions, generate.WithVersionContract(versionContract)) + } + + if cOps.kubePrismPort != constants.DefaultKubePrismPort { + genOptions = append(genOptions, + generate.WithKubePrismPort(cOps.kubePrismPort), + ) + } + + if cOps.controlPlanePort != constants.DefaultControlPlanePort { + genOptions = append(genOptions, + generate.WithLocalAPIServerPort(cOps.controlPlanePort), + ) + } + + if cOps.enableKubeSpan { + genOptions = append(genOptions, + generate.WithNetworkOptions( + v1alpha1.WithKubeSpan(), + ), + ) + } + + return genOptions, versionContract, nil +} + +func getEnpointListGenOption(cOps CommonOps, endpointList []string, ips [][]netip.Addr) []generate.Option { + genOptions := []generate.Option{} + switch { + case cOps.forceEndpoint != "": + // using non-default endpoints, provision additional cert SANs and fix endpoint list + endpointList = []string{cOps.forceEndpoint} + genOptions = append(genOptions, generate.WithAdditionalSubjectAltNames(endpointList)) + case cOps.forceInitNodeAsEndpoint: + endpointList = []string{ips[0][0].String()} + case len(endpointList) > 0: + for _, endpointHostPort := range endpointList { + endpointHost, _, err := net.SplitHostPort(endpointHostPort) + if err != nil { + endpointHost = endpointHostPort + } + + genOptions = append(genOptions, generate.WithAdditionalSubjectAltNames([]string{endpointHost})) + } + case endpointList == nil: + // use control plane nodes as endpoints, client-side load-balancing + for i := range cOps.controlplanes { + endpointList = append(endpointList, ips[0][i].String()) + } + } + return append(genOptions, generate.WithEndpointList(endpointList)) +} + +func getCommonConfigBundleOps(cOps CommonOps, gatewayIP string) ([]bundle.Option, error) { + var configBundleOpts []bundle.Option + addConfigPatch := func(configPatches []string, configOpt func([]configpatcher.Patch) bundle.Option) error { + var patches []configpatcher.Patch + patches, err := configpatcher.LoadPatches(configPatches) + if err != nil { + return fmt.Errorf("error parsing config JSON patch: %w", err) + } + configBundleOpts = append(configBundleOpts, configOpt(patches)) + return nil + } + + if err := addConfigPatch(cOps.configPatch, bundle.WithPatch); err != nil { + return configBundleOpts, err + } + if err := addConfigPatch(cOps.configPatchControlPlane, bundle.WithPatchControlPlane); err != nil { + return configBundleOpts, err + } + if err := addConfigPatch(cOps.configPatchWorker, bundle.WithPatchWorker); err != nil { + return configBundleOpts, err + } + + if cOps.withJSONLogs { + cfg := container.NewV1Alpha1( + &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineLogging: &v1alpha1.LoggingConfig{ + LoggingDestinations: []v1alpha1.LoggingDestination{ + { + LoggingEndpoint: &v1alpha1.Endpoint{ + URL: &url.URL{ + Scheme: "tcp", + Host: nethelpers.JoinHostPort(gatewayIP, jsonLogsPort), + }, + }, + LoggingFormat: "json_lines", + }, + }, + }, + }, + }) + configBundleOpts = append(configBundleOpts, bundle.WithPatch([]configpatcher.Patch{configpatcher.NewStrategicMergePatch(cfg)})) + } + + return configBundleOpts, nil +} + +func getNewConfigBundle(configBundleOpts []bundle.Option, cOps CommonOps, inClusterEndpoint string, genOptions []generate.Option) []bundle.Option { + configBundleOpts = append(configBundleOpts, + bundle.WithInputOptions( + &bundle.InputOptions{ + ClusterName: cOps.rootOps.ClusterName, + Endpoint: inClusterEndpoint, + KubeVersion: strings.TrimPrefix(cOps.kubernetesVersion, "v"), + GenOptions: genOptions, + }), + ) + return configBundleOpts +} + +func getCommonProvisionOps(cOps CommonOps, gatewayIP string) []provision.Option { + provisionOptions := []provision.Option{} + if cOps.withJSONLogs { + provisionOptions = append(provisionOptions, provision.WithJSONLogs(nethelpers.JoinHostPort(gatewayIP, jsonLogsPort))) + } + return provisionOptions +} + +func getConfigBundle(cOps CommonOps, configBundleOpts []bundle.Option) (configBundle *bundle.Bundle, bundleTalosconfig *clientconfig.Config, err error) { + configBundle, err = bundle.NewBundle(configBundleOpts...) + if err != nil { + return nil, nil, err + } + + bundleTalosconfig = configBundle.TalosConfig() + if bundleTalosconfig == nil { + if cOps.clusterWait { + return nil, nil, errors.New("no talosconfig in the config bundle: cannot wait for cluster") + } + + if cOps.applyConfigEnabled { + return nil, nil, errors.New("no talosconfig in the config bundle: cannot apply config") + } + } + + if cOps.skipInjectingConfig { + types := []machine.Type{machine.TypeControlPlane, machine.TypeWorker} + + if cOps.withInitNode { + types = slices.Insert(types, 0, machine.TypeInit) + } + + if err = configBundle.Write(".", encoder.CommentsAll, types...); err != nil { + return nil, nil, err + } + } + + return +} + +func getNodeIp(CIDRs []netip.Prefix, ips [][]netip.Addr, nodeIndex int) []netip.Addr { + nodeIPs := make([]netip.Addr, len(CIDRs)) + for j := range nodeIPs { + nodeIPs[j] = ips[j][nodeIndex] + } + return nodeIPs +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/create.go b/cmd/talosctl/cmd/mgmt/cluster/create/create.go new file mode 100644 index 0000000000..03ca31376a --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/create.go @@ -0,0 +1,598 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package create + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "slices" + "time" + + stdruntime "runtime" + + "github.com/docker/cli/opts" + "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster" + "github.com/siderolabs/talos/cmd/talosctl/pkg/mgmt/helpers" + "github.com/siderolabs/talos/pkg/cli" + "github.com/siderolabs/talos/pkg/images" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/version" + "github.com/siderolabs/talos/pkg/provision/providers" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +type CommonOps struct { + // rootOps are the options from the root cluster command + rootOps *cluster.ClusterCmdOps + talosconfig string + registryMirrors []string + registryInsecure []string + kubernetesVersion string + applyConfigEnabled bool + configDebug bool + networkCIDR string + networkMTU int + networkIPv4 bool + dnsDomain string + workers int + controlplanes int + controlPlaneCpus string + workersCpus string + controlPlaneMemory int + workersMemory int + clusterWait bool + clusterWaitTimeout time.Duration + forceInitNodeAsEndpoint bool + forceEndpoint string + inputDir string + controlPlanePort int + withInitNode bool + customCNIUrl string + crashdumpOnFailure bool + skipKubeconfig bool + skipInjectingConfig bool + talosVersion string + enableKubeSpan bool + enableClusterDiscovery bool + configPatch []string + configPatchControlPlane []string + configPatchWorker []string + kubePrismPort int + skipK8sNodeReadinessCheck bool + withJSONLogs bool + wireguardCIDR string +} + +type QemuOps struct { + nodeInstallImage string + nodeVmlinuzPath string + nodeInitramfsPath string + nodeISOPath string + nodeDiskImagePath string + nodeIPXEBootScript string + bootloaderEnabled bool + uefiEnabled bool + tpm2Enabled bool + extraUEFISearchPaths []string + networkNoMasqueradeCIDRs []string + networkIPv6 bool + nameservers []string + clusterDiskSize int + clusterDiskPreallocate bool + clusterDisks []string + extraDisks int + extraDiskSize int + extraDisksDrivers []string + targetArch string + cniBinPath []string + cniConfDir string + cniCacheDir string + cniBundleURL string + encryptStatePartition bool + encryptEphemeralPartition bool + useVIP bool + badRTC bool + extraBootKernelArgs string + dhcpSkipHostname bool + networkChaos bool + jitter time.Duration + latency time.Duration + packetLoss float64 + packetReorder float64 + packetCorrupt float64 + bandwidth int + diskEncryptionKeyTypes []string + withFirewall string + withUUIDHostnames bool + withSiderolinkAgent agentFlag + debugShellEnabled bool + configInjectionMethodFlagVal string +} + +type DockerOps struct { + dockerHostIP string + dockerDisableIPv6 bool + mountOpts opts.MountOpt + ports string + nodeImage string +} + +func init() { + const ( + // Docker + dockerHostIPFlag = "docker-host-ip" + nodeImageFlag = "image" + portsFlag = "exposed-ports" + dockerDisableIPv6Flag = "docker-disable-ipv6" + mountOptsFlag = "mount" + + // TODO sort + inputDirFlag = "input-dir" + networkIPv4Flag = "ipv4" + networkIPv6Flag = "ipv6" + networkMTUFlag = "mtu" + networkCIDRFlag = "cidr" + networkNoMasqueradeCIDRsFlag = "no-masquerade-cidrs" + nameserversFlag = "nameservers" + clusterDiskPreallocateFlag = "disk-preallocate" + clusterDisksFlag = "user-disk" + clusterDiskSizeFlag = "disk" + useVIPFlag = "use-vip" + bootloaderEnabledFlag = "with-bootloader" + controlPlanePortFlag = "control-plane-port" + firewallFlag = "with-firewall" + tpm2EnabledFlag = "with-tpm2" + withDebugShellFlag = "with-debug-shell" + talosconfigFlag = "talosconfig" + applyConfigEnabledFlag = "with-apply-config" + wireguardCIDRFlag = "wireguard-cidr" + workersFlag = "workers" + mastersFlag = "masters" + controlplanesFlag = "controlplanes" + controlPlaneCpusFlag = "cpus" + workersCpusFlag = "cpus-workers" + controlPlaneMemoryFlag = "memory" + workersMemoryFlag = "memory-workers" + clusterWaitFlag = "wait" + clusterWaitTimeoutFlag = "wait-timeout" + forceInitNodeAsEndpointFlag = "init-node-as-endpoint" + kubernetesVersionFlag = "kubernetes-version" + withInitNodeFlag = "with-init-node" + crashdumpOnFailureFlag = "crashdump" + skipKubeconfigFlag = "skip-kubeconfig" + skipInjectingConfigFlag = "skip-injecting-config" + configPatchFlag = "config-patch" + configPatchControlPlaneFlag = "config-patch-control-plane" + configPatchWorkerFlag = "config-patch-worker" + skipK8sNodeReadinessCheckFlag = "skip-k8s-node-readiness-check" + withJSONLogsFlag = "with-json-logs" + + // qemu + nodeVmlinuzPathFlag = "vmlinuz-path" + nodeISOPathFlag = "iso-path" + nodeInitramfsPathFlag = "initrd-path" + nodeDiskImagePathFlag = "disk-image-path" + nodeIPXEBootScriptFlag = "ipxe-boot-script" + uefiEnabledFlag = "with-uefi" + extraUEFISearchPathsFlag = "extra-uefi-search-paths" + extraDisksFlag = "extra-disks" + extraDisksDriversFlag = "extra-disks-drivers" + extraDiskSizeFlag = "extra-disks-size" + targetArchFlag = "arch" + cniBinPathFlag = "cni-bin-path" + cniConfDirFlag = "cni-conf-dir" + cniCacheDirFlag = "cni-cache-dir" + cniBundleURLFlag = "cni-bundle-url" + badRTCFlag = "bad-rtc" + extraBootKernelArgsFlag = "extra-boot-kernel-args" + dhcpSkipHostnameFlag = "disable-dhcp-hostname" + networkChaosFlag = "with-network-chaos" + jitterFlag = "with-network-jitter" + latencyFlag = "with-network-latency" + packetLossFlag = "with-network-packet-loss" + packetReorderFlag = "with-network-packet-reorder" + packetCorruptFlag = "with-network-packet-corrupt" + bandwidthFlag = "with-network-bandwidth" + withUUIDHostnamesFlag = "with-uuid-hostnames" + withSiderolinkAgentFlag = "with-siderolink" + configInjectionMethodFlag = "config-injection-method" + + // The following flags are the gen options - the options that are only used in machine configuration (i.e., not during the qemu/docker provisioning). + // They are not applicable when no machine configuration is generated, hence mutually exclusive with the --input-dir flag. + + nodeInstallImageFlag = "install-image" + configDebugFlag = "with-debug" + dnsDomainFlag = "dns-domain" + withClusterDiscoveryFlag = "with-cluster-discovery" + registryMirrorFlag = "registry-mirror" + registryInsecureFlag = "registry-insecure-skip-verify" + customCNIUrlFlag = "custom-cni-url" + talosVersionFlag = "talos-version" + encryptStatePartitionFlag = "encrypt-state" + encryptEphemeralPartitionFlag = "encrypt-ephemeral" + enableKubeSpanFlag = "with-kubespan" + forceEndpointFlag = "endpoint" + kubePrismFlag = "kubeprism-port" + diskEncryptionKeyTypesFlag = "disk-encryption-key-types" + ) + + unImplementedQemuFlagsDarwin := []string{ + nodeIPXEBootScriptFlag, + bootloaderEnabledFlag, + tpm2EnabledFlag, + withDebugShellFlag, + networkNoMasqueradeCIDRsFlag, + networkIPv6Flag, + nameserversFlag, + wireguardCIDRFlag, + cniBinPathFlag, + cniConfDirFlag, + cniCacheDirFlag, + cniBundleURLFlag, + useVIPFlag, + badRTCFlag, + dhcpSkipHostnameFlag, + networkChaosFlag, + jitterFlag, + latencyFlag, + packetLossFlag, + packetReorderFlag, + packetCorruptFlag, + bandwidthFlag, + firewallFlag, + withUUIDHostnamesFlag, + withSiderolinkAgentFlag, + configInjectionMethodFlag, + } + unImplementedQemuFlagsLinux := []string{} + + var commonOps = CommonOps{} + commonOps.rootOps = &cluster.Flags + var qemuOps = QemuOps{} + var dockerOps = DockerOps{} + + getDockerFlags := func() *pflag.FlagSet { + dockerFlags := pflag.NewFlagSet("", pflag.PanicOnError) + // Flags specific to the Docker provider + dockerFlags.StringVar(&dockerOps.dockerHostIP, dockerHostIPFlag, "0.0.0.0", "Host IP to forward exposed ports to") + dockerFlags.StringVar(&dockerOps.nodeImage, nodeImageFlag, helpers.DefaultImage(images.DefaultTalosImageRepository), "the image to use") + dockerFlags.StringVarP(&dockerOps.ports, portsFlag, "p", "", + "Comma-separated list of ports/protocols to expose on init node. Ex -p :/") + dockerFlags.BoolVar(&dockerOps.dockerDisableIPv6, dockerDisableIPv6Flag, false, "skip enabling IPv6 in containers") + dockerFlags.Var(&dockerOps.mountOpts, mountOptsFlag, "attach a mount to the container (Docker only)") + + dockerFlags.VisitAll(func(f *pflag.Flag) { + f.Usage = "(docker only) " + f.Usage + }) + return dockerFlags + } + + getQemuFlags := func() *pflag.FlagSet { + qemuFlags := pflag.NewFlagSet("", pflag.PanicOnError) + // Flags specific to the QEMU provider + qemuFlags.StringVar(&qemuOps.nodeInstallImage, nodeInstallImageFlag, helpers.DefaultImage(images.DefaultInstallerImageRepository), "the installer image to use") + qemuFlags.StringVar(&qemuOps.nodeVmlinuzPath, nodeVmlinuzPathFlag, helpers.ArtifactPath(constants.KernelAssetWithArch), "the compressed kernel image to use") + qemuFlags.StringVar(&qemuOps.nodeISOPath, nodeISOPathFlag, "", "the ISO path to use for the initial boot") + qemuFlags.StringVar(&qemuOps.nodeInitramfsPath, nodeInitramfsPathFlag, helpers.ArtifactPath(constants.InitramfsAssetWithArch), "initramfs image to use") + qemuFlags.StringVar(&qemuOps.nodeDiskImagePath, nodeDiskImagePathFlag, "", "disk image to use") + qemuFlags.StringVar(&qemuOps.nodeIPXEBootScript, nodeIPXEBootScriptFlag, "", "iPXE boot script (URL) to use") + qemuFlags.BoolVar(&qemuOps.bootloaderEnabled, bootloaderEnabledFlag, true, "enable bootloader to load kernel and initramfs from disk image after install") + qemuFlags.BoolVar(&qemuOps.uefiEnabled, uefiEnabledFlag, true, "enable UEFI on x86_64 architecture") + qemuFlags.BoolVar(&qemuOps.tpm2Enabled, tpm2EnabledFlag, false, "enable TPM2 emulation support using swtpm") + qemuFlags.BoolVar(&qemuOps.debugShellEnabled, withDebugShellFlag, false, "drop talos into a maintenance shell on boot, this is for advanced debugging for developers only") + qemuFlags.MarkHidden("with-debug-shell") //nolint:errcheck + qemuFlags.StringSliceVar(&qemuOps.extraUEFISearchPaths, extraUEFISearchPathsFlag, []string{}, "additional search paths for UEFI firmware (only applies when UEFI is enabled)") + qemuFlags.StringSliceVar(&qemuOps.networkNoMasqueradeCIDRs, networkNoMasqueradeCIDRsFlag, []string{}, "list of CIDRs to exclude from NAT") + qemuFlags.BoolVar(&qemuOps.networkIPv6, networkIPv6Flag, false, "enable IPv6 network in the cluster") + qemuFlags.StringSliceVar(&qemuOps.nameservers, nameserversFlag, []string{"8.8.8.8", "1.1.1.1", "2001:4860:4860::8888", "2606:4700:4700::1111"}, "list of nameservers to use") + qemuFlags.IntVar(&qemuOps.clusterDiskSize, clusterDiskSizeFlag, 6*1024, "default limit on disk size in MB (each VM)") + qemuFlags.BoolVar(&qemuOps.clusterDiskPreallocate, clusterDiskPreallocateFlag, true, "whether disk space should be preallocated") + qemuFlags.StringSliceVar(&qemuOps.clusterDisks, clusterDisksFlag, []string{}, "list of disks to create for each VM in format: :::") + qemuFlags.IntVar(&qemuOps.extraDisks, extraDisksFlag, 0, "number of extra disks to create for each worker VM") + qemuFlags.StringSliceVar(&qemuOps.extraDisksDrivers, extraDisksDriversFlag, nil, "driver for each extra disk (virtio, ide, ahci, scsi, nvme)") + qemuFlags.IntVar(&qemuOps.extraDiskSize, extraDiskSizeFlag, 5*1024, "default limit on disk size in MB (each VM)") + qemuFlags.StringVar(&qemuOps.targetArch, targetArchFlag, stdruntime.GOARCH, "cluster architecture") + qemuFlags.StringSliceVar(&qemuOps.cniBinPath, cniBinPathFlag, []string{filepath.Join(commonOps.rootOps.DefaultCNIDir, "bin")}, "search path for CNI binaries") + qemuFlags.StringVar(&qemuOps.cniConfDir, cniConfDirFlag, filepath.Join(commonOps.rootOps.DefaultCNIDir, "conf.d"), "CNI config directory path") + qemuFlags.StringVar(&qemuOps.cniCacheDir, cniCacheDirFlag, filepath.Join(commonOps.rootOps.DefaultCNIDir, "cache"), "CNI cache directory path") + qemuFlags.StringVar(&qemuOps.cniBundleURL, cniBundleURLFlag, fmt.Sprintf("https://github.com/%s/talos/releases/download/%s/talosctl-cni-bundle-%s.tar.gz", + images.Username, version.Trim(version.Tag), constants.ArchVariable), "URL to download CNI bundle from") + qemuFlags.BoolVar(&qemuOps.encryptStatePartition, encryptStatePartitionFlag, false, "enable state partition encryption") + qemuFlags.BoolVar(&qemuOps.encryptEphemeralPartition, encryptEphemeralPartitionFlag, false, "enable ephemeral partition encryption") + qemuFlags.StringArrayVar(&qemuOps.diskEncryptionKeyTypes, diskEncryptionKeyTypesFlag, []string{"uuid"}, "encryption key types to use for disk encryption (uuid, kms)") + qemuFlags.BoolVar(&qemuOps.useVIP, useVIPFlag, false, "use a virtual IP for the controlplane endpoint instead of the loadbalancer") + qemuFlags.BoolVar(&qemuOps.badRTC, badRTCFlag, false, "launch VM with bad RTC state") + qemuFlags.StringVar(&qemuOps.extraBootKernelArgs, extraBootKernelArgsFlag, "", "add extra kernel args to the initial boot from vmlinuz and initramfs") + qemuFlags.BoolVar(&qemuOps.dhcpSkipHostname, dhcpSkipHostnameFlag, false, "skip announcing hostname via DHCP") + qemuFlags.BoolVar(&qemuOps.networkChaos, networkChaosFlag, false, "enable network chaos parameters when creating a qemu cluster") + qemuFlags.DurationVar(&qemuOps.jitter, jitterFlag, 0, "specify jitter on the bridge interface") + qemuFlags.DurationVar(&qemuOps.latency, latencyFlag, 0, "specify latency on the bridge interface") + qemuFlags.Float64Var(&qemuOps.packetLoss, packetLossFlag, 0.0, "specify percent of packet loss on the bridge interface. e.g. 50% = 0.50 (default: 0.0)") + qemuFlags.Float64Var(&qemuOps.packetReorder, packetReorderFlag, 0.0, "specify percent of reordered packets on the bridge interface. e.g. 50% = 0.50 (default: 0.0)") + qemuFlags.Float64Var(&qemuOps.packetCorrupt, packetCorruptFlag, 0.0, "specify percent of corrupt packets on the bridge interface. e.g. 50% = 0.50 (default: 0.0)") + qemuFlags.IntVar(&qemuOps.bandwidth, bandwidthFlag, 0, "specify bandwidth restriction (in kbps) on the bridge interface") + qemuFlags.StringVar(&qemuOps.withFirewall, firewallFlag, "", "inject firewall rules into the cluster, value is default policy - accept/block") + qemuFlags.BoolVar(&qemuOps.withUUIDHostnames, withUUIDHostnamesFlag, false, "use machine UUIDs as default hostnames") + qemuFlags.Var(&qemuOps.withSiderolinkAgent, withSiderolinkAgentFlag, "enables the use of siderolink agent as configuration apply mechanism. `true` or `wireguard` enables the agent, `tunnel` enables the agent with grpc tunneling") //nolint:lll + qemuFlags.StringVar(&qemuOps.configInjectionMethodFlagVal, configInjectionMethodFlag, "", "a method to inject machine config: default is HTTP server, 'metal-iso' to mount an ISO") + + return qemuFlags + } + + getCommonFlags := func() *pflag.FlagSet { + commonFlags := pflag.NewFlagSet("", pflag.PanicOnError) + commonFlags.StringVar(&commonOps.talosconfig, talosconfigFlag, "", + fmt.Sprintf("The path to the Talos configuration file. Defaults to '%s' env variable if set, otherwise '%s' and '%s' in order.", + constants.TalosConfigEnvVar, + filepath.Join("$HOME", constants.TalosDir, constants.TalosconfigFilename), + filepath.Join(constants.ServiceAccountMountPath, constants.TalosconfigFilename), + ), + ) + commonFlags.BoolVar(&commonOps.applyConfigEnabled, applyConfigEnabledFlag, false, "enable apply config when the VM is starting in maintenance mode") + commonFlags.StringSliceVar(&commonOps.registryMirrors, registryMirrorFlag, []string{}, "list of registry mirrors to use in format: =") + commonFlags.StringSliceVar(&commonOps.registryInsecure, registryInsecureFlag, []string{}, "list of registry hostnames to skip TLS verification for") + commonFlags.BoolVar(&commonOps.configDebug, configDebugFlag, false, "enable debug in Talos config to send service logs to the console") + commonFlags.IntVar(&commonOps.networkMTU, networkMTUFlag, 1500, "MTU of the cluster network") + commonFlags.StringVar(&commonOps.networkCIDR, networkCIDRFlag, "10.5.0.0/24", "CIDR of the cluster network (IPv4, ULA network for IPv6 is derived in automated way)") + commonFlags.IntVar(&commonOps.controlPlanePort, controlPlanePortFlag, constants.DefaultControlPlanePort, "control plane port (load balancer and local API port)") + commonFlags.BoolVar(&commonOps.networkIPv4, networkIPv4Flag, true, "enable IPv4 network in the cluster") + commonFlags.StringVar(&commonOps.wireguardCIDR, wireguardCIDRFlag, "", "CIDR of the wireguard network") + commonFlags.IntVar(&commonOps.workers, workersFlag, 1, "the number of workers to create") + commonFlags.IntVar(&commonOps.controlplanes, mastersFlag, 1, "the number of masters to create") + commonFlags.MarkDeprecated("commonOps.masters", "use --controlplanes instead") //nolint:errcheck + commonFlags.IntVar(&commonOps.controlplanes, controlplanesFlag, 1, "the number of controlplanes to create") + commonFlags.StringVar(&commonOps.controlPlaneCpus, controlPlaneCpusFlag, "2.0", "the share of CPUs as fraction (each control plane/VM)") + commonFlags.StringVar(&commonOps.workersCpus, workersCpusFlag, "2.0", "the share of CPUs as fraction (each worker/VM)") + commonFlags.IntVar(&commonOps.controlPlaneMemory, controlPlaneMemoryFlag, 2048, "the limit on memory usage in MB (each control plane/VM)") + commonFlags.IntVar(&commonOps.workersMemory, workersMemoryFlag, 2048, "the limit on memory usage in MB (each worker/VM)") + commonFlags.BoolVar(&commonOps.clusterWait, clusterWaitFlag, true, "wait for the cluster to be ready before returning") + commonFlags.DurationVar(&commonOps.clusterWaitTimeout, clusterWaitTimeoutFlag, 20*time.Minute, "timeout to wait for the cluster to be ready") + commonFlags.BoolVar(&commonOps.forceInitNodeAsEndpoint, forceInitNodeAsEndpointFlag, false, "use init node as endpoint instead of any load balancer endpoint") + commonFlags.StringVar(&commonOps.forceEndpoint, forceEndpointFlag, "", "use endpoint instead of provider defaults") + commonFlags.StringVar(&commonOps.kubernetesVersion, kubernetesVersionFlag, constants.DefaultKubernetesVersion, "desired kubernetes version to run") + commonFlags.StringVarP(&commonOps.inputDir, inputDirFlag, "i", "", "location of pre-generated config files") + commonFlags.BoolVar(&commonOps.withInitNode, withInitNodeFlag, false, "create the cluster with an init node") + commonFlags.StringVar(&commonOps.customCNIUrl, customCNIUrlFlag, "", "install custom CNI from the URL (Talos cluster)") + commonFlags.StringVar(&commonOps.dnsDomain, dnsDomainFlag, "cluster.local", "the dns domain to use for cluster") + commonFlags.BoolVar(&commonOps.crashdumpOnFailure, crashdumpOnFailureFlag, false, "generate support zip when cluster startup fails") + commonFlags.BoolVar(&commonOps.skipKubeconfig, skipKubeconfigFlag, false, "skip merging kubeconfig from the created cluster") + commonFlags.BoolVar(&commonOps.skipInjectingConfig, skipInjectingConfigFlag, false, "skip injecting config from embedded metadata server, write config files to current directory") + commonFlags.StringVar(&commonOps.talosVersion, talosVersionFlag, "", "the desired Talos version to generate config for (if not set, defaults to image version)") + commonFlags.BoolVar(&commonOps.enableClusterDiscovery, withClusterDiscoveryFlag, true, "enable cluster discovery") + commonFlags.BoolVar(&commonOps.enableKubeSpan, enableKubeSpanFlag, false, "enable KubeSpan system") + commonFlags.StringArrayVar(&commonOps.configPatch, configPatchFlag, nil, "patch generated machineconfigs (applied to all node types), use @file to read a patch from file") + commonFlags.StringArrayVar(&commonOps.configPatchControlPlane, configPatchControlPlaneFlag, nil, "patch generated machineconfigs (applied to 'init' and 'controlplane' types)") + commonFlags.StringArrayVar(&commonOps.configPatchWorker, configPatchWorkerFlag, nil, "patch generated machineconfigs (applied to 'worker' type)") + commonFlags.IntVar(&commonOps.kubePrismPort, kubePrismFlag, constants.DefaultKubePrismPort, "KubePrism port (set to 0 to disable)") + commonFlags.BoolVar(&commonOps.skipK8sNodeReadinessCheck, skipK8sNodeReadinessCheckFlag, false, "skip k8s node readiness checks") + commonFlags.BoolVar(&commonOps.withJSONLogs, withJSONLogsFlag, false, "enable JSON logs receiver and configure Talos to send logs there") + + return commonFlags + } + + var commonFlags = getCommonFlags() + var qemuFlags = getQemuFlags() + var dockerFlags = getDockerFlags() + var rootCmdQemuFlags = getQemuFlags() + + // validateProviderFlags checks if flags not applicable for the given provisioner and OS are passed + validateProviderFlags := func(flags *pflag.FlagSet) error { + var unImplemented []string = make([]string, 0) + var invalidFlags *pflag.FlagSet + switch commonOps.rootOps.ProvisionerName { + case providers.DockerProviderName: + invalidFlags = rootCmdQemuFlags + case providers.QemuProviderName: + invalidFlags = dockerFlags + if currentOs == "darwin" { + unImplemented = unImplementedQemuFlagsDarwin + } else { + unImplemented = unImplementedQemuFlagsLinux + } + } + errLogged := false + invalidFlags.VisitAll(func(invalidFlag *pflag.Flag) { + if invalidFlag.Changed { + fmt.Printf("%s flag has been set but has no effect with the %s provisioner\n", invalidFlag.Name, commonOps.rootOps.ProvisionerName) + errLogged = true + } + }) + if errLogged { + fmt.Println() + return fmt.Errorf("Invalid provisioner flags founds") + } + + err := validateFlags(flags, unImplemented, currentOs) + return err + } + + var createQemuCmd = &cobra.Command{ + Use: providers.QemuProviderName, + Short: "Creates a local qemu based kubernetes cluster", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + validateCmdProvisioner(commonOps.rootOps.ProvisionerName, providers.QemuProviderName) + commonOps.rootOps.ProvisionerName = providers.QemuProviderName + if err := validateProviderFlags(cmd.Flags()); err != nil { + return err + } + return cli.WithContext(context.Background(), func(ctx context.Context) error { + return CreateQemuCluster(ctx, commonOps, qemuOps) + }) + }, + } + + var createDockerCmd = &cobra.Command{ + Use: providers.DockerProviderName, + Short: "Creates a local docker based kubernetes cluster", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + validateCmdProvisioner(commonOps.rootOps.ProvisionerName, providers.DockerProviderName) + if err := validateProviderFlags(cmd.Flags()); err != nil { + return err + } + return cli.WithContext(context.Background(), func(ctx context.Context) error { + return CreateDockerCluster(ctx, commonOps, dockerOps) + }) + }, + } + + // createCmd represents the cluster up command. + var createCmd = &cobra.Command{ + Use: "create", + Short: "Creates a local docker-based or QEMU-based kubernetes cluster", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if err := providers.IsValidProvider(commonOps.rootOps.ProvisionerName); err != nil { + return err + } + if err := validateProviderFlags(cmd.Flags()); err != nil { + return err + } + + return cli.WithContext(context.Background(), func(ctx context.Context) error { + if commonOps.rootOps.ProvisionerName == providers.DockerProviderName { + return CreateDockerCluster(ctx, commonOps, dockerOps) + } else { + return CreateQemuCluster(ctx, commonOps, qemuOps) + } + }) + }, + } + + qemuFlags.VisitAll(func(f *pflag.Flag) { + isUnimplementedOnDarwin := slices.Contains(unImplementedQemuFlagsDarwin, f.Name) + if isUnimplementedOnDarwin && currentOs == "darwin" { + f.Usage = "(linux only) " + f.Usage + } + }) + rootCmdQemuFlags.VisitAll(func(f *pflag.Flag) { + isUnimplemented := slices.Contains(unImplementedQemuFlagsDarwin, f.Name) + if isUnimplemented { + f.Usage = "(QEMU with linux only) " + f.Usage + } else { + f.Usage = "(QEMU only) " + f.Usage + } + }) + + createCmd.Flags().AddFlagSet(commonFlags) + createCmd.Flags().AddFlagSet(dockerFlags) + createCmd.Flags().AddFlagSet(rootCmdQemuFlags) + + // The individual flagsets are still sorted + createCmd.Flags().SortFlags = false + createDockerCmd.Flags().SortFlags = false + createQemuCmd.Flags().SortFlags = false + + markInputDirFlagsExclusive := func(cmd *cobra.Command) { + exclusiveFlags := []string{ + nodeInstallImageFlag, + configDebugFlag, + dnsDomainFlag, + withClusterDiscoveryFlag, + registryMirrorFlag, + registryInsecureFlag, + customCNIUrlFlag, + talosVersionFlag, + encryptStatePartitionFlag, + encryptEphemeralPartitionFlag, + enableKubeSpanFlag, + forceEndpointFlag, + kubePrismFlag, + diskEncryptionKeyTypesFlag} + + for _, f := range exclusiveFlags { + if cmd.Flag(f) != nil { + cmd.MarkFlagsMutuallyExclusive(inputDirFlag, f) + } + } + } + markInputDirFlagsExclusive(createCmd) + markInputDirFlagsExclusive(createQemuCmd) + markInputDirFlagsExclusive(createDockerCmd) + + createDockerCmd.Flags().AddFlagSet(commonFlags) + createDockerCmd.Flags().AddFlagSet(dockerFlags) + createQemuCmd.Flags().AddFlagSet(commonFlags) + createQemuCmd.Flags().AddFlagSet(qemuFlags) + + createCmd.AddCommand(createDockerCmd) + createCmd.AddCommand(createQemuCmd) + + cluster.Cmd.AddCommand(createCmd) +} + +// validateCmdProvisioner checks if the passed provisioner matches the command provisioner +func validateCmdProvisioner(passedProvisioner, cmdProvisioner string) error { + if passedProvisioner == "" { + return nil + } + if err := providers.IsValidProvider(passedProvisioner); err != nil { + return err + } + + if cmdProvisioner != passedProvisioner { + return fmt.Errorf("Invalid provisioner: \"%s\"\n--provisioner must be %s when using cluster create %s\n", passedProvisioner, cmdProvisioner, cmdProvisioner) + } + return nil +} + +type agentFlag uint8 + +func (a *agentFlag) String() string { + switch *a { + case 1: + return "wireguard" + case 2: + return "grpc-tunnel" + case 3: + return "wireguard+tls" + case 4: + return "grpc-tunnel+tls" + default: + return "none" + } +} + +func (a *agentFlag) Set(s string) error { + switch s { + case "true", "wireguard": + *a = 1 + case "tunnel": + *a = 2 + case "wireguard+tls": + *a = 3 + case "grpc-tunnel+tls": + *a = 4 + default: + return fmt.Errorf("unknown type: %s, possible values: 'true', 'wireguard' for the usual WG; 'tunnel' for WG over GRPC, add '+tls' to enable TLS for API", s) + } + + return nil +} + +func (a *agentFlag) Type() string { return "agent" } +func (a *agentFlag) IsEnabled() bool { return *a != 0 } +func (a *agentFlag) IsTunnel() bool { return *a == 2 || *a == 4 } +func (a *agentFlag) IsTLS() bool { return *a == 3 || *a == 4 } + +func validateFlags(flags *pflag.FlagSet, unImplementedFlags []string, currentOs string) error { + invalidFound := false + errMsg := "" + flags.VisitAll(func(f *pflag.Flag) { + if f.Changed && slices.Contains(unImplementedFlags, f.Name) { + errMsg += fmt.Sprintf("%s flag is not supported on %s\n", f.Name, currentOs) + invalidFound = true + } + }) + if invalidFound { + errMsg += "error: unsupported flags found" + return errors.New(errMsg) + } + return nil +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/docker.go b/cmd/talosctl/cmd/mgmt/cluster/create/docker.go new file mode 100644 index 0000000000..b208a0339d --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/docker.go @@ -0,0 +1,160 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package create + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/siderolabs/gen/xslices" + clustercmd "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster" + clusterpkg "github.com/siderolabs/talos/pkg/cluster" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/bundle" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/provision" + "github.com/siderolabs/talos/pkg/provision/access" + "github.com/siderolabs/talos/pkg/provision/providers/docker" +) + +func CreateDockerCluster(ctx context.Context, cOps CommonOps, dOps DockerOps) error { + clusterReqBase, provisionOptions, _, err := getBase(cOps) + networkRequestBase := clusterReqBase.Network + + ips, err := getIps(networkRequestBase.CIDRs, cOps) + if err != nil { + return fmt.Errorf("failed to get ips: %w", err) + } + + provisionOptions = append(provisionOptions, provision.WithDockerPortsHostIP(dOps.dockerHostIP)) + + portList := []string{} + if dOps.ports != "" { + portList = strings.Split(dOps.ports, ",") + provisionOptions = append(provisionOptions, provision.WithDockerPorts(portList)) + } + + if cOps.talosVersion == "" { + parts := strings.Split(dOps.nodeImage, ":") + + cOps.talosVersion = parts[len(parts)-1] + } + + var configBundleOpts []bundle.Option + + provisioner, err := docker.NewDockerProvisioner(ctx) + if err != nil { + return err + } + defer func() { + if err := provisioner.Close(); err != nil { + fmt.Printf("failed to close docker provisioner: %v", err) + } + }() + + request := docker.ClusterRequest{ + ClusterRequestBase: clusterReqBase, + Image: dOps.nodeImage, + Network: docker.NetworkRequest{ + NetworkRequestBase: networkRequestBase, + DockerDisableIPv6: dOps.dockerDisableIPv6, + }, + Nodes: docker.NodeRequests{}, + } + + if cOps.inputDir != "" { + configBundleOpts = append(configBundleOpts, bundle.WithExistingConfigs(cOps.inputDir)) + } else { + genOptions, _, err := getCommonGenOptions(cOps) + if err != nil { + return err + } + genOptions = append(genOptions, provisioner.GenOptions(networkRequestBase)...) + externalKubernetesEndpoint := provisioner.GetExternalKubernetesControlPlaneEndpoint(networkRequestBase) + provisionOptions = append(provisionOptions, provision.WithKubernetesEndpoint(externalKubernetesEndpoint)) + endpointList := provisioner.GetTalosAPIEndpoints(networkRequestBase) + genOptions = append(genOptions, getEnpointListGenOption(cOps, endpointList, ips)...) + inClusterEndpoint := provisioner.GetInClusterKubernetesControlPlaneEndpoint(networkRequestBase, cOps.controlPlanePort) + configBundleOpts = getNewConfigBundle(configBundleOpts, cOps, inClusterEndpoint, genOptions) + } + commonConfigBundleOps, err := getCommonConfigBundleOps(cOps, networkRequestBase.GatewayAddrs[0].String()) + if err != nil { + return err + } + configBundleOpts = append(configBundleOpts, commonConfigBundleOps...) + configBundle, bundleTalosconfig, err := getConfigBundle(cOps, configBundleOpts) + if err != nil { + return err + } + // Add talosconfig to provision options, so we'll have it to parse there + provisionOptions = append(provisionOptions, provision.WithTalosConfig(configBundle.TalosConfig())) + + baseNodes := append(clusterReqBase.Controlplanes, clusterReqBase.Workers...) + for _, n := range baseNodes { + var cfg config.Provider + + nodeIPs := getNodeIp(networkRequestBase.CIDRs, ips, n.Index) + + node := docker.NodeRequest{ + NodeRequestBase: n, + Mounts: dOps.mountOpts.Value(), + IPs: nodeIPs, + Ports: portList, + } + + if cOps.withInitNode && n.Index == 0 { + cfg = configBundle.Init() + node.Type = machine.TypeInit + } else if node.Type == machine.TypeControlPlane { + cfg = configBundle.ControlPlane() + } else if node.Type == machine.TypeWorker { + cfg = configBundle.Worker() + } + node.Config = cfg + request.Nodes = append(request.Nodes, node) + } + + cluster, err := provisioner.Create(ctx, request, provisionOptions...) + if err != nil { + return err + } + + // No talosconfig in the bundle - skip the operations below + if bundleTalosconfig == nil { + return nil + } + clusterAccess := access.NewAdapter(cluster, provisionOptions...) + + // Create and save the talosctl configuration file. + if err = saveConfig(bundleTalosconfig, cOps); err != nil { + return err + } + + if cOps.applyConfigEnabled { + nodeApplyCfgs := xslices.Map(request.Nodes, func(n docker.NodeRequest) clusterpkg.NodeApplyConfig { + return clusterpkg.NodeApplyConfig{NodeAddress: clusterpkg.NodeAddress{IP: n.IPs[0]}, Config: n.Config} + }) + err = clusterAccess.ApplyConfig(ctx, nodeApplyCfgs, nil, os.Stdout) + if err != nil { + return err + } + } + + defer clusterAccess.Close() //nolint:errcheck + if err = postCreate(ctx, clusterAccess, cOps); err != nil { + if cOps.crashdumpOnFailure { + provisioner.CrashDump(ctx, cluster, os.Stderr) + } + + return err + } + + return clustercmd.ShowCluster(cluster) +} + +func init() { +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/qemu.go b/cmd/talosctl/cmd/mgmt/cluster/create/qemu.go new file mode 100644 index 0000000000..9745a79e46 --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/qemu.go @@ -0,0 +1,934 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package create + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "net" + "net/netip" + "net/url" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "github.com/dustin/go-humanize" + "github.com/google/uuid" + "github.com/hashicorp/go-getter/v2" + "github.com/klauspost/compress/zstd" + "github.com/siderolabs/crypto/x509" + "github.com/siderolabs/gen/maps" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-blockdevice/v2/encryption" + "github.com/siderolabs/go-pointer" + "github.com/siderolabs/go-procfs/procfs" + sideronet "github.com/siderolabs/net" + clustercmd "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster" + "github.com/siderolabs/talos/cmd/talosctl/cmd/mgmt/cluster/internal/firewallpatch" + "github.com/siderolabs/talos/cmd/talosctl/pkg/mgmt/helpers" + clusterpkg "github.com/siderolabs/talos/pkg/cluster" + clientconfig "github.com/siderolabs/talos/pkg/machinery/client/config" + "github.com/siderolabs/talos/pkg/machinery/config" + "github.com/siderolabs/talos/pkg/machinery/config/bundle" + "github.com/siderolabs/talos/pkg/machinery/config/configloader" + "github.com/siderolabs/talos/pkg/machinery/config/configpatcher" + "github.com/siderolabs/talos/pkg/machinery/config/encoder" + "github.com/siderolabs/talos/pkg/machinery/config/generate" + "github.com/siderolabs/talos/pkg/machinery/config/machine" + "github.com/siderolabs/talos/pkg/machinery/config/types/security" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/nethelpers" + "github.com/siderolabs/talos/pkg/provision" + "github.com/siderolabs/talos/pkg/provision/access" + "github.com/siderolabs/talos/pkg/provision/providers/qemu" + "github.com/siderolabs/talos/pkg/provision/providers/vm" +) + +func CreateQemuCluster(ctx context.Context, cOps CommonOps, qOps QemuOps) error { + clusterReqBase, provisionOptions, cidr4, err := getBase(cOps) + networkRequestBase := clusterReqBase.Network + cidrs := networkRequestBase.CIDRs + gatewayIPs := networkRequestBase.GatewayAddrs + + fmt.Fprintln(os.Stderr, "validating CIDR and reserving IPs") + ips, err := getIps(networkRequestBase.CIDRs, cOps) + if err != nil { + return fmt.Errorf("failed to get ips: %w", err) + } + + // Virtual (shared) IP at the vipOffset IP in range, ex. 192.168.0.50 + var vip netip.Addr + // vipOffset is the offset from the network address of the CIDR to use for allocating the Virtual (shared) IP address, if enabled. + const vipOffset = 50 + if qOps.useVIP { + vip, err = sideronet.NthIPInNetwork(cidrs[0], vipOffset) + if err != nil { + return fmt.Errorf("failed to get virtual IP: %w", err) + } + } + + // use ULA IPv6 network fd00::/8, add 'TAL' in hex to build /32 network, add IPv4 CIDR to build /64 unique network + cidr6, err := netip.ParsePrefix( + fmt.Sprintf( + "fd74:616c:%02x%02x:%02x%02x::/64", + cidr4.Addr().As4()[0], cidr4.Addr().As4()[1], cidr4.Addr().As4()[2], cidr4.Addr().As4()[3], + ), + ) + if err != nil { + return fmt.Errorf("error validating cidr IPv6 block: %w", err) + } + + if cOps.talosVersion == "" { + parts := strings.Split(qOps.nodeInstallImage, ":") + + cOps.talosVersion = parts[len(parts)-1] + } + + if qOps.networkIPv6 { + networkRequestBase.CIDRs = append(networkRequestBase.CIDRs, cidr6) + } + + if len(networkRequestBase.CIDRs) == 0 { + return errors.New("neither IPv4 nor IPv6 network was enabled") + } + + // Validate network chaos flags + if !qOps.networkChaos { + if qOps.jitter != 0 || qOps.latency != 0 || qOps.packetLoss != 0 || qOps.packetReorder != 0 || qOps.packetCorrupt != 0 || qOps.bandwidth != 0 { + return errors.New("network chaos flags can only be used with --with-network-chaos") + } + } + + provisioner, err := qemu.NewQemuProvisioner(ctx) + if err != nil { + return err + } + + vmNetworkRequestBase := vm.NetworkRequestBase{ + NetworkRequestBase: networkRequestBase, + LoadBalancerPorts: []int{cOps.controlPlanePort}, + // DockerDisableIPv6: dockerOps.dockerDisableIPv6, + } + networkRequest, err := getNetworkRequest(vmNetworkRequestBase, qOps) + if err != nil { + return err + } + + defer func() { + if err := provisioner.Close(); err != nil { + fmt.Printf("failed to close qemu provisioner: %v", err) + } + }() + // Craft cluster and node requests + request := vm.ClusterRequest{ + ClusterRequestBase: clusterReqBase, + Network: networkRequest, + + // Image: dockerOps.nodeImage, + KernelPath: qOps.nodeVmlinuzPath, + InitramfsPath: qOps.nodeInitramfsPath, + ISOPath: qOps.nodeISOPath, + IPXEBootScript: qOps.nodeIPXEBootScript, + DiskImagePath: qOps.nodeDiskImagePath, + } + + provisionOptions = append(provisionOptions, + provision.WithBootlader(qOps.bootloaderEnabled), + provision.WithUEFI(qOps.uefiEnabled), + provision.WithTPM2(qOps.tpm2Enabled), + provision.WithDebugShell(qOps.debugShellEnabled), + provision.WithExtraUEFISearchPaths(qOps.extraUEFISearchPaths), + provision.WithTargetArch(qOps.targetArch), + provision.WithSiderolinkAgent(qOps.withSiderolinkAgent.IsEnabled()), + ) + + var configBundleOpts []bundle.Option + + provisionerName := "" + if qOps.debugShellEnabled { + if provisionerName != "qemu" { + return errors.New("debug shell only supported with qemu provisioner") + } + } + + disks, err := getDisks(qOps) + if err != nil { + return err + } + + if cOps.inputDir != "" { + configBundleOpts = append(configBundleOpts, bundle.WithExistingConfigs(cOps.inputDir)) + } else { + genOptions, versionContract, err := getCommonGenOptions(cOps) + if err != nil { + return err + } + genOptions = append(genOptions, generate.WithInstallImage(qOps.nodeInstallImage)) + genOptions = append(genOptions, provisioner.GenOptions(networkRequestBase)...) + + if len(disks) > 1 { + // convert provision disks to machine disks + machineDisks := make([]*v1alpha1.MachineDisk, len(disks)-1) + for i, disk := range disks[1:] { + machineDisks[i] = &v1alpha1.MachineDisk{ + DeviceName: provisioner.UserDiskName(i + 1), + DiskPartitions: disk.Partitions, + } + } + + genOptions = append(genOptions, generate.WithUserDisks(machineDisks)) + } + + if qOps.encryptStatePartition || qOps.encryptEphemeralPartition { + diskEncryptionConfig := &v1alpha1.SystemDiskEncryptionConfig{} + + var keys []*v1alpha1.EncryptionKey + + for i, key := range qOps.diskEncryptionKeyTypes { + switch key { + case "uuid": + keys = append(keys, &v1alpha1.EncryptionKey{ + KeyNodeID: &v1alpha1.EncryptionKeyNodeID{}, + KeySlot: i, + }) + case "kms": + var ip netip.Addr + + // get bridge IP + ip, err = sideronet.NthIPInNetwork(cidr4, 1) + if err != nil { + return err + } + + const port = 4050 + + keys = append(keys, &v1alpha1.EncryptionKey{ + KeyKMS: &v1alpha1.EncryptionKeyKMS{ + KMSEndpoint: "grpc://" + nethelpers.JoinHostPort(ip.String(), port), + }, + KeySlot: i, + }) + + provisionOptions = append(provisionOptions, provision.WithKMS(nethelpers.JoinHostPort("0.0.0.0", port))) + case "tpm": + keyTPM := &v1alpha1.EncryptionKeyTPM{} + + if versionContract.SecureBootEnrollEnforcementSupported() { + keyTPM.TPMCheckSecurebootStatusOnEnroll = pointer.To(true) + } + + keys = append(keys, &v1alpha1.EncryptionKey{ + KeyTPM: keyTPM, + KeySlot: i, + }) + default: + return fmt.Errorf("unknown key type %q", key) + } + } + + if len(keys) == 0 { + return errors.New("no disk encryption key types enabled") + } + + if qOps.encryptStatePartition { + diskEncryptionConfig.StatePartition = &v1alpha1.EncryptionConfig{ + EncryptionProvider: encryption.LUKS2, + EncryptionKeys: keys, + } + } + + if qOps.encryptEphemeralPartition { + diskEncryptionConfig.EphemeralPartition = &v1alpha1.EncryptionConfig{ + EncryptionProvider: encryption.LUKS2, + EncryptionKeys: keys, + } + } + + genOptions = append(genOptions, generate.WithSystemDiskEncryption(diskEncryptionConfig)) + } + + if qOps.useVIP { + genOptions = append(genOptions, + generate.WithNetworkOptions( + v1alpha1.WithNetworkInterfaceVirtualIP(provisioner.GetFirstInterface(), vip.String()), + ), + ) + } + + if !qOps.bootloaderEnabled { + // disable kexec, as this would effectively use the bootloader + genOptions = append(genOptions, + generate.WithSysctls(map[string]string{ + "kernel.kexec_load_disabled": "1", + }), + ) + } + + externalKubernetesEndpoint := provisioner.GetExternalKubernetesControlPlaneEndpoint(networkRequestBase, cOps.controlPlanePort) + + if qOps.useVIP { + externalKubernetesEndpoint = "https://" + nethelpers.JoinHostPort(vip.String(), cOps.controlPlanePort) + } + + provisionOptions = append(provisionOptions, provision.WithKubernetesEndpoint(externalKubernetesEndpoint)) + + endpointList := provisioner.GetTalosAPIEndpoints(networkRequestBase) + genOptions = append(genOptions, getEnpointListGenOption(cOps, endpointList, ips)...) + + inClusterEndpoint := provisioner.GetInClusterKubernetesControlPlaneEndpoint(networkRequestBase, cOps.controlPlanePort) + + if qOps.useVIP { + inClusterEndpoint = "https://" + nethelpers.JoinHostPort(vip.String(), cOps.controlPlanePort) + } + + configBundleOpts = getNewConfigBundle(configBundleOpts, cOps, inClusterEndpoint, genOptions) + } + commonConfigBundleOps, err := getCommonConfigBundleOps(cOps, gatewayIPs[0].String()) + if err != nil { + return err + } + configBundleOpts = append(configBundleOpts, commonConfigBundleOps...) + + if qOps.withFirewall != "" { + var defaultAction nethelpers.DefaultAction + + defaultAction, err = nethelpers.DefaultActionString(qOps.withFirewall) + if err != nil { + return err + } + + var controlplaneIPs []netip.Addr + + for i := range ips { + controlplaneIPs = append(controlplaneIPs, ips[i][:cOps.controlplanes]...) + } + + configBundleOpts = append(configBundleOpts, + bundle.WithPatchControlPlane([]configpatcher.Patch{firewallpatch.ControlPlane(defaultAction, cidrs, gatewayIPs, controlplaneIPs)}), + bundle.WithPatchWorker([]configpatcher.Patch{firewallpatch.Worker(defaultAction, cidrs, gatewayIPs)}), + ) + } + + var slb *siderolinkBuilder + + if qOps.withSiderolinkAgent.IsEnabled() { + slb, err = newSiderolinkBuilder(gatewayIPs[0].String(), qOps.withSiderolinkAgent.IsTLS()) + if err != nil { + return err + } + } + + if trustedRootsConfig := slb.TrustedRootsConfig(); trustedRootsConfig != nil { + trustedRootsPatch, err := configloader.NewFromBytes(trustedRootsConfig) + if err != nil { + return fmt.Errorf("error loading trusted roots config: %w", err) + } + + configBundleOpts = append(configBundleOpts, bundle.WithPatch([]configpatcher.Patch{configpatcher.NewStrategicMergePatch(trustedRootsPatch)})) + } + + configBundle, bundleTalosconfig, err := getConfigBundle(cOps, configBundleOpts) + if err != nil { + return err + } + + // Wireguard configuration. + var wireguardConfigBundle *helpers.WireguardConfigBundle + if cOps.wireguardCIDR != "" { + wireguardConfigBundle, err = helpers.NewWireguardConfigBundle(ips[0], cOps.wireguardCIDR, 51111, cOps.controlplanes) + if err != nil { + return err + } + } + + var extraKernelArgs *procfs.Cmdline + + if qOps.extraBootKernelArgs != "" || qOps.withSiderolinkAgent.IsEnabled() { + extraKernelArgs = procfs.NewCmdline(qOps.extraBootKernelArgs) + } + + err = slb.SetKernelArgs(extraKernelArgs, qOps.withSiderolinkAgent.IsTunnel()) + if err != nil { + return err + } + + // Add talosconfig to provision options, so we'll have it to parse there + provisionOptions = append(provisionOptions, provision.WithTalosConfig(configBundle.TalosConfig())) + + var configInjectionMethod vm.ConfigInjectionMethod + + switch qOps.configInjectionMethodFlagVal { + case "", "default", "http": + configInjectionMethod = vm.ConfigInjectionMethodHTTP + case "metal-iso": + configInjectionMethod = vm.ConfigInjectionMethodMetalISO + default: + return fmt.Errorf("unknown config injection method %q", configInjectionMethod) + } + + // Create the controlplane nodes. + for i, n := range clusterReqBase.Controlplanes { + var cfg config.Provider + + nodeIPs := getNodeIp(networkRequest.CIDRs, ips, i) + + nodeUUID := uuid.New() + + err = slb.DefineIPv6ForUUID(nodeUUID) + if err != nil { + return err + } + + n.Name = getQemuNodeName(cOps.rootOps.ClusterName, "controlplane", i+1, nodeUUID, qOps) + + nodeReq := vm.NodeRequest{ + NodeRequestBase: vm.NodeRequestBase{ + NodeRequestBase: n, + Disks: disks, + ConfigInjectionMethod: configInjectionMethod, + BadRTC: qOps.badRTC, + ExtraKernelArgs: extraKernelArgs, + UUID: pointer.To(nodeUUID), + }, + // IPs: nodeIPs, + } + + if cOps.withInitNode && n.Index == 0 { + cfg = configBundle.Init() + nodeReq.Type = machine.TypeInit + } else { + cfg = configBundle.ControlPlane() + } + + if wireguardConfigBundle != nil { + cfg, err = wireguardConfigBundle.PatchConfig(nodeIPs[0], cfg) + if err != nil { + return err + } + } + + nodeReq.Config = cfg + + request.Nodes = append(request.Nodes, nodeReq) + } + + for i, n := range clusterReqBase.Workers { + cfg := configBundle.Worker() + + nodeIPs := getNodeIp(networkRequest.CIDRs, ips, i) + + if wireguardConfigBundle != nil { + cfg, err = wireguardConfigBundle.PatchConfig(nodeIPs[0], cfg) + if err != nil { + return err + } + } + + nodeUUID := uuid.New() + + err = slb.DefineIPv6ForUUID(nodeUUID) + if err != nil { + return err + } + + n.Config = cfg + n.Name = getQemuNodeName(cOps.rootOps.ClusterName, "controlplane", i+1, nodeUUID, qOps) + + request.Nodes = append(request.Nodes, + vm.NodeRequest{ + NodeRequestBase: vm.NodeRequestBase{ + NodeRequestBase: n, + Disks: disks, + ConfigInjectionMethod: configInjectionMethod, + BadRTC: qOps.badRTC, + ExtraKernelArgs: extraKernelArgs, + }, + // Mounts: dockerOps.mountOpts.Value(), + // IPs: nodeIPs, + }) + } + + // append extra disks + for i := range qOps.extraDisks { + driver := "ide" + + // ide driver is not supported on arm64 + if qOps.targetArch == "arm64" { + driver = "virtio" + } + + if i < len(qOps.extraDisksDrivers) { + driver = qOps.extraDisksDrivers[i] + } + + disks = append(disks, &vm.Disk{ + Size: uint64(qOps.extraDiskSize) * 1024 * 1024, + SkipPreallocate: !qOps.clusterDiskPreallocate, + Driver: driver, + }) + } + + request.SiderolinkRequest = slb.SiderolinkRequest() + + cluster, err := provisioner.Create(ctx, request, provisionOptions...) + if err != nil { + return err + } + + if qOps.debugShellEnabled { + fmt.Println("You can now connect to debug shell on any node using these commands:") + + for _, node := range request.Nodes { + talosDir, err := clientconfig.GetTalosDirectory() + if err != nil { + return nil + } + + fmt.Printf("socat - UNIX-CONNECT:%s\n", filepath.Join(talosDir, "clusters", cOps.rootOps.ClusterName, node.Name+".serial")) + } + + return nil + } + + // No talosconfig in the bundle - skip the operations below + if bundleTalosconfig == nil { + return nil + } + clusterAccess := access.NewAdapter(cluster, provisionOptions...) + + // Create and save the talosctl configuration file. + if err = saveConfig(bundleTalosconfig, cOps); err != nil { + return err + } + + if cOps.applyConfigEnabled { + nodeApplyCfgs := xslices.Map(request.Nodes, func(n vm.NodeRequest) clusterpkg.NodeApplyConfig { + // TODO: Pass IP + return clusterpkg.NodeApplyConfig{NodeAddress: clusterpkg.NodeAddress{UUID: n.UUID}, Config: n.Config} + }) + err = clusterAccess.ApplyConfig(ctx, nodeApplyCfgs, &request.SiderolinkRequest, os.Stdout) + if err != nil { + return err + } + } + + defer clusterAccess.Close() //nolint:errcheck + if err = postCreate(ctx, clusterAccess, cOps); err != nil { + if cOps.crashdumpOnFailure { + provisioner.CrashDump(ctx, cluster, os.Stderr) + } + + return err + } + + return clustercmd.ShowCluster(cluster) +} + +//nolint:gocyclo +func downloadBootAssets(ctx context.Context, qOps QemuOps) error { + // download & cache images if provides as URLs + for _, downloadableImage := range []struct { + path *string + disableArchive bool + }{ + { + path: &qOps.nodeVmlinuzPath, + }, + { + path: &qOps.nodeInitramfsPath, + disableArchive: true, + }, + { + path: &qOps.nodeISOPath, + }, + { + path: &qOps.nodeDiskImagePath, + }, + } { + if *downloadableImage.path == "" { + continue + } + + u, err := url.Parse(*downloadableImage.path) + if err != nil || !(u.Scheme == "http" || u.Scheme == "https") { + // not a URL + continue + } + + defaultStateDir, err := clientconfig.GetTalosDirectory() + if err != nil { + return err + } + + cacheDir := filepath.Join(defaultStateDir, "cache") + + if os.MkdirAll(cacheDir, 0o755) != nil { + return err + } + + destPath := strings.ReplaceAll( + strings.ReplaceAll(u.String(), "/", "-"), + ":", "-") + + _, err = os.Stat(filepath.Join(cacheDir, destPath)) + if err == nil { + *downloadableImage.path = filepath.Join(cacheDir, destPath) + + // already cached + continue + } + + fmt.Fprintf(os.Stderr, "downloading asset from %q to %q\n", u.String(), filepath.Join(cacheDir, destPath)) + + client := getter.Client{ + Getters: []getter.Getter{ + &getter.HttpGetter{ + HeadFirstTimeout: 30 * time.Minute, + ReadTimeout: 30 * time.Minute, + }, + }, + } + + if downloadableImage.disableArchive { + q := u.Query() + + q.Set("archive", "false") + + u.RawQuery = q.Encode() + } + + _, err = client.Get(ctx, &getter.Request{ + Src: u.String(), + Dst: filepath.Join(cacheDir, destPath), + GetMode: getter.ModeFile, + }) + if err != nil { + // clean up the destination on failure + os.Remove(filepath.Join(cacheDir, destPath)) //nolint:errcheck + + return err + } + + *downloadableImage.path = filepath.Join(cacheDir, destPath) + } + + return nil +} + +func getDisks(qemuOps QemuOps) ([]*vm.Disk, error) { + // should have at least a single primary disk + disks := []*vm.Disk{ + { + Size: uint64(qemuOps.clusterDiskSize) * 1024 * 1024, + SkipPreallocate: !qemuOps.clusterDiskPreallocate, + Driver: "virtio", + }, + } + + for _, disk := range qemuOps.clusterDisks { + var ( + partitions = strings.Split(disk, ":") + diskPartitions = make([]*v1alpha1.DiskPartition, len(partitions)/2) + diskSize uint64 + ) + + if len(partitions)%2 != 0 { + return nil, errors.New("failed to parse malformed partition definitions") + } + + partitionIndex := 0 + + for j := 0; j < len(partitions); j += 2 { + partitionPath := partitions[j] + + if !strings.HasPrefix(partitionPath, "/var") { + return nil, errors.New("user disk partitions can only be mounted into /var folder") + } + + value, e := strconv.ParseInt(partitions[j+1], 10, 0) + partitionSize := uint64(value) + + if e != nil { + partitionSize, e = humanize.ParseBytes(partitions[j+1]) + + if e != nil { + return nil, errors.New("failed to parse partition size") + } + } + + diskPartitions[partitionIndex] = &v1alpha1.DiskPartition{ + DiskSize: v1alpha1.DiskSize(partitionSize), + DiskMountPoint: partitionPath, + } + diskSize += partitionSize + partitionIndex++ + } + + disks = append(disks, &vm.Disk{ + // add 1 MB to make extra room for GPT and alignment + Size: diskSize + 2*1024*1024, + Partitions: diskPartitions, + SkipPreallocate: !qemuOps.clusterDiskPreallocate, + Driver: "ide", + }) + } + + return disks, nil +} + +func getQemuNodeName(clusterName, role string, index int, uuid uuid.UUID, qemuOps QemuOps) string { + if qemuOps.withUUIDHostnames { + return fmt.Sprintf("machine-%s", uuid) + } + + return fmt.Sprintf("%s-%s-%d", clusterName, role, index) +} + +func newSiderolinkBuilder(wgHost string, useTLS bool) (*siderolinkBuilder, error) { + prefix, err := networkPrefix("") + if err != nil { + return nil, err + } + + result := &siderolinkBuilder{ + wgHost: wgHost, + binds: map[uuid.UUID]netip.Addr{}, + prefix: prefix, + nodeIPv6Addr: prefix.Addr().Next().String(), + } + + if useTLS { + ca, err := x509.NewSelfSignedCertificateAuthority(x509.ECDSA(true), x509.IPAddresses([]net.IP{net.ParseIP(wgHost)})) + if err != nil { + return nil, err + } + + result.apiCert = ca.CrtPEM + result.apiKey = ca.KeyPEM + } + + var resultErr error + + for range 10 { + for _, d := range []struct { + field *int + net string + what string + }{ + {&result.wgPort, "udp", "WireGuard"}, + {&result.apiPort, "tcp", "gRPC API"}, + {&result.sinkPort, "tcp", "Event Sink"}, + {&result.logPort, "tcp", "Log Receiver"}, + } { + var err error + + *d.field, err = getDynamicPort(d.net) + if err != nil { + return nil, fmt.Errorf("failed to get dynamic port for %s: %w", d.what, err) + } + } + + resultErr = checkPortsDontOverlap(result.wgPort, result.apiPort, result.sinkPort, result.logPort) + if resultErr == nil { + break + } + } + + if resultErr != nil { + return nil, fmt.Errorf("failed to get non-overlapping dynamic ports in 10 attempts: %w", resultErr) + } + + return result, nil +} + +type siderolinkBuilder struct { + wgHost string + + binds map[uuid.UUID]netip.Addr + prefix netip.Prefix + nodeIPv6Addr string + wgPort int + apiPort int + sinkPort int + logPort int + + apiCert []byte + apiKey []byte +} + +// DefineIPv6ForUUID defines an IPv6 address for a given UUID. It is safe to call this method on a nil pointer. +func (slb *siderolinkBuilder) DefineIPv6ForUUID(id uuid.UUID) error { + if slb == nil { + return nil + } + + result, err := generateRandomNodeAddr(slb.prefix) + if err != nil { + return err + } + + slb.binds[id] = result.Addr() + + return nil +} + +// SiderolinkRequest returns a SiderolinkRequest based on the current state of the builder. +// It is safe to call this method on a nil pointer. +func (slb *siderolinkBuilder) SiderolinkRequest() provision.SiderolinkRequest { + if slb == nil { + return provision.SiderolinkRequest{} + } + + return provision.SiderolinkRequest{ + WireguardEndpoint: net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.wgPort)), + APIEndpoint: ":" + strconv.Itoa(slb.apiPort), + APICertificate: slb.apiCert, + APIKey: slb.apiKey, + SinkEndpoint: ":" + strconv.Itoa(slb.sinkPort), + LogEndpoint: ":" + strconv.Itoa(slb.logPort), + SiderolinkBind: maps.ToSlice(slb.binds, func(k uuid.UUID, v netip.Addr) provision.SiderolinkBind { + return provision.SiderolinkBind{ + UUID: k, + Addr: v, + } + }), + } +} + +// TrustedRootsConfig returns the trusted roots config for the current builder. +func (slb *siderolinkBuilder) TrustedRootsConfig() []byte { + if slb == nil || slb.apiCert == nil { + return nil + } + + trustedRootsConfig := security.NewTrustedRootsConfigV1Alpha1() + trustedRootsConfig.MetaName = "siderolink-ca" + trustedRootsConfig.Certificates = string(slb.apiCert) + + marshaled, err := encoder.NewEncoder(trustedRootsConfig, encoder.WithComments(encoder.CommentsDisabled)).Encode() + if err != nil { + panic(fmt.Sprintf("failed to marshal trusted roots config: %s", err)) + } + + return marshaled +} + +// SetKernelArgs sets the kernel arguments for the current builder. It is safe to call this method on a nil pointer. +func (slb *siderolinkBuilder) SetKernelArgs(extraKernelArgs *procfs.Cmdline, tunnel bool) error { + switch { + case slb == nil: + return nil + case extraKernelArgs.Get("siderolink.api") != nil, + extraKernelArgs.Get("talos.events.sink") != nil, + extraKernelArgs.Get("talos.logging.kernel") != nil: + return errors.New("siderolink kernel arguments are already set, cannot run with --with-siderolink") + default: + scheme := "grpc://" + + if slb.apiCert != nil { + scheme = "https://" + } + + apiLink := scheme + net.JoinHostPort(slb.wgHost, strconv.Itoa(slb.apiPort)) + "?jointoken=foo" + + if tunnel { + apiLink += "&grpc_tunnel=true" + } + + extraKernelArgs.Append("siderolink.api", apiLink) + extraKernelArgs.Append("talos.events.sink", net.JoinHostPort(slb.nodeIPv6Addr, strconv.Itoa(slb.sinkPort))) + extraKernelArgs.Append("talos.logging.kernel", "tcp://"+net.JoinHostPort(slb.nodeIPv6Addr, strconv.Itoa(slb.logPort))) + + if trustedRootsConfig := slb.TrustedRootsConfig(); trustedRootsConfig != nil { + var buf bytes.Buffer + + zencoder, err := zstd.NewWriter(&buf) + if err != nil { + return fmt.Errorf("failed to create zstd encoder: %w", err) + } + + _, err = zencoder.Write(trustedRootsConfig) + if err != nil { + return fmt.Errorf("failed to write zstd data: %w", err) + } + + if err = zencoder.Close(); err != nil { + return fmt.Errorf("failed to close zstd encoder: %w", err) + } + + extraKernelArgs.Append(constants.KernelParamConfigInline, base64.StdEncoding.EncodeToString(buf.Bytes())) + } + + return nil + } +} + +func getDynamicPort(network string) (int, error) { + var ( + closeFn func() error + addrFn func() net.Addr + ) + + switch network { + case "tcp", "tcp4", "tcp6": + l, err := net.Listen(network, "127.0.0.1:0") + if err != nil { + return 0, err + } + + addrFn, closeFn = l.Addr, l.Close + case "udp", "udp4", "udp6": + l, err := net.ListenPacket(network, "127.0.0.1:0") + if err != nil { + return 0, err + } + + addrFn, closeFn = l.LocalAddr, l.Close + default: + return 0, fmt.Errorf("unsupported network: %s", network) + } + + _, portStr, err := net.SplitHostPort(addrFn().String()) + if err != nil { + return 0, handleCloseErr(err, closeFn()) + } + + port, err := strconv.Atoi(portStr) + if err != nil { + return 0, err + } + + return port, handleCloseErr(nil, closeFn()) +} + +func handleCloseErr(err error, closeErr error) error { + switch { + case err != nil && closeErr != nil: + return fmt.Errorf("error: %w, close error: %w", err, closeErr) + case err == nil && closeErr != nil: + return closeErr + case err != nil && closeErr == nil: + return err + default: + return nil + } +} + +func checkPortsDontOverlap(ports ...int) error { + slices.Sort(ports) + + if len(ports) != len(slices.Compact(ports)) { + return errors.New("generated ports overlap") + } + + return nil +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create_other.go b/cmd/talosctl/cmd/mgmt/cluster/create/qemu_darwin.go similarity index 63% rename from cmd/talosctl/cmd/mgmt/cluster/create_other.go rename to cmd/talosctl/cmd/mgmt/cluster/create/qemu_darwin.go index ce5d868081..c3b896c417 100644 --- a/cmd/talosctl/cmd/mgmt/cluster/create_other.go +++ b/cmd/talosctl/cmd/mgmt/cluster/create/qemu_darwin.go @@ -4,13 +4,17 @@ //go:build !linux -package cluster +package create import ( "errors" "net/netip" + + "github.com/siderolabs/talos/pkg/provision/providers/vm" ) +const currentOs = "darwin" + func generateRandomNodeAddr(prefix netip.Prefix) (netip.Prefix, error) { return netip.Prefix{}, nil } @@ -18,3 +22,9 @@ func generateRandomNodeAddr(prefix netip.Prefix) (netip.Prefix, error) { func networkPrefix(prefix string) (netip.Prefix, error) { return netip.Prefix{}, errors.New("unsupported platform") } + +func getNetworkRequest(base vm.NetworkRequestBase, qemuOps QemuOps) (req vm.NetworkRequest, err error) { + return vm.NetworkRequest{ + NetworkRequestBase: base, + }, nil +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create/qemu_linux.go b/cmd/talosctl/cmd/mgmt/cluster/create/qemu_linux.go new file mode 100644 index 0000000000..9f87077428 --- /dev/null +++ b/cmd/talosctl/cmd/mgmt/cluster/create/qemu_linux.go @@ -0,0 +1,70 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package create + +import ( + "fmt" + "net/netip" + + "github.com/siderolabs/siderolink/pkg/wireguard" + "github.com/siderolabs/talos/pkg/provision/providers/vm" +) + +const currentOs = "linux" + +func generateRandomNodeAddr(prefix netip.Prefix) (netip.Prefix, error) { + return wireguard.GenerateRandomNodeAddr(prefix) +} + +func networkPrefix(prefix string) (netip.Prefix, error) { + return wireguard.NetworkPrefix(prefix), nil +} + +func getNetworkRequest(base vm.NetworkRequestBase, qemuOps QemuOps) (req vm.NetworkRequest, err error) { + // Parse nameservers + nameserverIPs := make([]netip.Addr, len(qemuOps.nameservers)) + + for i := range nameserverIPs { + nameserverIPs[i], err = netip.ParseAddr(qemuOps.nameservers[i]) + if err != nil { + return req, fmt.Errorf("failed parsing nameserver IP %q: %w", qemuOps.nameservers[i], err) + } + } + + noMasqueradeCIDRs := make([]netip.Prefix, 0, len(qemuOps.networkNoMasqueradeCIDRs)) + + for _, cidr := range qemuOps.networkNoMasqueradeCIDRs { + var parsedCIDR netip.Prefix + + parsedCIDR, err = netip.ParsePrefix(cidr) + if err != nil { + return req, fmt.Errorf("error parsing non-masquerade CIDR %q: %w", cidr, err) + } + + noMasqueradeCIDRs = append(noMasqueradeCIDRs, parsedCIDR) + } + + return vm.NetworkRequest{ + NetworkRequestBase: base, + Nameservers: nameserverIPs, + CNI: vm.CNIConfig{ + BinPath: qemuOps.cniBinPath, + ConfDir: qemuOps.cniConfDir, + CacheDir: qemuOps.cniCacheDir, + + BundleURL: qemuOps.cniBundleURL, + }, + DHCPSkipHostname: qemuOps.dhcpSkipHostname, + NetworkChaos: qemuOps.networkChaos, + Jitter: qemuOps.jitter, + Latency: qemuOps.latency, + PacketLoss: qemuOps.packetLoss, + PacketReorder: qemuOps.packetReorder, + PacketCorrupt: qemuOps.packetCorrupt, + Bandwidth: qemuOps.bandwidth, + NoMasqueradeCIDRs: noMasqueradeCIDRs, + }, nil + +} diff --git a/cmd/talosctl/cmd/mgmt/cluster/create_linux.go b/cmd/talosctl/cmd/mgmt/cluster/create_linux.go deleted file mode 100644 index 8d07e6048a..0000000000 --- a/cmd/talosctl/cmd/mgmt/cluster/create_linux.go +++ /dev/null @@ -1,19 +0,0 @@ -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package cluster - -import ( - "net/netip" - - "github.com/siderolabs/siderolink/pkg/wireguard" -) - -func generateRandomNodeAddr(prefix netip.Prefix) (netip.Prefix, error) { - return wireguard.GenerateRandomNodeAddr(prefix) -} - -func networkPrefix(prefix string) (netip.Prefix, error) { - return wireguard.NetworkPrefix(prefix), nil -} diff --git a/cmd/talosctl/cmd/mgmt/cluster/destroy.go b/cmd/talosctl/cmd/mgmt/cluster/destroy.go index 680e1b6d94..a5b4a6a844 100644 --- a/cmd/talosctl/cmd/mgmt/cluster/destroy.go +++ b/cmd/talosctl/cmd/mgmt/cluster/destroy.go @@ -31,14 +31,14 @@ var destroyCmd = &cobra.Command{ } func destroy(ctx context.Context) error { - provisioner, err := providers.Factory(ctx, provisionerName) + provisioner, err := providers.Factory(ctx, Flags.ProvisionerName) if err != nil { return err } defer provisioner.Close() //nolint:errcheck - cluster, err := provisioner.Reflect(ctx, clusterName, stateDir) + cluster, err := provisioner.Reflect(ctx, Flags.ClusterName, Flags.StateDir) if err != nil { return err } diff --git a/cmd/talosctl/cmd/mgmt/cluster/show.go b/cmd/talosctl/cmd/mgmt/cluster/show.go index cfc088af7d..df1e556c8f 100644 --- a/cmd/talosctl/cmd/mgmt/cluster/show.go +++ b/cmd/talosctl/cmd/mgmt/cluster/show.go @@ -35,22 +35,22 @@ var showCmd = &cobra.Command{ } func show(ctx context.Context) error { - provisioner, err := providers.Factory(ctx, provisionerName) + provisioner, err := providers.Factory(ctx, Flags.ProvisionerName) if err != nil { return err } defer provisioner.Close() //nolint:errcheck - cluster, err := provisioner.Reflect(ctx, clusterName, stateDir) + cluster, err := provisioner.Reflect(ctx, Flags.ClusterName, Flags.StateDir) if err != nil { return err } - return showCluster(cluster) + return ShowCluster(cluster) } -func showCluster(cluster provision.Cluster) error { +func ShowCluster(cluster provision.Cluster) error { w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) fmt.Fprintf(w, "PROVISIONER\t%s\n", cluster.Provisioner()) fmt.Fprintf(w, "NAME\t%s\n", cluster.Info().ClusterName) diff --git a/cmd/talosctl/cmd/mgmt/qemu_launch_linux.go b/cmd/talosctl/cmd/mgmt/qemu_launch.go similarity index 96% rename from cmd/talosctl/cmd/mgmt/qemu_launch_linux.go rename to cmd/talosctl/cmd/mgmt/qemu_launch.go index dc9ca27d76..ff5d7b7e92 100644 --- a/cmd/talosctl/cmd/mgmt/qemu_launch_linux.go +++ b/cmd/talosctl/cmd/mgmt/qemu_launch.go @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -//go:build linux +//go:build linux || darwin package mgmt diff --git a/go.mod b/go.mod index dfb0b403b0..afdb3a597a 100644 --- a/go.mod +++ b/go.mod @@ -247,6 +247,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect github.com/cyphar/filepath-securejoin v0.3.4 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e // indirect github.com/docker/distribution v2.8.3+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect diff --git a/go.sum b/go.sum index 34f3b97d21..246e82029e 100644 --- a/go.sum +++ b/go.sum @@ -183,6 +183,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e h1:lj77EKYUpYXTd8CD/+QMIf8b6OIOTsfEBSXiAzuEHTU= +github.com/detailyang/go-fallocate v0.0.0-20180908115635-432fa640bd2e/go.mod h1:3ZQK6DMPSz/QZ73jlWxBtUhNA8xZx7LzUFSq/OfP8vk= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= diff --git a/pkg/cluster/apply-config.go b/pkg/cluster/apply-config.go index ec60e8c84e..0d92dc10e5 100644 --- a/pkg/cluster/apply-config.go +++ b/pkg/cluster/apply-config.go @@ -9,12 +9,15 @@ import ( "crypto/tls" "fmt" "io" + "net/netip" "time" + "github.com/google/uuid" "github.com/siderolabs/go-retry/retry" machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine" "github.com/siderolabs/talos/pkg/machinery/client" + "github.com/siderolabs/talos/pkg/machinery/config" "github.com/siderolabs/talos/pkg/provision" ) @@ -24,16 +27,28 @@ type ApplyConfigClient struct { Info } -// ApplyConfig on the node via the API using insecure mode. -func (s *APIBootstrapper) ApplyConfig(ctx context.Context, nodes []provision.NodeRequest, sl provision.SiderolinkRequest, out io.Writer) error { +type NodeAddress struct { + IP netip.Addr + UUID *uuid.UUID +} + +type NodeApplyConfig struct { + NodeAddress NodeAddress + Config config.Provider +} + +// ApplyConfig on the node via the API using insecure mode. If UUID is set attempts to apply via SideroLink +func (s *APIBootstrapper) ApplyConfig(ctx context.Context, nodes []NodeApplyConfig, sl *provision.SiderolinkRequest, out io.Writer) error { for _, node := range nodes { configureNode := func() error { - ep := node.IPs[0].String() + ep := node.NodeAddress.IP.String() - if addr, ok := sl.GetAddr(node.UUID); ok { - fmt.Fprintln(out, "using SideroLink node address for 'with-apply-config'", node.UUID, "=", addr.String()) + if sl != nil { + if addr, ok := sl.GetAddr(node.NodeAddress.UUID); ok { + fmt.Fprintln(out, "using SideroLink node address for 'with-apply-config'", node.NodeAddress.UUID, "=", addr.String()) - ep = addr.String() + ep = addr.String() + } } c, err := client.New(ctx, client.WithTLSConfig(&tls.Config{ diff --git a/pkg/provision/providers/docker/crashdump.go b/pkg/provision/providers/docker/crashdump.go index 954cbd67e5..d8afedb2c4 100644 --- a/pkg/provision/providers/docker/crashdump.go +++ b/pkg/provision/providers/docker/crashdump.go @@ -19,7 +19,7 @@ import ( ) // CrashDump produces debug information to help with debugging failures. -func (p *provisioner) CrashDump(ctx context.Context, cluster provision.Cluster, logWriter io.Writer) { +func (p *DockerProvisioner) CrashDump(ctx context.Context, cluster provision.Cluster, logWriter io.Writer) { containers, err := p.listNodes(ctx, cluster.Info().ClusterName) if err != nil { fmt.Fprintf(logWriter, "error listing containers: %s\n", err) diff --git a/pkg/provision/providers/docker/create.go b/pkg/provision/providers/docker/create.go index 52972de8f1..b9871419c3 100644 --- a/pkg/provision/providers/docker/create.go +++ b/pkg/provision/providers/docker/create.go @@ -10,12 +10,12 @@ import ( "os" "path/filepath" - "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/gen/xslices" "github.com/siderolabs/talos/pkg/provision" ) // Create Talos cluster as a set of docker containers on docker network. -func (p *provisioner) Create(ctx context.Context, request provision.ClusterRequest, opts ...provision.Option) (provision.Cluster, error) { +func (p *DockerProvisioner) Create(ctx context.Context, request ClusterRequest, opts ...provision.Option) (provision.Cluster, error) { var err error options := provision.DefaultOptions() @@ -48,7 +48,8 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque fmt.Fprintln(options.LogWriter, "creating controlplane nodes") - if nodeInfo, err = p.createNodes(ctx, request, request.Nodes.ControlPlaneNodes(), &options, true); err != nil { + controlplanes := xslices.Filter(request.Nodes, func(n NodeRequest) bool { return n.Type.IsControlPlane() }) + if nodeInfo, err = p.createNodes(ctx, request, controlplanes, &options, true); err != nil { return nil, err } @@ -56,7 +57,8 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque var workerNodeInfo []provision.NodeInfo - if workerNodeInfo, err = p.createNodes(ctx, request, request.Nodes.WorkerNodes(), &options, false); err != nil { + workers := xslices.Filter(request.Nodes, func(n NodeRequest) bool { return !n.Type.IsControlPlane() }) + if workerNodeInfo, err = p.createNodes(ctx, request, workers, &options, false); err != nil { return nil, err } @@ -72,7 +74,7 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque MTU: request.Network.MTU, }, Nodes: nodeInfo, - KubernetesEndpoint: p.GetExternalKubernetesControlPlaneEndpoint(request.Network, constants.DefaultControlPlanePort), + KubernetesEndpoint: p.GetExternalKubernetesControlPlaneEndpoint(request.Network.NetworkRequestBase), }, statePath: statePath, } diff --git a/pkg/provision/providers/docker/destroy.go b/pkg/provision/providers/docker/destroy.go index c09994da25..c2d73fcb8a 100644 --- a/pkg/provision/providers/docker/destroy.go +++ b/pkg/provision/providers/docker/destroy.go @@ -16,7 +16,7 @@ import ( // Destroy Talos cluster as set of Docker nodes. // // Only cluster.Info().ClusterName and cluster.Info().Network.Name is being used. -func (p *provisioner) Destroy(ctx context.Context, cluster provision.Cluster, opts ...provision.Option) error { +func (p *DockerProvisioner) Destroy(ctx context.Context, cluster provision.Cluster, opts ...provision.Option) error { options := provision.DefaultOptions() for _, opt := range opts { diff --git a/pkg/provision/providers/docker/docker.go b/pkg/provision/providers/docker/docker.go index 758538b737..c23bcd548b 100644 --- a/pkg/provision/providers/docker/docker.go +++ b/pkg/provision/providers/docker/docker.go @@ -19,7 +19,7 @@ import ( "github.com/siderolabs/talos/pkg/provision" ) -type provisioner struct { +type DockerProvisioner struct { client *client.Client mappedKubernetesPort, mappedTalosAPIPort int @@ -52,31 +52,36 @@ func getAvailableTCPPort() (int, error) { } // NewProvisioner initializes docker provisioner. -func NewProvisioner(ctx context.Context) (provision.Provisioner, error) { - p := &provisioner{} +func NewDockerProvisioner(ctx context.Context) (*DockerProvisioner, error) { + p := &DockerProvisioner{} var err error p.client, err = client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) if err != nil { - return nil, err + return p, err } p.mappedKubernetesPort, err = getAvailableTCPPort() if err != nil { - return nil, fmt.Errorf("failed to get available port for Kubernetes API: %w", err) + return p, fmt.Errorf("failed to get available port for Kubernetes API: %w", err) } p.mappedTalosAPIPort, err = getAvailableTCPPort() if err != nil { - return nil, fmt.Errorf("failed to get available port for Talos API: %w", err) + return p, fmt.Errorf("failed to get available port for Talos API: %w", err) } return p, nil } +func NewProvisioner(ctx context.Context) (provision.Provisioner, error) { + p, err := NewDockerProvisioner(ctx) + return provision.Provisioner(p), err +} + // Close and release resources. -func (p *provisioner) Close() error { +func (p *DockerProvisioner) Close() error { if p.client != nil { return p.client.Close() } @@ -85,7 +90,7 @@ func (p *provisioner) Close() error { } // GenOptions provides a list of additional config generate options. -func (p *provisioner) GenOptions(networkReq provision.NetworkRequest) []generate.Option { +func (p *DockerProvisioner) GenOptions(networkReq provision.NetworkRequestBase) []generate.Option { return []generate.Option{ generate.WithNetworkOptions( v1alpha1.WithNetworkInterfaceIgnore(v1alpha1.IfaceByName("eth0")), @@ -95,29 +100,29 @@ func (p *provisioner) GenOptions(networkReq provision.NetworkRequest) []generate } // GetInClusterKubernetesControlPlaneEndpoint returns the Kubernetes control plane endpoint. -func (p *provisioner) GetInClusterKubernetesControlPlaneEndpoint(networkReq provision.NetworkRequest, controlPlanePort int) string { +func (p *DockerProvisioner) GetInClusterKubernetesControlPlaneEndpoint(networkReq provision.NetworkRequestBase, controlPlanePort int) string { // Docker doesn't have a loadbalancer, so use the first container IP. return "https://" + nethelpers.JoinHostPort(networkReq.CIDRs[0].Addr().Next().Next().String(), controlPlanePort) } // GetExternalKubernetesControlPlaneEndpoint returns the Kubernetes control plane endpoint. -func (p *provisioner) GetExternalKubernetesControlPlaneEndpoint(provision.NetworkRequest, int) string { +func (p *DockerProvisioner) GetExternalKubernetesControlPlaneEndpoint(provision.NetworkRequestBase) string { // return a mapped to the localhost first container Kubernetes API endpoint. return "https://" + nethelpers.JoinHostPort("127.0.0.1", p.mappedKubernetesPort) } // GetTalosAPIEndpoints returns a list of Talos API endpoints. -func (p *provisioner) GetTalosAPIEndpoints(provision.NetworkRequest) []string { +func (p *DockerProvisioner) GetTalosAPIEndpoints(provision.NetworkRequestBase) []string { // return a mapped to the localhost first container Talos API endpoint. return []string{nethelpers.JoinHostPort("127.0.0.1", p.mappedTalosAPIPort)} } // UserDiskName not implemented for docker. -func (p *provisioner) UserDiskName(index int) string { +func (p *DockerProvisioner) UserDiskName(index int) string { return "" } // GetFirstInterface returns first network interface name. -func (p *provisioner) GetFirstInterface() v1alpha1.IfaceSelector { +func (p *DockerProvisioner) GetFirstInterface() v1alpha1.IfaceSelector { return v1alpha1.IfaceByName("eth0") } diff --git a/pkg/provision/providers/docker/image.go b/pkg/provision/providers/docker/image.go index 1ff7158d7c..da16c87aee 100644 --- a/pkg/provision/providers/docker/image.go +++ b/pkg/provision/providers/docker/image.go @@ -16,7 +16,7 @@ import ( "github.com/siderolabs/talos/pkg/provision" ) -func (p *provisioner) ensureImageExists(ctx context.Context, containerImage string, options *provision.Options) error { +func (p *DockerProvisioner) ensureImageExists(ctx context.Context, containerImage string, options *provision.Options) error { // In order to pull an image, the reference must be in canonical // format (e.g. domain/repo/image:tag). ref, err := reference.ParseNormalizedNamed(containerImage) diff --git a/pkg/provision/providers/docker/network.go b/pkg/provision/providers/docker/network.go index ea746999de..e40b29e221 100644 --- a/pkg/provision/providers/docker/network.go +++ b/pkg/provision/providers/docker/network.go @@ -12,13 +12,11 @@ import ( "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/network" "github.com/hashicorp/go-multierror" - - "github.com/siderolabs/talos/pkg/provision" ) // createNetwork will take a network request and check if a network with the same name + cidr exists. // If so, it simply returns without error and assumes we will re-use that network. Otherwise it will create a new one. -func (p *provisioner) createNetwork(ctx context.Context, req provision.NetworkRequest) error { +func (p *DockerProvisioner) createNetwork(ctx context.Context, req NetworkRequest) error { existingNet, err := p.listNetworks(ctx, req.Name) if err != nil { return err @@ -56,7 +54,7 @@ func (p *provisioner) createNetwork(ctx context.Context, req provision.NetworkRe return err } -func (p *provisioner) listNetworks(ctx context.Context, name string) ([]network.Inspect, error) { +func (p *DockerProvisioner) listNetworks(ctx context.Context, name string) ([]network.Inspect, error) { filters := filters.NewArgs() filters.Add("label", "talos.owned=true") filters.Add("label", "talos.cluster.name="+name) @@ -68,7 +66,7 @@ func (p *provisioner) listNetworks(ctx context.Context, name string) ([]network. return p.client.NetworkList(ctx, options) } -func (p *provisioner) destroyNetwork(ctx context.Context, name string) error { +func (p *DockerProvisioner) destroyNetwork(ctx context.Context, name string) error { networks, err := p.listNetworks(ctx, name) if err != nil { return err diff --git a/pkg/provision/providers/docker/node.go b/pkg/provision/providers/docker/node.go index 240bea7e33..eb91d16f7f 100644 --- a/pkg/provision/providers/docker/node.go +++ b/pkg/provision/providers/docker/node.go @@ -32,10 +32,10 @@ type portMap struct { portBindings nat.PortMap } -func (p *provisioner) createNodes( +func (p *DockerProvisioner) createNodes( ctx context.Context, - clusterReq provision.ClusterRequest, - nodeReqs []provision.NodeRequest, + clusterReq ClusterRequest, + nodeReqs []NodeRequest, options *provision.Options, isControlplane bool, ) ([]provision.NodeInfo, error) { @@ -43,7 +43,7 @@ func (p *provisioner) createNodes( nodeCh := make(chan provision.NodeInfo, len(nodeReqs)) for i, nodeReq := range nodeReqs { - go func(i int, nodeReq provision.NodeRequest) { + go func(i int, nodeReq NodeRequest) { if i == 0 && isControlplane { hostPrefix := "" @@ -88,7 +88,7 @@ func (p *provisioner) createNodes( } //nolint:gocyclo -func (p *provisioner) createNode(ctx context.Context, clusterReq provision.ClusterRequest, nodeReq provision.NodeRequest, options *provision.Options) (provision.NodeInfo, error) { +func (p *DockerProvisioner) createNode(ctx context.Context, clusterReq ClusterRequest, nodeReq NodeRequest, options *provision.Options) (provision.NodeInfo, error) { env := []string{ "PLATFORM=container", fmt.Sprintf("TALOSSKU=%dCPU-%dRAM", nodeReq.NanoCPUs/(1000*1000*1000), nodeReq.Memory/(1024*1024)), @@ -237,7 +237,7 @@ func (p *provisioner) createNode(ctx context.Context, clusterReq provision.Clust return nodeInfo, nil } -func (p *provisioner) listNodes(ctx context.Context, clusterName string) ([]types.Container, error) { +func (p *DockerProvisioner) listNodes(ctx context.Context, clusterName string) ([]types.Container, error) { filters := filters.NewArgs() filters.Add("label", "talos.owned=true") filters.Add("label", "talos.cluster.name="+clusterName) @@ -245,7 +245,7 @@ func (p *provisioner) listNodes(ctx context.Context, clusterName string) ([]type return p.client.ContainerList(ctx, container.ListOptions{All: true, Filters: filters}) } -func (p *provisioner) destroyNodes(ctx context.Context, clusterName string, options *provision.Options) error { +func (p *DockerProvisioner) destroyNodes(ctx context.Context, clusterName string, options *provision.Options) error { containers, err := p.listNodes(ctx, clusterName) if err != nil { return err diff --git a/pkg/provision/providers/docker/reflect.go b/pkg/provision/providers/docker/reflect.go index 2a737443e0..4c2267b586 100644 --- a/pkg/provision/providers/docker/reflect.go +++ b/pkg/provision/providers/docker/reflect.go @@ -17,7 +17,7 @@ import ( ) //nolint:gocyclo -func (p *provisioner) Reflect(ctx context.Context, clusterName, stateDirectory string) (provision.Cluster, error) { +func (p *DockerProvisioner) Reflect(ctx context.Context, clusterName, stateDirectory string) (provision.Cluster, error) { res := &result{ clusterInfo: provision.ClusterInfo{ ClusterName: clusterName, diff --git a/pkg/provision/providers/docker/request.go b/pkg/provision/providers/docker/request.go new file mode 100644 index 0000000000..3c6d3ef1e3 --- /dev/null +++ b/pkg/provision/providers/docker/request.go @@ -0,0 +1,38 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package docker implements Provisioner via docker. +package docker + +import ( + "net/netip" + + mounttypes "github.com/docker/docker/api/types/mount" + "github.com/siderolabs/talos/pkg/provision" +) + +type ClusterRequest struct { + provision.ClusterRequestBase + + // Docker specific parameters. + Image string + Network NetworkRequest + Nodes NodeRequests +} + +type NodeRequests []NodeRequest + +type NodeRequest struct { + provision.NodeRequestBase + + Mounts []mounttypes.Mount + Ports []string + IPs []netip.Addr +} + +type NetworkRequest struct { + provision.NetworkRequestBase + + DockerDisableIPv6 bool +} diff --git a/pkg/provision/providers/factory.go b/pkg/provision/providers/factory.go index 308d15c170..992038f793 100644 --- a/pkg/provision/providers/factory.go +++ b/pkg/provision/providers/factory.go @@ -12,14 +12,27 @@ import ( "github.com/siderolabs/talos/pkg/provision/providers/docker" ) +const QemuProviderName = "qemu" +const DockerProviderName = "docker" + // Factory instantiates provision provider by name. func Factory(ctx context.Context, name string) (provision.Provisioner, error) { + if err := IsValidProvider(name); err != nil { + return nil, err + } switch name { - case "docker": + case DockerProviderName: return docker.NewProvisioner(ctx) - case "qemu": + case QemuProviderName: return newQemu(ctx) - default: - return nil, fmt.Errorf("unsupported provisioner %q", name) } + return nil, nil +} + +// IsValidProvider returns an error if the passed provider doesn't exist +func IsValidProvider(name string) error { + if name != QemuProviderName && name != DockerProviderName { + return fmt.Errorf("unsupported provisioner %q", name) + } + return nil } diff --git a/pkg/provision/providers/qemu_linux.go b/pkg/provision/providers/qemu.go similarity index 94% rename from pkg/provision/providers/qemu_linux.go rename to pkg/provision/providers/qemu.go index 3dd51ebf5a..0ce7d073d6 100644 --- a/pkg/provision/providers/qemu_linux.go +++ b/pkg/provision/providers/qemu.go @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -//go:build linux +//go:build linux || darwin package providers diff --git a/pkg/provision/providers/qemu/arch.go b/pkg/provision/providers/qemu/arch.go index 3eb95a2f6f..b4ecdeeef8 100644 --- a/pkg/provision/providers/qemu/arch.go +++ b/pkg/provision/providers/qemu/arch.go @@ -83,6 +83,7 @@ func (arch Arch) PFlash(uefiEnabled bool, extraUEFISearchPaths []string) []PFlas "/usr/share/OVMF", "/usr/share/edk2/aarch64", // Fedora "/usr/share/edk2/experimental", // Fedora + "/opt/homebrew/share/qemu", // darwin } // Secure boot enabled firmware files @@ -96,12 +97,14 @@ func (arch Arch) PFlash(uefiEnabled bool, extraUEFISearchPaths []string) []PFlas "AAVMF_CODE.fd", "QEMU_EFI.fd", "OVMF.stateless.fd", + "edk2-aarch64-code.fd", } // Empty vars files uefiVarsFiles := []string{ "AAVMF_VARS.fd", "QEMU_VARS.fd", + "edk2-arm-vars.fd", } // Append extra search paths @@ -236,22 +239,33 @@ func (arch Arch) TPMDeviceArgs(socketPath string) []string { } } -// KVMArgs returns arguments for qemu to enable KVM. -func (arch Arch) KVMArgs(kvmEnabled bool) []string { - if !kvmEnabled { - return []string{"-machine", arch.QemuMachine()} +type MachineArgsParams struct { + kvmEnabled bool + hvfEnabled bool +} + +// MachineArgs returns arguments for the qemu machine +func (arch Arch) MachineArgs(params MachineArgsParams) []string { + if params.kvmEnabled && params.hvfEnabled { + panic("can't enable hvf and kvm simultaneously") } - machineArg := arch.QemuMachine() + ",accel=kvm" + args := []string{"-machine", arch.QemuMachine()} + + if params.kvmEnabled { + args[1] += ",accel=kvm" + } + if params.hvfEnabled { + args[1] += ",accel=hvf" + } switch arch { case ArchAmd64: - machineArg += ",smm=on" - - return []string{"-machine", machineArg} + args[1] += ",smm=on" + return args case ArchArm64: // smm is not supported on aarch64 - return []string{"-machine", machineArg} + return args default: panic("unsupported architecture") } diff --git a/pkg/provision/providers/qemu/create.go b/pkg/provision/providers/qemu/create.go index 1d69bf0b6a..c47eba344e 100644 --- a/pkg/provision/providers/qemu/create.go +++ b/pkg/provision/providers/qemu/create.go @@ -9,6 +9,7 @@ import ( "fmt" "path/filepath" + "github.com/siderolabs/gen/xslices" "github.com/siderolabs/talos/pkg/machinery/constants" "github.com/siderolabs/talos/pkg/provision" "github.com/siderolabs/talos/pkg/provision/providers/vm" @@ -17,7 +18,7 @@ import ( // Create Talos cluster as a set of qemu VMs. // //nolint:gocyclo,cyclop -func (p *provisioner) Create(ctx context.Context, request provision.ClusterRequest, opts ...provision.Option) (provision.Cluster, error) { +func (p *QemuProvisioner) Create(ctx context.Context, request vm.ClusterRequest, opts ...provision.Option) (provision.Cluster, error) { options := provision.DefaultOptions() for _, opt := range opts { @@ -51,7 +52,7 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque if options.SiderolinkEnabled { fmt.Fprintln(options.LogWriter, "creating siderolink agent") - if err = p.CreateSiderolinkAgent(state, request); err != nil { + if err = p.CreateSiderolinkAgent(state, vm.ClusterRequest(request)); err != nil { return nil, err } @@ -66,7 +67,11 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque fmt.Fprintln(options.LogWriter, "creating load balancer") - if err = p.CreateLoadBalancer(state, request); err != nil { + // TODO: Find a way to reuse provision.BaseNodeRequest filter methods + controlplanes := xslices.Filter(request.Nodes, func(n vm.NodeRequest) bool { return n.Type.IsControlPlane() }) + controlplaneIps := xslices.Map(controlplanes, func(n vm.NodeRequest) string { return "TODO: Pass IP" }) + + if err = p.CreateLoadBalancer(state, request, controlplaneIps); err != nil { return nil, fmt.Errorf("error creating loadbalancer: %w", err) } @@ -96,7 +101,7 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque fmt.Fprintln(options.LogWriter, "creating controlplane nodes") - if nodeInfo, err = p.createNodes(state, request, request.Nodes.ControlPlaneNodes(), &options); err != nil { + if nodeInfo, err = p.createNodes(state, request, controlplanes, &options); err != nil { return nil, err } @@ -104,7 +109,8 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque var workerNodeInfo []provision.NodeInfo - if workerNodeInfo, err = p.createNodes(state, request, request.Nodes.WorkerNodes(), &options); err != nil { + workers := xslices.Filter(request.Nodes, func(n vm.NodeRequest) bool { return !n.Type.IsControlPlane() }) + if workerNodeInfo, err = p.createNodes(state, request, workers, &options); err != nil { return nil, err } @@ -127,19 +133,18 @@ func (p *provisioner) Create(ctx context.Context, request provision.ClusterReque lbPort = request.Network.LoadBalancerPorts[0] } - state.ClusterInfo = provision.ClusterInfo{ + state.ClusterInfo = getPlatformClusterInfo(request, provision.ClusterInfo{ ClusterName: request.Name, Network: provision.NetworkInfo{ - Name: request.Network.Name, - CIDRs: request.Network.CIDRs, - NoMasqueradeCIDRs: request.Network.NoMasqueradeCIDRs, - GatewayAddrs: request.Network.GatewayAddrs, - MTU: request.Network.MTU, + Name: request.Network.Name, + CIDRs: request.Network.CIDRs, + GatewayAddrs: request.Network.GatewayAddrs, + MTU: request.Network.MTU, }, Nodes: nodeInfo, ExtraNodes: pxeNodeInfo, - KubernetesEndpoint: p.GetExternalKubernetesControlPlaneEndpoint(request.Network, lbPort), - } + KubernetesEndpoint: p.GetExternalKubernetesControlPlaneEndpoint(request.Network.NetworkRequestBase.NetworkRequestBase, lbPort), + }) err = state.Save() if err != nil { diff --git a/pkg/provision/providers/qemu/create_darwin.go b/pkg/provision/providers/qemu/create_darwin.go new file mode 100644 index 0000000000..efb625cc84 --- /dev/null +++ b/pkg/provision/providers/qemu/create_darwin.go @@ -0,0 +1,14 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package qemu + +import ( + "github.com/siderolabs/talos/pkg/provision" + "github.com/siderolabs/talos/pkg/provision/providers/vm" +) + +func getPlatformClusterInfo(request vm.ClusterRequest, base provision.ClusterInfo) provision.ClusterInfo { + return base +} diff --git a/pkg/provision/providers/qemu/create_linux.go b/pkg/provision/providers/qemu/create_linux.go new file mode 100644 index 0000000000..78f1a4b0ea --- /dev/null +++ b/pkg/provision/providers/qemu/create_linux.go @@ -0,0 +1,15 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package qemu + +import ( + "github.com/siderolabs/talos/pkg/provision" + "github.com/siderolabs/talos/pkg/provision/providers/vm" +) + +func getPlatformClusterInfo(request vm.ClusterRequest, base provision.ClusterInfo) provision.ClusterInfo { + base.Network.NoMasqueradeCIDRs = request.Network.NoMasqueradeCIDRs + return base +} diff --git a/pkg/provision/providers/qemu/destroy.go b/pkg/provision/providers/qemu/destroy.go index afaaa245cc..02cecde8b4 100644 --- a/pkg/provision/providers/qemu/destroy.go +++ b/pkg/provision/providers/qemu/destroy.go @@ -17,7 +17,7 @@ import ( // Destroy Talos cluster as set of qemu VMs. // //nolint:gocyclo -func (p *provisioner) Destroy(ctx context.Context, cluster provision.Cluster, opts ...provision.Option) error { +func (p *QemuProvisioner) Destroy(ctx context.Context, cluster provision.Cluster, opts ...provision.Option) error { options := provision.DefaultOptions() for _, opt := range opts { diff --git a/pkg/provision/providers/qemu/launch.go b/pkg/provision/providers/qemu/launch.go index efe9aa0554..8318b7dc40 100644 --- a/pkg/provision/providers/qemu/launch.go +++ b/pkg/provision/providers/qemu/launch.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "log" + "net" "net/netip" "os" "os/exec" @@ -16,21 +17,9 @@ import ( "strconv" "strings" - "github.com/alexflint/go-filemutex" "github.com/containernetworking/cni/libcni" - "github.com/containernetworking/cni/pkg/types" - types100 "github.com/containernetworking/cni/pkg/types/100" - "github.com/containernetworking/plugins/pkg/ns" - "github.com/containernetworking/plugins/pkg/testutils" - "github.com/containernetworking/plugins/pkg/utils" - "github.com/coreos/go-iptables/iptables" "github.com/google/uuid" - "github.com/siderolabs/gen/xslices" - "github.com/siderolabs/go-blockdevice/v2/blkid" - sideronet "github.com/siderolabs/net" - "github.com/siderolabs/talos/pkg/provision" - "github.com/siderolabs/talos/pkg/provision/internal/cniutils" "github.com/siderolabs/talos/pkg/provision/providers/vm" ) @@ -51,7 +40,6 @@ type LaunchConfig struct { KernelArgs string MonitorPath string DefaultBootOrder string - EnableKVM bool BootloaderEnabled bool TPM2Config tpm2Config NodeUUID uuid.UUID @@ -63,16 +51,13 @@ type LaunchConfig struct { Config string // Network - BridgeName string - NetworkConfig *libcni.NetworkConfigList - CNI provision.CNIConfig - IPs []netip.Addr - CIDRs []netip.Prefix - NoMasqueradeCIDRs []netip.Prefix - Hostname string - GatewayAddrs []netip.Addr - MTU int - Nameservers []netip.Addr + BridgeName string + NetworkConfig *libcni.NetworkConfigList + IPs []netip.Addr + CIDRs []netip.Prefix + Hostname string + GatewayAddrs []netip.Addr + VmMAC string // PXE TFTPServer string @@ -80,18 +65,16 @@ type LaunchConfig struct { IPXEBootFileName string // API - APIPort int - - // filled by CNI invocation - tapName string - vmMAC string - ns ns.NetNS + APIBindAddress *net.TCPAddr // signals c chan os.Signal // controller controller *Controller + + // platform specific options + PlatformOps platformOps } type tpm2Config struct { @@ -99,194 +82,7 @@ type tpm2Config struct { StateDir string } -// withCNIOperationLocked ensures that CNI operations don't run concurrently. -// -// There are race conditions in the CNI plugins that can cause a failure if called concurrently. -func withCNIOperationLocked[T any](config *LaunchConfig, f func() (T, error)) (T, error) { - var zeroT T - - lock, err := filemutex.New(filepath.Join(config.StatePath, "cni.lock")) - if err != nil { - return zeroT, fmt.Errorf("failed to create CNI lock: %w", err) - } - - if err = lock.Lock(); err != nil { - return zeroT, fmt.Errorf("failed to acquire CNI lock: %w", err) - } - - defer func() { - if err := lock.Close(); err != nil { - log.Printf("failed to release CNI lock: %s", err) - } - }() - - return f() -} - -// withCNIOperationLockedNoResult ensures that CNI operations don't run concurrently. -func withCNIOperationLockedNoResult(config *LaunchConfig, f func() error) error { - _, err := withCNIOperationLocked(config, func() (struct{}, error) { - return struct{}{}, f() - }) - - return err -} - -// withCNI creates network namespace, launches CNI and passes control to the next function -// filling config with netNS and interface details. -// -//nolint:gocyclo -func withCNI(ctx context.Context, config *LaunchConfig, f func(config *LaunchConfig) error) error { - // random ID for the CNI, maps to single VM - containerID := uuid.New().String() - - cniConfig := libcni.NewCNIConfigWithCacheDir(config.CNI.BinPath, config.CNI.CacheDir, nil) - - // create a network namespace - ns, err := testutils.NewNS() - if err != nil { - return err - } - - defer func() { - ns.Close() //nolint:errcheck - testutils.UnmountNS(ns) //nolint:errcheck - }() - - ips := make([]string, len(config.IPs)) - for j := range ips { - ips[j] = sideronet.FormatCIDR(config.IPs[j], config.CIDRs[j]) - } - - gatewayAddrs := xslices.Map(config.GatewayAddrs, netip.Addr.String) - - runtimeConf := libcni.RuntimeConf{ - ContainerID: containerID, - NetNS: ns.Path(), - IfName: "veth0", - Args: [][2]string{ - {"IP", strings.Join(ips, ",")}, - {"GATEWAY", strings.Join(gatewayAddrs, ",")}, - {"IgnoreUnknown", "1"}, - }, - } - - // attempt to clean up network in case it was deployed previously - err = withCNIOperationLockedNoResult( - config, - func() error { - return cniConfig.DelNetworkList(ctx, config.NetworkConfig, &runtimeConf) - }, - ) - if err != nil { - return fmt.Errorf("error deleting CNI network: %w", err) - } - - res, err := withCNIOperationLocked( - config, - func() (types.Result, error) { - return cniConfig.AddNetworkList(ctx, config.NetworkConfig, &runtimeConf) - }, - ) - if err != nil { - return fmt.Errorf("error provisioning CNI network: %w", err) - } - - defer func() { - if e := withCNIOperationLockedNoResult( - config, - func() error { - return cniConfig.DelNetworkList(ctx, config.NetworkConfig, &runtimeConf) - }, - ); e != nil { - log.Printf("error cleaning up CNI: %s", e) - } - }() - - currentResult, err := types100.NewResultFromResult(res) - if err != nil { - return fmt.Errorf("failed to parse cni result: %w", err) - } - - vmIface, tapIface, err := cniutils.VMTapPair(currentResult, containerID) - if err != nil { - return errors.New( - "failed to parse VM network configuration from CNI output, ensure CNI is configured with a plugin " + - "that supports automatic VM network configuration such as tc-redirect-tap") - } - - cniChain := utils.FormatChainName(config.NetworkConfig.Name, containerID) - - ipt, err := iptables.New() - if err != nil { - return fmt.Errorf("failed to initialize iptables: %w", err) - } - - // don't masquerade traffic with "broadcast" destination from the VM - // - // no need to clean up the rule, as CNI drops the whole chain - if err = ipt.InsertUnique("nat", cniChain, 1, "--destination", "255.255.255.255/32", "-j", "ACCEPT"); err != nil { - return fmt.Errorf("failed to insert iptables rule to allow broadcast traffic: %w", err) - } - - for _, cidr := range config.NoMasqueradeCIDRs { - if err = ipt.InsertUnique("nat", cniChain, 1, "--destination", cidr.String(), "-j", "ACCEPT"); err != nil { - return fmt.Errorf("failed to insert iptables rule to allow non-masquerade traffic to cidr %q: %w", cidr.String(), err) - } - } - - config.tapName = tapIface.Name - config.vmMAC = vmIface.Mac - config.ns = ns - - for j := range config.CIDRs { - nameservers := make([]netip.Addr, 0, len(config.Nameservers)) - - // filter nameservers by IPv4/IPv6 matching IPs - for i := range config.Nameservers { - if config.IPs[j].Is6() { - if config.Nameservers[i].Is6() { - nameservers = append(nameservers, config.Nameservers[i]) - } - } else { - if config.Nameservers[i].Is4() { - nameservers = append(nameservers, config.Nameservers[i]) - } - } - } - - // dump node IP/mac/hostname for dhcp - if err = vm.DumpIPAMRecord(config.StatePath, vm.IPAMRecord{ - IP: config.IPs[j], - Netmask: byte(config.CIDRs[j].Bits()), - MAC: vmIface.Mac, - Hostname: config.Hostname, - Gateway: config.GatewayAddrs[j], - MTU: config.MTU, - Nameservers: nameservers, - TFTPServer: config.TFTPServer, - IPXEBootFilename: config.IPXEBootFileName, - }); err != nil { - return err - } - } - - return f(config) -} - -func checkPartitions(config *LaunchConfig) (bool, error) { - info, err := blkid.ProbePath(config.DiskPaths[0]) - if err != nil { - return false, fmt.Errorf("error probing disk: %w", err) - } - - return info.Name == "gpt" && len(info.Parts) > 0, nil -} - -// launchVM runs qemu with args built based on config. -// -//nolint:gocyclo,cyclop -func launchVM(config *LaunchConfig) error { +func getArgs(config *LaunchConfig) (args []string, err error) { bootOrder := config.DefaultBootOrder if config.controller.ForcePXEBoot() { @@ -299,13 +95,12 @@ func launchVM(config *LaunchConfig) error { cpuArg += ",-kvmclock" } - args := []string{ + args = []string{ "-m", strconv.FormatInt(config.MemSize, 10), "-smp", fmt.Sprintf("cpus=%d", config.VCPUCount), "-cpu", cpuArg, "-nographic", - "-netdev", fmt.Sprintf("tap,id=net0,ifname=%s,script=no,downscript=no", config.tapName), - "-device", fmt.Sprintf("virtio-net-pci,netdev=net0,mac=%s", config.vmMAC), + "-device", fmt.Sprintf("virtio-net-pci,netdev=net0,mac=%s", config.VmMAC), // TODO: uncomment the following line to get another eth interface not connected to anything // "-nic", "tap,model=virtio-net-pci", "-device", "virtio-rng-pci", @@ -318,8 +113,7 @@ func launchVM(config *LaunchConfig) error { "-device", "virtio-serial", "-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0", "-device", "i6300esb,id=watchdog0", - "-watchdog-action", - "pause", + "-watchdog-action", "pause", } if config.WithDebugShell { @@ -379,12 +173,10 @@ func launchVM(config *LaunchConfig) error { "-device", fmt.Sprintf("nvme-ns,drive=nvme%d", i), ) default: - return fmt.Errorf("unsupported disk driver %q", driver) + return args, fmt.Errorf("unsupported disk driver %q", driver) } } - args = append(args, config.ArchitectureData.KVMArgs(config.EnableKVM)...) - pflashArgs := make([]string, 2*len(config.PFlashImages)) for i := range config.PFlashImages { pflashArgs[2*i] = "-drive" @@ -403,7 +195,7 @@ func launchVM(config *LaunchConfig) error { // check if disk is empty/wiped diskBootable, err := checkPartitions(config) if err != nil { - return err + return args, err } if config.TPM2Config.NodeName != "" { @@ -425,7 +217,7 @@ func launchVM(config *LaunchConfig) error { log.Printf("starting swtpm: %s", cmd.String()) if err := cmd.Start(); err != nil { - return err + return args, err } args = append(args, @@ -454,19 +246,33 @@ func launchVM(config *LaunchConfig) error { "base=2011-11-11T11:11:00,clock=rt", ) } + platformArgs, err := getPlatformSpecificArgs(*config) + if err != nil { + return args, err + } + args = append(args, platformArgs...) + return args, nil +} - fmt.Fprintf(os.Stderr, "starting %s with args:\n%s\n", config.ArchitectureData.QemuExecutable(), strings.Join(args, " ")) +// launchVM runs qemu with args built based on config. +// +//nolint:gocyclo,cyclop +func launchVM(config *LaunchConfig) error { + args, err := getArgs(config) + if err != nil { + return err + } + fmt.Fprintf(os.Stdout, "starting %s with args:\n%s\n", config.ArchitectureData.QemuExecutable(), strings.Join(args, " ")) cmd := exec.Command( config.ArchitectureData.QemuExecutable(), args..., ) + fmt.Fprintf(os.Stdout, "workind dir: %s", cmd.Dir) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - if err := ns.WithNetNSPath(config.ns.Path(), func(_ ns.NetNS) error { - return cmd.Start() - }); err != nil { + if err := cmdStartQemu(config, cmd); err != nil { return err } @@ -535,7 +341,11 @@ func Launch() error { config.c = vm.ConfigureSignals() config.controller = NewController() - httpServer, err := vm.NewHTTPServer(config.GatewayAddrs[0], config.APIPort, []byte(config.Config), config.controller) + addr, err := netip.ParseAddr(config.APIBindAddress.IP.String()) + if err != nil { + return err + } + httpServer, err := vm.NewHTTPServer(addr, config.APIBindAddress.Port, []byte(config.Config), config.controller) if err != nil { return err } @@ -543,10 +353,15 @@ func Launch() error { httpServer.Serve() defer httpServer.Shutdown(ctx) //nolint:errcheck + configServerAddr, err := getConfigServerAddr(httpServer.GetAddr(), config) + if err != nil { + return err + } + // patch kernel args - config.KernelArgs = strings.ReplaceAll(config.KernelArgs, "{TALOS_CONFIG_URL}", fmt.Sprintf("http://%s/config.yaml", httpServer.GetAddr())) + config.KernelArgs = strings.ReplaceAll(config.KernelArgs, "{TALOS_CONFIG_URL}", fmt.Sprintf("http://%s/config.yaml", configServerAddr)) - return withCNI(ctx, &config, func(config *LaunchConfig) error { + return withNetworkContext(ctx, &config, func(config *LaunchConfig) error { for { for config.controller.PowerState() != PoweredOn { select { diff --git a/pkg/provision/providers/qemu/launch_darwin.go b/pkg/provision/providers/qemu/launch_darwin.go new file mode 100644 index 0000000000..849d4f063d --- /dev/null +++ b/pkg/provision/providers/qemu/launch_darwin.go @@ -0,0 +1,89 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package qemu + +import ( + "context" + "crypto/rand" + "fmt" + "net" + "os/exec" + "strings" + + "go4.org/netipx" +) + +type platformOps struct { + // NetworkInitNode is the node that initialized the qemu network that other nodes join + NetworkInitNode bool +} + +// withNetworkContext on darwin just runs the f on the host network +func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config *LaunchConfig) error) error { + config.VmMAC = getRandomMacAddress() + return f(config) +} + +func checkPartitions(config *LaunchConfig) (bool, error) { + // todo: use qemu-img with both darwin and linux + return false, nil +} + +func cmdStartQemu(config *LaunchConfig, cmd *exec.Cmd) error { + return cmd.Start() +} + +func getPlatformSpecificArgs(config LaunchConfig) (args []string, err error) { + netDevArg := "vmnet-shared,id=net0" + if config.PlatformOps.NetworkInitNode { + cidr := config.CIDRs[0] + ipnet := netipx.PrefixIPNet(cidr) + ip := ipnet.IP.To4() + mask := net.IP(ipnet.Mask).To4() + n := len(ip) + broadcast := make(net.IP, n) + for i := range n { + broadcast[i] = ip[i] | ^mask[i] + } + m := net.CIDRMask(cidr.Bits(), 32) + subnetMask := fmt.Sprintf("%d.%d.%d.%d", m[0], m[1], m[2], m[3]) + // This ip will be assigned to the bridge + // The following ips will be assigned to the vms + startAddr := config.IPs[0].Prev() + netDevArg += fmt.Sprintf(",start-address=%s,end-address=%s,subnet-mask=%s", startAddr, broadcast, subnetMask) + } + args = []string{"-netdev", netDevArg} + args = append(args, config.ArchitectureData.MachineArgs(MachineArgsParams{kvmEnabled: false, hvfEnabled: true})...) + + return args, nil +} + +// getConfigServerAddr returns the ip of the config file accessible to the VM +// hostAddrs is the address on which the server is accessible from the host network +func getConfigServerAddr(hostAddrs net.Addr, config LaunchConfig) (net.Addr, error) { + split := strings.Split(hostAddrs.String(), ":") + port := split[len(split)-1] + gateway := config.IPs[0].Prev() // has access to host through this IP + addr, err := net.ResolveTCPAddr("tcp", gateway.String()+":"+port) + if err != nil { + return nil, fmt.Errorf("failed resolving config server address: %e", err) + } + return addr, err +} + +// getRandomMacAddress generates a random local MAC address +// https://stackoverflow.com/a/21027407/10938317 +func getRandomMacAddress() string { + const ( + local = 0b10 + multicast = 0b1 + ) + + buf := make([]byte, 6) + rand.Read(buf) + // clear multicast bit (&^), ensure local bit (|) + buf[0] = buf[0]&^multicast | local + return fmt.Sprintf("%02x:%02x:%02x:%02x:%02x:%02x", buf[0], buf[1], buf[2], buf[3], buf[4], buf[5]) +} diff --git a/pkg/provision/providers/qemu/launch_linux.go b/pkg/provision/providers/qemu/launch_linux.go new file mode 100644 index 0000000000..47c5cd57c5 --- /dev/null +++ b/pkg/provision/providers/qemu/launch_linux.go @@ -0,0 +1,249 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package qemu + +import ( + "context" + "errors" + "fmt" + "log" + "net" + "net/netip" + "os/exec" + "path/filepath" + "strings" + + "github.com/alexflint/go-filemutex" + "github.com/containernetworking/cni/libcni" + "github.com/containernetworking/cni/pkg/types" + types100 "github.com/containernetworking/cni/pkg/types/100" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containernetworking/plugins/pkg/testutils" + "github.com/containernetworking/plugins/pkg/utils" + "github.com/coreos/go-iptables/iptables" + "github.com/google/uuid" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-blockdevice/v2/blkid" + sideronet "github.com/siderolabs/net" + "github.com/siderolabs/talos/pkg/provision/internal/cniutils" + "github.com/siderolabs/talos/pkg/provision/providers/vm" +) + +type platformOps struct { + CNI vm.CNIConfig + Nameservers []netip.Addr + MTU int + EnableKVM bool + NoMasqueradeCIDRs []netip.Prefix + + // filled by CNI invocation + tapName string + ns ns.NetNS +} + +func getPlatformSpecificArgs(config LaunchConfig) (args []string, err error) { + args = []string{"-netdev", fmt.Sprintf("tap,id=net0,ifname=%s,script=no,downscript=no", config.PlatformOps.tapName)} + args = append(args, config.ArchitectureData.MachineArgs( + MachineArgsParams{kvmEnabled: config.PlatformOps.EnableKVM, hvfEnabled: false})..., + ) + return args, nil +} + +// withCNIOperationLocked ensures that CNI operations don't run concurrently. +// +// There are race conditions in the CNI plugins that can cause a failure if called concurrently. +func withCNIOperationLocked[T any](config *LaunchConfig, f func() (T, error)) (T, error) { + var zeroT T + + lock, err := filemutex.New(filepath.Join(config.StatePath, "cni.lock")) + if err != nil { + return zeroT, fmt.Errorf("failed to create CNI lock: %w", err) + } + + if err = lock.Lock(); err != nil { + return zeroT, fmt.Errorf("failed to acquire CNI lock: %w", err) + } + + defer func() { + if err := lock.Close(); err != nil { + log.Printf("failed to release CNI lock: %s", err) + } + }() + + return f() +} + +// withCNIOperationLockedNoResult ensures that CNI operations don't run concurrently. +func withCNIOperationLockedNoResult(config *LaunchConfig, f func() error) error { + _, err := withCNIOperationLocked(config, func() (struct{}, error) { + return struct{}{}, f() + }) + + return err +} + +// withNetworkContext on linux creates a network namespace, launches CNI and passes control to the next function +// filling config with netNS and interface details. +// +//nolint:gocyclo +func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config *LaunchConfig) error) error { + // random ID for the CNI, maps to single VM + containerID := uuid.New().String() + + cniConfig := libcni.NewCNIConfigWithCacheDir(config.PlatformOps.CNI.BinPath, config.PlatformOps.CNI.CacheDir, nil) + + // create a network namespace + ns, err := testutils.NewNS() + if err != nil { + return err + } + + defer func() { + ns.Close() //nolint:errcheck + testutils.UnmountNS(ns) //nolint:errcheck + }() + + ips := make([]string, len(config.IPs)) + for j := range ips { + ips[j] = sideronet.FormatCIDR(config.IPs[j], config.CIDRs[j]) + } + + gatewayAddrs := xslices.Map(config.GatewayAddrs, netip.Addr.String) + + runtimeConf := libcni.RuntimeConf{ + ContainerID: containerID, + NetNS: ns.Path(), + IfName: "veth0", + Args: [][2]string{ + {"IP", strings.Join(ips, ",")}, + {"GATEWAY", strings.Join(gatewayAddrs, ",")}, + {"IgnoreUnknown", "1"}, + }, + } + + // attempt to clean up network in case it was deployed previously + err = withCNIOperationLockedNoResult( + config, + func() error { + return cniConfig.DelNetworkList(ctx, config.NetworkConfig, &runtimeConf) + }, + ) + if err != nil { + return fmt.Errorf("error deleting CNI network: %w", err) + } + + res, err := withCNIOperationLocked( + config, + func() (types.Result, error) { + return cniConfig.AddNetworkList(ctx, config.NetworkConfig, &runtimeConf) + }, + ) + if err != nil { + return fmt.Errorf("error provisioning CNI network: %w", err) + } + + defer func() { + if e := withCNIOperationLockedNoResult( + config, + func() error { + return cniConfig.DelNetworkList(ctx, config.NetworkConfig, &runtimeConf) + }, + ); e != nil { + log.Printf("error cleaning up CNI: %s", e) + } + }() + + currentResult, err := types100.NewResultFromResult(res) + if err != nil { + return fmt.Errorf("failed to parse cni result: %w", err) + } + + vmIface, tapIface, err := cniutils.VMTapPair(currentResult, containerID) + if err != nil { + return errors.New( + "failed to parse VM network configuration from CNI output, ensure CNI is configured with a plugin " + + "that supports automatic VM network configuration such as tc-redirect-tap") + } + + cniChain := utils.FormatChainName(config.NetworkConfig.Name, containerID) + + ipt, err := iptables.New() + if err != nil { + return fmt.Errorf("failed to initialize iptables: %w", err) + } + + // don't masquerade traffic with "broadcast" destination from the VM + // + // no need to clean up the rule, as CNI drops the whole chain + if err = ipt.InsertUnique("nat", cniChain, 1, "--destination", "255.255.255.255/32", "-j", "ACCEPT"); err != nil { + return fmt.Errorf("failed to insert iptables rule to allow broadcast traffic: %w", err) + } + + for _, cidr := range config.PlatformOps.NoMasqueradeCIDRs { + if err = ipt.InsertUnique("nat", cniChain, 1, "--destination", cidr.String(), "-j", "ACCEPT"); err != nil { + return fmt.Errorf("failed to insert iptables rule to allow non-masquerade traffic to cidr %q: %w", cidr.String(), err) + } + } + + config.PlatformOps.tapName = tapIface.Name + config.VmMAC = vmIface.Mac + config.PlatformOps.ns = ns + + for j := range config.CIDRs { + nameservers := make([]netip.Addr, 0, len(config.PlatformOps.Nameservers)) + + // filter nameservers by IPv4/IPv6 matching IPs + for i := range config.PlatformOps.Nameservers { + if config.IPs[j].Is6() { + if config.PlatformOps.Nameservers[i].Is6() { + nameservers = append(nameservers, config.PlatformOps.Nameservers[i]) + } + } else { + if config.PlatformOps.Nameservers[i].Is4() { + nameservers = append(nameservers, config.PlatformOps.Nameservers[i]) + } + } + } + + // dump node IP/mac/hostname for dhcp + if err = vm.DumpIPAMRecord(config.StatePath, vm.IPAMRecord{ + IP: config.IPs[j], + Netmask: byte(config.CIDRs[j].Bits()), + MAC: vmIface.Mac, + Hostname: config.Hostname, + Gateway: config.GatewayAddrs[j], + MTU: config.PlatformOps.MTU, + Nameservers: nameservers, + TFTPServer: config.TFTPServer, + IPXEBootFilename: config.IPXEBootFileName, + }); err != nil { + return err + } + } + + return f(config) +} + +func checkPartitions(config *LaunchConfig) (bool, error) { + info, err := blkid.ProbePath(config.DiskPaths[0]) + if err != nil { + return false, fmt.Errorf("error probing disk: %w", err) + } + + return info.Name == "gpt" && len(info.Parts) > 0, nil +} + +func cmdStartQemu(config *LaunchConfig, cmd *exec.Cmd) error { + if err := ns.WithNetNSPath(config.PlatformOps.ns.Path(), func(_ ns.NetNS) error { + return cmd.Start() + }); err != nil { + return err + } + return nil +} + +func getConfigServerAddr(hostAddrs net.Addr, config LaunchConfig) (net.Addr, error) { + return hostAddrs, nil +} diff --git a/pkg/provision/providers/qemu/node.go b/pkg/provision/providers/qemu/node.go index 75493376b1..8f18294cce 100644 --- a/pkg/provision/providers/qemu/node.go +++ b/pkg/provision/providers/qemu/node.go @@ -9,11 +9,9 @@ import ( "fmt" "io" "math" - "net" "os" "os/exec" "path/filepath" - "runtime" "strconv" "strings" "syscall" @@ -31,7 +29,7 @@ import ( ) //nolint:gocyclo,cyclop -func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRequest, nodeReq provision.NodeRequest, opts *provision.Options) (provision.NodeInfo, error) { +func (p *QemuProvisioner) createNode(state *vm.State, clusterReq vm.ClusterRequest, nodeReq vm.NodeRequest, opts *provision.Options) (provision.NodeInfo, error) { arch := Arch(opts.TargetArch) pidPath := state.GetRelativePath(fmt.Sprintf("%s.pid", nodeReq.Name)) @@ -113,9 +111,9 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe } switch nodeReq.ConfigInjectionMethod { - case provision.ConfigInjectionMethodHTTP: + case vm.ConfigInjectionMethodHTTP: cmdline.Append("talos.config", "{TALOS_CONFIG_URL}") // to be patched by launcher - case provision.ConfigInjectionMethodMetalISO: + case vm.ConfigInjectionMethodMetalISO: cmdline.Append("talos.config", "metal-iso") extraISOPath, err = p.createMetalConfigISO(state, nodeReq.Name, nodeConfig) @@ -130,7 +128,7 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe nodeUUID = *nodeReq.UUID } - apiPort, err := p.findBridgeListenPort(clusterReq) + apiBindAddress, err := p.findAPIBind(clusterReq) if err != nil { return provision.NodeInfo{}, fmt.Errorf("error finding listen address for the API: %w", err) } @@ -156,7 +154,7 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe launchConfig := LaunchConfig{ ArchitectureData: arch, DiskPaths: diskPaths, - DiskDrivers: xslices.Map(nodeReq.Disks, func(disk *provision.Disk) string { + DiskDrivers: xslices.Map(nodeReq.Disks, func(disk *vm.Disk) string { return disk.Driver }), VCPUCount: vcpuCount, @@ -165,7 +163,6 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe ExtraISOPath: extraISOPath, PFlashImages: pflashImages, MonitorPath: state.GetRelativePath(fmt.Sprintf("%s.monitor", nodeReq.Name)), - EnableKVM: opts.TargetArch == runtime.GOARCH, BadRTC: nodeReq.BadRTC, DefaultBootOrder: defaultBootOrder, BootloaderEnabled: opts.BootloaderEnabled, @@ -173,17 +170,14 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe Config: nodeConfig, BridgeName: state.BridgeName, NetworkConfig: state.VMCNIConfig, - CNI: clusterReq.Network.CNI, CIDRs: clusterReq.Network.CIDRs, - NoMasqueradeCIDRs: clusterReq.Network.NoMasqueradeCIDRs, - IPs: nodeReq.IPs, - GatewayAddrs: clusterReq.Network.GatewayAddrs, - MTU: clusterReq.Network.MTU, - Nameservers: clusterReq.Network.Nameservers, - TFTPServer: nodeReq.TFTPServer, - IPXEBootFileName: nodeReq.IPXEBootFilename, - APIPort: apiPort, - WithDebugShell: opts.WithDebugShell, + // TODO + // IPs: nodeReq.IPs, + GatewayAddrs: clusterReq.Network.GatewayAddrs, + TFTPServer: nodeReq.TFTPServer, + IPXEBootFileName: nodeReq.IPXEBootFilename, + APIBindAddress: apiBindAddress, + WithDebugShell: opts.WithDebugShell, } if clusterReq.IPXEBootScript != "" { @@ -201,9 +195,10 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe Memory: nodeReq.Memory, DiskSize: nodeReq.Disks[0].Size, - IPs: nodeReq.IPs, + // TODO + // IPs: nodeReq.IPs, - APIPort: apiPort, + APIAddress: apiBindAddress, } if opts.TPM2Enabled { @@ -216,8 +211,11 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe nodeInfo.TPM2StateDir = tpm2.StateDir } - if !clusterReq.Network.DHCPSkipHostname { - launchConfig.Hostname = nodeReq.Name + launchConfig.Hostname = nodeReq.Name + + launchConfig, err = addPlatformOpts(clusterReq, launchConfig, nodeReq, *opts) + if err != nil { + return provision.NodeInfo{}, err } if !(nodeReq.PXEBooted || launchConfig.IPXEBootFileName != "") { @@ -267,12 +265,12 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe return nodeInfo, nil } -func (p *provisioner) createNodes(state *vm.State, clusterReq provision.ClusterRequest, nodeReqs []provision.NodeRequest, opts *provision.Options) ([]provision.NodeInfo, error) { +func (p *QemuProvisioner) createNodes(state *vm.State, clusterReq vm.ClusterRequest, nodeReqs []vm.NodeRequest, opts *provision.Options) ([]provision.NodeInfo, error) { errCh := make(chan error) nodeCh := make(chan provision.NodeInfo, len(nodeReqs)) for _, nodeReq := range nodeReqs { - go func(nodeReq provision.NodeRequest) { + go func(nodeReq vm.NodeRequest) { nodeInfo, err := p.createNode(state, clusterReq, nodeReq, opts) if err == nil { nodeCh <- nodeInfo @@ -299,18 +297,7 @@ func (p *provisioner) createNodes(state *vm.State, clusterReq provision.ClusterR return nodesInfo, multiErr.ErrorOrNil() } -func (p *provisioner) findBridgeListenPort(clusterReq provision.ClusterRequest) (int, error) { - l, err := net.Listen("tcp", net.JoinHostPort(clusterReq.Network.GatewayAddrs[0].String(), "0")) - if err != nil { - return 0, err - } - - port := l.Addr().(*net.TCPAddr).Port - - return port, l.Close() -} - -func (p *provisioner) populateSystemDisk(disks []string, clusterReq provision.ClusterRequest) error { +func (p *QemuProvisioner) populateSystemDisk(disks []string, clusterReq vm.ClusterRequest) error { if len(disks) > 0 && clusterReq.DiskImagePath != "" { disk, err := os.OpenFile(disks[0], os.O_RDWR, 0o755) if err != nil { @@ -332,7 +319,7 @@ func (p *provisioner) populateSystemDisk(disks []string, clusterReq provision.Cl return nil } -func (p *provisioner) createMetalConfigISO(state *vm.State, nodeName, config string) (string, error) { +func (p *QemuProvisioner) createMetalConfigISO(state *vm.State, nodeName, config string) (string, error) { isoPath := state.GetRelativePath(nodeName + "-metal-config.iso") tmpDir, err := os.MkdirTemp("", "talos-metal-config-iso") diff --git a/pkg/provision/providers/qemu/node_darwin.go b/pkg/provision/providers/qemu/node_darwin.go new file mode 100644 index 0000000000..29f1ff50b4 --- /dev/null +++ b/pkg/provision/providers/qemu/node_darwin.go @@ -0,0 +1,29 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package qemu + +import ( + "net" + + "github.com/siderolabs/talos/pkg/provision" + "github.com/siderolabs/talos/pkg/provision/providers/vm" +) + +func (p *QemuProvisioner) findAPIBind(clusterReq vm.ClusterRequest) (*net.TCPAddr, error) { + l, err := net.Listen("tcp", net.JoinHostPort("0.0.0.0", "0")) + if err != nil { + return nil, err + } + + return l.Addr().(*net.TCPAddr), l.Close() +} + +// addPlatformOpts returns a modified launchConfig based on platform specific options +func addPlatformOpts(clusterReq vm.ClusterRequest, launchConfig LaunchConfig, nodeReq vm.NodeRequest, opts provision.Options) (LaunchConfig, error) { + if nodeReq.Index == 0 { + launchConfig.PlatformOps.NetworkInitNode = true + } + return launchConfig, nil +} diff --git a/pkg/provision/providers/qemu/node_linux.go b/pkg/provision/providers/qemu/node_linux.go new file mode 100644 index 0000000000..fec89040c6 --- /dev/null +++ b/pkg/provision/providers/qemu/node_linux.go @@ -0,0 +1,57 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package qemu + +import ( + "fmt" + "net" + "os" + "runtime" + + "github.com/siderolabs/talos/pkg/provision" + "github.com/siderolabs/talos/pkg/provision/providers/vm" +) + +func (p *QemuProvisioner) findAPIBind(clusterReq vm.ClusterRequest) (*net.TCPAddr, error) { + l, err := net.Listen("tcp", net.JoinHostPort(clusterReq.Network.GatewayAddrs[0].String(), "0")) + if err != nil { + return nil, err + } + + return l.Addr().(*net.TCPAddr), l.Close() +} + +// addPlatformOpts returns a modified launchConfig based on platform specific options +func addPlatformOpts(clusterReq vm.ClusterRequest, launchConfig LaunchConfig, nodeReq vm.NodeRequest, opts provision.Options) (LaunchConfig, error) { + if clusterReq.Network.DHCPSkipHostname { + launchConfig.Hostname = "" + } + + kvmErr := checkKVM() + if kvmErr != nil { + fmt.Println(kvmErr) + fmt.Println("running without KVM") + } + platformOps := platformOps{ + Nameservers: clusterReq.Network.Nameservers, + CNI: clusterReq.Network.CNI, + MTU: clusterReq.Network.MTU, + NoMasqueradeCIDRs: clusterReq.Network.NoMasqueradeCIDRs, + EnableKVM: kvmErr == nil && opts.TargetArch == runtime.GOARCH, + } + + launchConfig.PlatformOps = platformOps + + return launchConfig, nil +} + +func checkKVM() (err error) { + f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) + defer f.Close() + if err != nil { + return fmt.Errorf("error opening /dev/kvm, please make sure KVM support is enabled in Linux kernel: %w", err) + } + return nil +} diff --git a/pkg/provision/providers/qemu/pflash.go b/pkg/provision/providers/qemu/pflash.go index 1383fa9eed..7da5f1f082 100644 --- a/pkg/provision/providers/qemu/pflash.go +++ b/pkg/provision/providers/qemu/pflash.go @@ -13,7 +13,7 @@ import ( ) //nolint:gocyclo -func (p *provisioner) createPFlashImages(state *vm.State, nodeName string, pflashSpec []PFlash) ([]string, error) { +func (p *QemuProvisioner) createPFlashImages(state *vm.State, nodeName string, pflashSpec []PFlash) ([]string, error) { var images []string for i, pflash := range pflashSpec { diff --git a/pkg/provision/providers/qemu/preflight.go b/pkg/provision/providers/qemu/preflight.go index 4c6fddac93..d55190bd5f 100644 --- a/pkg/provision/providers/qemu/preflight.go +++ b/pkg/provision/providers/qemu/preflight.go @@ -9,20 +9,12 @@ import ( "errors" "fmt" "os" - "os/exec" - "path/filepath" - "runtime" - "slices" - "strings" - "github.com/coreos/go-iptables/iptables" - "github.com/hashicorp/go-getter/v2" - - "github.com/siderolabs/talos/pkg/machinery/constants" "github.com/siderolabs/talos/pkg/provision" + "github.com/siderolabs/talos/pkg/provision/providers/vm" ) -func (p *provisioner) preflightChecks(ctx context.Context, request provision.ClusterRequest, options provision.Options, arch Arch) error { +func (p *QemuProvisioner) preflightChecks(ctx context.Context, request vm.ClusterRequest, options provision.Options, arch Arch) error { checkContext := preflightCheckContext{ request: request, options: options, @@ -31,46 +23,32 @@ func (p *provisioner) preflightChecks(ctx context.Context, request provision.Clu for _, check := range []func(ctx context.Context) error{ checkContext.verifyRoot, - checkContext.checkKVM, checkContext.qemuExecutable, checkContext.checkFlashImages, - checkContext.swtpmExecutable, - checkContext.cniDirectories, - checkContext.cniBundle, - checkContext.checkIptables, } { if err := check(ctx); err != nil { return err } } - return nil + return checkContext.verifyPlatformSpecific(ctx) } type preflightCheckContext struct { - request provision.ClusterRequest + request vm.ClusterRequest options provision.Options arch Arch } -func (check *preflightCheckContext) verifyRoot(context.Context) error { +func (check *preflightCheckContext) verifyRoot(ctx context.Context) error { if os.Geteuid() != 0 { - return errors.New("error: please run as root user (CNI requirement), we recommend running with `sudo -E`") + return errors.New("error: please run as root user (CNI, qemu hvf requirement), we recommend running with `sudo -E`") } return nil } -func (check *preflightCheckContext) checkKVM(context.Context) error { - f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) - if err != nil { - return fmt.Errorf("error opening /dev/kvm, please make sure KVM support is enabled in Linux kernel: %w", err) - } - - return f.Close() -} - -func (check *preflightCheckContext) qemuExecutable(context.Context) error { +func (check *preflightCheckContext) qemuExecutable(ctx context.Context) error { if check.arch.QemuExecutable() == "" { return fmt.Errorf("QEMU executable (qemu-system-%s or qemu-kvm) not found, please install QEMU with package manager", check.arch.QemuArch()) } @@ -78,7 +56,7 @@ func (check *preflightCheckContext) qemuExecutable(context.Context) error { return nil } -func (check *preflightCheckContext) checkFlashImages(context.Context) error { +func (check *preflightCheckContext) checkFlashImages(ctx context.Context) error { for _, flashImage := range check.arch.PFlash(check.options.UEFIEnabled, check.options.ExtraUEFISearchPaths) { if len(flashImage.SourcePaths) == 0 { continue @@ -103,102 +81,3 @@ func (check *preflightCheckContext) checkFlashImages(context.Context) error { return nil } - -func (check *preflightCheckContext) swtpmExecutable(context.Context) error { - if check.options.TPM2Enabled { - if _, err := exec.LookPath("swtpm"); err != nil { - return fmt.Errorf("swtpm not found in PATH, please install swtpm-tools with the package manager: %w", err) - } - } - - return nil -} - -func (check *preflightCheckContext) cniDirectories(context.Context) error { - cniDirs := slices.Clone(check.request.Network.CNI.BinPath) - cniDirs = append(cniDirs, check.request.Network.CNI.CacheDir, check.request.Network.CNI.ConfDir) - - for _, cniDir := range cniDirs { - st, err := os.Stat(cniDir) - if err != nil { - if !os.IsNotExist(err) { - return fmt.Errorf("error checking CNI directory %q: %w", cniDir, err) - } - - fmt.Fprintf(check.options.LogWriter, "creating %q\n", cniDir) - - err = os.MkdirAll(cniDir, 0o777) - if err != nil { - return err - } - - continue - } - - if !st.IsDir() { - return fmt.Errorf("CNI path %q exists, but it's not a directory", cniDir) - } - } - - return nil -} - -func (check *preflightCheckContext) cniBundle(ctx context.Context) error { - var missing bool - - requiredCNIPlugins := []string{"bridge", "firewall", "static", "tc-redirect-tap"} - - for _, cniPlugin := range requiredCNIPlugins { - missing = true - - for _, binPath := range check.request.Network.CNI.BinPath { - _, err := os.Stat(filepath.Join(binPath, cniPlugin)) - if err == nil { - missing = false - - break - } - } - - if missing { - break - } - } - - if !missing { - return nil - } - - if check.request.Network.CNI.BundleURL == "" { - return fmt.Errorf("error: required CNI plugins %q were not found in %q", requiredCNIPlugins, check.request.Network.CNI.BinPath) - } - - pwd, err := os.Getwd() - if err != nil { - return err - } - - client := getter.Client{} - src := strings.ReplaceAll(check.request.Network.CNI.BundleURL, constants.ArchVariable, runtime.GOARCH) - dst := check.request.Network.CNI.BinPath[0] - - fmt.Fprintf(check.options.LogWriter, "downloading CNI bundle from %q to %q\n", src, dst) - - _, err = client.Get(ctx, &getter.Request{ - Src: src, - Dst: dst, - Pwd: pwd, - GetMode: getter.ModeDir, - }) - - return err -} - -func (check *preflightCheckContext) checkIptables(ctx context.Context) error { - _, err := iptables.New() - if err != nil { - return fmt.Errorf("error accessing iptables: %w", err) - } - - return nil -} diff --git a/pkg/provision/providers/qemu/preflight_darwin.go b/pkg/provision/providers/qemu/preflight_darwin.go new file mode 100644 index 0000000000..3cb2040f54 --- /dev/null +++ b/pkg/provision/providers/qemu/preflight_darwin.go @@ -0,0 +1,17 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package qemu + +import ( + "context" +) + +func (checkContext *preflightCheckContext) verifyPlatformSpecific(ctx context.Context) error { + return nil +} + +func checkKVM() (useKvm bool, err error) { + return false, nil +} diff --git a/pkg/provision/providers/qemu/preflight_linux.go b/pkg/provision/providers/qemu/preflight_linux.go new file mode 100644 index 0000000000..80f2d76f25 --- /dev/null +++ b/pkg/provision/providers/qemu/preflight_linux.go @@ -0,0 +1,134 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package qemu + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/coreos/go-iptables/iptables" + "github.com/hashicorp/go-getter/v2" + + "github.com/siderolabs/talos/pkg/machinery/constants" +) + +func (checkContext *preflightCheckContext) verifyPlatformSpecific(ctx context.Context) error { + for _, check := range []func(ctx context.Context) error{ + checkContext.cniDirectories, + checkContext.cniBundle, + checkContext.checkIptables, + checkContext.swtpmExecutable, + } { + if err := check(ctx); err != nil { + return err + } + } + + return nil +} + +func (check *preflightCheckContext) cniDirectories(ctx context.Context) error { + cniDirs := append([]string{}, check.request.Network.CNI.BinPath...) + cniDirs = append(cniDirs, check.request.Network.CNI.CacheDir, check.request.Network.CNI.ConfDir) + + for _, cniDir := range cniDirs { + st, err := os.Stat(cniDir) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("error checking CNI directory %q: %w", cniDir, err) + } + + fmt.Fprintf(check.options.LogWriter, "creating %q\n", cniDir) + + err = os.MkdirAll(cniDir, 0o777) + if err != nil { + return err + } + + continue + } + + if !st.IsDir() { + return fmt.Errorf("CNI path %q exists, but it's not a directory", cniDir) + } + } + + return nil +} + +func (check *preflightCheckContext) cniBundle(ctx context.Context) error { + var missing bool + + requiredCNIPlugins := []string{"bridge", "firewall", "static", "tc-redirect-tap"} + + for _, cniPlugin := range requiredCNIPlugins { + missing = true + + for _, binPath := range check.request.Network.CNI.BinPath { + _, err := os.Stat(filepath.Join(binPath, cniPlugin)) + if err == nil { + missing = false + + break + } + } + + if missing { + break + } + } + + if !missing { + return nil + } + + if check.request.Network.CNI.BundleURL == "" { + return fmt.Errorf("error: required CNI plugins %q were not found in %q", requiredCNIPlugins, check.request.Network.CNI.BinPath) + } + + pwd, err := os.Getwd() + if err != nil { + return err + } + + client := getter.Client{} + src := strings.ReplaceAll(check.request.Network.CNI.BundleURL, constants.ArchVariable, runtime.GOARCH) + dst := check.request.Network.CNI.BinPath[0] + + fmt.Fprintf(check.options.LogWriter, "downloading CNI bundle from %q to %q\n", src, dst) + + _, err = client.Get(ctx, &getter.Request{ + Src: src, + Dst: dst, + Pwd: pwd, + GetMode: getter.ModeDir, + }) + + return err +} + +func (check *preflightCheckContext) checkIptables(ctx context.Context) error { + _, err := iptables.New() + if err != nil { + return fmt.Errorf("error accessing iptables: %w", err) + } + + return nil +} + +func (check *preflightCheckContext) swtpmExecutable(ctx context.Context) error { + if check.options.TPM2Enabled { + if _, err := exec.LookPath("swtpm"); err != nil { + return fmt.Errorf("swtpm not found in PATH, please install swtpm-tools with the package manager: %w", err) + } + } + + return nil +} diff --git a/pkg/provision/providers/qemu/qemu.go b/pkg/provision/providers/qemu/qemu.go index 98b2a3d213..ee4c83d431 100644 --- a/pkg/provision/providers/qemu/qemu.go +++ b/pkg/provision/providers/qemu/qemu.go @@ -14,13 +14,23 @@ import ( "github.com/siderolabs/talos/pkg/provision/providers/vm" ) -type provisioner struct { +type QemuProvisioner struct { vm.Provisioner } +func NewQemuProvisioner(ctx context.Context) (QemuProvisioner, error) { + p := QemuProvisioner{ + Provisioner: vm.Provisioner{ + Name: "qemu", + }, + } + + return p, nil +} + // NewProvisioner initializes qemu provisioner. func NewProvisioner(ctx context.Context) (provision.Provisioner, error) { - p := &provisioner{ + p := &QemuProvisioner{ vm.Provisioner{ Name: "qemu", }, @@ -30,12 +40,12 @@ func NewProvisioner(ctx context.Context) (provision.Provisioner, error) { } // Close and release resources. -func (p *provisioner) Close() error { +func (p *QemuProvisioner) Close() error { return nil } // GenOptions provides a list of additional config generate options. -func (p *provisioner) GenOptions(networkReq provision.NetworkRequest) []generate.Option { +func (p *QemuProvisioner) GenOptions(networkReq provision.NetworkRequestBase) []generate.Option { hasIPv4 := false hasIPv6 := false @@ -71,25 +81,25 @@ func (p *provisioner) GenOptions(networkReq provision.NetworkRequest) []generate } // GetInClusterKubernetesControlPlaneEndpoint returns the Kubernetes control plane endpoint. -func (p *provisioner) GetInClusterKubernetesControlPlaneEndpoint(networkReq provision.NetworkRequest, controlPlanePort int) string { +func (p *QemuProvisioner) GetInClusterKubernetesControlPlaneEndpoint(networkReq provision.NetworkRequestBase, controlPlanePort int) string { // QEMU provisioner always runs TCP loadbalancer on the bridge IP and port 6443. return "https://" + nethelpers.JoinHostPort(networkReq.GatewayAddrs[0].String(), controlPlanePort) } // GetExternalKubernetesControlPlaneEndpoint returns the Kubernetes control plane endpoint. -func (p *provisioner) GetExternalKubernetesControlPlaneEndpoint(networkReq provision.NetworkRequest, controlPlanePort int) string { +func (p *QemuProvisioner) GetExternalKubernetesControlPlaneEndpoint(networkReq provision.NetworkRequestBase, controlPlanePort int) string { // for QEMU, external and in-cluster endpoints are same. return p.GetInClusterKubernetesControlPlaneEndpoint(networkReq, controlPlanePort) } // GetTalosAPIEndpoints returns a list of Talos API endpoints. -func (p *provisioner) GetTalosAPIEndpoints(provision.NetworkRequest) []string { +func (p *QemuProvisioner) GetTalosAPIEndpoints(provision.NetworkRequestBase) []string { // nil means that the API of controlplane endpoints should be used return nil } // GetFirstInterface returns first network interface name. -func (p *provisioner) GetFirstInterface() v1alpha1.IfaceSelector { +func (p *QemuProvisioner) GetFirstInterface() v1alpha1.IfaceSelector { return v1alpha1.IfaceBySelector(v1alpha1.NetworkDeviceSelector{ NetworkDeviceKernelDriver: "virtio_net", }) diff --git a/pkg/provision/providers/qemu/tpm2.go b/pkg/provision/providers/qemu/tpm2.go index 6a61fdc6e5..2af10c512f 100644 --- a/pkg/provision/providers/qemu/tpm2.go +++ b/pkg/provision/providers/qemu/tpm2.go @@ -15,7 +15,7 @@ import ( "github.com/siderolabs/talos/pkg/provision/providers/vm" ) -func (p *provisioner) createVirtualTPM2State(state *vm.State, nodeName string) (tpm2Config, error) { +func (p *QemuProvisioner) createVirtualTPM2State(state *vm.State, nodeName string) (tpm2Config, error) { tpm2StateDir := state.GetRelativePath(fmt.Sprintf("%s-tpm2", nodeName)) if err := os.MkdirAll(tpm2StateDir, 0o755); err != nil { @@ -28,7 +28,7 @@ func (p *provisioner) createVirtualTPM2State(state *vm.State, nodeName string) ( }, nil } -func (p *provisioner) destroyVirtualTPM2s(cluster provision.ClusterInfo) error { +func (p *QemuProvisioner) destroyVirtualTPM2s(cluster provision.ClusterInfo) error { errCh := make(chan error) nodes := append([]provision.NodeInfo{}, cluster.Nodes...) @@ -58,6 +58,6 @@ func (p *provisioner) destroyVirtualTPM2s(cluster provision.ClusterInfo) error { return multiErr.ErrorOrNil() } -func (p *provisioner) destroyVirtualTPM2(pid string) error { +func (p *QemuProvisioner) destroyVirtualTPM2(pid string) error { return vm.StopProcessByPidfile(pid) } diff --git a/pkg/provision/providers/qemu_other.go b/pkg/provision/providers/qemu_other.go index eaf9ba4505..27b1cebf61 100644 --- a/pkg/provision/providers/qemu_other.go +++ b/pkg/provision/providers/qemu_other.go @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at http://mozilla.org/MPL/2.0/. -//go:build !linux +//go:build !linux && !darwin package providers diff --git a/pkg/provision/providers/vm/dhcpd_darwin.go b/pkg/provision/providers/vm/dhcpd_darwin.go new file mode 100644 index 0000000000..79e77e5137 --- /dev/null +++ b/pkg/provision/providers/vm/dhcpd_darwin.go @@ -0,0 +1,15 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package vm + +// CreateDHCPd creates DHCPd. +func (p *Provisioner) CreateDHCPd(state *State, clusterReq ClusterRequest) error { + return nil +} + +// DestroyDHCPd destoys load balancer. +func (p *Provisioner) DestroyDHCPd(state *State) error { + return nil +} diff --git a/pkg/provision/providers/vm/dhcpd.go b/pkg/provision/providers/vm/dhcpd_linux.go similarity index 98% rename from pkg/provision/providers/vm/dhcpd.go rename to pkg/provision/providers/vm/dhcpd_linux.go index bb5fd07d95..659c872a2f 100644 --- a/pkg/provision/providers/vm/dhcpd.go +++ b/pkg/provision/providers/vm/dhcpd_linux.go @@ -23,8 +23,6 @@ import ( "github.com/insomniacslk/dhcp/iana" "github.com/siderolabs/gen/xslices" "golang.org/x/sync/errgroup" - - "github.com/siderolabs/talos/pkg/provision" ) //nolint:gocyclo @@ -271,7 +269,7 @@ const ( ) // CreateDHCPd creates DHCPd. -func (p *Provisioner) CreateDHCPd(state *State, clusterReq provision.ClusterRequest) error { +func (p *Provisioner) CreateDHCPd(state *State, clusterReq ClusterRequest) error { pidPath := state.GetRelativePath(dhcpPid) logFile, err := os.OpenFile(state.GetRelativePath(dhcpLog), os.O_APPEND|os.O_CREATE|os.O_RDWR, 0o666) diff --git a/pkg/provision/providers/vm/disk.go b/pkg/provision/providers/vm/disk.go index d19ebf3128..c04d19357e 100644 --- a/pkg/provision/providers/vm/disk.go +++ b/pkg/provision/providers/vm/disk.go @@ -8,9 +8,8 @@ import ( "errors" "fmt" "os" - "syscall" - "github.com/siderolabs/talos/pkg/provision" + "github.com/detailyang/go-fallocate" ) // UserDiskName returns disk device path. @@ -23,7 +22,7 @@ func (p *Provisioner) UserDiskName(index int) string { } // CreateDisks creates empty disk files for each disk. -func (p *Provisioner) CreateDisks(state *State, nodeReq provision.NodeRequest) (diskPaths []string, err error) { +func (p *Provisioner) CreateDisks(state *State, nodeReq NodeRequest) (diskPaths []string, err error) { diskPaths = make([]string, len(nodeReq.Disks)) for i, disk := range nodeReq.Disks { @@ -43,7 +42,7 @@ func (p *Provisioner) CreateDisks(state *State, nodeReq provision.NodeRequest) ( } if !disk.SkipPreallocate { - if err = syscall.Fallocate(int(diskF.Fd()), 0, 0, int64(disk.Size)); err != nil { + if err = fallocate.Fallocate(diskF, 0, int64(disk.Size)); err != nil { fmt.Fprintf(os.Stderr, "WARNING: failed to preallocate disk space for %q (size %d): %s", diskPath, disk.Size, err) } } diff --git a/pkg/provision/providers/vm/json-logs.go b/pkg/provision/providers/vm/json-logs.go index e8f61e37fb..a02f10ab57 100644 --- a/pkg/provision/providers/vm/json-logs.go +++ b/pkg/provision/providers/vm/json-logs.go @@ -22,7 +22,7 @@ const ( ) // CreateJSONLogs creates JSON logs server. -func (p *Provisioner) CreateJSONLogs(state *State, clusterReq provision.ClusterRequest, options provision.Options) error { +func (p *Provisioner) CreateJSONLogs(state *State, clusterReq ClusterRequest, options provision.Options) error { pidPath := state.GetRelativePath(jsonLogsPid) logFile, err := os.OpenFile(state.GetRelativePath(jsonLogsLog), os.O_APPEND|os.O_CREATE|os.O_RDWR, 0o666) diff --git a/pkg/provision/providers/vm/kms.go b/pkg/provision/providers/vm/kms.go index 1542c70859..e1945d3b53 100644 --- a/pkg/provision/providers/vm/kms.go +++ b/pkg/provision/providers/vm/kms.go @@ -23,7 +23,7 @@ const ( ) // CreateKMS creates KMS server. -func (p *Provisioner) CreateKMS(state *State, clusterReq provision.ClusterRequest, options provision.Options) error { +func (p *Provisioner) CreateKMS(state *State, clusterReq ClusterRequest, options provision.Options) error { pidPath := state.GetRelativePath(kmsPid) logFile, err := os.OpenFile(state.GetRelativePath(kmsLog), os.O_APPEND|os.O_CREATE|os.O_RDWR, 0o666) diff --git a/pkg/provision/providers/vm/loadbalancer.go b/pkg/provision/providers/vm/loadbalancer.go index 635fb273e7..2e17175a3b 100644 --- a/pkg/provision/providers/vm/loadbalancer.go +++ b/pkg/provision/providers/vm/loadbalancer.go @@ -13,8 +13,6 @@ import ( "syscall" "github.com/siderolabs/gen/xslices" - - "github.com/siderolabs/talos/pkg/provision" ) const ( @@ -23,7 +21,7 @@ const ( ) // CreateLoadBalancer creates load balancer. -func (p *Provisioner) CreateLoadBalancer(state *State, clusterReq provision.ClusterRequest) error { +func (p *Provisioner) CreateLoadBalancer(state *State, clusterReq ClusterRequest, controlPlaneIPs []string) error { pidPath := state.GetRelativePath(lbPid) logFile, err := os.OpenFile(state.GetRelativePath(lbLog), os.O_APPEND|os.O_CREATE|os.O_RDWR, 0o666) @@ -33,12 +31,11 @@ func (p *Provisioner) CreateLoadBalancer(state *State, clusterReq provision.Clus defer logFile.Close() //nolint:errcheck - controlPlaneIPs := xslices.Map(clusterReq.Nodes.ControlPlaneNodes(), func(req provision.NodeRequest) string { return req.IPs[0].String() }) ports := xslices.Map(clusterReq.Network.LoadBalancerPorts, strconv.Itoa) args := []string{ "loadbalancer-launch", - "--loadbalancer-addr", clusterReq.Network.GatewayAddrs[0].String(), + "--loadbalancer-addr", getLbBindIp(clusterReq.Network.GatewayAddrs[0]), "--loadbalancer-upstreams", strings.Join(controlPlaneIPs, ","), } diff --git a/pkg/provision/providers/vm/loadbalancer_darwin.go b/pkg/provision/providers/vm/loadbalancer_darwin.go new file mode 100644 index 0000000000..08cb60902a --- /dev/null +++ b/pkg/provision/providers/vm/loadbalancer_darwin.go @@ -0,0 +1,7 @@ +package vm + +import "net/netip" + +func getLbBindIp(gateway netip.Addr) string { + return "0.0.0.0" +} diff --git a/pkg/provision/providers/vm/loadbalancer_linux.go b/pkg/provision/providers/vm/loadbalancer_linux.go new file mode 100644 index 0000000000..d21a4cee3b --- /dev/null +++ b/pkg/provision/providers/vm/loadbalancer_linux.go @@ -0,0 +1,7 @@ +package vm + +import "net/netip" + +func getLbBindIp(gateway netip.Addr) string { + return gateway.String() +} diff --git a/pkg/provision/providers/vm/network_darwin.go b/pkg/provision/providers/vm/network_darwin.go new file mode 100644 index 0000000000..f5f1e24a0d --- /dev/null +++ b/pkg/provision/providers/vm/network_darwin.go @@ -0,0 +1,21 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package vm + +import ( + "context" + + "github.com/siderolabs/talos/pkg/provision" +) + +// CreateNetwork +func (p *Provisioner) CreateNetwork(ctx context.Context, state *State, network NetworkRequest, options provision.Options) error { + return nil +} + +// DestroyNetwork destroy bridge interface by name to clean up. +func (p *Provisioner) DestroyNetwork(state *State) error { + return nil +} diff --git a/pkg/provision/providers/vm/network.go b/pkg/provision/providers/vm/network_linux.go similarity index 98% rename from pkg/provision/providers/vm/network.go rename to pkg/provision/providers/vm/network_linux.go index d8406c655c..34a5520194 100644 --- a/pkg/provision/providers/vm/network.go +++ b/pkg/provision/providers/vm/network_linux.go @@ -39,7 +39,7 @@ import ( // different bridge interfaces. // //nolint:gocyclo -func (p *Provisioner) CreateNetwork(ctx context.Context, state *State, network provision.NetworkRequest, options provision.Options) error { +func (p *Provisioner) CreateNetwork(ctx context.Context, state *State, network NetworkRequest, options provision.Options) error { networkNameHash := sha256.Sum256([]byte(network.Name)) state.BridgeName = fmt.Sprintf("%s%s", "talos", hex.EncodeToString(networkNameHash[:])[:8]) @@ -233,7 +233,7 @@ func getTicksInUsec() (float64, error) { } //nolint:gocyclo -func (p *Provisioner) configureNetworkChaos(network provision.NetworkRequest, state *State, options provision.Options) error { +func (p *Provisioner) configureNetworkChaos(network NetworkRequest, state *State, options provision.Options) error { if (network.Bandwidth != 0) && (network.Latency != 0 || network.Jitter != 0 || network.PacketLoss != 0 || network.PacketReorder != 0 || network.PacketCorrupt != 0) { return errors.New("bandwidth and other chaos options cannot be used together") } diff --git a/pkg/provision/providers/vm/request.go b/pkg/provision/providers/vm/request.go new file mode 100644 index 0000000000..a288e9fb83 --- /dev/null +++ b/pkg/provision/providers/vm/request.go @@ -0,0 +1,117 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package docker implements Provisioner via docker. +package vm + +import ( + "github.com/google/uuid" + "github.com/siderolabs/gen/xslices" + "github.com/siderolabs/go-procfs/procfs" + "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" + "github.com/siderolabs/talos/pkg/provision" +) + +type ClusterRequest struct { + provision.ClusterRequestBase + + Network NetworkRequest + Nodes NodeRequests + + // Boot options + KernelPath string + InitramfsPath string + ISOPath string + DiskImagePath string + IPXEBootScript string + + // Encryption + KMSEndpoint string + SiderolinkRequest provision.SiderolinkRequest +} + +// ConfigInjectionMethod describes how to inject configuration into the node. +type ConfigInjectionMethod int + +const ( + // ConfigInjectionMethodHTTP injects configuration via HTTP. + ConfigInjectionMethodHTTP ConfigInjectionMethod = iota + // ConfigInjectionMethodMetalISO injects configuration via Metal ISO. + ConfigInjectionMethodMetalISO +) + +// Disk represents a disk size and name in NodeRequest. +type Disk struct { + // Size in bytes. + Size uint64 + // Whether to skip preallocating the disk space. + SkipPreallocate bool + // Partitions represents the list of partitions. + Partitions []*v1alpha1.DiskPartition + // Driver for the disk. + // + // Supported types: "virtio", "ide", "ahci", "scsi", "nvme". + Driver string +} + +type NodeRequests []NodeRequest + +func (n NodeRequests) GetBase() provision.BaseNodeRequests { + base := xslices.Map(n, func(n NodeRequest) provision.NodeRequestBase { + return provision.NodeRequestBase(n.NodeRequestBase.NodeRequestBase) + }) + return provision.BaseNodeRequests(base) +} + +// PXENodes returns subset of nodes which are PXE booted. +func (reqs NodeRequests) PXENodes() (nodes NodeRequests) { + for i := range reqs { + if reqs[i].PXEBooted { + nodes = append(nodes, reqs[i]) + } + } + + return +} + +// TODO: think of a different name: currently it's a layer 2 base +type NodeRequestBase struct { + provision.NodeRequestBase + + ConfigInjectionMethod ConfigInjectionMethod + // Disks (volumes), if applicable (VM only) + Disks []*Disk + + // DefaultBootOrder overrides default boot order "cn" (disk, then network boot). + // + // BootOrder can be forced to be "nc" (PXE boot) via the API in QEMU provisioner. + DefaultBootOrder string + + // ExtraKernelArgs passes additional kernel args + // to the initial boot from initramfs and vmlinuz. + // + // This doesn't apply to boots from ISO or from the disk image. + ExtraKernelArgs *procfs.Cmdline + + // UUID allows to specify the UUID of the node (VMs only). + // + // If not specified, a random UUID will be generated. + UUID *uuid.UUID + + // Testing features + + // BadRTC resets RTC to well known time in the past (QEMU provisioner). + BadRTC bool + + // PXE-booted VMs + PXEBooted bool + TFTPServer string + IPXEBootFilename string +} + +// TODO: think of a different name: currently it's a layer 2 base +type NetworkRequestBase struct { + provision.NetworkRequestBase + LoadBalancerPorts []int +} diff --git a/pkg/provision/providers/vm/request_darwin.go b/pkg/provision/providers/vm/request_darwin.go new file mode 100644 index 0000000000..84e9b60eaa --- /dev/null +++ b/pkg/provision/providers/vm/request_darwin.go @@ -0,0 +1,14 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package vm + +// NetworkRequest describes cluster network. +type NetworkRequest struct { + NetworkRequestBase +} + +type NodeRequest struct { + NodeRequestBase +} diff --git a/pkg/provision/providers/vm/request_linux.go b/pkg/provision/providers/vm/request_linux.go new file mode 100644 index 0000000000..751793d276 --- /dev/null +++ b/pkg/provision/providers/vm/request_linux.go @@ -0,0 +1,48 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +package vm + +import ( + "net/netip" + "time" +) + +// NetworkRequest describes cluster network. +type NetworkRequest struct { + NetworkRequestBase + + MTU int + Nameservers []netip.Addr + NoMasqueradeCIDRs []netip.Prefix + + // CNI-specific parameters. + CNI CNIConfig + + // DHCP options + DHCPSkipHostname bool + + // Network chaos parameters. + NetworkChaos bool + Jitter time.Duration + Latency time.Duration + PacketLoss float64 + PacketReorder float64 + PacketCorrupt float64 + Bandwidth int +} + +type NodeRequest struct { + NodeRequestBase + IPs []netip.Addr +} + +// CNIConfig describes CNI part of NetworkRequest. +type CNIConfig struct { + BinPath []string + ConfDir string + CacheDir string + + BundleURL string +} diff --git a/pkg/provision/providers/vm/siderolink-agent.go b/pkg/provision/providers/vm/siderolink-agent.go index 95162abc59..63832413fd 100644 --- a/pkg/provision/providers/vm/siderolink-agent.go +++ b/pkg/provision/providers/vm/siderolink-agent.go @@ -11,8 +11,6 @@ import ( "os/exec" "strconv" "syscall" - - "github.com/siderolabs/talos/pkg/provision" ) const ( @@ -23,7 +21,7 @@ const ( ) // CreateSiderolinkAgent creates siderlink agent. -func (p *Provisioner) CreateSiderolinkAgent(state *State, clusterReq provision.ClusterRequest) error { +func (p *Provisioner) CreateSiderolinkAgent(state *State, clusterReq ClusterRequest) error { pidPath := state.GetRelativePath(siderolinkAgentPid) logFile, err := os.OpenFile(state.GetRelativePath(siderolinkAgentLog), os.O_APPEND|os.O_CREATE|os.O_RDWR, 0o666) diff --git a/pkg/provision/provision.go b/pkg/provision/provision.go index e5577430ba..815f331477 100644 --- a/pkg/provision/provision.go +++ b/pkg/provision/provision.go @@ -17,18 +17,16 @@ import ( // //nolint:interfacebloat type Provisioner interface { - Create(context.Context, ClusterRequest, ...Option) (Cluster, error) Destroy(context.Context, Cluster, ...Option) error CrashDump(context.Context, Cluster, io.Writer) Reflect(ctx context.Context, clusterName, stateDirectory string) (Cluster, error) - GenOptions(NetworkRequest) []generate.Option + GenOptions(NetworkRequestBase) []generate.Option - GetInClusterKubernetesControlPlaneEndpoint(req NetworkRequest, controlPlanePort int) string - GetExternalKubernetesControlPlaneEndpoint(req NetworkRequest, controlPlanePort int) string - GetTalosAPIEndpoints(NetworkRequest) []string + GetInClusterKubernetesControlPlaneEndpoint(req NetworkRequestBase, controlPlanePort int) string + GetTalosAPIEndpoints(NetworkRequestBase) []string GetFirstInterface() v1alpha1.IfaceSelector diff --git a/pkg/provision/request.go b/pkg/provision/request.go index d8f831319f..349cc8349e 100644 --- a/pkg/provision/request.go +++ b/pkg/provision/request.go @@ -8,90 +8,41 @@ import ( "errors" "net/netip" "slices" - "time" - mounttypes "github.com/docker/docker/api/types/mount" "github.com/google/uuid" - "github.com/siderolabs/go-procfs/procfs" "github.com/siderolabs/talos/pkg/machinery/config" "github.com/siderolabs/talos/pkg/machinery/config/machine" - "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1" ) -// ClusterRequest is the root object describing cluster to be provisioned. -type ClusterRequest struct { +// ClusterRequestBase is the base options common across providers for the cluster to be created +type ClusterRequestBase struct { Name string - Network NetworkRequest - Nodes NodeRequests - - // Docker specific parameters. - Image string - - // Boot options (QEMU). - KernelPath string - InitramfsPath string - ISOPath string - DiskImagePath string - IPXEBootScript string - - // Encryption - KMSEndpoint string + Network NetworkRequestBase + Workers BaseNodeRequests + Controlplanes BaseNodeRequests // Path to talosctl executable to re-execute itself as needed. SelfExecutable string // Path to root of state directory (~/.talos/clusters by default). StateDirectory string - - SiderolinkRequest SiderolinkRequest } -// CNIConfig describes CNI part of NetworkRequest. -type CNIConfig struct { - BinPath []string - ConfDir string - CacheDir string - - BundleURL string +// NetworkRequestBase describes cluster network parameters common accross OSs and providers. +type NetworkRequestBase struct { + CIDRs []netip.Prefix + Name string + MTU int + GatewayAddrs []netip.Addr } -// NetworkRequest describes cluster network. -type NetworkRequest struct { - Name string - CIDRs []netip.Prefix - NoMasqueradeCIDRs []netip.Prefix - GatewayAddrs []netip.Addr - MTU int - Nameservers []netip.Addr - - LoadBalancerPorts []int - - // CNI-specific parameters. - CNI CNIConfig - - // DHCP options - DHCPSkipHostname bool - - // Docker-specific parameters. - DockerDisableIPv6 bool - - // Network chaos parameters. - NetworkChaos bool - Jitter time.Duration - Latency time.Duration - PacketLoss float64 - PacketReorder float64 - PacketCorrupt float64 - Bandwidth int -} - -// NodeRequests is a list of NodeRequest. -type NodeRequests []NodeRequest +// BaseNodeRequests is a list of NodeRequest. +type BaseNodeRequests []NodeRequestBase // FindInitNode looks up init node, it returns an error if no init node is present or if it's duplicate. -func (reqs NodeRequests) FindInitNode() (req NodeRequest, err error) { +func (reqs BaseNodeRequests) FindInitNode() (req NodeRequestBase, err error) { found := false for i := range reqs { @@ -119,7 +70,7 @@ func (reqs NodeRequests) FindInitNode() (req NodeRequest, err error) { } // ControlPlaneNodes returns subset of nodes which are Init/ControlPlane type. -func (reqs NodeRequests) ControlPlaneNodes() (nodes []NodeRequest) { +func (reqs BaseNodeRequests) ControlPlaneNodes() (nodes []NodeRequestBase) { for i := range reqs { if reqs[i].Type == machine.TypeInit || reqs[i].Type == machine.TypeControlPlane { nodes = append(nodes, reqs[i]) @@ -130,7 +81,7 @@ func (reqs NodeRequests) ControlPlaneNodes() (nodes []NodeRequest) { } // WorkerNodes returns subset of nodes which are Init/ControlPlane type. -func (reqs NodeRequests) WorkerNodes() (nodes []NodeRequest) { +func (reqs BaseNodeRequests) WorkerNodes() (nodes []NodeRequestBase) { for i := range reqs { if reqs[i].Type == machine.TypeWorker { nodes = append(nodes, reqs[i]) @@ -140,87 +91,21 @@ func (reqs NodeRequests) WorkerNodes() (nodes []NodeRequest) { return } -// PXENodes returns subset of nodes which are PXE booted. -func (reqs NodeRequests) PXENodes() (nodes []NodeRequest) { - for i := range reqs { - if reqs[i].PXEBooted { - nodes = append(nodes, reqs[i]) - } - } - - return -} - -// Disk represents a disk size and name in NodeRequest. -type Disk struct { - // Size in bytes. - Size uint64 - // Whether to skip preallocating the disk space. - SkipPreallocate bool - // Partitions represents the list of partitions. - Partitions []*v1alpha1.DiskPartition - // Driver for the disk. - // - // Supported types: "virtio", "ide", "ahci", "scsi", "nvme". - Driver string -} - -// ConfigInjectionMethod describes how to inject configuration into the node. -type ConfigInjectionMethod int - -const ( - // ConfigInjectionMethodHTTP injects configuration via HTTP. - ConfigInjectionMethodHTTP ConfigInjectionMethod = iota - // ConfigInjectionMethodMetalISO injects configuration via Metal ISO. - ConfigInjectionMethodMetalISO -) - // NodeRequest describes a request for a node. -type NodeRequest struct { - Name string - IPs []netip.Addr - Type machine.Type +type NodeRequestBase struct { + Name string + Type machine.Type + Index int - Config config.Provider - ConfigInjectionMethod ConfigInjectionMethod + Config config.Provider // Share of CPUs, in 1e-9 fractions NanoCPUs int64 // Memory limit in bytes Memory int64 - // Disks (volumes), if applicable (VM only) - Disks []*Disk - // Mounts (containers only) - Mounts []mounttypes.Mount - // Ports - Ports []string + // SkipInjectingConfig disables reading configuration from http server SkipInjectingConfig bool - // DefaultBootOrder overrides default boot order "cn" (disk, then network boot). - // - // BootOrder can be forced to be "nc" (PXE boot) via the API in QEMU provisioner. - DefaultBootOrder string - - // ExtraKernelArgs passes additional kernel args - // to the initial boot from initramfs and vmlinuz. - // - // This doesn't apply to boots from ISO or from the disk image. - ExtraKernelArgs *procfs.Cmdline - - // UUID allows to specify the UUID of the node (VMs only). - // - // If not specified, a random UUID will be generated. - UUID *uuid.UUID - - // Testing features - - // BadRTC resets RTC to well known time in the past (QEMU provisioner). - BadRTC bool - - // PXE-booted VMs - PXEBooted bool - TFTPServer string - IPXEBootFilename string } // SiderolinkRequest describes a request for SideroLink agent. diff --git a/pkg/provision/result.go b/pkg/provision/result.go index f46e33044a..80126f68e9 100644 --- a/pkg/provision/result.go +++ b/pkg/provision/result.go @@ -5,6 +5,7 @@ package provision import ( + "net" "net/netip" "github.com/google/uuid" @@ -61,6 +62,6 @@ type NodeInfo struct { IPs []netip.Addr - APIPort int + APIAddress *net.TCPAddr TPM2StateDir string }