Skip to content

Commit

Permalink
feat(k3d): Auto detect ip when setting 0.0.0.0 for API Port field
Browse files Browse the repository at this point in the history
When running autok3s in docker, 0.0.0.0 won't work as bridge network is used.
Now the autok3s will detect docker host IP and set it to tls-san for k3d cluster
Also when creating cluster via UI, the request hostname will also added to tls-san when running inside container.
  • Loading branch information
orangedeng committed Oct 12, 2024
1 parent 66d8b5a commit 7901720
Showing 4 changed files with 178 additions and 13 deletions.
135 changes: 135 additions & 0 deletions pkg/providers/k3d/docker.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
package k3d

import (
"context"
"fmt"
"net"
"net/url"
"os"
"strings"
"sync"

"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/k3d-io/k3d/v5/pkg/runtimes"
"github.com/k3d-io/k3d/v5/pkg/runtimes/docker"
"github.com/sirupsen/logrus"
clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
)

var (
dockerHost string
once sync.Once
)

func getDockerHost() string {
once.Do(func() {
var err error
if runtimes.SelectedRuntime.ID() != "docker" {
logrus.Debugf("runtime not docker")
return
}
// TODO find a better way to get container id via docker.sock
// Hostname as container id is not 100% reliable.
hostname := os.Getenv("HOSTNAME_OVERRIDE")
if hostname == "" {
hostname, err = os.Hostname()
if err != nil {
logrus.Debugf("failed to get hostname, %v", err)
return
}
}

runtime := runtimes.Docker

if runtime.GetHost() != "" {
logrus.Debugf("runtime host is %s", runtime.GetHost())
return
}

if _, err = os.Stat(runtime.GetRuntimePath()); err != nil {
logrus.Debugf("runtime path %s doesn't exist", runtime.GetRuntimePath())
return
}
nodes, err := getContainersByLabel(context.Background(), map[string]string{
"org.opencontainers.image.title": "autok3s",
})
if err != nil {
logrus.Debugf("failed to get container from runtime %s, %v", runtime.ID(), err)
return
}
if len(nodes) == 0 {
logrus.Debug("autok3s docker container not found. Skip finding docker host IP.")
return
}
var currentContainer *types.Container
for i := range nodes {
node := nodes[i]
if strings.HasPrefix(node.ID, hostname) {
currentContainer = &node
break
}
}
if currentContainer == nil {
logrus.Debugf("no container found for hostname %s", hostname)
return
}
if currentContainer.HostConfig.NetworkMode == "host" {
logrus.Debug("do nothing when running host network")
return
}
logrus.Debugf("found container %s", currentContainer.ID)
gw, err := runtime.GetHostIP(context.Background(), currentContainer.HostConfig.NetworkMode)
if err != nil {
logrus.Debugf("failed to get gateway ip for network %s, %v", currentContainer.HostConfig.NetworkMode, err)
return
}
dockerHost = gw.String()
logrus.Infof("found docker host IP %s", dockerHost)
})
return dockerHost
}

func getContainersByLabel(ctx context.Context, labels map[string]string) ([]types.Container, error) {
// (0) create docker client
docker, err := docker.GetDockerClient()
if err != nil {
return nil, fmt.Errorf("Failed to create docker client. %+v", err)
}
defer docker.Close()

filters := filters.NewArgs()
for k, v := range labels {
filters.Add("label", fmt.Sprintf("%s=%s", k, v))
}

containers, err := docker.ContainerList(ctx, container.ListOptions{
Filters: filters,
All: true,
})
if err != nil {
return nil, fmt.Errorf("failed to list containers: %w", err)
}

return containers, nil
}

func OverrideK3dKubeConfigServer(from, to string, config *clientcmdapi.Config) {
if to == "" {
return
}
if from == "" && getDockerHost() != "" {
from = getDockerHost()
}
for key := range config.Clusters {
cluster := config.Clusters[key]
serverURL, _ := url.Parse(cluster.Server)
if serverURL.Hostname() == from {
_, port, _ := net.SplitHostPort(serverURL.Host)
serverURL.Host = fmt.Sprintf("%s:%s", to, port)
}
cluster.Server = serverURL.String()
return
}
}
34 changes: 23 additions & 11 deletions pkg/providers/k3d/k3d.go
Original file line number Diff line number Diff line change
@@ -45,6 +45,8 @@ var (
type K3d struct {
*cluster.ProviderBase `json:",inline"`
typesk3d.Options `json:",inline"`

dockerHost, additionalHost string
}

func init() {
@@ -62,6 +64,7 @@ func newProvider() *K3d {
APIPort: k3dAPIPort,
Image: k3dImage,
},
dockerHost: getDockerHost(),
}
}

@@ -173,6 +176,11 @@ func (p *K3d) GetProviderOptions(opt []byte) (interface{}, error) {

// SetConfig set cluster config.
func (p *K3d) SetConfig(config []byte) error {
tmpHost := struct {
AdditionalHost string `json:"additionalHost,omitempty"`
}{}
_ = json.Unmarshal(config, &tmpHost)

c, err := p.SetClusterConfig(config)
if err != nil {
return err
@@ -189,7 +197,7 @@ func (p *K3d) SetConfig(config []byte) error {
}
targetOption := reflect.ValueOf(opt).Elem()
utils.MergeConfig(sourceOption, targetOption)

p.additionalHost = tmpHost.AdditionalHost
return nil
}

@@ -326,6 +334,8 @@ func (p *K3d) obtainKubeCfg() (kubeCfg, ip string, err error) {
return
}

OverrideK3dKubeConfigServer(k3d.DefaultAPIHost, p.dockerHost, kubeConfig)

bytes, err := clientcmd.Write(*kubeConfig)
if err != nil {
return
@@ -611,20 +621,15 @@ func (p *K3d) wrapCliFlags(masters, workers int) (*k3dconf.ClusterConfig, error)
}

if p.APIPort != "" {
exposeAPI, err := k3dutil.ParsePortExposureSpec(p.APIPort, k3d.DefaultAPIPort)
apiPort := p.APIPort
if strings.HasSuffix(apiPort, ":0") {
apiPort = strings.TrimSuffix(apiPort, ":0") + ":random"
}
exposeAPI, err := k3dutil.ParsePortExposureSpec(apiPort, k3d.DefaultAPIPort)
if err != nil {
return nil, fmt.Errorf("[%s] cluster %s parse port config failed: %w", p.GetProviderName(), p.Name, err)
}

cfg.ExposeAPI.HostIP = exposeAPI.Binding.HostIP

if exposeAPI.Binding.HostPort == "0" {
exposeAPI, err = k3dutil.ParsePortExposureSpec("random", k3d.DefaultAPIPort)
if err != nil {
return nil, fmt.Errorf("[%s] cluster %s parse random port config failed: %w", p.GetProviderName(), p.Name, err)
}
}

cfg.ExposeAPI.HostPort = exposeAPI.Binding.HostPort
p.APIPort = fmt.Sprintf("%s:%s", cfg.ExposeAPI.HostIP, cfg.ExposeAPI.HostPort)
}
@@ -645,6 +650,13 @@ func (p *K3d) wrapCliFlags(masters, workers int) (*k3dconf.ClusterConfig, error)
cfg.Options.Runtime.AgentsMemory = p.WorkersMemory
}

for _, host := range []string{p.additionalHost, p.dockerHost} {
if host == "" {
continue
}
p.MasterExtraArgs = strings.TrimPrefix(p.MasterExtraArgs+" --tls-san="+host, " ")
}

if p.MasterExtraArgs != "" {
cfg.Options.K3sOptions.ExtraArgs = []k3dconf.K3sArgWithNodeFilters{}
for _, arg := range strings.Split(p.MasterExtraArgs, " ") {
12 changes: 12 additions & 0 deletions pkg/server/store/cluster/action.go
Original file line number Diff line number Diff line change
@@ -5,16 +5,20 @@ import (
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"path/filepath"
"strings"

"github.com/cnrancher/autok3s/pkg/common"
"github.com/cnrancher/autok3s/pkg/providers"
"github.com/cnrancher/autok3s/pkg/providers/k3d"
autok3stypes "github.com/cnrancher/autok3s/pkg/types/apis"

"github.com/gorilla/mux"
"github.com/rancher/apiserver/pkg/apierror"
"github.com/rancher/apiserver/pkg/types"
"github.com/rancher/apiserver/pkg/urlbuilder"
"github.com/rancher/wrangler/v2/pkg/schemas/validation"
"github.com/sirupsen/logrus"
"k8s.io/apimachinery/pkg/runtime"
@@ -256,6 +260,14 @@ func (d downloadKubeconfig) ServeHTTP(_ http.ResponseWriter, req *http.Request)
}
}

if strings.HasPrefix(clusterID, "k3d-") {
host, _, _ := net.SplitHostPort(urlbuilder.GetHost(req, ""))
// When parsing empty string as from parameter, the dockerHost will be used as the origin server
// if the dockerHost is empty(e.g. DOCKER_HOST is set), this function will do nothing as the k3d already use the
// proper cluster server address in kubeconfig
k3d.OverrideK3dKubeConfigServer("", host, &currentCfg)
}

result, err := clientcmd.Write(currentCfg)
if err != nil {
apiRequest.WriteError(apierror.NewAPIError(validation.ServerError, err.Error()))
10 changes: 8 additions & 2 deletions pkg/server/store/cluster/store.go
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@ package cluster
import (
"encoding/json"
"fmt"
"net"
"strings"

"github.com/cnrancher/autok3s/pkg/cluster"
@@ -14,6 +15,7 @@ import (
"github.com/rancher/apiserver/pkg/apierror"
"github.com/rancher/apiserver/pkg/store/empty"
"github.com/rancher/apiserver/pkg/types"
"github.com/rancher/apiserver/pkg/urlbuilder"
"github.com/rancher/wrangler/v2/pkg/schemas/validation"
"github.com/sirupsen/logrus"
)
@@ -24,9 +26,13 @@ type Store struct {
}

// Create creates cluster based on the request data.
func (c *Store) Create(_ *types.APIRequest, schema *types.APISchema, data types.APIObject) (types.APIObject, error) {
func (c *Store) Create(req *types.APIRequest, schema *types.APISchema, data types.APIObject) (types.APIObject, error) {
providerName := data.Data().String("provider")
b, err := json.Marshal(data.Data())
// for k3d to add the request host for additional tls-san
objMap := data.Data()
host, _, _ := net.SplitHostPort(urlbuilder.GetHost(req.Request, ""))
objMap.Set("additionalHost", host)
b, err := json.Marshal(objMap)
if err != nil {
return types.APIObject{}, err
}

0 comments on commit 7901720

Please sign in to comment.