gregandev
/
k3d
mirror of https://github.com/k3d-io/k3d
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[Enhancement/Fix] Properly use env/runtime info and inject dns accordingly (#758)

Browse Source 
- make use of environment and runtime info
- DfD: use host.docker.internal
- All other cases: use Docker network Gateway
- k3d-tools: based on alpine to have `getent` present
pull/767/head
Thorsten Klein 3 years ago committed by iwilltry42
parent f801e46e9e
commit 67d8c8c84f
No known key found for this signature in database
GPG Key ID: 7BA57AD1CFF16110
7 changed files with 132 additions and 96 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 58
      pkg/client/cluster.go
  2. 15
      pkg/client/environment.go
  3. 79
      pkg/client/host.go
  4. 54
      pkg/client/tools.go
  5. 13
      pkg/runtimes/docker/host.go
  6. 2
      pkg/types/types.go
  7. 7
      tools/Dockerfile

58
pkg/client/cluster.go
Unescape View File

@ -60,6 +60,9 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi
return fmt.Errorf("Failed Cluster Preparation: %+v", err)
}
// Create tools-node for later steps
go EnsureToolsNode(ctx, runtime, &clusterConfig.Cluster)
/*
* Step 1: Create Containers
*/
@ -295,6 +298,7 @@ func ClusterPrepImageVolume(ctx context.Context, runtime k3drt.Runtime, cluster
}
clusterCreateOpts.GlobalLabels[k3d.LabelImageVolume] = imageVolumeName
cluster.ImageVolume = imageVolumeName
// attach volume to nodes
for _, node := range cluster.Nodes {
@ -812,12 +816,12 @@ func GenerateNodeName(cluster string, role k3d.Role, suffix int) string {
}
// ClusterStart starts a whole cluster (i.e. all nodes of the cluster)
func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, startClusterOpts types.ClusterStartOpts) error {
func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterStartOpts types.ClusterStartOpts) error {
l.Log().Infof("Starting cluster '%s'", cluster.Name)
if startClusterOpts.Timeout > 0*time.Second {
if clusterStartOpts.Timeout > 0*time.Second {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, startClusterOpts.Timeout)
ctx, cancel = context.WithTimeout(ctx, clusterStartOpts.Timeout)
defer cancel()
}
@ -860,9 +864,9 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
l.Log().Infoln("Starting the initializing server...")
if err := NodeStart(ctx, runtime, initNode, &k3d.NodeStartOpts{
Wait: true, // always wait for the init node
NodeHooks: startClusterOpts.NodeHooks,
NodeHooks: clusterStartOpts.NodeHooks,
ReadyLogMessage: "Running kube-apiserver", // initNode means, that we're using etcd -> this will need quorum, so "k3s is up and running" won't happen right now
EnvironmentInfo: startClusterOpts.EnvironmentInfo,
EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
}); err != nil {
return fmt.Errorf("Failed to start initializing server node: %+v", err)
}
@ -875,8 +879,8 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
for _, serverNode := range servers {
if err := NodeStart(ctx, runtime, serverNode, &k3d.NodeStartOpts{
Wait: true,
NodeHooks: startClusterOpts.NodeHooks,
EnvironmentInfo: startClusterOpts.EnvironmentInfo,
NodeHooks: clusterStartOpts.NodeHooks,
EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
}); err != nil {
return fmt.Errorf("Failed to start server %s: %+v", serverNode.Name, err)
}
@ -894,8 +898,8 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
agentWG.Go(func() error {
return NodeStart(aCtx, runtime, currentAgentNode, &k3d.NodeStartOpts{
Wait: true,
NodeHooks: startClusterOpts.NodeHooks,
EnvironmentInfo: startClusterOpts.EnvironmentInfo,
NodeHooks: clusterStartOpts.NodeHooks,
EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
})
})
}
@ -915,7 +919,7 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
helperWG.Go(func() error {
nodeStartOpts := &k3d.NodeStartOpts{
NodeHooks: currentHelperNode.HookActions,
EnvironmentInfo: startClusterOpts.EnvironmentInfo,
EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
}
if currentHelperNode.Role == k3d.LoadBalancerRole {
nodeStartOpts.Wait = true
@ -936,7 +940,7 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
/*** DNS ***/
// add /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system
if err := prepInjectHostIP(ctx, runtime, cluster); err != nil {
if err := prepInjectHostIP(ctx, runtime, cluster, &clusterStartOpts); err != nil {
return fmt.Errorf("failed to inject host IP: %w", err)
}
@ -1007,33 +1011,27 @@ func corednsAddHost(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clu
}
// prepInjectHostIP adds /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system
func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterStartOpts *k3d.ClusterStartOpts) error {
if cluster.Network.Name == "host" {
l.Log().Tracef("Not injecting hostIP as clusternetwork is 'host'")
return nil
}
l.Log().Infoln("Trying to get IP of the docker host and inject it into the cluster as 'host.k3d.internal' for easy access")
hostIP, err := GetHostIP(ctx, runtime, cluster)
if err != nil {
l.Log().Warnf("Failed to get HostIP to inject as host.k3d.internal: %+v", err)
return nil
}
if hostIP != nil {
hostsEntry := fmt.Sprintf("%s %s", hostIP.String(), k3d.DefaultK3dInternalHostRecord)
l.Log().Debugf("Adding extra host entry '%s'...", hostsEntry)
for _, node := range cluster.Nodes {
if err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", hostsEntry)}); err != nil {
return fmt.Errorf("failed to add extra entry '%s' to /etc/hosts in node '%s': %w", hostsEntry, node.Name, err)
}
hostIP := clusterStartOpts.EnvironmentInfo.HostGateway
hostsEntry := fmt.Sprintf("%s %s", hostIP.String(), k3d.DefaultK3dInternalHostRecord)
l.Log().Debugf("Adding extra host entry '%s'...", hostsEntry)
for _, node := range cluster.Nodes {
if err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", hostsEntry)}); err != nil {
return fmt.Errorf("failed to add extra entry '%s' to /etc/hosts in node '%s': %w", hostsEntry, node.Name, err)
}
l.Log().Debugf("Successfully added host record \"%s\" to /etc/hosts in all nodes", hostsEntry)
}
l.Log().Debugf("Successfully added host record \"%s\" to /etc/hosts in all nodes", hostsEntry)
err := corednsAddHost(ctx, runtime, cluster, hostIP.String(), k3d.DefaultK3dInternalHostRecord)
if err != nil {
return fmt.Errorf("failed to inject host record \"%s\" into CoreDNS ConfigMap: %w", hostsEntry, err)
}
l.Log().Debugf("Successfully added host record \"%s\" to the CoreDNS ConfigMap ", hostsEntry)
err := corednsAddHost(ctx, runtime, cluster, hostIP.String(), k3d.DefaultK3dInternalHostRecord)
if err != nil {
return fmt.Errorf("failed to inject host record \"%s\" into CoreDNS ConfigMap: %w", hostsEntry, err)
}
l.Log().Debugf("Successfully added host record \"%s\" to the CoreDNS ConfigMap ", hostsEntry)
return nil
}

15
pkg/client/environment.go
Unescape View File

@ -32,14 +32,23 @@ import (
)
func GatherEnvironmentInfo(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster) (*k3d.EnvironmentInfo, error) {
envInfo := &k3d.EnvironmentInfo{}
rtimeInfo, err := runtime.Info()
if err != nil {
return nil, err
}
envInfo.RuntimeInfo = *rtimeInfo
l.Log().Infof("Using the k3d-tools node to gather environment information")
toolsNode, err := EnsureToolsNode(ctx, runtime, cluster)
if err != nil {
return nil, err
}
defer NodeDelete(ctx, runtime, toolsNode, k3d.NodeDeleteOpts{SkipLBUpdate: true})
envInfo := &k3d.EnvironmentInfo{}
defer func() {
go NodeDelete(ctx, runtime, toolsNode, k3d.NodeDeleteOpts{SkipLBUpdate: true})
}()
hostIP, err := GetHostIP(ctx, runtime, cluster)
if err != nil {

79
pkg/client/host.go
Unescape View File

@ -27,51 +27,74 @@ import (
"fmt"
"net"
"regexp"
"runtime"
goruntime "runtime"
"strings"
l "github.com/rancher/k3d/v5/pkg/logger"
rt "github.com/rancher/k3d/v5/pkg/runtimes"
"github.com/rancher/k3d/v5/pkg/runtimes"
k3d "github.com/rancher/k3d/v5/pkg/types"
"github.com/rancher/k3d/v5/pkg/util"
)
var nsLookupAddressRegexp = regexp.MustCompile(`^Address:\s+(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$`)
type ResolveHostCmd struct {
Cmd string
LogMatcher *regexp.Regexp
}
var (
ResolveHostCmdNSLookup = ResolveHostCmd{
Cmd: "nslookup %s",
LogMatcher: regexp.MustCompile(`^Address:\s+(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$`),
}
ResolveHostCmdGetEnt = ResolveHostCmd{
Cmd: "getent ahostsv4 '%s'",
LogMatcher: regexp.MustCompile(`(?P<ip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s+STREAM.+`), // e.g. `192.168.47.4 STREAM host.docker.internal`,
}
)
// GetHostIP returns the routable IP address to be able to access services running on the host system from inside the cluster.
// This depends on the Operating System and the chosen Runtime.
func GetHostIP(ctx context.Context, rtime rt.Runtime, cluster *k3d.Cluster) (net.IP, error) {
func GetHostIP(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster) (net.IP, error) {
// Docker Runtime
if rtime == rt.Docker {
rtimeInfo, err := runtime.Info()
if err != nil {
return nil, err
}
l.Log().Tracef("Runtime GOOS: %s", runtime.GOOS)
l.Log().Tracef("GOOS: %s / Runtime OS: %s (%s)", goruntime.GOOS, rtimeInfo.OSType, rtimeInfo.OS)
// "native" Docker on Linux
if runtime.GOOS == "linux" {
ip, err := rtime.GetHostIP(ctx, cluster.Network.Name)
if err != nil {
return nil, fmt.Errorf("runtime failed to get host IP: %w", err)
}
return ip, nil
}
isDockerDesktop := func(os string) bool {
return strings.ToLower(os) == "docker desktop"
}
// Docker Runtime
if runtime == runtimes.Docker {
// Docker (for Desktop) on MacOS or Windows
if runtime.GOOS == "windows" || runtime.GOOS == "darwin" {
if isDockerDesktop(rtimeInfo.OS) {
toolsNode, err := EnsureToolsNode(ctx, rtime, cluster)
toolsNode, err := EnsureToolsNode(ctx, runtime, cluster)
if err != nil {
return nil, fmt.Errorf("failed to ensure that k3d-tools node is running to get host IP :%w", err)
}
ip, err := resolveHostnameFromInside(ctx, rtime, toolsNode, "host.docker.internal")
if err != nil {
return nil, fmt.Errorf("failed to resolve 'host.docker.internal' from inside the k3d-tools node: %w", err)
ip, err := resolveHostnameFromInside(ctx, runtime, toolsNode, "host.docker.internal", ResolveHostCmdGetEnt)
if err == nil {
return ip, nil
}
return ip, nil
l.Log().Warnf("failed to resolve 'host.docker.internal' from inside the k3d-tools node: %v", err)
}
l.Log().Infof("HostIP: using network gateway...")
ip, err := runtime.GetHostIP(ctx, cluster.Network.Name)
if err != nil {
return nil, fmt.Errorf("runtime failed to get host IP: %w", err)
}
// Catch all other GOOS cases
return nil, fmt.Errorf("GetHostIP only implemented for Linux, MacOS (Darwin) and Windows")
return ip, nil
}
@ -80,9 +103,9 @@ func GetHostIP(ctx context.Context, rtime rt.Runtime, cluster *k3d.Cluster) (net
}
func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d.Node, hostname string) (net.IP, error) {
func resolveHostnameFromInside(ctx context.Context, rtime runtimes.Runtime, node *k3d.Node, hostname string, cmd ResolveHostCmd) (net.IP, error) {
logreader, execErr := rtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", fmt.Sprintf("nslookup %s", hostname)})
logreader, execErr := rtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", fmt.Sprintf(cmd.Cmd, hostname)})
if logreader == nil {
if execErr != nil {
@ -105,12 +128,12 @@ func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d.
}
for scanner.Scan() {
l.Log().Tracef("Scanning Log Line '%s'", scanner.Text())
match := nsLookupAddressRegexp.FindStringSubmatch(scanner.Text())
match := cmd.LogMatcher.FindStringSubmatch(scanner.Text())
if len(match) == 0 {
continue
}
l.Log().Tracef("-> Match(es): '%+v'", match)
submatches = util.MapSubexpNames(nsLookupAddressRegexp.SubexpNames(), match)
submatches = util.MapSubexpNames(cmd.LogMatcher.SubexpNames(), match)
l.Log().Tracef(" -> Submatch(es): %+v", submatches)
break
}
@ -118,7 +141,7 @@ func resolveHostnameFromInside(ctx context.Context, rtime rt.Runtime, node *k3d.
if execErr != nil {
l.Log().Errorln(execErr)
}
return nil, fmt.Errorf("Failed to read address for '%s' from nslookup response", hostname)
return nil, fmt.Errorf("Failed to read address for '%s' from command output", hostname)
}
l.Log().Debugf("Hostname '%s' -> Address '%s'", hostname, submatches["ip"])

54
pkg/client/tools.go
Unescape View File

@ -256,35 +256,41 @@ func runToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cl
}
func EnsureToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster) (*k3d.Node, error) {
var err error
cluster, err = ClusterGet(ctx, runtime, cluster)
if err != nil {
return nil, fmt.Errorf("failed to retrieve cluster '%s': %w", cluster.Name, err)
}
var toolsNode *k3d.Node
toolsNode, err := runtime.GetNode(ctx, &k3d.Node{Name: fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name)})
if err != nil || toolsNode == nil {
if cluster.Network.Name == "" {
return nil, fmt.Errorf("failed to get network for cluster '%s'", cluster.Name)
}
// Get more info on the cluster, if required
var imageVolume string
if cluster.Network.Name == "" || cluster.ImageVolume == "" {
l.Log().Debugf("Gathering some more info about the cluster before creating the tools node...")
var err error
cluster, err = ClusterGet(ctx, runtime, cluster)
if err != nil {
return nil, fmt.Errorf("failed to retrieve cluster: %w", err)
}
var imageVolume string
var ok bool
for _, node := range cluster.Nodes {
if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole {
if imageVolume, ok = node.RuntimeLabels[k3d.LabelImageVolume]; ok {
break
if cluster.Network.Name == "" {
return nil, fmt.Errorf("failed to get network for cluster '%s'", cluster.Name)
}
}
}
if imageVolume == "" {
return nil, fmt.Errorf("Failed to find image volume for cluster '%s'", cluster.Name)
}
l.Log().Debugf("Attaching to cluster's image volume '%s'", imageVolume)
var ok bool
for _, node := range cluster.Nodes {
if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole {
if imageVolume, ok = node.RuntimeLabels[k3d.LabelImageVolume]; ok {
break
}
}
}
if imageVolume == "" {
return nil, fmt.Errorf("Failed to find image volume for cluster '%s'", cluster.Name)
}
l.Log().Debugf("Attaching to cluster's image volume '%s'", imageVolume)
cluster.ImageVolume = imageVolume
}
var toolsNode *k3d.Node
toolsNode, err = runtime.GetNode(ctx, &k3d.Node{Name: fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name)})
if err != nil || toolsNode == nil {
// start tools node
l.Log().Infoln("Starting new tools node...")
toolsNode, err = runToolsNode(
ctx,
@ -292,7 +298,7 @@ func EnsureToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d
cluster,
cluster.Network.Name,
[]string{
fmt.Sprintf("%s:%s", imageVolume, k3d.DefaultImageVolumeMountPath),
fmt.Sprintf("%s:%s", cluster.ImageVolume, k3d.DefaultImageVolumeMountPath),
fmt.Sprintf("%s:%s", runtime.GetRuntimePath(), runtime.GetRuntimePath()),
})
if err != nil {

13
pkg/runtimes/docker/host.go
Unescape View File

@ -25,19 +25,14 @@ import (
"context"
"fmt"
"net"
"runtime"
)
// GetHostIP returns the IP of the docker host (routable from inside the containers)
func (d Docker) GetHostIP(ctx context.Context, network string) (net.IP, error) {
if runtime.GOOS == "linux" {
ip, err := GetGatewayIP(ctx, network)
if err != nil {
return nil, fmt.Errorf("failed to get gateway IP of docker network '%s': %w", network, err)
}
return ip, nil
ip, err := GetGatewayIP(ctx, network)
if err != nil {
return nil, fmt.Errorf("failed to get gateway IP of docker network '%s': %w", network, err)
}
return nil, fmt.Errorf("Docker Runtime: GetHostIP only implemented for Linux")
return ip, nil
}

2
pkg/types/types.go
Unescape View File

@ -28,6 +28,7 @@ import (
"time"
"github.com/docker/go-connections/nat"
runtimeTypes "github.com/rancher/k3d/v5/pkg/runtimes/types"
"github.com/rancher/k3d/v5/pkg/types/k3s"
"github.com/rancher/k3d/v5/version"
"inet.af/netaddr"
@ -424,4 +425,5 @@ type RegistryExternal struct {
type EnvironmentInfo struct {
HostGateway net.IP
RuntimeInfo runtimeTypes.RuntimeInfo
}

7
tools/Dockerfile
Unescape View File

@ -1,13 +1,16 @@
FROM golang:1.17 as builder
FROM golang:1.17-alpine as builder
ARG GIT_TAG
WORKDIR /app
COPY . .
RUN apk update && apk add make bash git
ENV GIT_TAG=${GIT_TAG}
ENV GO111MODULE=on
ENV CGO_ENABLED=0
RUN make build
FROM busybox:1.31
FROM alpine:3.14
RUN apk update && apk add bash
WORKDIR /app
COPY --from=builder /app/bin/k3d-tools .
ENTRYPOINT [ "/app/k3d-tools"]

Write Preview
Loading…
Cancel
Save