diff --git a/cmd/cluster/clusterCreate.go b/cmd/cluster/clusterCreate.go index ef77b501..a28d2f7b 100644 --- a/cmd/cluster/clusterCreate.go +++ b/cmd/cluster/clusterCreate.go @@ -191,7 +191,7 @@ func NewCmdClusterCreate() *cobra.Command { * Kubeconfig * **************/ - if clusterConfig.KubeconfigOpts.UpdateDefaultKubeconfig && clusterConfig.KubeconfigOpts.SwitchCurrentContext { + if !clusterConfig.KubeconfigOpts.UpdateDefaultKubeconfig && clusterConfig.KubeconfigOpts.SwitchCurrentContext { l.Log().Infoln("--kubeconfig-update-default=false --> sets --kubeconfig-switch-context=false") clusterConfig.KubeconfigOpts.SwitchCurrentContext = false } diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go index bf68f90a..c1e7527c 100644 --- a/pkg/client/cluster.go +++ b/pkg/client/cluster.go @@ -27,6 +27,7 @@ import ( "errors" "fmt" "io/ioutil" + "os" "sort" "strconv" "time" @@ -934,19 +935,31 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust } /* - * Additional Cluster Preparation + * Additional Cluster Preparation (post start) */ + postStartErrgrp, postStartErrgrpCtx := errgroup.WithContext(ctx) + /*** DNS ***/ - // add /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system - if err := prepInjectHostIP(ctx, runtime, cluster, &clusterStartOpts); err != nil { - return fmt.Errorf("failed to inject host IP: %w", err) - } + // add host.k3d.internal record to /etc/hosts in all nodes + postStartErrgrp.Go(func() error { + return prepInjectHostIP(postStartErrgrpCtx, runtime, cluster, &clusterStartOpts) + }) + + postStartErrgrp.Go(func() error { + // add host.k3d.internal record to the CoreDNS Configmap + l.Log().Infoln("Injecting record for host.k3d.internal into CoreDNS configmap...") + if err := corednsAddHost(postStartErrgrpCtx, runtime, cluster, clusterStartOpts.EnvironmentInfo.HostGateway.String(), k3d.DefaultK3dInternalHostRecord); err != nil { + return err + } + + // add records for other containers in the cluster network to the CoreDNS configmap (e.g. useful for using registries from within Pods inside the cluster) + return prepCoreDNSInjectNetworkMembers(postStartErrgrpCtx, runtime, cluster) + }) - // create host records in CoreDNS for external registries - if err := prepCoreDNSInjectNetworkMembers(ctx, runtime, cluster); err != nil { - return fmt.Errorf("failed to patch CoreDNS with network members: %w", err) + if err := postStartErrgrp.Wait(); err != nil { + return fmt.Errorf("error during post-start cluster preparation: %w", err) } return nil @@ -983,34 +996,55 @@ func SortClusters(clusters []*k3d.Cluster) []*k3d.Cluster { // corednsAddHost adds a host entry to the CoreDNS configmap if it doesn't exist (a host entry is a single line of the form "IP HOST") func corednsAddHost(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, ip string, name string) error { + retries := 3 + if v, ok := os.LookupEnv("K3D_DEBUG_COREDNS_RETRIES"); ok && v != "" { + l.Log().Debugf("Running with K3D_DEBUG_COREDNS_RETRIES=%s", v) + if r, err := strconv.Atoi(v); err == nil { + retries = r + } else { + return fmt.Errorf("Invalid value set for env var K3D_DEBUG_COREDNS_RETRIES (%s): %w", v, err) + } + } + + // select any server node + var node *k3d.Node + for _, n := range cluster.Nodes { + if n.Role == k3d.ServerRole { + node = n + } + } + hostsEntry := fmt.Sprintf("%s %s", ip, name) patchCmd := `patch=$(kubectl get cm coredns -n kube-system --template='{{.data.NodeHosts}}' | sed -n -E -e '/[0-9\.]{4,12}\s` + name + `$/!p' -e '$a` + hostsEntry + `' | tr '\n' '^' | busybox xargs -0 printf '{"data": {"NodeHosts":"%s"}}'| sed -E 's%\^%\\n%g') && kubectl patch cm coredns -n kube-system -p="$patch"` successInjectCoreDNSEntry := false - for _, node := range cluster.Nodes { - if node.Role == k3d.AgentRole || node.Role == k3d.ServerRole { - logreader, err := runtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", patchCmd}) - if err == nil { - successInjectCoreDNSEntry = true - break - } else { - msg := fmt.Sprintf("error patching the CoreDNS ConfigMap to include entry '%s': %+v", hostsEntry, err) - if logreader != nil { - readlogs, err := ioutil.ReadAll(logreader) - if err != nil { - l.Log().Debugf("error reading the logs from failed CoreDNS patch exec process in node %s: %v", node.Name, err) - } else { - msg += fmt.Sprintf("\nLogs: %s", string(readlogs)) - } + + // try 3 (or K3D_DEBUG_COREDNS_RETRIES value) times, as e.g. on cluster startup it may take some time for the Configmap to be available and the server to be responsive + for i := 0; i < retries; i++ { + l.Log().Debugf("Running CoreDNS patch in node %s to add %s (try %d/%d)...", node.Name, hostsEntry, i, retries) + logreader, err := runtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", patchCmd}) + if err == nil { + successInjectCoreDNSEntry = true + break + } else { + msg := fmt.Sprintf("(try %d/%d) error patching the CoreDNS ConfigMap to include entry '%s': %+v", i, retries, hostsEntry, err) + if logreader != nil { + readlogs, err := ioutil.ReadAll(logreader) + if err != nil { + l.Log().Debugf("(try %d/%d) error reading the logs from failed CoreDNS patch exec process in node %s: %v", i, retries, node.Name, err) } else { - l.Log().Debugf("no logreader returned for exec process") + msg += fmt.Sprintf("\nLogs: %s", string(readlogs)) } - l.Log().Debugln(msg) + } else { + l.Log().Debugf("(try %d/%d) error reading the logs from failed CoreDNS patch exec process in node %s: no logreader returned for exec process", i, retries, node.Name) } + l.Log().Debugln(msg) + time.Sleep(1 * time.Second) } } if !successInjectCoreDNSEntry { - return fmt.Errorf("Failed to patch CoreDNS ConfigMap to include entry '%s' (see debug logs)", hostsEntry) + return fmt.Errorf("failed to patch CoreDNS ConfigMap to include entry '%s' (%d tries, see debug logs)", hostsEntry, retries) } + l.Log().Debugf("Successfully patched CoreDNS Configmap with record '%s'", hostsEntry) return nil } @@ -1023,7 +1057,7 @@ func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.C hostIP := clusterStartOpts.EnvironmentInfo.HostGateway hostsEntry := fmt.Sprintf("%s %s", hostIP.String(), k3d.DefaultK3dInternalHostRecord) - l.Log().Infof("Injecting record '%s'...", hostsEntry) + l.Log().Infof("Injecting '%s' into /etc/hosts of all nodes...", hostsEntry) // entry in /etc/hosts errgrp, errgrpctx := errgroup.WithContext(ctx) @@ -1038,24 +1072,19 @@ func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.C } l.Log().Debugf("Successfully added host record \"%s\" to /etc/hosts in all nodes", hostsEntry) - err := corednsAddHost(ctx, runtime, cluster, hostIP.String(), k3d.DefaultK3dInternalHostRecord) - if err != nil { - return fmt.Errorf("failed to inject host record \"%s\" into CoreDNS ConfigMap: %w", hostsEntry, err) - } - l.Log().Debugf("Successfully added host record \"%s\" to the CoreDNS ConfigMap ", hostsEntry) return nil } func prepCoreDNSInjectNetworkMembers(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error { net, err := runtime.GetNetwork(ctx, &cluster.Network) if err != nil { - return fmt.Errorf("failed to get cluster network %s to inject host records into CoreDNS: %v", cluster.Network.Name, err) + return fmt.Errorf("failed to get cluster network %s to inject host records into CoreDNS: %w", cluster.Network.Name, err) } - l.Log().Debugf("Adding %d network members to coredns", len(net.Members)) + l.Log().Debugf("Adding %d network members to CoreDNS...", len(net.Members)) for _, member := range net.Members { hostsEntry := fmt.Sprintf("%s %s", member.IP.String(), member.Name) if err := corednsAddHost(ctx, runtime, cluster, member.IP.String(), member.Name); err != nil { - return fmt.Errorf("failed to add host entry \"%s\" into CoreDNS: %v", hostsEntry, err) + return fmt.Errorf("failed to add host entry \"%s\" into CoreDNS: %w", hostsEntry, err) } } return nil diff --git a/tests/common.sh b/tests/common.sh index 57778213..e5deac1c 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -181,4 +181,4 @@ k3s_assert_node_label() { # $1 = node name # $2 = label to assert kubectl get node "$1" --output go-template='{{ range $k, $v := .metadata.labels }}{{ printf "%s=%s\n" $k $v }}{{ end }}' | grep -qE "^$2$" -} \ No newline at end of file +} diff --git a/tests/test_full_lifecycle.sh b/tests/test_full_lifecycle.sh index f1002b1d..e51b2ee4 100755 --- a/tests/test_full_lifecycle.sh +++ b/tests/test_full_lifecycle.sh @@ -77,7 +77,8 @@ wait_for_pod_running_by_label "k8s-app=kube-dns" "kube-system" sleep 5 # 6. test host.k3d.internal -info "Checking DNS Lookup for host.k3d.internal" +info "Checking DNS Lookup for host.k3d.internal..." +kubectl describe cm coredns -n kube-system | grep "host.k3d.internal" > /dev/null 2>&1 || failed "Couldn't find host.k3d.internal in CoreDNS configmap" wait_for_pod_exec "testimage" "nslookup host.k3d.internal" 15 || failed "DNS Lookup for host.k3d.internal failed" # Cleanup