[Enhancement] Edit CoreDNS ConfigMap on disk for more reliability (#814)

pkg/actions/nodehooks.go

@@ -22,11 +22,16 @@ THE SOFTWARE.
 package actions
 
 import (
+	"bytes"
 	"context"
+	"fmt"
+	"io/ioutil"
 	"os"
 
 	"github.com/rancher/k3d/v5/pkg/runtimes"
 	k3d "github.com/rancher/k3d/v5/pkg/types"
+
+	l "github.com/rancher/k3d/v5/pkg/logger"
 )
 
 type WriteFileAction struct {
@@ -39,3 +44,35 @@ type WriteFileAction struct {
 func (act WriteFileAction) Run(ctx context.Context, node *k3d.Node) error {
 	return act.Runtime.WriteToNode(ctx, act.Content, act.Dest, act.Mode, node)
 }
+
+type RewriteFileAction struct {
+	Runtime     runtimes.Runtime
+	Path        string
+	RewriteFunc func([]byte) ([]byte, error)
+	Mode        os.FileMode
+}
+
+func (act RewriteFileAction) Run(ctx context.Context, node *k3d.Node) error {
+	reader, err := act.Runtime.ReadFromNode(ctx, act.Path, node)
+	if err != nil {
+		return fmt.Errorf("runtime failed to read '%s' from node '%s': %w", act.Path, node.Name, err)
+	}
+	defer reader.Close()
+
+	file, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return fmt.Errorf("failed to read file: %w", err)
+	}
+
+	file = bytes.Trim(file[512:], "\x00") // trim control characters, etc.
+
+	file, err = act.RewriteFunc(file)
+	if err != nil {
+		return fmt.Errorf("error while rewriting %s in %s: %w", act.Path, node.Name, err)
+	}
+
+	l.Log().Tracef("Rewritten:\n%s", string(file))
+
+	return act.Runtime.WriteToNode(ctx, file, act.Path, act.Mode, node)
+
+}

pkg/client/cluster.go

@@ -22,6 +22,7 @@ THE SOFTWARE.
 package client
 
 import (
+	"bytes"
 	"context"
 	_ "embed"
 	"errors"
@@ -30,6 +31,7 @@ import (
 	"os"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 
 	gort "runtime"
@@ -845,18 +847,10 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 		}
 	}
 
-	// TODO: remove trace logs below
-	l.Log().Traceln("Servers before sort:")
-	for i, n := range servers {
-		l.Log().Tracef("Server %d - %s", i, n.Name)
-	}
+	// sort list of servers for properly ordered sequential start
 	sort.Slice(servers, func(i, j int) bool {
 		return servers[i].Name < servers[j].Name
 	})
-	l.Log().Traceln("Servers after sort:")
-	for i, n := range servers {
-		l.Log().Tracef("Server %d - %s", i, n.Name)
-	}
 
 	/*
 	 * Init Node
@@ -880,7 +874,7 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 	for _, serverNode := range servers {
 		if err := NodeStart(ctx, runtime, serverNode, &k3d.NodeStartOpts{
 			Wait:            true,
-			NodeHooks:       clusterStartOpts.NodeHooks,
+			NodeHooks:       append(clusterStartOpts.NodeHooks, serverNode.HookActions...),
 			EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
 		}); err != nil {
 			return fmt.Errorf("Failed to start server %s: %+v", serverNode.Name, err)
@@ -948,14 +942,65 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 	})
 
 	postStartErrgrp.Go(func() error {
-		// add host.k3d.internal record to the CoreDNS Configmap
-		l.Log().Infoln("Injecting record for host.k3d.internal into CoreDNS configmap...")
-		if err := corednsAddHost(postStartErrgrpCtx, runtime, cluster, clusterStartOpts.EnvironmentInfo.HostGateway.String(), k3d.DefaultK3dInternalHostRecord); err != nil {
-			return err
+
+		hosts := fmt.Sprintf("%s %s\n", clusterStartOpts.EnvironmentInfo.HostGateway.String(), k3d.DefaultK3dInternalHostRecord)
+
+		net, err := runtime.GetNetwork(ctx, &cluster.Network)
+		if err != nil {
+			return fmt.Errorf("failed to get cluster network %s to inject host records into CoreDNS: %w", cluster.Network.Name, err)
+		}
+		for _, member := range net.Members {
+			hosts += fmt.Sprintf("%s %s\n", member.IP.String(), member.Name)
+		}
+
+		l.Log().Infof("Injecting records for host.k3d.internal and for %d network members into CoreDNS configmap...", len(net.Members))
+		act := actions.RewriteFileAction{
+			Runtime: runtime,
+			Path:    "/var/lib/rancher/k3s/server/manifests/coredns.yaml",
+			Mode:    0744,
+			RewriteFunc: func(input []byte) ([]byte, error) {
+				split, err := util.SplitYAML(input)
+				if err != nil {
+					return nil, fmt.Errorf("error splitting yaml: %w", err)
+				}
+
+				var outputBuf bytes.Buffer
+				outputEncoder := yaml.NewEncoder(&outputBuf)
+
+				for _, d := range split {
+					var doc map[string]interface{}
+					if err := yaml.Unmarshal(d, &doc); err != nil {
+						return nil, err
+					}
+					if kind, ok := doc["kind"]; ok {
+						if strings.ToLower(kind.(string)) == "configmap" {
+							configmapData := doc["data"].(map[interface{}]interface{})
+							configmapData["NodeHosts"] = hosts
+						}
+					}
+					if err := outputEncoder.Encode(doc); err != nil {
+						return nil, err
+					}
+				}
+				outputEncoder.Close()
+				return outputBuf.Bytes(), nil
+			},
 		}
 
-		// add records for other containers in the cluster network to the CoreDNS configmap (e.g. useful for using registries from within Pods inside the cluster)
-		return prepCoreDNSInjectNetworkMembers(postStartErrgrpCtx, runtime, cluster)
+		// get the first server in the list and run action on it once it's ready for it
+		for _, n := range cluster.Nodes {
+			if n.Role == k3d.ServerRole {
+				ts, err := time.Parse("2006-01-02T15:04:05.999999999Z", n.State.Started)
+				if err != nil {
+					return err
+				}
+				if err := NodeWaitForLogMessage(ctx, runtime, n, "Cluster dns configmap", ts.Truncate(time.Second)); err != nil {
+					return err
+				}
+				return act.Run(ctx, n)
+			}
+		}
+		return nil
 	})
 
 	if err := postStartErrgrp.Wait(); err != nil {
@@ -1075,21 +1120,6 @@ func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.C
 	return nil
 }
 
-func prepCoreDNSInjectNetworkMembers(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
-	net, err := runtime.GetNetwork(ctx, &cluster.Network)
-	if err != nil {
-		return fmt.Errorf("failed to get cluster network %s to inject host records into CoreDNS: %w", cluster.Network.Name, err)
-	}
-	l.Log().Debugf("Adding %d network members to CoreDNS...", len(net.Members))
-	for _, member := range net.Members {
-		hostsEntry := fmt.Sprintf("%s %s", member.IP.String(), member.Name)
-		if err := corednsAddHost(ctx, runtime, cluster, member.IP.String(), member.Name); err != nil {
-			return fmt.Errorf("failed to add host entry \"%s\" into CoreDNS: %w", hostsEntry, err)
-		}
-	}
-	return nil
-}
-
 func prepCreateLocalRegistryHostingConfigMap(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
 	success := false
 	for _, node := range cluster.Nodes {

pkg/client/node.go

@@ -409,6 +409,16 @@ func NodeStart(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, no
 		}
 	}
 
+	// execute lifecycle hook actions
+	for _, hook := range nodeStartOpts.NodeHooks {
+		if hook.Stage == k3d.LifecycleStagePostStart {
+			l.Log().Tracef("Node %s: Executing postStartAction '%s'", node.Name, reflect.TypeOf(hook))
+			if err := hook.Action.Run(ctx, node); err != nil {
+				l.Log().Errorf("Node %s: Failed executing postStartAction '%+v': %+v", node.Name, hook, err)
+			}
+		}
+	}
+
 	return nil
 }
 
@@ -444,7 +454,7 @@ func enableFixes(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node,
 
 		// DNS Fix
 		if fixes.FixEnabled(fixes.EnvFixDNS) {
-			l.Log().Debugf("ENABLING DNS MAGIC!!!")
+			l.Log().Debugln(">>> enabling dns magic")
 
 			if nodeStartOpts.EnvironmentInfo == nil || nodeStartOpts.EnvironmentInfo.HostGateway == nil {
 				return fmt.Errorf("Cannot enable DNS fix, as Host Gateway IP is missing!")
@@ -465,7 +475,7 @@ func enableFixes(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node,
 
 		// CGroupsV2Fix
 		if fixes.FixEnabled(fixes.EnvFixCgroupV2) {
-			l.Log().Debugf("ENABLING CGROUPSV2 MAGIC!!!")
+			l.Log().Debugf(">>> enabling cgroupsv2 magic")
 
 			if nodeStartOpts.NodeHooks == nil {
 				nodeStartOpts.NodeHooks = []k3d.NodeHook{}

pkg/util/util.go

@@ -22,7 +22,13 @@ THE SOFTWARE.
 
 package util
 
-import "strings"
+import (
+	"bytes"
+	"io"
+	"strings"
+
+	"gopkg.in/yaml.v2"
+)
 
 func RemoveElementFromStringSlice(slice []string, index int) []string {
 	slice[index] = slice[len(slice)-1]
@@ -35,3 +41,26 @@ func ReplaceInAllElements(replacer *strings.Replacer, arr []string) []string {
 	}
 	return arr
 }
+
+func SplitYAML(resources []byte) ([][]byte, error) {
+
+	dec := yaml.NewDecoder(bytes.NewReader(resources))
+
+	var res [][]byte
+	for {
+		var value interface{}
+		err := dec.Decode(&value)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, err
+		}
+		valueBytes, err := yaml.Marshal(value)
+		if err != nil {
+			return nil, err
+		}
+		res = append(res, valueBytes)
+	}
+	return res, nil
+}

tests/test_full_lifecycle.sh

@@ -49,6 +49,8 @@ $EXE cluster start "$clustername" --wait --timeout 360s || failed "cluster didn'
 info "Checking that we have access to the cluster..."
 check_clusters "$clustername" || failed "error checking cluster"
 
+kubectl delete pod -n kube-system -l k8s-app=kube-dns  > /dev/null 2>&1 # delete coredns to force reload of config (reload plugin uses default 30s, which will make tests below fail)
+
 info "Checking that we have 2 nodes online..."
 check_multi_node "$clustername" 2 || failed "failed to verify number of nodes"