From 212979d0bb4231efcfb0cd00dfc7bae0a974d593 Mon Sep 17 00:00:00 2001 From: Thorsten Klein Date: Tue, 24 Aug 2021 10:18:53 +0200 Subject: [PATCH] [Enhancement] DNS Injection (#718) - remove`--no-hostip` flag and the related `disableHostIPInjection` config option - inject host IP on every cluster startup (except when hostnetwork is chosen)(/etc/hosts + CoreDNS) - inject host entries for every cluster network member container into the CoreDNS configmap --- cmd/cluster/clusterCreate.go | 3 - docs/usage/configfile.md | 1 - go.mod | 2 +- pkg/client/cluster.go | 122 +++++++++++++++++++------------ pkg/config/process.go | 3 - pkg/config/process_test.go | 2 - pkg/config/transform.go | 19 +++-- pkg/config/v1alpha3/schema.json | 4 - pkg/config/v1alpha3/types.go | 13 ++-- pkg/runtimes/docker/network.go | 33 +++++++-- pkg/runtimes/docker/translate.go | 38 ++++++++++ pkg/types/types.go | 31 ++++---- 12 files changed, 171 insertions(+), 100 deletions(-) diff --git a/cmd/cluster/clusterCreate.go b/cmd/cluster/clusterCreate.go index 94bef74d..d4c3a6cc 100644 --- a/cmd/cluster/clusterCreate.go +++ b/cmd/cluster/clusterCreate.go @@ -363,9 +363,6 @@ func NewCmdClusterCreate() *cobra.Command { cmd.Flags().Bool("no-rollback", false, "Disable the automatic rollback actions, if anything goes wrong") _ = cfgViper.BindPFlag("options.k3d.disablerollback", cmd.Flags().Lookup("no-rollback")) - cmd.Flags().Bool("no-hostip", false, "Disable the automatic injection of the Host IP as 'host.k3d.internal' into the containers and CoreDNS") - _ = cfgViper.BindPFlag("options.k3d.disablehostipinjection", cmd.Flags().Lookup("no-hostip")) - cmd.Flags().String("gpus", "", "GPU devices to add to the cluster node containers ('all' to pass all GPUs) [From docker]") _ = cfgViper.BindPFlag("options.runtime.gpurequest", cmd.Flags().Lookup("gpus")) diff --git a/docs/usage/configfile.md b/docs/usage/configfile.md index d4e79586..ddbe15b1 100644 --- a/docs/usage/configfile.md +++ b/docs/usage/configfile.md @@ -92,7 +92,6 @@ options: disableLoadbalancer: false # same as `--no-lb` disableImageVolume: false # same as `--no-image-volume` disableRollback: false # same as `--no-Rollback` - disableHostIPInjection: false # same as `--no-hostip` k3s: # options passed on to K3s itself extraArgs: # additional arguments passed to the `k3s server|agent` command; same as `--k3s-arg` - arg: --tls-san=my.host.domain diff --git a/go.mod b/go.mod index bb9d7fce..36a8336e 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3 // indirect github.com/Microsoft/hcsshim v0.8.14 // indirect github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68 // indirect - github.com/containerd/containerd v1.4.4 // indirect + github.com/containerd/containerd v1.4.4 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e // indirect github.com/docker/cli v20.10.7+incompatible github.com/docker/docker v20.10.7+incompatible diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go index 2beb29c5..723ff4fd 100644 --- a/pkg/client/cluster.go +++ b/pkg/client/cluster.go @@ -90,15 +90,6 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi * Additional Cluster Preparation * **********************************/ - /* - * Networking Magic - */ - - // add /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system - if !clusterConfig.ClusterCreateOpts.PrepDisableHostIPInjection { - prepInjectHostIP(ctx, runtime, &clusterConfig.Cluster) - } - // create the registry hosting configmap if len(clusterConfig.ClusterCreateOpts.Registries.Use) > 0 { if err := prepCreateLocalRegistryHostingConfigMap(ctx, runtime, &clusterConfig.Cluster); err != nil { @@ -933,6 +924,22 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust return fmt.Errorf("Failed to add one or more helper nodes: %w", err) } + /* + * Additional Cluster Preparation + */ + + /*** DNS ***/ + + // add /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system + if err := prepInjectHostIP(ctx, runtime, cluster); err != nil { + return err + } + + // create host records in CoreDNS for external registries + if err := prepCoreDNSInjectNetworkMembers(ctx, runtime, cluster); err != nil { + return err + } + return nil } @@ -963,60 +970,79 @@ func SortClusters(clusters []*k3d.Cluster) []*k3d.Cluster { return clusters } +// corednsAddHost adds a host entry to the CoreDNS configmap if it doesn't exist (a host entry is a single line of the form "IP HOST") +func corednsAddHost(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, ip string, name string) error { + hostsEntry := fmt.Sprintf("%s %s", ip, name) + patchCmd := `patch=$(kubectl get cm coredns -n kube-system --template='{{.data.NodeHosts}}' | sed -n -E -e '/[0-9\.]{4,12}\s` + name + `$/!p' -e '$a` + hostsEntry + `' | tr '\n' '^' | busybox xargs -0 printf '{"data": {"NodeHosts":"%s"}}'| sed -E 's%\^%\\n%g') && kubectl patch cm coredns -n kube-system -p="$patch"` + successInjectCoreDNSEntry := false + for _, node := range cluster.Nodes { + if node.Role == k3d.AgentRole || node.Role == k3d.ServerRole { + logreader, err := runtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", patchCmd}) + if err == nil { + successInjectCoreDNSEntry = true + break + } else { + msg := fmt.Sprintf("error patching the CoreDNS ConfigMap to include entry '%s': %+v", hostsEntry, err) + readlogs, err := ioutil.ReadAll(logreader) + if err != nil { + l.Log().Debugf("error reading the logs from failed CoreDNS patch exec process in node %s: %v", node.Name, err) + } else { + msg += fmt.Sprintf("\nLogs: %s", string(readlogs)) + } + l.Log().Debugln(msg) + } + } + } + if !successInjectCoreDNSEntry { + return fmt.Errorf("Failed to patch CoreDNS ConfigMap to include entry '%s' (see debug logs)", hostsEntry) + } + return nil +} + // prepInjectHostIP adds /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system -func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) { - l.Log().Infoln("(Optional) Trying to get IP of the docker host and inject it into the cluster as 'host.k3d.internal' for easy access") +func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error { + if cluster.Network.Name == "host" { + l.Log().Tracef("Not injecting hostIP as clusternetwork is 'host'") + return nil + } + l.Log().Infoln("Trying to get IP of the docker host and inject it into the cluster as 'host.k3d.internal' for easy access") hostIP, err := GetHostIP(ctx, runtime, cluster) if err != nil { - l.Log().Warnf("Failed to get HostIP: %+v", err) + l.Log().Warnf("Failed to get HostIP to inject as host.k3d.internal: %+v", err) + return nil } if hostIP != nil { - hostRecordSuccessMessage := "" - etcHostsFailureCount := 0 - hostsEntry := fmt.Sprintf("%s %s", hostIP, k3d.DefaultK3dInternalHostRecord) + hostsEntry := fmt.Sprintf("%s %s", hostIP.String(), k3d.DefaultK3dInternalHostRecord) l.Log().Debugf("Adding extra host entry '%s'...", hostsEntry) for _, node := range cluster.Nodes { if err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", hostsEntry)}); err != nil { - l.Log().Warnf("Failed to add extra entry '%s' to /etc/hosts in node '%s'", hostsEntry, node.Name) - etcHostsFailureCount++ + return fmt.Errorf("failed to add extra entry '%s' to /etc/hosts in node '%s': %w", hostsEntry, node.Name, err) } } - if etcHostsFailureCount < len(cluster.Nodes) { - hostRecordSuccessMessage += fmt.Sprintf("Successfully added host record to /etc/hosts in %d/%d nodes", (len(cluster.Nodes) - etcHostsFailureCount), len(cluster.Nodes)) - } - - patchCmd := `patch=$(kubectl get cm coredns -n kube-system --template='{{.data.NodeHosts}}' | sed -n -E -e '/[0-9\.]{4,12}\s+host\.k3d\.internal$/!p' -e '$a` + hostsEntry + `' | tr '\n' '^' | busybox xargs -0 printf '{"data": {"NodeHosts":"%s"}}'| sed -E 's%\^%\\n%g') && kubectl patch cm coredns -n kube-system -p="$patch"` - successInjectCoreDNSEntry := false - for _, node := range cluster.Nodes { + l.Log().Debugf("Successfully added host record \"%s\" to /etc/hosts in all nodes", hostsEntry) - if node.Role == k3d.AgentRole || node.Role == k3d.ServerRole { - logreader, err := runtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", patchCmd}) - if err == nil { - successInjectCoreDNSEntry = true - break - } else { - msg := fmt.Sprintf("error patching the CoreDNS ConfigMap to include entry '%s': %+v", hostsEntry, err) - readlogs, err := ioutil.ReadAll(logreader) - if err != nil { - l.Log().Debugf("error reading the logs from failed CoreDNS patch exec process in node %s: %v", node.Name, err) - } else { - msg += fmt.Sprintf("\nLogs: %s", string(readlogs)) - } - l.Log().Debugln(msg) - } - } - } - if successInjectCoreDNSEntry == false { - l.Log().Warnf("Failed to patch CoreDNS ConfigMap to include entry '%s' (see debug logs)", hostsEntry) - } else { - hostRecordSuccessMessage += " and to the CoreDNS ConfigMap" + err := corednsAddHost(ctx, runtime, cluster, hostIP.String(), k3d.DefaultK3dInternalHostRecord) + if err != nil { + return fmt.Errorf("failed to inject host record \"%s\" into CoreDNS ConfigMap: %w", hostsEntry, err) } + l.Log().Debugf("Successfully added host record \"%s\" to the CoreDNS ConfigMap ", hostsEntry) + } + return nil +} - if hostRecordSuccessMessage != "" { - l.Log().Infoln(hostRecordSuccessMessage) +func prepCoreDNSInjectNetworkMembers(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error { + net, err := runtime.GetNetwork(ctx, &cluster.Network) + if err != nil { + return fmt.Errorf("failed to get cluster network %s to inject host records into CoreDNS: %v", cluster.Network.Name, err) + } + l.Log().Debugf("Adding %d network members to coredns", len(net.Members)) + for _, member := range net.Members { + hostsEntry := fmt.Sprintf("%s %s", member.IP.String(), member.Name) + if err := corednsAddHost(ctx, runtime, cluster, member.IP.String(), member.Name); err != nil { + return fmt.Errorf("failed to add host entry \"%s\" into CoreDNS: %v", hostsEntry, err) } - } + return nil } func prepCreateLocalRegistryHostingConfigMap(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error { diff --git a/pkg/config/process.go b/pkg/config/process.go index 244c72b3..a2dda855 100644 --- a/pkg/config/process.go +++ b/pkg/config/process.go @@ -38,9 +38,6 @@ func ProcessClusterConfig(clusterConfig conf.ClusterConfig) (*conf.ClusterConfig l.Log().Debugf("Host network was chosen, changing provided/random api port to k3s:%s", k3sPort) cluster.KubeAPI.PortMapping.Binding.HostPort = k3sPort - // if network is host, dont inject docker host into the cluster - clusterConfig.ClusterCreateOpts.PrepDisableHostIPInjection = true - // if network is host, disable load balancer // serverlb not supported in hostnetwork mode due to port collisions with server node clusterConfig.ClusterCreateOpts.DisableLoadBalancer = true diff --git a/pkg/config/process_test.go b/pkg/config/process_test.go index c4d890ed..4d68355e 100644 --- a/pkg/config/process_test.go +++ b/pkg/config/process_test.go @@ -55,7 +55,6 @@ func TestProcessClusterConfig(t *testing.T) { clusterCfg, err = ProcessClusterConfig(*clusterCfg) assert.Assert(t, clusterCfg.ClusterCreateOpts.DisableLoadBalancer == false, "The load balancer should be enabled") - assert.Assert(t, clusterCfg.ClusterCreateOpts.PrepDisableHostIPInjection == false, "The host ip injection should be enabled") t.Logf("\n===== Resulting Cluster Config (non-host network) =====\n%+v\n===============\n", clusterCfg) @@ -64,7 +63,6 @@ func TestProcessClusterConfig(t *testing.T) { clusterCfg.Cluster.Network.Name = "host" clusterCfg, err = ProcessClusterConfig(*clusterCfg) assert.Assert(t, clusterCfg.ClusterCreateOpts.DisableLoadBalancer == true, "The load balancer should be disabled") - assert.Assert(t, clusterCfg.ClusterCreateOpts.PrepDisableHostIPInjection == true, "The host ip injection should be disabled") t.Logf("\n===== Resulting Cluster Config (host network) =====\n%+v\n===============\n", clusterCfg) diff --git a/pkg/config/transform.go b/pkg/config/transform.go index 8d5a35bf..949448f7 100644 --- a/pkg/config/transform.go +++ b/pkg/config/transform.go @@ -261,16 +261,15 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim **************************/ clusterCreateOpts := k3d.ClusterCreateOpts{ - PrepDisableHostIPInjection: simpleConfig.Options.K3dOptions.PrepDisableHostIPInjection, - DisableImageVolume: simpleConfig.Options.K3dOptions.DisableImageVolume, - WaitForServer: simpleConfig.Options.K3dOptions.Wait, - Timeout: simpleConfig.Options.K3dOptions.Timeout, - DisableLoadBalancer: simpleConfig.Options.K3dOptions.DisableLoadbalancer, - GPURequest: simpleConfig.Options.Runtime.GPURequest, - ServersMemory: simpleConfig.Options.Runtime.ServersMemory, - AgentsMemory: simpleConfig.Options.Runtime.AgentsMemory, - GlobalLabels: map[string]string{}, // empty init - GlobalEnv: []string{}, // empty init + DisableImageVolume: simpleConfig.Options.K3dOptions.DisableImageVolume, + WaitForServer: simpleConfig.Options.K3dOptions.Wait, + Timeout: simpleConfig.Options.K3dOptions.Timeout, + DisableLoadBalancer: simpleConfig.Options.K3dOptions.DisableLoadbalancer, + GPURequest: simpleConfig.Options.Runtime.GPURequest, + ServersMemory: simpleConfig.Options.Runtime.ServersMemory, + AgentsMemory: simpleConfig.Options.Runtime.AgentsMemory, + GlobalLabels: map[string]string{}, // empty init + GlobalEnv: []string{}, // empty init } // ensure, that we have the default object labels diff --git a/pkg/config/v1alpha3/schema.json b/pkg/config/v1alpha3/schema.json index 93a81b75..5672dc47 100644 --- a/pkg/config/v1alpha3/schema.json +++ b/pkg/config/v1alpha3/schema.json @@ -137,10 +137,6 @@ "disableRollback": { "type": "boolean", "default": false - }, - "disableHostIPInjection": { - "type": "boolean", - "default": false } }, "additionalProperties": false diff --git a/pkg/config/v1alpha3/types.go b/pkg/config/v1alpha3/types.go index 9783d1c1..d15c2823 100644 --- a/pkg/config/v1alpha3/types.go +++ b/pkg/config/v1alpha3/types.go @@ -102,13 +102,12 @@ type SimpleConfigOptionsRuntime struct { } type SimpleConfigOptionsK3d struct { - Wait bool `mapstructure:"wait" yaml:"wait"` - Timeout time.Duration `mapstructure:"timeout" yaml:"timeout"` - DisableLoadbalancer bool `mapstructure:"disableLoadbalancer" yaml:"disableLoadbalancer"` - DisableImageVolume bool `mapstructure:"disableImageVolume" yaml:"disableImageVolume"` - NoRollback bool `mapstructure:"disableRollback" yaml:"disableRollback"` - PrepDisableHostIPInjection bool `mapstructure:"disableHostIPInjection" yaml:"disableHostIPInjection"` - NodeHookActions []k3d.NodeHookAction `mapstructure:"nodeHookActions" yaml:"nodeHookActions,omitempty"` + Wait bool `mapstructure:"wait" yaml:"wait"` + Timeout time.Duration `mapstructure:"timeout" yaml:"timeout"` + DisableLoadbalancer bool `mapstructure:"disableLoadbalancer" yaml:"disableLoadbalancer"` + DisableImageVolume bool `mapstructure:"disableImageVolume" yaml:"disableImageVolume"` + NoRollback bool `mapstructure:"disableRollback" yaml:"disableRollback"` + NodeHookActions []k3d.NodeHookAction `mapstructure:"nodeHookActions" yaml:"nodeHookActions,omitempty"` } type SimpleConfigOptionsK3s struct { diff --git a/pkg/runtimes/docker/network.go b/pkg/runtimes/docker/network.go index 8b79ccd9..43751857 100644 --- a/pkg/runtimes/docker/network.go +++ b/pkg/runtimes/docker/network.go @@ -73,25 +73,31 @@ func (d Docker) GetNetwork(ctx context.Context, searchNet *k3d.ClusterNetwork) ( return nil, runtimeErr.ErrRuntimeNetworkNotExists } + targetNetwork, err := docker.NetworkInspect(ctx, networkList[0].ID, types.NetworkInspectOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to inspect network %s: %w", networkList[0].Name, err) + } + l.Log().Debugf("Found network %+v", targetNetwork) + network := &k3d.ClusterNetwork{ - Name: networkList[0].Name, - ID: networkList[0].ID, + Name: targetNetwork.Name, + ID: targetNetwork.ID, } // for networks that have an IPAM config, we inspect that as well (e.g. "host" network doesn't have it) - if len(networkList[0].IPAM.Config) > 0 { - network.IPAM, err = d.parseIPAM(networkList[0].IPAM.Config[0]) + if len(targetNetwork.IPAM.Config) > 0 { + network.IPAM, err = d.parseIPAM(targetNetwork.IPAM.Config[0]) if err != nil { return nil, err } - for _, container := range networkList[0].Containers { + for _, container := range targetNetwork.Containers { if container.IPv4Address != "" { - ip, err := netaddr.ParseIP(container.IPv4Address) + prefix, err := netaddr.ParseIPPrefix(container.IPv4Address) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to parse IP of container %s: %w", container.Name, err) } - network.IPAM.IPsUsed = append(network.IPAM.IPsUsed, ip) + network.IPAM.IPsUsed = append(network.IPAM.IPsUsed, prefix.IP) } } @@ -105,6 +111,17 @@ func (d Docker) GetNetwork(ctx context.Context, searchNet *k3d.ClusterNetwork) ( l.Log().Debugf("Network %s does not have an IPAM config", network.Name) } + for _, container := range targetNetwork.Containers { + prefix, err := netaddr.ParseIPPrefix(container.IPv4Address) + if err != nil { + return nil, fmt.Errorf("failed to parse IP Prefix of network \"%s\"'s member %s: %v", network.Name, container.Name, err) + } + network.Members = append(network.Members, &k3d.NetworkMember{ + Name: container.Name, + IP: prefix.IP, + }) + } + // Only one Network allowed, but some functions don't care about this, so they can ignore the error and just use the first one returned if len(networkList) > 1 { return network, runtimeErr.ErrRuntimeNetworkMultiSameName diff --git a/pkg/runtimes/docker/translate.go b/pkg/runtimes/docker/translate.go index ef27bdc0..ff4c7370 100644 --- a/pkg/runtimes/docker/translate.go +++ b/pkg/runtimes/docker/translate.go @@ -27,6 +27,7 @@ import ( "fmt" "strings" + "github.com/containerd/containerd/log" "github.com/docker/docker/api/types" docker "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/network" @@ -35,6 +36,7 @@ import ( runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors" k3d "github.com/rancher/k3d/v4/pkg/types" "github.com/rancher/k3d/v4/pkg/types/fixes" + "inet.af/netaddr" dockercliopts "github.com/docker/cli/opts" dockerunits "github.com/docker/go-units" @@ -263,6 +265,41 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d memoryStr = "" } + // IP + var nodeIP k3d.NodeIP + var clusterNet *network.EndpointSettings + if netLabel, ok := labels[k3d.LabelNetwork]; ok { + for netName, net := range containerDetails.NetworkSettings.Networks { + if netName == netLabel { + clusterNet = net + } + } + } else { + l.Log().Debugf("no netlabel present on container %s", containerDetails.Name) + } + if clusterNet != nil { + parsedIP, err := netaddr.ParseIP(clusterNet.IPAddress) + if err != nil { + if nodeState.Running { + return nil, fmt.Errorf("failed to parse IP '%s' for container '%s': %s\nStatus: %v\n%+v", clusterNet.IPAddress, containerDetails.Name, err, nodeState.Status, containerDetails.NetworkSettings) + } else { + log.L.Debugf("failed to parse IP '%s' for container '%s', likely because it's not running: %v", clusterNet.IPAddress, containerDetails.Name, err) + } + } + isStaticIP := false + if staticIPLabel, ok := labels[k3d.LabelNodeStaticIP]; ok && staticIPLabel != "" { + isStaticIP = true + } + if !parsedIP.IsZero() { + nodeIP = k3d.NodeIP{ + IP: parsedIP, + Static: isStaticIP, + } + } + } else { + l.Log().Debugf("failed to get IP for container %s as we couldn't find the cluster network", containerDetails.Name) + } + node := &k3d.Node{ Name: strings.TrimPrefix(containerDetails.Name, "/"), // container name with leading '/' cut off Role: k3d.NodeRoles[containerDetails.Config.Labels[k3d.LabelRole]], @@ -280,6 +317,7 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d AgentOpts: k3d.AgentOpts{}, State: nodeState, Memory: memoryStr, + IP: nodeIP, // only valid for the cluster network } return node, nil } diff --git a/pkg/types/types.go b/pkg/types/types.go index 63092502..0cf0cd91 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -164,18 +164,17 @@ var DoNotCopyServerFlags = []string{ // ClusterCreateOpts describe a set of options one can set when creating a cluster type ClusterCreateOpts struct { - PrepDisableHostIPInjection bool `yaml:"prepDisableHostIPInjection" json:"prepDisableHostIPInjection,omitempty"` - DisableImageVolume bool `yaml:"disableImageVolume" json:"disableImageVolume,omitempty"` - WaitForServer bool `yaml:"waitForServer" json:"waitForServer,omitempty"` - Timeout time.Duration `yaml:"timeout" json:"timeout,omitempty"` - DisableLoadBalancer bool `yaml:"disableLoadbalancer" json:"disableLoadbalancer,omitempty"` - GPURequest string `yaml:"gpuRequest" json:"gpuRequest,omitempty"` - ServersMemory string `yaml:"serversMemory" json:"serversMemory,omitempty"` - AgentsMemory string `yaml:"agentsMemory" json:"agentsMemory,omitempty"` - NodeHooks []NodeHook `yaml:"nodeHooks,omitempty" json:"nodeHooks,omitempty"` - GlobalLabels map[string]string `yaml:"globalLabels,omitempty" json:"globalLabels,omitempty"` - GlobalEnv []string `yaml:"globalEnv,omitempty" json:"globalEnv,omitempty"` - Registries struct { + DisableImageVolume bool `yaml:"disableImageVolume" json:"disableImageVolume,omitempty"` + WaitForServer bool `yaml:"waitForServer" json:"waitForServer,omitempty"` + Timeout time.Duration `yaml:"timeout" json:"timeout,omitempty"` + DisableLoadBalancer bool `yaml:"disableLoadbalancer" json:"disableLoadbalancer,omitempty"` + GPURequest string `yaml:"gpuRequest" json:"gpuRequest,omitempty"` + ServersMemory string `yaml:"serversMemory" json:"serversMemory,omitempty"` + AgentsMemory string `yaml:"agentsMemory" json:"agentsMemory,omitempty"` + NodeHooks []NodeHook `yaml:"nodeHooks,omitempty" json:"nodeHooks,omitempty"` + GlobalLabels map[string]string `yaml:"globalLabels,omitempty" json:"globalLabels,omitempty"` + GlobalEnv []string `yaml:"globalEnv,omitempty" json:"globalEnv,omitempty"` + Registries struct { Create *Registry `yaml:"create,omitempty" json:"create,omitempty"` Use []*Registry `yaml:"use,omitempty" json:"use,omitempty"` Config *k3s.Registry `yaml:"config,omitempty" json:"config,omitempty"` // registries.yaml (k3s config for containerd registry override) @@ -246,12 +245,18 @@ type IPAM struct { Managed bool // IPAM is done by k3d } +type NetworkMember struct { + Name string + IP netaddr.IP +} + // ClusterNetwork describes a network which a cluster is running in type ClusterNetwork struct { Name string `yaml:"name" json:"name,omitempty"` ID string `yaml:"id" json:"id"` // may be the same as name, but e.g. docker only differentiates by random ID, not by name External bool `yaml:"external" json:"isExternal,omitempty"` IPAM IPAM `yaml:"ipam" json:"ipam,omitempty"` + Members []*NetworkMember } // Cluster describes a k3d cluster @@ -323,7 +328,7 @@ type Node struct { GPURequest string // filled automatically Memory string // filled automatically State NodeState // filled automatically - IP NodeIP // filled automatically + IP NodeIP // filled automatically -> refers solely to the cluster network HookActions []NodeHook `yaml:"hooks" json:"hooks,omitempty"` }