[Enhancement] Refactoring: normalize label flags (k3s node & runtime) (#598, @ejose19)

cmd/cluster/clusterCreate.go

@@ -283,8 +283,11 @@ func NewCmdClusterCreate() *cobra.Command {
 	cmd.Flags().StringArrayP("port", "p", nil, "Map ports from the node containers to the host (Format: `[HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]`)\n - Example: `k3d cluster create --agents 2 -p 8080:80@agent[0] -p 8081@agent[1]`")
 	_ = ppViper.BindPFlag("cli.ports", cmd.Flags().Lookup("port"))
 
-	cmd.Flags().StringArrayP("label", "l", nil, "Add label to node container (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -l \"my.label@agent[0,1]\" -l \"other.label=somevalue@server[0]\"`")
-	_ = ppViper.BindPFlag("cli.labels", cmd.Flags().Lookup("label"))
+	cmd.Flags().StringArrayP("k3s-node-label", "", nil, "Add label to k3s node (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --k3s-node-label \"my.label@agent[0,1]\" --k3s-node-label \"other.label=somevalue@server[0]\"`")
+	_ = ppViper.BindPFlag("cli.k3s-node-labels", cmd.Flags().Lookup("k3s-node-label"))
+
+	cmd.Flags().StringArrayP("runtime-label", "", nil, "Add label to container runtime (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --runtime-label \"my.label@agent[0,1]\" --runtime-label \"other.label=somevalue@server[0]\"`")
+	_ = ppViper.BindPFlag("cli.runtime-labels", cmd.Flags().Lookup("runtime-label"))
 
 	/* k3s */
 	cmd.Flags().StringArray("k3s-arg", nil, "Additional args passed to k3s command (Format: `ARG@NODEFILTER[;@NODEFILTER]`)\n - Example: `k3d cluster create --k3s-arg \"--disable=traefik@server[0]\"")
@@ -481,10 +484,38 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 
 	log.Tracef("PortFilterMap: %+v", portFilterMap)
 
-	// --label
-	// labelFilterMap will add container label to applied node filters
-	labelFilterMap := make(map[string][]string, 1)
-	for _, labelFlag := range ppViper.GetStringSlice("cli.labels") {
+	// --k3s-node-label
+	// k3sNodeLabelFilterMap will add k3s node label to applied node filters
+	k3sNodeLabelFilterMap := make(map[string][]string, 1)
+	for _, labelFlag := range ppViper.GetStringSlice("cli.k3s-node-labels") {
+
+		// split node filter from the specified label
+		label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag)
+		if err != nil {
+			log.Fatalln(err)
+		}
+
+		// create new entry or append filter to existing entry
+		if _, exists := k3sNodeLabelFilterMap[label]; exists {
+			k3sNodeLabelFilterMap[label] = append(k3sNodeLabelFilterMap[label], nodeFilters...)
+		} else {
+			k3sNodeLabelFilterMap[label] = nodeFilters
+		}
+	}
+
+	for label, nodeFilters := range k3sNodeLabelFilterMap {
+		cfg.Options.K3sOptions.NodeLabels = append(cfg.Options.K3sOptions.NodeLabels, conf.LabelWithNodeFilters{
+			Label:       label,
+			NodeFilters: nodeFilters,
+		})
+	}
+
+	log.Tracef("K3sNodeLabelFilterMap: %+v", k3sNodeLabelFilterMap)
+
+	// --runtime-label
+	// runtimeLabelFilterMap will add container runtime label to applied node filters
+	runtimeLabelFilterMap := make(map[string][]string, 1)
+	for _, labelFlag := range ppViper.GetStringSlice("cli.runtime-labels") {
 
 		// split node filter from the specified label
 		label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag)
@@ -492,22 +523,24 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 			log.Fatalln(err)
 		}
 
+		cliutil.ValidateRuntimeLabelKey(strings.Split(label, "=")[0])
+
 		// create new entry or append filter to existing entry
-		if _, exists := labelFilterMap[label]; exists {
-			labelFilterMap[label] = append(labelFilterMap[label], nodeFilters...)
+		if _, exists := runtimeLabelFilterMap[label]; exists {
+			runtimeLabelFilterMap[label] = append(runtimeLabelFilterMap[label], nodeFilters...)
 		} else {
-			labelFilterMap[label] = nodeFilters
+			runtimeLabelFilterMap[label] = nodeFilters
 		}
 	}
 
-	for label, nodeFilters := range labelFilterMap {
-		cfg.Labels = append(cfg.Labels, conf.LabelWithNodeFilters{
+	for label, nodeFilters := range runtimeLabelFilterMap {
+		cfg.Options.Runtime.Labels = append(cfg.Options.Runtime.Labels, conf.LabelWithNodeFilters{
 			Label:       label,
 			NodeFilters: nodeFilters,
 		})
 	}
 
-	log.Tracef("LabelFilterMap: %+v", labelFilterMap)
+	log.Tracef("RuntimeLabelFilterMap: %+v", runtimeLabelFilterMap)
 
 	// --env
 	// envFilterMap will add container env vars to applied node filters

cmd/node/nodeCreate.go

@@ -30,6 +30,7 @@ import (
 
 	dockerunits "github.com/docker/go-units"
 	"github.com/rancher/k3d/v4/cmd/util"
+	cliutil "github.com/rancher/k3d/v4/cmd/util"
 	k3dc "github.com/rancher/k3d/v4/pkg/client"
 	"github.com/rancher/k3d/v4/pkg/runtimes"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
@@ -74,6 +75,7 @@ func NewCmdNodeCreate() *cobra.Command {
 	cmd.Flags().BoolVar(&createNodeOpts.Wait, "wait", false, "Wait for the node(s) to be ready before returning.")
 	cmd.Flags().DurationVar(&createNodeOpts.Timeout, "timeout", 0*time.Second, "Maximum waiting time for '--wait' before canceling/returning.")
 
+	cmd.Flags().StringSliceP("runtime-label", "", []string{}, "Specify container runtime labels in format \"foo=bar\"")
 	cmd.Flags().StringSliceP("k3s-node-label", "", []string{}, "Specify k3s node labels in format \"foo=bar\"")
 
 	// done
@@ -127,9 +129,30 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
 		log.Errorf("Provided memory limit value is invalid")
 	}
 
+	// --runtime-label
+	runtimeLabelsFlag, err := cmd.Flags().GetStringSlice("runtime-label")
+	if err != nil {
+		log.Errorln("No runtime-label specified")
+		log.Fatalln(err)
+	}
+
+	runtimeLabels := make(map[string]string, len(runtimeLabelsFlag)+1)
+	for _, label := range runtimeLabelsFlag {
+		labelSplitted := strings.Split(label, "=")
+		if len(labelSplitted) != 2 {
+			log.Fatalf("unknown runtime-label format format: %s, use format \"foo=bar\"", label)
+		}
+		cliutil.ValidateRuntimeLabelKey(labelSplitted[0])
+		runtimeLabels[labelSplitted[0]] = labelSplitted[1]
+	}
+
+	// Internal k3d runtime labels take precedence over user-defined labels
+	runtimeLabels[k3d.LabelRole] = roleStr
+
+	// --k3s-node-label
 	k3sNodeLabelsFlag, err := cmd.Flags().GetStringSlice("k3s-node-label")
 	if err != nil {
-		log.Errorln("No node-label specified")
+		log.Errorln("No k3s-node-label specified")
 		log.Fatalln(err)
 	}
 
@@ -137,7 +160,7 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
 	for _, label := range k3sNodeLabelsFlag {
 		labelSplitted := strings.Split(label, "=")
 		if len(labelSplitted) != 2 {
-			log.Fatalf("unknown label format format: %s, use format \"foo=bar\"", label)
+			log.Fatalf("unknown k3s-node-label format format: %s, use format \"foo=bar\"", label)
 		}
 		k3sNodeLabels[labelSplitted[0]] = labelSplitted[1]
 	}
@@ -146,13 +169,11 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
 	nodes := []*k3d.Node{}
 	for i := 0; i < replicas; i++ {
 		node := &k3d.Node{
-			Name:  fmt.Sprintf("%s-%s-%d", k3d.DefaultObjectNamePrefix, args[0], i),
-			Role:  role,
-			Image: image,
-			Labels: map[string]string{
-				k3d.LabelRole: roleStr,
-			},
+			Name:          fmt.Sprintf("%s-%s-%d", k3d.DefaultObjectNamePrefix, args[0], i),
+			Role:          role,
+			Image:         image,
 			K3sNodeLabels: k3sNodeLabels,
+			RuntimeLabels: runtimeLabels,
 			Restart:       true,
 			Memory:        memory,
 		}

cmd/node/nodeList.go

@@ -88,7 +88,7 @@ func NewCmdNodeList() *cobra.Command {
 					fmt.Fprintf(tabwriter, "%s\t%s\t%s\t%s\n",
 						strings.TrimPrefix(node.Name, "/"),
 						string(node.Role),
-						node.Labels[k3d.LabelClusterName],
+						node.RuntimeLabels[k3d.LabelClusterName],
 						node.State.Status)
 				}))
 		},

cmd/registry/registryList.go

@@ -88,8 +88,8 @@ func NewCmdRegistryList() *cobra.Command {
 			util.PrintNodes(existingNodes, registryListFlags.output,
 				headers, util.NodePrinterFunc(func(tabwriter *tabwriter.Writer, node *k3d.Node) {
 					cluster := "*"
-					if _, ok := node.Labels[k3d.LabelClusterName]; ok {
-						cluster = node.Labels[k3d.LabelClusterName]
+					if _, ok := node.RuntimeLabels[k3d.LabelClusterName]; ok {
+						cluster = node.RuntimeLabels[k3d.LabelClusterName]
 					}
 					fmt.Fprintf(tabwriter, "%s\t%s\t%s\t%s\n",
 						strings.TrimPrefix(node.Name, "/"),

cmd/util/runtimeLabels.go

@@ -0,0 +1,35 @@
+/*
+Copyright © 2020 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package util
+
+import (
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// validateRuntimeLabelKey validates a given label key is not reserved for internal k3d usage
+func ValidateRuntimeLabelKey(labelKey string) {
+	if strings.HasPrefix(labelKey, "k3s.") || strings.HasPrefix(labelKey, "k3d.") || labelKey == "app" {
+		log.Fatalf("runtime label \"%s\" is reserved for internal usage", labelKey)
+	}
+}

pkg/client/cluster.go

@@ -381,13 +381,13 @@ ClusterCreatOpts:
 
 	nodeSetup := func(node *k3d.Node, suffix int) error {
 		// cluster specific settings
-		if node.Labels == nil {
-			node.Labels = make(map[string]string) // TODO: maybe create an init function?
+		if node.RuntimeLabels == nil {
+			node.RuntimeLabels = make(map[string]string) // TODO: maybe create an init function?
 		}
 
 		// ensure global labels
 		for k, v := range clusterCreateOpts.GlobalLabels {
-			node.Labels[k] = v
+			node.RuntimeLabels[k] = v
 		}
 
 		// ensure global env
@@ -404,7 +404,7 @@ ClusterCreatOpts:
 				cluster.Network.IPAM.IPsUsed = append(cluster.Network.IPAM.IPsUsed, ip) // make sure that we're not reusing the same IP next time
 				node.IP.Static = true
 				node.IP.IP = ip
-				node.Labels[k3d.LabelNodeStaticIP] = ip.String()
+				node.RuntimeLabels[k3d.LabelNodeStaticIP] = ip.String()
 			}
 
 			node.ServerOpts.KubeAPI = cluster.KubeAPI
@@ -412,7 +412,7 @@ ClusterCreatOpts:
 			// the cluster has an init server node, but its not this one, so connect it to the init node
 			if cluster.InitNode != nil && !node.ServerOpts.IsInit {
 				node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", connectionURL))
-				node.Labels[k3d.LabelServerIsInit] = "false" // set label, that this server node is not the init server
+				node.RuntimeLabels[k3d.LabelServerIsInit] = "false" // set label, that this server node is not the init server
 			}
 
 		} else if node.Role == k3d.AgentRole {
@@ -446,10 +446,10 @@ ClusterCreatOpts:
 	if cluster.InitNode != nil {
 		log.Infoln("Creating initializing server node")
 		cluster.InitNode.Args = append(cluster.InitNode.Args, "--cluster-init")
-		if cluster.InitNode.Labels == nil {
-			cluster.InitNode.Labels = map[string]string{}
+		if cluster.InitNode.RuntimeLabels == nil {
+			cluster.InitNode.RuntimeLabels = map[string]string{}
 		}
-		cluster.InitNode.Labels[k3d.LabelServerIsInit] = "true" // set label, that this server node is the init server
+		cluster.InitNode.RuntimeLabels[k3d.LabelServerIsInit] = "true" // set label, that this server node is the init server
 
 		// in case the LoadBalancer was disabled, expose the API Port on the initializing server node
 		if clusterCreateOpts.DisableLoadBalancer {
@@ -547,10 +547,10 @@ ClusterCreatOpts:
 				fmt.Sprintf("PORTS=%s", strings.Join(ports, ",")),
 				fmt.Sprintf("WORKER_PROCESSES=%d", len(ports)),
 			},
-			Role:     k3d.LoadBalancerRole,
-			Labels:   clusterCreateOpts.GlobalLabels, // TODO: createLoadBalancer: add more expressive labels
-			Networks: []string{cluster.Network.Name},
-			Restart:  true,
+			Role:          k3d.LoadBalancerRole,
+			RuntimeLabels: clusterCreateOpts.GlobalLabels, // TODO: createLoadBalancer: add more expressive labels
+			Networks:      []string{cluster.Network.Name},
+			Restart:       true,
 		}
 		if len(udp_ports) > 0 {
 			lbNode.Env = append(lbNode.Env, fmt.Sprintf("UDP_PORTS=%s", strings.Join(udp_ports, ",")))
@@ -673,7 +673,7 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 // ClusterList returns a list of all existing clusters
 func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, error) {
 	log.Traceln("Listing Clusters...")
-	nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultObjectLabels)
+	nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultRuntimeLabels)
 	if err != nil {
 		log.Errorln("Failed to get clusters")
 		return nil, err
@@ -691,7 +691,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er
 	log.Tracef("Found %d cluster-internal nodes", len(nodes))
 	if log.GetLevel() == log.TraceLevel {
 		for _, node := range nodes {
-			log.Tracef("Found cluster-internal node %s of role %s belonging to cluster %s", node.Name, node.Role, node.Labels[k3d.LabelClusterName])
+			log.Tracef("Found cluster-internal node %s of role %s belonging to cluster %s", node.Name, node.Role, node.RuntimeLabels[k3d.LabelClusterName])
 		}
 	}
 
@@ -700,7 +700,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er
 	for _, node := range nodes {
 		clusterExists := false
 		for _, cluster := range clusters {
-			if node.Labels[k3d.LabelClusterName] == cluster.Name { // TODO: handle case, where this label doesn't exist
+			if node.RuntimeLabels[k3d.LabelClusterName] == cluster.Name { // TODO: handle case, where this label doesn't exist
 				cluster.Nodes = append(cluster.Nodes, node)
 				clusterExists = true
 				break
@@ -709,7 +709,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er
 		// cluster is not in the list yet, so we add it with the current node as its first member
 		if !clusterExists {
 			clusters = append(clusters, &k3d.Cluster{
-				Name:  node.Labels[k3d.LabelClusterName],
+				Name:  node.RuntimeLabels[k3d.LabelClusterName],
 				Nodes: []*k3d.Node{node},
 			})
 		}
@@ -734,7 +734,7 @@ func populateClusterFieldsFromLabels(cluster *k3d.Cluster) error {
 
 		// get the name of the cluster network
 		if cluster.Network.Name == "" {
-			if networkName, ok := node.Labels[k3d.LabelNetwork]; ok {
+			if networkName, ok := node.RuntimeLabels[k3d.LabelNetwork]; ok {
 				cluster.Network.Name = networkName
 			}
 		}
@@ -742,7 +742,7 @@ func populateClusterFieldsFromLabels(cluster *k3d.Cluster) error {
 		// check if the network is external
 		// since the struct value is a bool, initialized as false, we cannot check if it's unset
 		if !cluster.Network.External && !networkExternalSet {
-			if networkExternalString, ok := node.Labels[k3d.LabelNetworkExternal]; ok {
+			if networkExternalString, ok := node.RuntimeLabels[k3d.LabelNetworkExternal]; ok {
 				if networkExternal, err := strconv.ParseBool(networkExternalString); err == nil {
 					cluster.Network.External = networkExternal
 					networkExternalSet = true
@@ -752,14 +752,14 @@ func populateClusterFieldsFromLabels(cluster *k3d.Cluster) error {
 
 		// get image volume // TODO: enable external image volumes the same way we do it with networks
 		if cluster.ImageVolume == "" {
-			if imageVolumeName, ok := node.Labels[k3d.LabelImageVolume]; ok {
+			if imageVolumeName, ok := node.RuntimeLabels[k3d.LabelImageVolume]; ok {
 				cluster.ImageVolume = imageVolumeName
 			}
 		}
 
 		// get k3s cluster's token
 		if cluster.Token == "" {
-			if token, ok := node.Labels[k3d.LabelClusterToken]; ok {
+			if token, ok := node.RuntimeLabels[k3d.LabelClusterToken]; ok {
 				cluster.Token = token
 			}
 		}

pkg/client/kubeconfig.go

@@ -131,11 +131,11 @@ func KubeconfigGet(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.C
 	APIHost := k3d.DefaultAPIHost
 
 	for _, server := range serverNodes {
-		if _, ok := server.Labels[k3d.LabelServerAPIPort]; ok {
+		if _, ok := server.RuntimeLabels[k3d.LabelServerAPIPort]; ok {
 			chosenServer = server
-			APIPort = server.Labels[k3d.LabelServerAPIPort]
-			if _, ok := server.Labels[k3d.LabelServerAPIHost]; ok {
-				APIHost = server.Labels[k3d.LabelServerAPIHost]
+			APIPort = server.RuntimeLabels[k3d.LabelServerAPIPort]
+			if _, ok := server.RuntimeLabels[k3d.LabelServerAPIHost]; ok {
+				APIHost = server.RuntimeLabels[k3d.LabelServerAPIHost]
 			}
 			break
 		}

pkg/client/node.go

@@ -59,8 +59,8 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
 	node.Networks = []string{cluster.Network.Name}
 
 	// skeleton
-	if node.Labels == nil {
-		node.Labels = map[string]string{
+	if node.RuntimeLabels == nil {
+		node.RuntimeLabels = map[string]string{
 			k3d.LabelRole: string(node.Role),
 		}
 	}
@@ -141,7 +141,7 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
 		}
 	}
 	if !k3sURLFound {
-		if url, ok := node.Labels[k3d.LabelClusterURL]; ok {
+		if url, ok := node.RuntimeLabels[k3d.LabelClusterURL]; ok {
 			node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", url))
 		} else {
 			log.Warnln("Failed to find K3S_URL value!")
@@ -381,18 +381,22 @@ func NodeCreate(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, c
 
 	// ### Labels ###
 	labels := make(map[string]string)
-	for k, v := range k3d.DefaultObjectLabels {
+	for k, v := range k3d.DefaultRuntimeLabels {
 		labels[k] = v
 	}
-	for k, v := range k3d.DefaultObjectLabelsVar {
+	for k, v := range k3d.DefaultRuntimeLabelsVar {
 		labels[k] = v
 	}
-	for k, v := range node.Labels {
+	for k, v := range node.RuntimeLabels {
 		labels[k] = v
 	}
-	node.Labels = labels
+	node.RuntimeLabels = labels
 	// second most important: the node role label
-	node.Labels[k3d.LabelRole] = string(node.Role)
+	node.RuntimeLabels[k3d.LabelRole] = string(node.Role)
+
+	for k, v := range node.K3sNodeLabels {
+		node.Args = append(node.Args, "--node-label", fmt.Sprintf("%s=%s", k, v))
+	}
 
 	// ### Environment ###
 	node.Env = append(node.Env, k3d.DefaultNodeEnv...) // append default node env vars
@@ -469,7 +473,7 @@ func NodeDelete(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, o
 
 	// update the server loadbalancer
 	if !opts.SkipLBUpdate && (node.Role == k3d.ServerRole || node.Role == k3d.AgentRole) {
-		cluster, err := ClusterGet(ctx, runtime, &k3d.Cluster{Name: node.Labels[k3d.LabelClusterName]})
+		cluster, err := ClusterGet(ctx, runtime, &k3d.Cluster{Name: node.RuntimeLabels[k3d.LabelClusterName]})
 		if err != nil {
 			log.Errorf("Failed to find cluster for node '%s'", node.Name)
 			return err
@@ -493,10 +497,6 @@ func patchAgentSpec(node *k3d.Node) error {
 		node.Cmd = []string{"agent"}
 	}
 
-	for k, v := range node.K3sNodeLabels {
-		node.Args = append(node.Args, "--node-label", fmt.Sprintf("%s=%s", k, v))
-	}
-
 	return nil
 }
 
@@ -509,9 +509,9 @@ func patchServerSpec(node *k3d.Node, runtime runtimes.Runtime) error {
 
 	// Add labels and TLS SAN for the exposed API
 	// FIXME: For now, the labels concerning the API on the server nodes are only being used for configuring the kubeconfig
-	node.Labels[k3d.LabelServerAPIHostIP] = node.ServerOpts.KubeAPI.Binding.HostIP // TODO: maybe get docker machine IP here
-	node.Labels[k3d.LabelServerAPIHost] = node.ServerOpts.KubeAPI.Host
-	node.Labels[k3d.LabelServerAPIPort] = node.ServerOpts.KubeAPI.Binding.HostPort
+	node.RuntimeLabels[k3d.LabelServerAPIHostIP] = node.ServerOpts.KubeAPI.Binding.HostIP // TODO: maybe get docker machine IP here
+	node.RuntimeLabels[k3d.LabelServerAPIHost] = node.ServerOpts.KubeAPI.Host
+	node.RuntimeLabels[k3d.LabelServerAPIPort] = node.ServerOpts.KubeAPI.Binding.HostPort
 
 	// If the runtime is docker, attempt to use the docker host
 	if runtime == runtimes.Docker {
@@ -519,19 +519,19 @@ func patchServerSpec(node *k3d.Node, runtime runtimes.Runtime) error {
 		if dockerHost != "" {
 			dockerHost = strings.Split(dockerHost, ":")[0] // remove the port
 			log.Tracef("Using docker host %s", dockerHost)
-			node.Labels[k3d.LabelServerAPIHostIP] = dockerHost
-			node.Labels[k3d.LabelServerAPIHost] = dockerHost
+			node.RuntimeLabels[k3d.LabelServerAPIHostIP] = dockerHost
+			node.RuntimeLabels[k3d.LabelServerAPIHost] = dockerHost
 		}
 	}
 
-	node.Args = append(node.Args, "--tls-san", node.Labels[k3d.LabelServerAPIHost]) // add TLS SAN for non default host name
+	node.Args = append(node.Args, "--tls-san", node.RuntimeLabels[k3d.LabelServerAPIHost]) // add TLS SAN for non default host name
 
 	return nil
 }
 
 // NodeList returns a list of all existing clusters
 func NodeList(ctx context.Context, runtime runtimes.Runtime) ([]*k3d.Node, error) {
-	nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultObjectLabels)
+	nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultRuntimeLabels)
 	if err != nil {
 		log.Errorln("Failed to get nodes")
 		return nil, err

pkg/client/registry.go

@@ -77,7 +77,7 @@ func RegistryCreate(ctx context.Context, runtime runtimes.Runtime, reg *k3d.Regi
 	}
 
 	// setup the node labels
-	registryNode.Labels = map[string]string{
+	registryNode.RuntimeLabels = map[string]string{
 		k3d.LabelClusterName:          reg.ClusterRef,
 		k3d.LabelRole:                 string(k3d.RegistryRole),
 		k3d.LabelRegistryHost:         reg.ExposureOpts.Host, // TODO: docker machine host?
@@ -85,11 +85,11 @@ func RegistryCreate(ctx context.Context, runtime runtimes.Runtime, reg *k3d.Regi
 		k3d.LabelRegistryPortExternal: reg.ExposureOpts.Binding.HostPort,
 		k3d.LabelRegistryPortInternal: reg.ExposureOpts.Port.Port(),
 	}
-	for k, v := range k3d.DefaultObjectLabels {
-		registryNode.Labels[k] = v
+	for k, v := range k3d.DefaultRuntimeLabels {
+		registryNode.RuntimeLabels[k] = v
 	}
-	for k, v := range k3d.DefaultObjectLabelsVar {
-		registryNode.Labels[k] = v
+	for k, v := range k3d.DefaultRuntimeLabelsVar {
+		registryNode.RuntimeLabels[k] = v
 	}
 
 	// port

pkg/config/config_test.go

@@ -64,12 +64,6 @@ func TestReadSimpleConfig(t *testing.T) {
 				NodeFilters: []string{"loadbalancer"},
 			},
 		},
-		Labels: []conf.LabelWithNodeFilters{
-			{
-				Label:       "foo=bar",
-				NodeFilters: []string{"server[0]", "loadbalancer"},
-			},
-		},
 		Env: []conf.EnvVarWithNodeFilters{
 			{
 				EnvVar:      "bar=baz",
@@ -90,11 +84,25 @@ func TestReadSimpleConfig(t *testing.T) {
 						NodeFilters: []string{"server[*]"},
 					},
 				},
+				NodeLabels: []conf.LabelWithNodeFilters{
+					{
+						Label:       "foo=bar",
+						NodeFilters: []string{"server[0]", "loadbalancer"},
+					},
+				},
 			},
 			KubeconfigOptions: conf.SimpleConfigOptionsKubeconfig{
 				UpdateDefaultKubeconfig: true,
 				SwitchCurrentContext:    true,
 			},
+			Runtime: conf.SimpleConfigOptionsRuntime{
+				Labels: []conf.LabelWithNodeFilters{
+					{
+						Label:       "foo=bar",
+						NodeFilters: []string{"server[0]", "loadbalancer"},
+					},
+				},
+			},
 		},
 	}
 

pkg/config/test_assets/config_test_simple.yaml

@@ -22,11 +22,6 @@ env:
   - envVar: bar=baz
     nodeFilters:
       - all
-labels:
-  - label: foo=bar
-    nodeFilters:
-      - "server[0]"
-      - loadbalancer
 
 options:
   k3d:
@@ -39,6 +34,17 @@ options:
       - arg: --tls-san=127.0.0.1
         nodeFilters:
           - "server[*]"
+    nodeLabels:
+      - label: foo=bar
+        nodeFilters:
+          - server[0]
+          - loadbalancer
   kubeconfig:
     updateDefaultKubeconfig: true
     switchCurrentContext: true
+  runtime:
+    labels:
+      - label: foo=bar
+        nodeFilters:
+          - server[0]
+          - loadbalancer

pkg/config/test_assets/config_test_simple_invalid_servers.yaml

@@ -22,11 +22,6 @@ env:
   - envVar: bar=baz
     nodeFilters:
       - all
-labels:
-  - label: foo=bar
-    nodeFilters:
-      - server[0]
-      - loadbalancer
 
 options:
   k3d:
@@ -39,6 +34,17 @@ options:
       - arg: --tls-san=127.0.0.1
         nodeFilters:
           - "server[*]"
+    nodeLabels:
+      - label: foo=bar
+        nodeFilters:
+          - server[0]
+          - loadbalancer
   kubeconfig:
     updateDefaultKubeconfig: true
-    switchCurrentContext: true
+    switchCurrentContext: true
+  runtime:
+    labels:
+      - label: foo=bar
+        nodeFilters:
+          - server[0]
+          - loadbalancer

pkg/config/transform.go

@@ -190,23 +190,47 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
 		}
 	}
 
-	// -> LABELS
-	for _, labelWithNodeFilters := range simpleConfig.Labels {
-		if len(labelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
-			return nil, fmt.Errorf("Labelmapping '%s' lacks a node filter, but there's more than one node", labelWithNodeFilters.Label)
+	// -> K3S NODE LABELS
+	for _, k3sNodeLabelWithNodeFilters := range simpleConfig.Options.K3sOptions.NodeLabels {
+		if len(k3sNodeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
+			return nil, fmt.Errorf("K3sNodeLabelmapping '%s' lacks a node filter, but there's more than one node", k3sNodeLabelWithNodeFilters.Label)
 		}
 
-		nodes, err := util.FilterNodes(nodeList, labelWithNodeFilters.NodeFilters)
+		nodes, err := util.FilterNodes(nodeList, k3sNodeLabelWithNodeFilters.NodeFilters)
 		if err != nil {
 			return nil, err
 		}
 
 		for _, node := range nodes {
-			if node.Labels == nil {
-				node.Labels = make(map[string]string) // ensure that the map is initialized
+			if node.K3sNodeLabels == nil {
+				node.K3sNodeLabels = make(map[string]string) // ensure that the map is initialized
 			}
-			k, v := util.SplitLabelKeyValue(labelWithNodeFilters.Label)
-			node.Labels[k] = v
+			k, v := util.SplitLabelKeyValue(k3sNodeLabelWithNodeFilters.Label)
+			node.K3sNodeLabels[k] = v
+
+		}
+	}
+
+	// -> RUNTIME LABELS
+	for _, runtimeLabelWithNodeFilters := range simpleConfig.Options.Runtime.Labels {
+		if len(runtimeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
+			return nil, fmt.Errorf("RuntimeLabelmapping '%s' lacks a node filter, but there's more than one node", runtimeLabelWithNodeFilters.Label)
+		}
+
+		nodes, err := util.FilterNodes(nodeList, runtimeLabelWithNodeFilters.NodeFilters)
+		if err != nil {
+			return nil, err
+		}
+
+		for _, node := range nodes {
+			if node.RuntimeLabels == nil {
+				node.RuntimeLabels = make(map[string]string) // ensure that the map is initialized
+			}
+			k, v := util.SplitLabelKeyValue(runtimeLabelWithNodeFilters.Label)
+
+			cliutil.ValidateRuntimeLabelKey(k)
+
+			node.RuntimeLabels[k] = v
 		}
 	}
 
@@ -260,7 +284,7 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
 	}
 
 	// ensure, that we have the default object labels
-	for k, v := range k3d.DefaultObjectLabels {
+	for k, v := range k3d.DefaultRuntimeLabels {
 		clusterCreateOpts.GlobalLabels[k] = v
 	}
 

pkg/config/v1alpha3/migrations.go

@@ -49,6 +49,15 @@ func MigrateV1Alpha2(input configtypes.Config) (configtypes.Config, error) {
 			return nil, err
 		}
 
+		cfg.Options.Runtime.Labels = []LabelWithNodeFilters{}
+
+		for _, label := range input.(v1alpha2.SimpleConfig).Labels {
+			cfg.Options.Runtime.Labels = append(cfg.Options.Runtime.Labels, LabelWithNodeFilters{
+				Label:       label.Label,
+				NodeFilters: label.NodeFilters,
+			})
+		}
+
 		cfg.Options.K3sOptions.ExtraArgs = []K3sArgWithNodeFilters{}
 
 		for _, arg := range input.(v1alpha2.SimpleConfig).Options.K3sOptions.ExtraServerArgs {

pkg/config/v1alpha3/schema.json

@@ -108,21 +108,6 @@
         "additionalProperties": false
       }
     },
-    "labels": {
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "label": {
-            "type": "string"
-          },
-          "nodeFilters": {  
-            "$ref": "#/definitions/nodeFilters"
-          }
-        },
-        "additionalProperties": false
-      }
-    },
     "options": {
       "type": "object",
       "properties": {
@@ -170,12 +155,24 @@
                 "properties": {
                   "arg": {
                     "type": "string",
-                    "examples": [
-                      "--tls-san=127.0.0.1",
-                      "--disable=traefik"
-                    ]
+                    "examples": ["--tls-san=127.0.0.1", "--disable=traefik"]
                   },
-                  "nodeFilters": {  
+                  "nodeFilters": {
+                    "$ref": "#/definitions/nodeFilters"
+                  }
+                },
+                "additionalProperties": false
+              }
+            },
+            "nodeLabels": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "label": {
+                    "type": "string"
+                  },
+                  "nodeFilters": {
                     "$ref": "#/definitions/nodeFilters"
                   }
                 },
@@ -210,6 +207,21 @@
             },
             "agentsMemory": {
               "type": "string"
+            },
+            "labels": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "label": {
+                    "type": "string"
+                  },
+                  "nodeFilters": {
+                    "$ref": "#/definitions/nodeFilters"
+                  }
+                },
+                "additionalProperties": false
+              }
             }
           }
         }

pkg/config/v1alpha3/types.go

@@ -95,9 +95,10 @@ type SimpleConfigOptions struct {
 }
 
 type SimpleConfigOptionsRuntime struct {
-	GPURequest    string `mapstructure:"gpuRequest" yaml:"gpuRequest"`
-	ServersMemory string `mapstructure:"serversMemory" yaml:"serversMemory"`
-	AgentsMemory  string `mapstructure:"agentsMemory" yaml:"agentsMemory"`
+	GPURequest    string                 `mapstructure:"gpuRequest" yaml:"gpuRequest"`
+	ServersMemory string                 `mapstructure:"serversMemory" yaml:"serversMemory"`
+	AgentsMemory  string                 `mapstructure:"agentsMemory" yaml:"agentsMemory"`
+	Labels        []LabelWithNodeFilters `mapstructure:"labels" yaml:"labels"`
 }
 
 type SimpleConfigOptionsK3d struct {
@@ -111,7 +112,8 @@ type SimpleConfigOptionsK3d struct {
 }
 
 type SimpleConfigOptionsK3s struct {
-	ExtraArgs []K3sArgWithNodeFilters `mapstructure:"extraArgs" yaml:"extraArgs"`
+	ExtraArgs  []K3sArgWithNodeFilters `mapstructure:"extraArgs" yaml:"extraArgs"`
+	NodeLabels []LabelWithNodeFilters  `mapstructure:"nodeLabels" yaml:"nodeLabels"`
 }
 
 // SimpleConfig describes the toplevel k3d configuration file.
@@ -127,7 +129,6 @@ type SimpleConfig struct {
 	ClusterToken    string                  `mapstructure:"token" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated
 	Volumes         []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes" json:"volumes,omitempty"`
 	Ports           []PortWithNodeFilters   `mapstructure:"ports" yaml:"ports" json:"ports,omitempty"`
-	Labels          []LabelWithNodeFilters  `mapstructure:"labels" yaml:"labels" json:"labels,omitempty"`
 	Options         SimpleConfigOptions     `mapstructure:"options" yaml:"options" json:"options,omitempty"`
 	Env             []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"`
 	Registries      struct {

pkg/runtimes/docker/container.go

@@ -151,7 +151,7 @@ func getNodeContainer(ctx context.Context, node *k3d.Node) (*types.Container, er
 
 	// (1) list containers which have the default k3d labels attached
 	filters := filters.NewArgs()
-	for k, v := range node.Labels {
+	for k, v := range node.RuntimeLabels {
 		filters.Add("label", fmt.Sprintf("%s=%s", k, v))
 	}
 

pkg/runtimes/docker/network.go

@@ -147,7 +147,7 @@ func (d Docker) CreateNetworkIfNotPresent(ctx context.Context, inNet *k3d.Cluste
 	// (3) Create a new network
 	netCreateOpts := types.NetworkCreate{
 		CheckDuplicate: true,
-		Labels:         k3d.DefaultObjectLabels,
+		Labels:         k3d.DefaultRuntimeLabels,
 	}
 
 	// we want a managed (user-defined) network, but user didn't specify a subnet, so we try to auto-generate one

pkg/runtimes/docker/node.go

@@ -178,7 +178,7 @@ func getContainersByLabel(ctx context.Context, labels map[string]string) ([]type
 
 	// (1) list containers which have the default k3d labels attached
 	filters := filters.NewArgs()
-	for k, v := range k3d.DefaultObjectLabels {
+	for k, v := range k3d.DefaultRuntimeLabels {
 		filters.Add("label", fmt.Sprintf("%s=%s", k, v))
 	}
 	for k, v := range labels {

pkg/runtimes/docker/translate.go

@@ -73,7 +73,7 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) {
 	containerConfig.Env = node.Env
 
 	/* Labels */
-	containerConfig.Labels = node.Labels // has to include the role
+	containerConfig.Labels = node.RuntimeLabels // has to include the role
 
 	/* Auto-Restart */
 	if node.Restart {
@@ -162,10 +162,10 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) {
 // TranslateContainerToNode translates a docker container object into a k3d node representation
 func TranslateContainerToNode(cont *types.Container) (*k3d.Node, error) {
 	node := &k3d.Node{
-		Name:   strings.TrimPrefix(cont.Names[0], "/"), // container name with leading '/' cut off
-		Image:  cont.Image,
-		Labels: cont.Labels,
-		Role:   k3d.NodeRoles[cont.Labels[k3d.LabelRole]],
+		Name:          strings.TrimPrefix(cont.Names[0], "/"), // container name with leading '/' cut off
+		Image:         cont.Image,
+		RuntimeLabels: cont.Labels,
+		Role:          k3d.NodeRoles[cont.Labels[k3d.LabelRole]],
 		// TODO: all the rest
 	}
 	return node, nil
@@ -175,7 +175,7 @@ func TranslateContainerToNode(cont *types.Container) (*k3d.Node, error) {
 func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d.Node, error) {
 
 	// first, make sure, that it's actually a k3d managed container by checking if it has all the default labels
-	for k, v := range k3d.DefaultObjectLabels {
+	for k, v := range k3d.DefaultRuntimeLabels {
 		log.Tracef("TranslateContainerDetailsToNode: Checking for default object label %s=%s on container %s", k, v, containerDetails.Name)
 		found := false
 		for lk, lv := range containerDetails.Config.Labels {
@@ -273,22 +273,22 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d
 	}
 
 	node := &k3d.Node{
-		Name:       strings.TrimPrefix(containerDetails.Name, "/"), // container name with leading '/' cut off
-		Role:       k3d.NodeRoles[containerDetails.Config.Labels[k3d.LabelRole]],
-		Image:      containerDetails.Image,
-		Volumes:    containerDetails.HostConfig.Binds,
-		Env:        env,
-		Cmd:        containerDetails.Config.Cmd,
-		Args:       []string{}, // empty, since Cmd already contains flags
-		Ports:      containerDetails.HostConfig.PortBindings,
-		Restart:    restart,
-		Created:    containerDetails.Created,
-		Labels:     labels,
-		Networks:   orderedNetworks,
-		ServerOpts: serverOpts,
-		AgentOpts:  k3d.AgentOpts{},
-		State:      nodeState,
-		Memory:     memoryStr,
+		Name:          strings.TrimPrefix(containerDetails.Name, "/"), // container name with leading '/' cut off
+		Role:          k3d.NodeRoles[containerDetails.Config.Labels[k3d.LabelRole]],
+		Image:         containerDetails.Image,
+		Volumes:       containerDetails.HostConfig.Binds,
+		Env:           env,
+		Cmd:           containerDetails.Config.Cmd,
+		Args:          []string{}, // empty, since Cmd already contains flags
+		Ports:         containerDetails.HostConfig.PortBindings,
+		Restart:       restart,
+		Created:       containerDetails.Created,
+		RuntimeLabels: labels,
+		Networks:      orderedNetworks,
+		ServerOpts:    serverOpts,
+		AgentOpts:     k3d.AgentOpts{},
+		State:         nodeState,
+		Memory:        memoryStr,
 	}
 	return node, nil
 }

pkg/runtimes/docker/translate_test.go

@@ -52,9 +52,9 @@ func TestTranslateNodeToContainer(t *testing.T) {
 				},
 			},
 		},
-		Restart:  true,
-		Labels:   map[string]string{k3d.LabelRole: string(k3d.ServerRole), "test_key_1": "test_val_1"},
-		Networks: []string{"mynet"},
+		Restart:       true,
+		RuntimeLabels: map[string]string{k3d.LabelRole: string(k3d.ServerRole), "test_key_1": "test_val_1"},
+		Networks:      []string{"mynet"},
 	}
 
 	init := true

pkg/runtimes/docker/util.go

@@ -44,7 +44,7 @@ import (
 // GetDefaultObjectLabelsFilter returns docker type filters created from k3d labels
 func GetDefaultObjectLabelsFilter(clusterName string) filters.Args {
 	filters := filters.NewArgs()
-	for key, value := range k3d.DefaultObjectLabels {
+	for key, value := range k3d.DefaultRuntimeLabels {
 		filters.Add("label", fmt.Sprintf("%s=%s", key, value))
 	}
 	filters.Add("label", fmt.Sprintf("%s=%s", k3d.LabelClusterName, clusterName))

pkg/runtimes/docker/volume.go

@@ -49,10 +49,10 @@ func (d Docker) CreateVolume(ctx context.Context, name string, labels map[string
 		DriverOpts: map[string]string{},
 	}
 
-	for k, v := range k3d.DefaultObjectLabels {
+	for k, v := range k3d.DefaultRuntimeLabels {
 		volumeCreateOptions.Labels[k] = v
 	}
-	for k, v := range k3d.DefaultObjectLabelsVar {
+	for k, v := range k3d.DefaultRuntimeLabelsVar {
 		volumeCreateOptions.Labels[k] = v
 	}
 

pkg/tools/tools.go

@@ -65,7 +65,7 @@ func ImageImportIntoClusterMulti(ctx context.Context, runtime runtimes.Runtime,
 	var ok bool
 	for _, node := range cluster.Nodes {
 		if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole {
-			if imageVolume, ok = node.Labels[k3d.LabelImageVolume]; ok {
+			if imageVolume, ok = node.RuntimeLabels[k3d.LabelImageVolume]; ok {
 				break
 			}
 		}
@@ -276,23 +276,23 @@ func containsVersionPart(imageTag string) bool {
 // runToolsNode will start a new k3d tools container and connect it to the network of the chosen cluster
 func runToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster, network string, volumes []string) (*k3d.Node, error) {
 	labels := map[string]string{}
-	for k, v := range k3d.DefaultObjectLabels {
+	for k, v := range k3d.DefaultRuntimeLabels {
 		labels[k] = v
 	}
-	for k, v := range k3d.DefaultObjectLabelsVar {
+	for k, v := range k3d.DefaultRuntimeLabelsVar {
 		labels[k] = v
 	}
 	node := &k3d.Node{
-		Name:     fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name),
-		Image:    fmt.Sprintf("%s:%s", k3d.DefaultToolsImageRepo, version.GetHelperImageVersion()),
-		Role:     k3d.NoRole,
-		Volumes:  volumes,
-		Networks: []string{network},
-		Cmd:      []string{},
-		Args:     []string{"noop"},
-		Labels:   k3d.DefaultObjectLabels,
+		Name:          fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name),
+		Image:         fmt.Sprintf("%s:%s", k3d.DefaultToolsImageRepo, version.GetHelperImageVersion()),
+		Role:          k3d.NoRole,
+		Volumes:       volumes,
+		Networks:      []string{network},
+		Cmd:           []string{},
+		Args:          []string{"noop"},
+		RuntimeLabels: k3d.DefaultRuntimeLabels,
 	}
-	node.Labels[k3d.LabelClusterName] = cluster.Name
+	node.RuntimeLabels[k3d.LabelClusterName] = cluster.Name
 	if err := k3dc.NodeRun(ctx, runtime, node, k3d.NodeCreateOpts{}); err != nil {
 		log.Errorf("Failed to create tools container for cluster '%s'", cluster.Name)
 		return node, err

pkg/types/types.go

@@ -105,13 +105,13 @@ var ClusterExternalNodeRoles = []Role{
 	RegistryRole,
 }
 
-// DefaultObjectLabels specifies a set of labels that will be attached to k3d objects by default
-var DefaultObjectLabels = map[string]string{
+// DefaultRuntimeLabels specifies a set of labels that will be attached to k3d runtime objects by default
+var DefaultRuntimeLabels = map[string]string{
 	"app": "k3d",
 }
 
-// DefaultObjectLabelsVar specifies a set of labels that will be attached to k3d objects by default but are not static (e.g. across k3d versions)
-var DefaultObjectLabelsVar = map[string]string{
+// DefaultRuntimeLabelsVar specifies a set of labels that will be attached to k3d runtime objects by default but are not static (e.g. across k3d versions)
+var DefaultRuntimeLabelsVar = map[string]string{
 	"k3d.version": version.GetVersion(),
 }
 
@@ -339,7 +339,7 @@ type Node struct {
 	Ports         nat.PortMap       `yaml:"portMappings" json:"portMappings,omitempty"`
 	Restart       bool              `yaml:"restart" json:"restart,omitempty"`
 	Created       string            `yaml:"created" json:"created,omitempty"`
-	Labels        map[string]string // filled automatically
+	RuntimeLabels map[string]string `yaml:"runtimeLabels" json:"runtimeLabels,omitempty"`
 	K3sNodeLabels map[string]string `yaml:"k3sNodeLabels" json:"k3sNodeLabels,omitempty"`
 	Networks      []string          // filled automatically
 	ExtraHosts    []string          // filled automatically

tests/assets/config_test_simple.yaml

@@ -22,11 +22,6 @@ env:
   - envVar: bar=baz,bob
     nodeFilters:
       - all
-labels:
-  - label: foo=bar
-    nodeFilters:
-      - server[0]
-      - loadbalancer
 registries:
   create: true
   use: []
@@ -47,6 +42,17 @@ options:
       - arg: --tls-san=127.0.0.1
         nodeFilters:
           - server[*]
+    nodeLabels:
+      - label: foo=bar
+        nodeFilters:
+          - server[0]
+          - loadbalancer
   kubeconfig:
     updateDefaultKubeconfig: true
-    switchCurrentContext: true
+    switchCurrentContext: true
+  runtime:
+    labels:
+      - label: foo=bar
+        nodeFilters:
+          - server[0]
+          - loadbalancer

tests/assets/config_test_simple_migration_v1alpha3.yaml

@@ -22,11 +22,6 @@ env:
   - envVar: bar=baz,bob
     nodeFilters:
       - all
-labels:
-  - label: foo=bar
-    nodeFilters:
-      - server[0]
-      - loadbalancer
 registries:
   create: true
   use: []
@@ -49,4 +44,10 @@ options:
           - server[*]
   kubeconfig:
     updateDefaultKubeconfig: true
-    switchCurrentContext: true
+    switchCurrentContext: true
+  runtime:
+    labels:
+      - label: foo=bar
+        nodeFilters:
+          - server[0]
+          - loadbalancer

tests/common.sh

@@ -175,4 +175,10 @@ docker_assert_container_label() {
   # $1 = container/node name
   # $2 = label to assert
   docker inspect --format '{{ range $k, $v := .Config.Labels }}{{ printf "%s=%s\n" $k $v }}{{ end }}' "$1" | grep -E "^$2$"
+}
+
+k3s_assert_node_label() {
+  # $1 = node name
+  # $2 = label to assert
+  kubectl get node "$1" --output go-template='{{ range $k, $v := .metadata.labels }}{{ printf "%s=%s\n" $k $v }}{{ end }}' | grep -E "^$2$"
 }

tests/test_config_file.sh

@@ -18,7 +18,7 @@ fi
 export CURRENT_STAGE="Test | config-file | $K3S_IMAGE_TAG"
 
 
-clustername="ConfigTest"
+clustername="configtest"
 
 highlight "[START] ConfigTest $EXTRA_TITLE"
 
@@ -45,6 +45,10 @@ exec_in_node "k3d-$clustername-server-0" "env" | grep "bar=baz,bob" || failed "E
 info "Ensuring that container labels have been set as stated in the config"
 docker_assert_container_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on container/node k3d-$clustername-server-0"
 
+## K3s Node Labels
+info "Ensuring that k3s node labels have been set as stated in the config"
+k3s_assert_node_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on node k3d-$clustername-server-0"
+
 ## Registry Node
 info "Ensuring, that we have a registry node present"
 $EXE node list "k3d-$clustername-registry" || failed "Expected k3d-$clustername-registry to be present"

tests/test_config_with_overrides.sh

@@ -44,6 +44,10 @@ exec_in_node "k3d-$clustername-agent-1" "env" | grep "x=y" || failed "Expected e
 info "Ensuring that container labels have been set as stated in the config"
 docker_assert_container_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on container/node k3d-$clustername-server-0"
 
+## K3s Node Labels
+info "Ensuring that k3s node labels have been set as stated in the config"
+k3s_assert_node_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on node k3d-$clustername-server-0"
+
 ## Registry Node
 info "Ensuring, that we DO NOT have a registry node present"
 $EXE node list "k3d-$clustername-registry" && failed "Expected k3d-$clustername-registry to NOT be present"