diff --git a/cmd/cluster/clusterCreate.go b/cmd/cluster/clusterCreate.go index 574db83d..89a2fbae 100644 --- a/cmd/cluster/clusterCreate.go +++ b/cmd/cluster/clusterCreate.go @@ -283,8 +283,11 @@ func NewCmdClusterCreate() *cobra.Command { cmd.Flags().StringArrayP("port", "p", nil, "Map ports from the node containers to the host (Format: `[HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]`)\n - Example: `k3d cluster create --agents 2 -p 8080:80@agent[0] -p 8081@agent[1]`") _ = ppViper.BindPFlag("cli.ports", cmd.Flags().Lookup("port")) - cmd.Flags().StringArrayP("label", "l", nil, "Add label to node container (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -l \"my.label@agent[0,1]\" -l \"other.label=somevalue@server[0]\"`") - _ = ppViper.BindPFlag("cli.labels", cmd.Flags().Lookup("label")) + cmd.Flags().StringArrayP("k3s-node-label", "", nil, "Add label to k3s node (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --k3s-node-label \"my.label@agent[0,1]\" --k3s-node-label \"other.label=somevalue@server[0]\"`") + _ = ppViper.BindPFlag("cli.k3s-node-labels", cmd.Flags().Lookup("k3s-node-label")) + + cmd.Flags().StringArrayP("runtime-label", "", nil, "Add label to container runtime (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --runtime-label \"my.label@agent[0,1]\" --runtime-label \"other.label=somevalue@server[0]\"`") + _ = ppViper.BindPFlag("cli.runtime-labels", cmd.Flags().Lookup("runtime-label")) /* k3s */ cmd.Flags().StringArray("k3s-arg", nil, "Additional args passed to k3s command (Format: `ARG@NODEFILTER[;@NODEFILTER]`)\n - Example: `k3d cluster create --k3s-arg \"--disable=traefik@server[0]\"") @@ -481,10 +484,38 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) { log.Tracef("PortFilterMap: %+v", portFilterMap) - // --label - // labelFilterMap will add container label to applied node filters - labelFilterMap := make(map[string][]string, 1) - for _, labelFlag := range ppViper.GetStringSlice("cli.labels") { + // --k3s-node-label + // k3sNodeLabelFilterMap will add k3s node label to applied node filters + k3sNodeLabelFilterMap := make(map[string][]string, 1) + for _, labelFlag := range ppViper.GetStringSlice("cli.k3s-node-labels") { + + // split node filter from the specified label + label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag) + if err != nil { + log.Fatalln(err) + } + + // create new entry or append filter to existing entry + if _, exists := k3sNodeLabelFilterMap[label]; exists { + k3sNodeLabelFilterMap[label] = append(k3sNodeLabelFilterMap[label], nodeFilters...) + } else { + k3sNodeLabelFilterMap[label] = nodeFilters + } + } + + for label, nodeFilters := range k3sNodeLabelFilterMap { + cfg.Options.K3sOptions.NodeLabels = append(cfg.Options.K3sOptions.NodeLabels, conf.LabelWithNodeFilters{ + Label: label, + NodeFilters: nodeFilters, + }) + } + + log.Tracef("K3sNodeLabelFilterMap: %+v", k3sNodeLabelFilterMap) + + // --runtime-label + // runtimeLabelFilterMap will add container runtime label to applied node filters + runtimeLabelFilterMap := make(map[string][]string, 1) + for _, labelFlag := range ppViper.GetStringSlice("cli.runtime-labels") { // split node filter from the specified label label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag) @@ -492,22 +523,24 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) { log.Fatalln(err) } + cliutil.ValidateRuntimeLabelKey(strings.Split(label, "=")[0]) + // create new entry or append filter to existing entry - if _, exists := labelFilterMap[label]; exists { - labelFilterMap[label] = append(labelFilterMap[label], nodeFilters...) + if _, exists := runtimeLabelFilterMap[label]; exists { + runtimeLabelFilterMap[label] = append(runtimeLabelFilterMap[label], nodeFilters...) } else { - labelFilterMap[label] = nodeFilters + runtimeLabelFilterMap[label] = nodeFilters } } - for label, nodeFilters := range labelFilterMap { - cfg.Labels = append(cfg.Labels, conf.LabelWithNodeFilters{ + for label, nodeFilters := range runtimeLabelFilterMap { + cfg.Options.Runtime.Labels = append(cfg.Options.Runtime.Labels, conf.LabelWithNodeFilters{ Label: label, NodeFilters: nodeFilters, }) } - log.Tracef("LabelFilterMap: %+v", labelFilterMap) + log.Tracef("RuntimeLabelFilterMap: %+v", runtimeLabelFilterMap) // --env // envFilterMap will add container env vars to applied node filters diff --git a/cmd/node/nodeCreate.go b/cmd/node/nodeCreate.go index cc7c2b10..8169170c 100644 --- a/cmd/node/nodeCreate.go +++ b/cmd/node/nodeCreate.go @@ -30,6 +30,7 @@ import ( dockerunits "github.com/docker/go-units" "github.com/rancher/k3d/v4/cmd/util" + cliutil "github.com/rancher/k3d/v4/cmd/util" k3dc "github.com/rancher/k3d/v4/pkg/client" "github.com/rancher/k3d/v4/pkg/runtimes" k3d "github.com/rancher/k3d/v4/pkg/types" @@ -74,6 +75,7 @@ func NewCmdNodeCreate() *cobra.Command { cmd.Flags().BoolVar(&createNodeOpts.Wait, "wait", false, "Wait for the node(s) to be ready before returning.") cmd.Flags().DurationVar(&createNodeOpts.Timeout, "timeout", 0*time.Second, "Maximum waiting time for '--wait' before canceling/returning.") + cmd.Flags().StringSliceP("runtime-label", "", []string{}, "Specify container runtime labels in format \"foo=bar\"") cmd.Flags().StringSliceP("k3s-node-label", "", []string{}, "Specify k3s node labels in format \"foo=bar\"") // done @@ -127,9 +129,30 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl log.Errorf("Provided memory limit value is invalid") } + // --runtime-label + runtimeLabelsFlag, err := cmd.Flags().GetStringSlice("runtime-label") + if err != nil { + log.Errorln("No runtime-label specified") + log.Fatalln(err) + } + + runtimeLabels := make(map[string]string, len(runtimeLabelsFlag)+1) + for _, label := range runtimeLabelsFlag { + labelSplitted := strings.Split(label, "=") + if len(labelSplitted) != 2 { + log.Fatalf("unknown runtime-label format format: %s, use format \"foo=bar\"", label) + } + cliutil.ValidateRuntimeLabelKey(labelSplitted[0]) + runtimeLabels[labelSplitted[0]] = labelSplitted[1] + } + + // Internal k3d runtime labels take precedence over user-defined labels + runtimeLabels[k3d.LabelRole] = roleStr + + // --k3s-node-label k3sNodeLabelsFlag, err := cmd.Flags().GetStringSlice("k3s-node-label") if err != nil { - log.Errorln("No node-label specified") + log.Errorln("No k3s-node-label specified") log.Fatalln(err) } @@ -137,7 +160,7 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl for _, label := range k3sNodeLabelsFlag { labelSplitted := strings.Split(label, "=") if len(labelSplitted) != 2 { - log.Fatalf("unknown label format format: %s, use format \"foo=bar\"", label) + log.Fatalf("unknown k3s-node-label format format: %s, use format \"foo=bar\"", label) } k3sNodeLabels[labelSplitted[0]] = labelSplitted[1] } @@ -146,13 +169,11 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl nodes := []*k3d.Node{} for i := 0; i < replicas; i++ { node := &k3d.Node{ - Name: fmt.Sprintf("%s-%s-%d", k3d.DefaultObjectNamePrefix, args[0], i), - Role: role, - Image: image, - Labels: map[string]string{ - k3d.LabelRole: roleStr, - }, + Name: fmt.Sprintf("%s-%s-%d", k3d.DefaultObjectNamePrefix, args[0], i), + Role: role, + Image: image, K3sNodeLabels: k3sNodeLabels, + RuntimeLabels: runtimeLabels, Restart: true, Memory: memory, } diff --git a/cmd/node/nodeList.go b/cmd/node/nodeList.go index fd0698f8..c4abde01 100644 --- a/cmd/node/nodeList.go +++ b/cmd/node/nodeList.go @@ -88,7 +88,7 @@ func NewCmdNodeList() *cobra.Command { fmt.Fprintf(tabwriter, "%s\t%s\t%s\t%s\n", strings.TrimPrefix(node.Name, "/"), string(node.Role), - node.Labels[k3d.LabelClusterName], + node.RuntimeLabels[k3d.LabelClusterName], node.State.Status) })) }, diff --git a/cmd/registry/registryList.go b/cmd/registry/registryList.go index f41d5f89..30fa66f0 100644 --- a/cmd/registry/registryList.go +++ b/cmd/registry/registryList.go @@ -88,8 +88,8 @@ func NewCmdRegistryList() *cobra.Command { util.PrintNodes(existingNodes, registryListFlags.output, headers, util.NodePrinterFunc(func(tabwriter *tabwriter.Writer, node *k3d.Node) { cluster := "*" - if _, ok := node.Labels[k3d.LabelClusterName]; ok { - cluster = node.Labels[k3d.LabelClusterName] + if _, ok := node.RuntimeLabels[k3d.LabelClusterName]; ok { + cluster = node.RuntimeLabels[k3d.LabelClusterName] } fmt.Fprintf(tabwriter, "%s\t%s\t%s\t%s\n", strings.TrimPrefix(node.Name, "/"), diff --git a/cmd/util/runtimeLabels.go b/cmd/util/runtimeLabels.go new file mode 100644 index 00000000..e603cef1 --- /dev/null +++ b/cmd/util/runtimeLabels.go @@ -0,0 +1,35 @@ +/* +Copyright © 2020 The k3d Author(s) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package util + +import ( + "strings" + + log "github.com/sirupsen/logrus" +) + +// validateRuntimeLabelKey validates a given label key is not reserved for internal k3d usage +func ValidateRuntimeLabelKey(labelKey string) { + if strings.HasPrefix(labelKey, "k3s.") || strings.HasPrefix(labelKey, "k3d.") || labelKey == "app" { + log.Fatalf("runtime label \"%s\" is reserved for internal usage", labelKey) + } +} diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go index 4aa29fcb..e04484c1 100644 --- a/pkg/client/cluster.go +++ b/pkg/client/cluster.go @@ -381,13 +381,13 @@ ClusterCreatOpts: nodeSetup := func(node *k3d.Node, suffix int) error { // cluster specific settings - if node.Labels == nil { - node.Labels = make(map[string]string) // TODO: maybe create an init function? + if node.RuntimeLabels == nil { + node.RuntimeLabels = make(map[string]string) // TODO: maybe create an init function? } // ensure global labels for k, v := range clusterCreateOpts.GlobalLabels { - node.Labels[k] = v + node.RuntimeLabels[k] = v } // ensure global env @@ -404,7 +404,7 @@ ClusterCreatOpts: cluster.Network.IPAM.IPsUsed = append(cluster.Network.IPAM.IPsUsed, ip) // make sure that we're not reusing the same IP next time node.IP.Static = true node.IP.IP = ip - node.Labels[k3d.LabelNodeStaticIP] = ip.String() + node.RuntimeLabels[k3d.LabelNodeStaticIP] = ip.String() } node.ServerOpts.KubeAPI = cluster.KubeAPI @@ -412,7 +412,7 @@ ClusterCreatOpts: // the cluster has an init server node, but its not this one, so connect it to the init node if cluster.InitNode != nil && !node.ServerOpts.IsInit { node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", connectionURL)) - node.Labels[k3d.LabelServerIsInit] = "false" // set label, that this server node is not the init server + node.RuntimeLabels[k3d.LabelServerIsInit] = "false" // set label, that this server node is not the init server } } else if node.Role == k3d.AgentRole { @@ -446,10 +446,10 @@ ClusterCreatOpts: if cluster.InitNode != nil { log.Infoln("Creating initializing server node") cluster.InitNode.Args = append(cluster.InitNode.Args, "--cluster-init") - if cluster.InitNode.Labels == nil { - cluster.InitNode.Labels = map[string]string{} + if cluster.InitNode.RuntimeLabels == nil { + cluster.InitNode.RuntimeLabels = map[string]string{} } - cluster.InitNode.Labels[k3d.LabelServerIsInit] = "true" // set label, that this server node is the init server + cluster.InitNode.RuntimeLabels[k3d.LabelServerIsInit] = "true" // set label, that this server node is the init server // in case the LoadBalancer was disabled, expose the API Port on the initializing server node if clusterCreateOpts.DisableLoadBalancer { @@ -547,10 +547,10 @@ ClusterCreatOpts: fmt.Sprintf("PORTS=%s", strings.Join(ports, ",")), fmt.Sprintf("WORKER_PROCESSES=%d", len(ports)), }, - Role: k3d.LoadBalancerRole, - Labels: clusterCreateOpts.GlobalLabels, // TODO: createLoadBalancer: add more expressive labels - Networks: []string{cluster.Network.Name}, - Restart: true, + Role: k3d.LoadBalancerRole, + RuntimeLabels: clusterCreateOpts.GlobalLabels, // TODO: createLoadBalancer: add more expressive labels + Networks: []string{cluster.Network.Name}, + Restart: true, } if len(udp_ports) > 0 { lbNode.Env = append(lbNode.Env, fmt.Sprintf("UDP_PORTS=%s", strings.Join(udp_ports, ","))) @@ -673,7 +673,7 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus // ClusterList returns a list of all existing clusters func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, error) { log.Traceln("Listing Clusters...") - nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultObjectLabels) + nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultRuntimeLabels) if err != nil { log.Errorln("Failed to get clusters") return nil, err @@ -691,7 +691,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er log.Tracef("Found %d cluster-internal nodes", len(nodes)) if log.GetLevel() == log.TraceLevel { for _, node := range nodes { - log.Tracef("Found cluster-internal node %s of role %s belonging to cluster %s", node.Name, node.Role, node.Labels[k3d.LabelClusterName]) + log.Tracef("Found cluster-internal node %s of role %s belonging to cluster %s", node.Name, node.Role, node.RuntimeLabels[k3d.LabelClusterName]) } } @@ -700,7 +700,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er for _, node := range nodes { clusterExists := false for _, cluster := range clusters { - if node.Labels[k3d.LabelClusterName] == cluster.Name { // TODO: handle case, where this label doesn't exist + if node.RuntimeLabels[k3d.LabelClusterName] == cluster.Name { // TODO: handle case, where this label doesn't exist cluster.Nodes = append(cluster.Nodes, node) clusterExists = true break @@ -709,7 +709,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er // cluster is not in the list yet, so we add it with the current node as its first member if !clusterExists { clusters = append(clusters, &k3d.Cluster{ - Name: node.Labels[k3d.LabelClusterName], + Name: node.RuntimeLabels[k3d.LabelClusterName], Nodes: []*k3d.Node{node}, }) } @@ -734,7 +734,7 @@ func populateClusterFieldsFromLabels(cluster *k3d.Cluster) error { // get the name of the cluster network if cluster.Network.Name == "" { - if networkName, ok := node.Labels[k3d.LabelNetwork]; ok { + if networkName, ok := node.RuntimeLabels[k3d.LabelNetwork]; ok { cluster.Network.Name = networkName } } @@ -742,7 +742,7 @@ func populateClusterFieldsFromLabels(cluster *k3d.Cluster) error { // check if the network is external // since the struct value is a bool, initialized as false, we cannot check if it's unset if !cluster.Network.External && !networkExternalSet { - if networkExternalString, ok := node.Labels[k3d.LabelNetworkExternal]; ok { + if networkExternalString, ok := node.RuntimeLabels[k3d.LabelNetworkExternal]; ok { if networkExternal, err := strconv.ParseBool(networkExternalString); err == nil { cluster.Network.External = networkExternal networkExternalSet = true @@ -752,14 +752,14 @@ func populateClusterFieldsFromLabels(cluster *k3d.Cluster) error { // get image volume // TODO: enable external image volumes the same way we do it with networks if cluster.ImageVolume == "" { - if imageVolumeName, ok := node.Labels[k3d.LabelImageVolume]; ok { + if imageVolumeName, ok := node.RuntimeLabels[k3d.LabelImageVolume]; ok { cluster.ImageVolume = imageVolumeName } } // get k3s cluster's token if cluster.Token == "" { - if token, ok := node.Labels[k3d.LabelClusterToken]; ok { + if token, ok := node.RuntimeLabels[k3d.LabelClusterToken]; ok { cluster.Token = token } } diff --git a/pkg/client/kubeconfig.go b/pkg/client/kubeconfig.go index 957f593d..a016f153 100644 --- a/pkg/client/kubeconfig.go +++ b/pkg/client/kubeconfig.go @@ -131,11 +131,11 @@ func KubeconfigGet(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.C APIHost := k3d.DefaultAPIHost for _, server := range serverNodes { - if _, ok := server.Labels[k3d.LabelServerAPIPort]; ok { + if _, ok := server.RuntimeLabels[k3d.LabelServerAPIPort]; ok { chosenServer = server - APIPort = server.Labels[k3d.LabelServerAPIPort] - if _, ok := server.Labels[k3d.LabelServerAPIHost]; ok { - APIHost = server.Labels[k3d.LabelServerAPIHost] + APIPort = server.RuntimeLabels[k3d.LabelServerAPIPort] + if _, ok := server.RuntimeLabels[k3d.LabelServerAPIHost]; ok { + APIHost = server.RuntimeLabels[k3d.LabelServerAPIHost] } break } diff --git a/pkg/client/node.go b/pkg/client/node.go index ebbf28f0..a00f4666 100644 --- a/pkg/client/node.go +++ b/pkg/client/node.go @@ -59,8 +59,8 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N node.Networks = []string{cluster.Network.Name} // skeleton - if node.Labels == nil { - node.Labels = map[string]string{ + if node.RuntimeLabels == nil { + node.RuntimeLabels = map[string]string{ k3d.LabelRole: string(node.Role), } } @@ -141,7 +141,7 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N } } if !k3sURLFound { - if url, ok := node.Labels[k3d.LabelClusterURL]; ok { + if url, ok := node.RuntimeLabels[k3d.LabelClusterURL]; ok { node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", url)) } else { log.Warnln("Failed to find K3S_URL value!") @@ -381,18 +381,22 @@ func NodeCreate(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, c // ### Labels ### labels := make(map[string]string) - for k, v := range k3d.DefaultObjectLabels { + for k, v := range k3d.DefaultRuntimeLabels { labels[k] = v } - for k, v := range k3d.DefaultObjectLabelsVar { + for k, v := range k3d.DefaultRuntimeLabelsVar { labels[k] = v } - for k, v := range node.Labels { + for k, v := range node.RuntimeLabels { labels[k] = v } - node.Labels = labels + node.RuntimeLabels = labels // second most important: the node role label - node.Labels[k3d.LabelRole] = string(node.Role) + node.RuntimeLabels[k3d.LabelRole] = string(node.Role) + + for k, v := range node.K3sNodeLabels { + node.Args = append(node.Args, "--node-label", fmt.Sprintf("%s=%s", k, v)) + } // ### Environment ### node.Env = append(node.Env, k3d.DefaultNodeEnv...) // append default node env vars @@ -469,7 +473,7 @@ func NodeDelete(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, o // update the server loadbalancer if !opts.SkipLBUpdate && (node.Role == k3d.ServerRole || node.Role == k3d.AgentRole) { - cluster, err := ClusterGet(ctx, runtime, &k3d.Cluster{Name: node.Labels[k3d.LabelClusterName]}) + cluster, err := ClusterGet(ctx, runtime, &k3d.Cluster{Name: node.RuntimeLabels[k3d.LabelClusterName]}) if err != nil { log.Errorf("Failed to find cluster for node '%s'", node.Name) return err @@ -493,10 +497,6 @@ func patchAgentSpec(node *k3d.Node) error { node.Cmd = []string{"agent"} } - for k, v := range node.K3sNodeLabels { - node.Args = append(node.Args, "--node-label", fmt.Sprintf("%s=%s", k, v)) - } - return nil } @@ -509,9 +509,9 @@ func patchServerSpec(node *k3d.Node, runtime runtimes.Runtime) error { // Add labels and TLS SAN for the exposed API // FIXME: For now, the labels concerning the API on the server nodes are only being used for configuring the kubeconfig - node.Labels[k3d.LabelServerAPIHostIP] = node.ServerOpts.KubeAPI.Binding.HostIP // TODO: maybe get docker machine IP here - node.Labels[k3d.LabelServerAPIHost] = node.ServerOpts.KubeAPI.Host - node.Labels[k3d.LabelServerAPIPort] = node.ServerOpts.KubeAPI.Binding.HostPort + node.RuntimeLabels[k3d.LabelServerAPIHostIP] = node.ServerOpts.KubeAPI.Binding.HostIP // TODO: maybe get docker machine IP here + node.RuntimeLabels[k3d.LabelServerAPIHost] = node.ServerOpts.KubeAPI.Host + node.RuntimeLabels[k3d.LabelServerAPIPort] = node.ServerOpts.KubeAPI.Binding.HostPort // If the runtime is docker, attempt to use the docker host if runtime == runtimes.Docker { @@ -519,19 +519,19 @@ func patchServerSpec(node *k3d.Node, runtime runtimes.Runtime) error { if dockerHost != "" { dockerHost = strings.Split(dockerHost, ":")[0] // remove the port log.Tracef("Using docker host %s", dockerHost) - node.Labels[k3d.LabelServerAPIHostIP] = dockerHost - node.Labels[k3d.LabelServerAPIHost] = dockerHost + node.RuntimeLabels[k3d.LabelServerAPIHostIP] = dockerHost + node.RuntimeLabels[k3d.LabelServerAPIHost] = dockerHost } } - node.Args = append(node.Args, "--tls-san", node.Labels[k3d.LabelServerAPIHost]) // add TLS SAN for non default host name + node.Args = append(node.Args, "--tls-san", node.RuntimeLabels[k3d.LabelServerAPIHost]) // add TLS SAN for non default host name return nil } // NodeList returns a list of all existing clusters func NodeList(ctx context.Context, runtime runtimes.Runtime) ([]*k3d.Node, error) { - nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultObjectLabels) + nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultRuntimeLabels) if err != nil { log.Errorln("Failed to get nodes") return nil, err diff --git a/pkg/client/registry.go b/pkg/client/registry.go index 4df7ff7a..9e1dc86c 100644 --- a/pkg/client/registry.go +++ b/pkg/client/registry.go @@ -77,7 +77,7 @@ func RegistryCreate(ctx context.Context, runtime runtimes.Runtime, reg *k3d.Regi } // setup the node labels - registryNode.Labels = map[string]string{ + registryNode.RuntimeLabels = map[string]string{ k3d.LabelClusterName: reg.ClusterRef, k3d.LabelRole: string(k3d.RegistryRole), k3d.LabelRegistryHost: reg.ExposureOpts.Host, // TODO: docker machine host? @@ -85,11 +85,11 @@ func RegistryCreate(ctx context.Context, runtime runtimes.Runtime, reg *k3d.Regi k3d.LabelRegistryPortExternal: reg.ExposureOpts.Binding.HostPort, k3d.LabelRegistryPortInternal: reg.ExposureOpts.Port.Port(), } - for k, v := range k3d.DefaultObjectLabels { - registryNode.Labels[k] = v + for k, v := range k3d.DefaultRuntimeLabels { + registryNode.RuntimeLabels[k] = v } - for k, v := range k3d.DefaultObjectLabelsVar { - registryNode.Labels[k] = v + for k, v := range k3d.DefaultRuntimeLabelsVar { + registryNode.RuntimeLabels[k] = v } // port diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index eb15f152..41d76422 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -64,12 +64,6 @@ func TestReadSimpleConfig(t *testing.T) { NodeFilters: []string{"loadbalancer"}, }, }, - Labels: []conf.LabelWithNodeFilters{ - { - Label: "foo=bar", - NodeFilters: []string{"server[0]", "loadbalancer"}, - }, - }, Env: []conf.EnvVarWithNodeFilters{ { EnvVar: "bar=baz", @@ -90,11 +84,25 @@ func TestReadSimpleConfig(t *testing.T) { NodeFilters: []string{"server[*]"}, }, }, + NodeLabels: []conf.LabelWithNodeFilters{ + { + Label: "foo=bar", + NodeFilters: []string{"server[0]", "loadbalancer"}, + }, + }, }, KubeconfigOptions: conf.SimpleConfigOptionsKubeconfig{ UpdateDefaultKubeconfig: true, SwitchCurrentContext: true, }, + Runtime: conf.SimpleConfigOptionsRuntime{ + Labels: []conf.LabelWithNodeFilters{ + { + Label: "foo=bar", + NodeFilters: []string{"server[0]", "loadbalancer"}, + }, + }, + }, }, } diff --git a/pkg/config/test_assets/config_test_simple.yaml b/pkg/config/test_assets/config_test_simple.yaml index 4e132176..f8f873cb 100644 --- a/pkg/config/test_assets/config_test_simple.yaml +++ b/pkg/config/test_assets/config_test_simple.yaml @@ -22,11 +22,6 @@ env: - envVar: bar=baz nodeFilters: - all -labels: - - label: foo=bar - nodeFilters: - - "server[0]" - - loadbalancer options: k3d: @@ -39,6 +34,17 @@ options: - arg: --tls-san=127.0.0.1 nodeFilters: - "server[*]" + nodeLabels: + - label: foo=bar + nodeFilters: + - server[0] + - loadbalancer kubeconfig: updateDefaultKubeconfig: true switchCurrentContext: true + runtime: + labels: + - label: foo=bar + nodeFilters: + - server[0] + - loadbalancer diff --git a/pkg/config/test_assets/config_test_simple_invalid_servers.yaml b/pkg/config/test_assets/config_test_simple_invalid_servers.yaml index b9e75fb6..7b9bc8a0 100644 --- a/pkg/config/test_assets/config_test_simple_invalid_servers.yaml +++ b/pkg/config/test_assets/config_test_simple_invalid_servers.yaml @@ -22,11 +22,6 @@ env: - envVar: bar=baz nodeFilters: - all -labels: - - label: foo=bar - nodeFilters: - - server[0] - - loadbalancer options: k3d: @@ -39,6 +34,17 @@ options: - arg: --tls-san=127.0.0.1 nodeFilters: - "server[*]" + nodeLabels: + - label: foo=bar + nodeFilters: + - server[0] + - loadbalancer kubeconfig: updateDefaultKubeconfig: true - switchCurrentContext: true \ No newline at end of file + switchCurrentContext: true + runtime: + labels: + - label: foo=bar + nodeFilters: + - server[0] + - loadbalancer diff --git a/pkg/config/transform.go b/pkg/config/transform.go index 0ec43686..59e674d9 100644 --- a/pkg/config/transform.go +++ b/pkg/config/transform.go @@ -190,23 +190,47 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim } } - // -> LABELS - for _, labelWithNodeFilters := range simpleConfig.Labels { - if len(labelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 { - return nil, fmt.Errorf("Labelmapping '%s' lacks a node filter, but there's more than one node", labelWithNodeFilters.Label) + // -> K3S NODE LABELS + for _, k3sNodeLabelWithNodeFilters := range simpleConfig.Options.K3sOptions.NodeLabels { + if len(k3sNodeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 { + return nil, fmt.Errorf("K3sNodeLabelmapping '%s' lacks a node filter, but there's more than one node", k3sNodeLabelWithNodeFilters.Label) } - nodes, err := util.FilterNodes(nodeList, labelWithNodeFilters.NodeFilters) + nodes, err := util.FilterNodes(nodeList, k3sNodeLabelWithNodeFilters.NodeFilters) if err != nil { return nil, err } for _, node := range nodes { - if node.Labels == nil { - node.Labels = make(map[string]string) // ensure that the map is initialized + if node.K3sNodeLabels == nil { + node.K3sNodeLabels = make(map[string]string) // ensure that the map is initialized } - k, v := util.SplitLabelKeyValue(labelWithNodeFilters.Label) - node.Labels[k] = v + k, v := util.SplitLabelKeyValue(k3sNodeLabelWithNodeFilters.Label) + node.K3sNodeLabels[k] = v + + } + } + + // -> RUNTIME LABELS + for _, runtimeLabelWithNodeFilters := range simpleConfig.Options.Runtime.Labels { + if len(runtimeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 { + return nil, fmt.Errorf("RuntimeLabelmapping '%s' lacks a node filter, but there's more than one node", runtimeLabelWithNodeFilters.Label) + } + + nodes, err := util.FilterNodes(nodeList, runtimeLabelWithNodeFilters.NodeFilters) + if err != nil { + return nil, err + } + + for _, node := range nodes { + if node.RuntimeLabels == nil { + node.RuntimeLabels = make(map[string]string) // ensure that the map is initialized + } + k, v := util.SplitLabelKeyValue(runtimeLabelWithNodeFilters.Label) + + cliutil.ValidateRuntimeLabelKey(k) + + node.RuntimeLabels[k] = v } } @@ -260,7 +284,7 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim } // ensure, that we have the default object labels - for k, v := range k3d.DefaultObjectLabels { + for k, v := range k3d.DefaultRuntimeLabels { clusterCreateOpts.GlobalLabels[k] = v } diff --git a/pkg/config/v1alpha3/migrations.go b/pkg/config/v1alpha3/migrations.go index 5c894ca1..fd1bc6dc 100644 --- a/pkg/config/v1alpha3/migrations.go +++ b/pkg/config/v1alpha3/migrations.go @@ -49,6 +49,15 @@ func MigrateV1Alpha2(input configtypes.Config) (configtypes.Config, error) { return nil, err } + cfg.Options.Runtime.Labels = []LabelWithNodeFilters{} + + for _, label := range input.(v1alpha2.SimpleConfig).Labels { + cfg.Options.Runtime.Labels = append(cfg.Options.Runtime.Labels, LabelWithNodeFilters{ + Label: label.Label, + NodeFilters: label.NodeFilters, + }) + } + cfg.Options.K3sOptions.ExtraArgs = []K3sArgWithNodeFilters{} for _, arg := range input.(v1alpha2.SimpleConfig).Options.K3sOptions.ExtraServerArgs { diff --git a/pkg/config/v1alpha3/schema.json b/pkg/config/v1alpha3/schema.json index 2b07c02c..1deb0060 100644 --- a/pkg/config/v1alpha3/schema.json +++ b/pkg/config/v1alpha3/schema.json @@ -108,21 +108,6 @@ "additionalProperties": false } }, - "labels": { - "type": "array", - "items": { - "type": "object", - "properties": { - "label": { - "type": "string" - }, - "nodeFilters": { - "$ref": "#/definitions/nodeFilters" - } - }, - "additionalProperties": false - } - }, "options": { "type": "object", "properties": { @@ -170,12 +155,24 @@ "properties": { "arg": { "type": "string", - "examples": [ - "--tls-san=127.0.0.1", - "--disable=traefik" - ] + "examples": ["--tls-san=127.0.0.1", "--disable=traefik"] }, - "nodeFilters": { + "nodeFilters": { + "$ref": "#/definitions/nodeFilters" + } + }, + "additionalProperties": false + } + }, + "nodeLabels": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "nodeFilters": { "$ref": "#/definitions/nodeFilters" } }, @@ -210,6 +207,21 @@ }, "agentsMemory": { "type": "string" + }, + "labels": { + "type": "array", + "items": { + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "nodeFilters": { + "$ref": "#/definitions/nodeFilters" + } + }, + "additionalProperties": false + } } } } diff --git a/pkg/config/v1alpha3/types.go b/pkg/config/v1alpha3/types.go index 22465c71..9783d1c1 100644 --- a/pkg/config/v1alpha3/types.go +++ b/pkg/config/v1alpha3/types.go @@ -95,9 +95,10 @@ type SimpleConfigOptions struct { } type SimpleConfigOptionsRuntime struct { - GPURequest string `mapstructure:"gpuRequest" yaml:"gpuRequest"` - ServersMemory string `mapstructure:"serversMemory" yaml:"serversMemory"` - AgentsMemory string `mapstructure:"agentsMemory" yaml:"agentsMemory"` + GPURequest string `mapstructure:"gpuRequest" yaml:"gpuRequest"` + ServersMemory string `mapstructure:"serversMemory" yaml:"serversMemory"` + AgentsMemory string `mapstructure:"agentsMemory" yaml:"agentsMemory"` + Labels []LabelWithNodeFilters `mapstructure:"labels" yaml:"labels"` } type SimpleConfigOptionsK3d struct { @@ -111,7 +112,8 @@ type SimpleConfigOptionsK3d struct { } type SimpleConfigOptionsK3s struct { - ExtraArgs []K3sArgWithNodeFilters `mapstructure:"extraArgs" yaml:"extraArgs"` + ExtraArgs []K3sArgWithNodeFilters `mapstructure:"extraArgs" yaml:"extraArgs"` + NodeLabels []LabelWithNodeFilters `mapstructure:"nodeLabels" yaml:"nodeLabels"` } // SimpleConfig describes the toplevel k3d configuration file. @@ -127,7 +129,6 @@ type SimpleConfig struct { ClusterToken string `mapstructure:"token" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated Volumes []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes" json:"volumes,omitempty"` Ports []PortWithNodeFilters `mapstructure:"ports" yaml:"ports" json:"ports,omitempty"` - Labels []LabelWithNodeFilters `mapstructure:"labels" yaml:"labels" json:"labels,omitempty"` Options SimpleConfigOptions `mapstructure:"options" yaml:"options" json:"options,omitempty"` Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"` Registries struct { diff --git a/pkg/runtimes/docker/container.go b/pkg/runtimes/docker/container.go index 6124d4e3..296c3070 100644 --- a/pkg/runtimes/docker/container.go +++ b/pkg/runtimes/docker/container.go @@ -151,7 +151,7 @@ func getNodeContainer(ctx context.Context, node *k3d.Node) (*types.Container, er // (1) list containers which have the default k3d labels attached filters := filters.NewArgs() - for k, v := range node.Labels { + for k, v := range node.RuntimeLabels { filters.Add("label", fmt.Sprintf("%s=%s", k, v)) } diff --git a/pkg/runtimes/docker/network.go b/pkg/runtimes/docker/network.go index 80f62f4c..b1661233 100644 --- a/pkg/runtimes/docker/network.go +++ b/pkg/runtimes/docker/network.go @@ -147,7 +147,7 @@ func (d Docker) CreateNetworkIfNotPresent(ctx context.Context, inNet *k3d.Cluste // (3) Create a new network netCreateOpts := types.NetworkCreate{ CheckDuplicate: true, - Labels: k3d.DefaultObjectLabels, + Labels: k3d.DefaultRuntimeLabels, } // we want a managed (user-defined) network, but user didn't specify a subnet, so we try to auto-generate one diff --git a/pkg/runtimes/docker/node.go b/pkg/runtimes/docker/node.go index b5e3b39a..c7c3b37d 100644 --- a/pkg/runtimes/docker/node.go +++ b/pkg/runtimes/docker/node.go @@ -178,7 +178,7 @@ func getContainersByLabel(ctx context.Context, labels map[string]string) ([]type // (1) list containers which have the default k3d labels attached filters := filters.NewArgs() - for k, v := range k3d.DefaultObjectLabels { + for k, v := range k3d.DefaultRuntimeLabels { filters.Add("label", fmt.Sprintf("%s=%s", k, v)) } for k, v := range labels { diff --git a/pkg/runtimes/docker/translate.go b/pkg/runtimes/docker/translate.go index b6861aad..c8331c75 100644 --- a/pkg/runtimes/docker/translate.go +++ b/pkg/runtimes/docker/translate.go @@ -73,7 +73,7 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) { containerConfig.Env = node.Env /* Labels */ - containerConfig.Labels = node.Labels // has to include the role + containerConfig.Labels = node.RuntimeLabels // has to include the role /* Auto-Restart */ if node.Restart { @@ -162,10 +162,10 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) { // TranslateContainerToNode translates a docker container object into a k3d node representation func TranslateContainerToNode(cont *types.Container) (*k3d.Node, error) { node := &k3d.Node{ - Name: strings.TrimPrefix(cont.Names[0], "/"), // container name with leading '/' cut off - Image: cont.Image, - Labels: cont.Labels, - Role: k3d.NodeRoles[cont.Labels[k3d.LabelRole]], + Name: strings.TrimPrefix(cont.Names[0], "/"), // container name with leading '/' cut off + Image: cont.Image, + RuntimeLabels: cont.Labels, + Role: k3d.NodeRoles[cont.Labels[k3d.LabelRole]], // TODO: all the rest } return node, nil @@ -175,7 +175,7 @@ func TranslateContainerToNode(cont *types.Container) (*k3d.Node, error) { func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d.Node, error) { // first, make sure, that it's actually a k3d managed container by checking if it has all the default labels - for k, v := range k3d.DefaultObjectLabels { + for k, v := range k3d.DefaultRuntimeLabels { log.Tracef("TranslateContainerDetailsToNode: Checking for default object label %s=%s on container %s", k, v, containerDetails.Name) found := false for lk, lv := range containerDetails.Config.Labels { @@ -273,22 +273,22 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d } node := &k3d.Node{ - Name: strings.TrimPrefix(containerDetails.Name, "/"), // container name with leading '/' cut off - Role: k3d.NodeRoles[containerDetails.Config.Labels[k3d.LabelRole]], - Image: containerDetails.Image, - Volumes: containerDetails.HostConfig.Binds, - Env: env, - Cmd: containerDetails.Config.Cmd, - Args: []string{}, // empty, since Cmd already contains flags - Ports: containerDetails.HostConfig.PortBindings, - Restart: restart, - Created: containerDetails.Created, - Labels: labels, - Networks: orderedNetworks, - ServerOpts: serverOpts, - AgentOpts: k3d.AgentOpts{}, - State: nodeState, - Memory: memoryStr, + Name: strings.TrimPrefix(containerDetails.Name, "/"), // container name with leading '/' cut off + Role: k3d.NodeRoles[containerDetails.Config.Labels[k3d.LabelRole]], + Image: containerDetails.Image, + Volumes: containerDetails.HostConfig.Binds, + Env: env, + Cmd: containerDetails.Config.Cmd, + Args: []string{}, // empty, since Cmd already contains flags + Ports: containerDetails.HostConfig.PortBindings, + Restart: restart, + Created: containerDetails.Created, + RuntimeLabels: labels, + Networks: orderedNetworks, + ServerOpts: serverOpts, + AgentOpts: k3d.AgentOpts{}, + State: nodeState, + Memory: memoryStr, } return node, nil } diff --git a/pkg/runtimes/docker/translate_test.go b/pkg/runtimes/docker/translate_test.go index 38c6f3a2..4243dc57 100644 --- a/pkg/runtimes/docker/translate_test.go +++ b/pkg/runtimes/docker/translate_test.go @@ -52,9 +52,9 @@ func TestTranslateNodeToContainer(t *testing.T) { }, }, }, - Restart: true, - Labels: map[string]string{k3d.LabelRole: string(k3d.ServerRole), "test_key_1": "test_val_1"}, - Networks: []string{"mynet"}, + Restart: true, + RuntimeLabels: map[string]string{k3d.LabelRole: string(k3d.ServerRole), "test_key_1": "test_val_1"}, + Networks: []string{"mynet"}, } init := true diff --git a/pkg/runtimes/docker/util.go b/pkg/runtimes/docker/util.go index 7720c1e5..f64c39a2 100644 --- a/pkg/runtimes/docker/util.go +++ b/pkg/runtimes/docker/util.go @@ -44,7 +44,7 @@ import ( // GetDefaultObjectLabelsFilter returns docker type filters created from k3d labels func GetDefaultObjectLabelsFilter(clusterName string) filters.Args { filters := filters.NewArgs() - for key, value := range k3d.DefaultObjectLabels { + for key, value := range k3d.DefaultRuntimeLabels { filters.Add("label", fmt.Sprintf("%s=%s", key, value)) } filters.Add("label", fmt.Sprintf("%s=%s", k3d.LabelClusterName, clusterName)) diff --git a/pkg/runtimes/docker/volume.go b/pkg/runtimes/docker/volume.go index 6c0f4c80..32d39634 100644 --- a/pkg/runtimes/docker/volume.go +++ b/pkg/runtimes/docker/volume.go @@ -49,10 +49,10 @@ func (d Docker) CreateVolume(ctx context.Context, name string, labels map[string DriverOpts: map[string]string{}, } - for k, v := range k3d.DefaultObjectLabels { + for k, v := range k3d.DefaultRuntimeLabels { volumeCreateOptions.Labels[k] = v } - for k, v := range k3d.DefaultObjectLabelsVar { + for k, v := range k3d.DefaultRuntimeLabelsVar { volumeCreateOptions.Labels[k] = v } diff --git a/pkg/tools/tools.go b/pkg/tools/tools.go index 56866ed3..62e68fd9 100644 --- a/pkg/tools/tools.go +++ b/pkg/tools/tools.go @@ -65,7 +65,7 @@ func ImageImportIntoClusterMulti(ctx context.Context, runtime runtimes.Runtime, var ok bool for _, node := range cluster.Nodes { if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole { - if imageVolume, ok = node.Labels[k3d.LabelImageVolume]; ok { + if imageVolume, ok = node.RuntimeLabels[k3d.LabelImageVolume]; ok { break } } @@ -276,23 +276,23 @@ func containsVersionPart(imageTag string) bool { // runToolsNode will start a new k3d tools container and connect it to the network of the chosen cluster func runToolsNode(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.Cluster, network string, volumes []string) (*k3d.Node, error) { labels := map[string]string{} - for k, v := range k3d.DefaultObjectLabels { + for k, v := range k3d.DefaultRuntimeLabels { labels[k] = v } - for k, v := range k3d.DefaultObjectLabelsVar { + for k, v := range k3d.DefaultRuntimeLabelsVar { labels[k] = v } node := &k3d.Node{ - Name: fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name), - Image: fmt.Sprintf("%s:%s", k3d.DefaultToolsImageRepo, version.GetHelperImageVersion()), - Role: k3d.NoRole, - Volumes: volumes, - Networks: []string{network}, - Cmd: []string{}, - Args: []string{"noop"}, - Labels: k3d.DefaultObjectLabels, + Name: fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name), + Image: fmt.Sprintf("%s:%s", k3d.DefaultToolsImageRepo, version.GetHelperImageVersion()), + Role: k3d.NoRole, + Volumes: volumes, + Networks: []string{network}, + Cmd: []string{}, + Args: []string{"noop"}, + RuntimeLabels: k3d.DefaultRuntimeLabels, } - node.Labels[k3d.LabelClusterName] = cluster.Name + node.RuntimeLabels[k3d.LabelClusterName] = cluster.Name if err := k3dc.NodeRun(ctx, runtime, node, k3d.NodeCreateOpts{}); err != nil { log.Errorf("Failed to create tools container for cluster '%s'", cluster.Name) return node, err diff --git a/pkg/types/types.go b/pkg/types/types.go index 23667855..4df5e434 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -105,13 +105,13 @@ var ClusterExternalNodeRoles = []Role{ RegistryRole, } -// DefaultObjectLabels specifies a set of labels that will be attached to k3d objects by default -var DefaultObjectLabels = map[string]string{ +// DefaultRuntimeLabels specifies a set of labels that will be attached to k3d runtime objects by default +var DefaultRuntimeLabels = map[string]string{ "app": "k3d", } -// DefaultObjectLabelsVar specifies a set of labels that will be attached to k3d objects by default but are not static (e.g. across k3d versions) -var DefaultObjectLabelsVar = map[string]string{ +// DefaultRuntimeLabelsVar specifies a set of labels that will be attached to k3d runtime objects by default but are not static (e.g. across k3d versions) +var DefaultRuntimeLabelsVar = map[string]string{ "k3d.version": version.GetVersion(), } @@ -339,7 +339,7 @@ type Node struct { Ports nat.PortMap `yaml:"portMappings" json:"portMappings,omitempty"` Restart bool `yaml:"restart" json:"restart,omitempty"` Created string `yaml:"created" json:"created,omitempty"` - Labels map[string]string // filled automatically + RuntimeLabels map[string]string `yaml:"runtimeLabels" json:"runtimeLabels,omitempty"` K3sNodeLabels map[string]string `yaml:"k3sNodeLabels" json:"k3sNodeLabels,omitempty"` Networks []string // filled automatically ExtraHosts []string // filled automatically diff --git a/tests/assets/config_test_simple.yaml b/tests/assets/config_test_simple.yaml index c1d05e2b..75588924 100755 --- a/tests/assets/config_test_simple.yaml +++ b/tests/assets/config_test_simple.yaml @@ -22,11 +22,6 @@ env: - envVar: bar=baz,bob nodeFilters: - all -labels: - - label: foo=bar - nodeFilters: - - server[0] - - loadbalancer registries: create: true use: [] @@ -47,6 +42,17 @@ options: - arg: --tls-san=127.0.0.1 nodeFilters: - server[*] + nodeLabels: + - label: foo=bar + nodeFilters: + - server[0] + - loadbalancer kubeconfig: updateDefaultKubeconfig: true - switchCurrentContext: true \ No newline at end of file + switchCurrentContext: true + runtime: + labels: + - label: foo=bar + nodeFilters: + - server[0] + - loadbalancer diff --git a/tests/assets/config_test_simple_migration_v1alpha3.yaml b/tests/assets/config_test_simple_migration_v1alpha3.yaml index c1d05e2b..98eccbde 100755 --- a/tests/assets/config_test_simple_migration_v1alpha3.yaml +++ b/tests/assets/config_test_simple_migration_v1alpha3.yaml @@ -22,11 +22,6 @@ env: - envVar: bar=baz,bob nodeFilters: - all -labels: - - label: foo=bar - nodeFilters: - - server[0] - - loadbalancer registries: create: true use: [] @@ -49,4 +44,10 @@ options: - server[*] kubeconfig: updateDefaultKubeconfig: true - switchCurrentContext: true \ No newline at end of file + switchCurrentContext: true + runtime: + labels: + - label: foo=bar + nodeFilters: + - server[0] + - loadbalancer diff --git a/tests/common.sh b/tests/common.sh index f1529fbb..4d4de061 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -175,4 +175,10 @@ docker_assert_container_label() { # $1 = container/node name # $2 = label to assert docker inspect --format '{{ range $k, $v := .Config.Labels }}{{ printf "%s=%s\n" $k $v }}{{ end }}' "$1" | grep -E "^$2$" +} + +k3s_assert_node_label() { + # $1 = node name + # $2 = label to assert + kubectl get node "$1" --output go-template='{{ range $k, $v := .metadata.labels }}{{ printf "%s=%s\n" $k $v }}{{ end }}' | grep -E "^$2$" } \ No newline at end of file diff --git a/tests/test_config_file.sh b/tests/test_config_file.sh index b367bac1..8793eb44 100755 --- a/tests/test_config_file.sh +++ b/tests/test_config_file.sh @@ -18,7 +18,7 @@ fi export CURRENT_STAGE="Test | config-file | $K3S_IMAGE_TAG" -clustername="ConfigTest" +clustername="configtest" highlight "[START] ConfigTest $EXTRA_TITLE" @@ -45,6 +45,10 @@ exec_in_node "k3d-$clustername-server-0" "env" | grep "bar=baz,bob" || failed "E info "Ensuring that container labels have been set as stated in the config" docker_assert_container_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on container/node k3d-$clustername-server-0" +## K3s Node Labels +info "Ensuring that k3s node labels have been set as stated in the config" +k3s_assert_node_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on node k3d-$clustername-server-0" + ## Registry Node info "Ensuring, that we have a registry node present" $EXE node list "k3d-$clustername-registry" || failed "Expected k3d-$clustername-registry to be present" diff --git a/tests/test_config_with_overrides.sh b/tests/test_config_with_overrides.sh index 294491b1..6b705a06 100755 --- a/tests/test_config_with_overrides.sh +++ b/tests/test_config_with_overrides.sh @@ -44,6 +44,10 @@ exec_in_node "k3d-$clustername-agent-1" "env" | grep "x=y" || failed "Expected e info "Ensuring that container labels have been set as stated in the config" docker_assert_container_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on container/node k3d-$clustername-server-0" +## K3s Node Labels +info "Ensuring that k3s node labels have been set as stated in the config" +k3s_assert_node_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on node k3d-$clustername-server-0" + ## Registry Node info "Ensuring, that we DO NOT have a registry node present" $EXE node list "k3d-$clustername-registry" && failed "Expected k3d-$clustername-registry to NOT be present"