/* Copyright © 2020-2021 The k3d Author(s) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ package config import ( "context" "fmt" "io/ioutil" "os" "strings" "github.com/docker/go-connections/nat" cliutil "github.com/rancher/k3d/v4/cmd/util" // TODO: move parseapiport to pkg "github.com/rancher/k3d/v4/pkg/client" conf "github.com/rancher/k3d/v4/pkg/config/v1alpha3" "github.com/rancher/k3d/v4/pkg/runtimes" k3d "github.com/rancher/k3d/v4/pkg/types" "github.com/rancher/k3d/v4/pkg/types/k3s" "github.com/rancher/k3d/v4/pkg/util" "github.com/rancher/k3d/v4/version" "gopkg.in/yaml.v2" "inet.af/netaddr" log "github.com/sirupsen/logrus" ) var ( DefaultTargetsNodefiltersPortMappings = []string{"servers:*:proxy", "agents:*:proxy"} ) // TransformSimpleToClusterConfig transforms a simple configuration to a full-fledged cluster configuration func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtime, simpleConfig conf.SimpleConfig) (*conf.ClusterConfig, error) { // set default cluster name if simpleConfig.Name == "" { simpleConfig.Name = k3d.DefaultClusterName } // fetch latest image if simpleConfig.Image == "latest" { simpleConfig.Image = version.GetK3sVersion(true) } clusterNetwork := k3d.ClusterNetwork{} if simpleConfig.Network != "" { clusterNetwork.Name = simpleConfig.Network clusterNetwork.External = true } else { clusterNetwork.Name = fmt.Sprintf("%s-%s", k3d.DefaultObjectNamePrefix, simpleConfig.Name) clusterNetwork.External = false } if simpleConfig.Subnet != "" { if simpleConfig.Subnet != "auto" { subnet, err := netaddr.ParseIPPrefix(simpleConfig.Subnet) if err != nil { return nil, fmt.Errorf("invalid subnet '%s': %w", simpleConfig.Subnet, err) } clusterNetwork.IPAM.IPPrefix = subnet } clusterNetwork.IPAM.Managed = true } // -> API if simpleConfig.ExposeAPI.HostIP == "" { simpleConfig.ExposeAPI.HostIP = k3d.DefaultAPIHost } if simpleConfig.ExposeAPI.Host == "" { simpleConfig.ExposeAPI.Host = simpleConfig.ExposeAPI.HostIP } kubeAPIExposureOpts := &k3d.ExposureOpts{ Host: simpleConfig.ExposeAPI.Host, } kubeAPIExposureOpts.Port = k3d.DefaultAPIPort kubeAPIExposureOpts.Binding = nat.PortBinding{ HostIP: simpleConfig.ExposeAPI.HostIP, HostPort: simpleConfig.ExposeAPI.HostPort, } // FILL CLUSTER CONFIG newCluster := k3d.Cluster{ Name: simpleConfig.Name, Network: clusterNetwork, Token: simpleConfig.ClusterToken, KubeAPI: kubeAPIExposureOpts, } // -> NODES newCluster.Nodes = []*k3d.Node{} if !simpleConfig.Options.K3dOptions.DisableLoadbalancer { newCluster.ServerLoadBalancer = k3d.NewLoadbalancer() var err error newCluster.ServerLoadBalancer.Node, err = client.LoadbalancerPrepare(ctx, runtime, &newCluster, nil) if err != nil { return nil, fmt.Errorf("error preparing the loadbalancer: %w", err) } newCluster.Nodes = append(newCluster.Nodes, newCluster.ServerLoadBalancer.Node) } else { log.Debugln("Disabling the load balancer") } /************* * Add Nodes * *************/ for i := 0; i < simpleConfig.Servers; i++ { serverNode := k3d.Node{ Name: client.GenerateNodeName(newCluster.Name, k3d.ServerRole, i), Role: k3d.ServerRole, Image: simpleConfig.Image, ServerOpts: k3d.ServerOpts{}, Memory: simpleConfig.Options.Runtime.ServersMemory, } // first server node will be init node if we have more than one server specified but no external datastore if i == 0 && simpleConfig.Servers > 1 { serverNode.ServerOpts.IsInit = true newCluster.InitNode = &serverNode } newCluster.Nodes = append(newCluster.Nodes, &serverNode) newCluster.ServerLoadBalancer.Config.Ports[fmt.Sprintf("%s.tcp", k3d.DefaultAPIPort)] = append(newCluster.ServerLoadBalancer.Config.Ports[fmt.Sprintf("%s.tcp", k3d.DefaultAPIPort)], serverNode.Name) } for i := 0; i < simpleConfig.Agents; i++ { agentNode := k3d.Node{ Name: client.GenerateNodeName(newCluster.Name, k3d.AgentRole, i), Role: k3d.AgentRole, Image: simpleConfig.Image, Memory: simpleConfig.Options.Runtime.AgentsMemory, } newCluster.Nodes = append(newCluster.Nodes, &agentNode) } /**************************** * Extra Node Configuration * ****************************/ // -> VOLUMES nodeCount := len(newCluster.Nodes) nodeList := newCluster.Nodes for _, volumeWithNodeFilters := range simpleConfig.Volumes { nodes, err := util.FilterNodes(nodeList, volumeWithNodeFilters.NodeFilters) if err != nil { return nil, err } for _, node := range nodes { node.Volumes = append(node.Volumes, volumeWithNodeFilters.Volume) } } // -> PORTS for _, portWithNodeFilters := range simpleConfig.Ports { log.Tracef("inspecting port mapping for %s with nodefilters %s", portWithNodeFilters.Port, portWithNodeFilters.NodeFilters) if len(portWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 { log.Infof("portmapping '%s' lacks a nodefilter, but there's more than one node: defaulting to %s", portWithNodeFilters.Port, DefaultTargetsNodefiltersPortMappings) portWithNodeFilters.NodeFilters = DefaultTargetsNodefiltersPortMappings } for _, f := range portWithNodeFilters.NodeFilters { if strings.HasPrefix(f, "loadbalancer") { log.Infof("portmapping '%s' targets the loadbalancer: defaulting to %s", portWithNodeFilters.Port, DefaultTargetsNodefiltersPortMappings) portWithNodeFilters.NodeFilters = DefaultTargetsNodefiltersPortMappings break } } filteredNodes, err := util.FilterNodesWithSuffix(nodeList, portWithNodeFilters.NodeFilters) if err != nil { return nil, err } nn := "" for _, n := range filteredNodes["proxy"] { nn = strings.Join([]string{nn, n.Name}, ",") } log.Debugf("Filtered nodes: %#v", nn) for suffix, nodes := range filteredNodes { portmappings, err := nat.ParsePortSpec(portWithNodeFilters.Port) if err != nil { return nil, fmt.Errorf("error parsing port spec '%s': %+v", portWithNodeFilters.Port, err) } if suffix == "proxy" || suffix == util.NodeFilterSuffixNone { // proxy is the default suffix for port mappings if newCluster.ServerLoadBalancer == nil { return nil, fmt.Errorf("port-mapping of type 'proxy' specified, but loadbalancer is disabled") } if err := addPortMappings(newCluster.ServerLoadBalancer.Node, portmappings); err != nil { return nil, err } for _, pm := range portmappings { if err := loadbalancerAddPortConfigs(newCluster.ServerLoadBalancer, pm, nodes); err != nil { return nil, err } } } else if suffix == "direct" { if len(nodes) > 1 { return nil, fmt.Errorf("error: cannot apply a direct port-mapping (%s) to more than one node", portmappings) } for _, node := range nodes { if err := addPortMappings(node, portmappings); err != nil { return nil, err } } } else if suffix != util.NodeFilterMapKeyAll { return nil, fmt.Errorf("error adding port mappings: unknown suffix %s", suffix) } } } // print generated loadbalancer config if log.GetLevel() >= log.DebugLevel { yamlized, err := yaml.Marshal(newCluster.ServerLoadBalancer.Config) if err != nil { log.Errorf("error printing loadbalancer config: %v", err) } else { log.Debugf("generated loadbalancer config:\n%s", string(yamlized)) } } // -> K3S NODE LABELS for _, k3sNodeLabelWithNodeFilters := range simpleConfig.Options.K3sOptions.NodeLabels { if len(k3sNodeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 { return nil, fmt.Errorf("K3sNodeLabelmapping '%s' lacks a node filter, but there's more than one node", k3sNodeLabelWithNodeFilters.Label) } nodes, err := util.FilterNodes(nodeList, k3sNodeLabelWithNodeFilters.NodeFilters) if err != nil { return nil, err } for _, node := range nodes { if node.K3sNodeLabels == nil { node.K3sNodeLabels = make(map[string]string) // ensure that the map is initialized } k, v := util.SplitLabelKeyValue(k3sNodeLabelWithNodeFilters.Label) node.K3sNodeLabels[k] = v } } // -> RUNTIME LABELS for _, runtimeLabelWithNodeFilters := range simpleConfig.Options.Runtime.Labels { if len(runtimeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 { return nil, fmt.Errorf("RuntimeLabelmapping '%s' lacks a node filter, but there's more than one node", runtimeLabelWithNodeFilters.Label) } nodes, err := util.FilterNodes(nodeList, runtimeLabelWithNodeFilters.NodeFilters) if err != nil { return nil, err } for _, node := range nodes { if node.RuntimeLabels == nil { node.RuntimeLabels = make(map[string]string) // ensure that the map is initialized } k, v := util.SplitLabelKeyValue(runtimeLabelWithNodeFilters.Label) cliutil.ValidateRuntimeLabelKey(k) node.RuntimeLabels[k] = v } } // -> ENV for _, envVarWithNodeFilters := range simpleConfig.Env { if len(envVarWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 { return nil, fmt.Errorf("EnvVarMapping '%s' lacks a node filter, but there's more than one node", envVarWithNodeFilters.EnvVar) } nodes, err := util.FilterNodes(nodeList, envVarWithNodeFilters.NodeFilters) if err != nil { return nil, err } for _, node := range nodes { node.Env = append(node.Env, envVarWithNodeFilters.EnvVar) } } // -> ARGS for _, argWithNodeFilters := range simpleConfig.Options.K3sOptions.ExtraArgs { if len(argWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 { return nil, fmt.Errorf("K3sExtraArg '%s' lacks a node filter, but there's more than one node", argWithNodeFilters.Arg) } nodes, err := util.FilterNodes(nodeList, argWithNodeFilters.NodeFilters) if err != nil { return nil, err } for _, node := range nodes { node.Args = append(node.Args, argWithNodeFilters.Arg) } } /************************** * Cluster Create Options * **************************/ clusterCreateOpts := k3d.ClusterCreateOpts{ PrepDisableHostIPInjection: simpleConfig.Options.K3dOptions.PrepDisableHostIPInjection, DisableImageVolume: simpleConfig.Options.K3dOptions.DisableImageVolume, WaitForServer: simpleConfig.Options.K3dOptions.Wait, Timeout: simpleConfig.Options.K3dOptions.Timeout, DisableLoadBalancer: simpleConfig.Options.K3dOptions.DisableLoadbalancer, GPURequest: simpleConfig.Options.Runtime.GPURequest, ServersMemory: simpleConfig.Options.Runtime.ServersMemory, AgentsMemory: simpleConfig.Options.Runtime.AgentsMemory, GlobalLabels: map[string]string{}, // empty init GlobalEnv: []string{}, // empty init } // ensure, that we have the default object labels for k, v := range k3d.DefaultRuntimeLabels { clusterCreateOpts.GlobalLabels[k] = v } /* * Registries */ if simpleConfig.Registries.Create { regPort, err := cliutil.ParsePortExposureSpec("random", k3d.DefaultRegistryPort) if err != nil { return nil, fmt.Errorf("Failed to get port for registry: %+v", err) } clusterCreateOpts.Registries.Create = &k3d.Registry{ ClusterRef: newCluster.Name, Host: fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, newCluster.Name), Image: fmt.Sprintf("%s:%s", k3d.DefaultRegistryImageRepo, k3d.DefaultRegistryImageTag), ExposureOpts: *regPort, } } for _, usereg := range simpleConfig.Registries.Use { reg, err := util.ParseRegistryRef(usereg) if err != nil { return nil, fmt.Errorf("Failed to parse use-registry string '%s': %+v", usereg, err) } log.Tracef("Parsed registry reference: %+v", reg) clusterCreateOpts.Registries.Use = append(clusterCreateOpts.Registries.Use, reg) } if simpleConfig.Registries.Config != "" { var k3sRegistry *k3s.Registry if strings.Contains(simpleConfig.Registries.Config, "\n") { // CASE 1: embedded registries.yaml (multiline string) log.Debugf("Found multiline registries config embedded in SimpleConfig:\n%s", simpleConfig.Registries.Config) if err := yaml.Unmarshal([]byte(simpleConfig.Registries.Config), &k3sRegistry); err != nil { return nil, fmt.Errorf("Failed to read embedded registries config: %+v", err) } } else { // CASE 2: registries.yaml file referenced by path (single line) registryConfigFile, err := os.Open(simpleConfig.Registries.Config) if err != nil { return nil, fmt.Errorf("Failed to open registry config file at %s: %+v", simpleConfig.Registries.Config, err) } configBytes, err := ioutil.ReadAll(registryConfigFile) if err != nil { return nil, fmt.Errorf("Failed to read registry config file at %s: %+v", registryConfigFile.Name(), err) } if err := yaml.Unmarshal(configBytes, &k3sRegistry); err != nil { return nil, fmt.Errorf("Failed to read registry configuration: %+v", err) } } log.Tracef("Registry: read config from input:\n%+v", k3sRegistry) clusterCreateOpts.Registries.Config = k3sRegistry } /********************** * Kubeconfig Options * **********************/ // Currently, the kubeconfig options for the cluster config are the same as for the simple config /****************************** * Create Full Cluster Config * ******************************/ clusterConfig := &conf.ClusterConfig{ Cluster: newCluster, ClusterCreateOpts: clusterCreateOpts, KubeconfigOpts: simpleConfig.Options.KubeconfigOptions, } return clusterConfig, nil } func addPortMappings(node *k3d.Node, portmappings []nat.PortMapping) error { if node.Ports == nil { node.Ports = nat.PortMap{} } for _, pm := range portmappings { if _, exists := node.Ports[pm.Port]; exists { node.Ports[pm.Port] = append(node.Ports[pm.Port], pm.Binding) } else { node.Ports[pm.Port] = []nat.PortBinding{pm.Binding} } } return nil } func loadbalancerAddPortConfigs(loadbalancer *k3d.Loadbalancer, pm nat.PortMapping, nodes []*k3d.Node) error { portconfig := fmt.Sprintf("%s.%s", pm.Port.Port(), pm.Port.Proto()) nodenames := []string{} for _, node := range nodes { if node.Role == k3d.LoadBalancerRole { return fmt.Errorf("error adding port config to loadbalancer: cannot add port config referencing the loadbalancer itself (loop)") } nodenames = append(nodenames, node.Name) } // entry for that port doesn't exist yet, so we simply create it with the list of node names if _, ok := loadbalancer.Config.Ports[portconfig]; !ok { loadbalancer.Config.Ports[portconfig] = nodenames return nil } nodenameLoop: for _, nodename := range nodenames { for _, existingNames := range loadbalancer.Config.Ports[portconfig] { if nodename == existingNames { continue nodenameLoop } loadbalancer.Config.Ports[portconfig] = append(loadbalancer.Config.Ports[portconfig], nodename) } } return nil }