k3d/pkg/config/transform.go

/*
Copyright © 2020-2021 The k3d Author(s)

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

package config

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/docker/go-connections/nat"
	cliutil "github.com/rancher/k3d/v4/cmd/util" // TODO: move parseapiport to pkg
	"github.com/rancher/k3d/v4/pkg/client"
	conf "github.com/rancher/k3d/v4/pkg/config/v1alpha3"
	"github.com/rancher/k3d/v4/pkg/runtimes"
	k3d "github.com/rancher/k3d/v4/pkg/types"
	"github.com/rancher/k3d/v4/pkg/types/k3s"
	"github.com/rancher/k3d/v4/pkg/util"
	"github.com/rancher/k3d/v4/version"
	"gopkg.in/yaml.v2"
	"inet.af/netaddr"

	log "github.com/sirupsen/logrus"
)

var (
	DefaultTargetsNodefiltersPortMappings = []string{"servers:*:proxy", "agents:*:proxy"}
)

// TransformSimpleToClusterConfig transforms a simple configuration to a full-fledged cluster configuration
func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtime, simpleConfig conf.SimpleConfig) (*conf.ClusterConfig, error) {

	// set default cluster name
	if simpleConfig.Name == "" {
		simpleConfig.Name = k3d.DefaultClusterName
	}

	// fetch latest image
	if simpleConfig.Image == "latest" {
		simpleConfig.Image = version.GetK3sVersion(true)
	}

	clusterNetwork := k3d.ClusterNetwork{}
	if simpleConfig.Network != "" {
		clusterNetwork.Name = simpleConfig.Network
		clusterNetwork.External = true
	} else {
		clusterNetwork.Name = fmt.Sprintf("%s-%s", k3d.DefaultObjectNamePrefix, simpleConfig.Name)
		clusterNetwork.External = false
	}

	if simpleConfig.Subnet != "" {
		if simpleConfig.Subnet != "auto" {
			subnet, err := netaddr.ParseIPPrefix(simpleConfig.Subnet)
			if err != nil {
				return nil, fmt.Errorf("invalid subnet '%s': %w", simpleConfig.Subnet, err)
			}
			clusterNetwork.IPAM.IPPrefix = subnet
		}
		clusterNetwork.IPAM.Managed = true
	}

	// -> API
	if simpleConfig.ExposeAPI.HostIP == "" {
		simpleConfig.ExposeAPI.HostIP = k3d.DefaultAPIHost
	}
	if simpleConfig.ExposeAPI.Host == "" {
		simpleConfig.ExposeAPI.Host = simpleConfig.ExposeAPI.HostIP
	}

	kubeAPIExposureOpts := &k3d.ExposureOpts{
		Host: simpleConfig.ExposeAPI.Host,
	}
	kubeAPIExposureOpts.Port = k3d.DefaultAPIPort
	kubeAPIExposureOpts.Binding = nat.PortBinding{
		HostIP:   simpleConfig.ExposeAPI.HostIP,
		HostPort: simpleConfig.ExposeAPI.HostPort,
	}

	// FILL CLUSTER CONFIG
	newCluster := k3d.Cluster{
		Name:    simpleConfig.Name,
		Network: clusterNetwork,
		Token:   simpleConfig.ClusterToken,
		KubeAPI: kubeAPIExposureOpts,
	}

	// -> NODES
	newCluster.Nodes = []*k3d.Node{}

	if !simpleConfig.Options.K3dOptions.DisableLoadbalancer {
		newCluster.ServerLoadBalancer = k3d.NewLoadbalancer()
		var err error
		newCluster.ServerLoadBalancer.Node, err = client.LoadbalancerPrepare(ctx, runtime, &newCluster, nil)
		if err != nil {
			return nil, fmt.Errorf("error preparing the loadbalancer: %w", err)
		}
		newCluster.Nodes = append(newCluster.Nodes, newCluster.ServerLoadBalancer.Node)
	} else {
		log.Debugln("Disabling the load balancer")
	}

	/*************
	 * Add Nodes *
	 *************/

	for i := 0; i < simpleConfig.Servers; i++ {
		serverNode := k3d.Node{
			Name:       client.GenerateNodeName(newCluster.Name, k3d.ServerRole, i),
			Role:       k3d.ServerRole,
			Image:      simpleConfig.Image,
			ServerOpts: k3d.ServerOpts{},
			Memory:     simpleConfig.Options.Runtime.ServersMemory,
		}

		// first server node will be init node if we have more than one server specified but no external datastore
		if i == 0 && simpleConfig.Servers > 1 {
			serverNode.ServerOpts.IsInit = true
			newCluster.InitNode = &serverNode
		}

		newCluster.Nodes = append(newCluster.Nodes, &serverNode)

		newCluster.ServerLoadBalancer.Config.Ports[fmt.Sprintf("%s.tcp", k3d.DefaultAPIPort)] = append(newCluster.ServerLoadBalancer.Config.Ports[fmt.Sprintf("%s.tcp", k3d.DefaultAPIPort)], serverNode.Name)
	}

	for i := 0; i < simpleConfig.Agents; i++ {
		agentNode := k3d.Node{
			Name:   client.GenerateNodeName(newCluster.Name, k3d.AgentRole, i),
			Role:   k3d.AgentRole,
			Image:  simpleConfig.Image,
			Memory: simpleConfig.Options.Runtime.AgentsMemory,
		}
		newCluster.Nodes = append(newCluster.Nodes, &agentNode)
	}

	/****************************
	 * Extra Node Configuration *
	 ****************************/

	// -> VOLUMES
	nodeCount := len(newCluster.Nodes)
	nodeList := newCluster.Nodes
	for _, volumeWithNodeFilters := range simpleConfig.Volumes {
		nodes, err := util.FilterNodes(nodeList, volumeWithNodeFilters.NodeFilters)
		if err != nil {
			return nil, err
		}

		for _, node := range nodes {
			node.Volumes = append(node.Volumes, volumeWithNodeFilters.Volume)
		}
	}

	// -> PORTS
	for _, portWithNodeFilters := range simpleConfig.Ports {
		log.Tracef("inspecting port mapping for %s with nodefilters %s", portWithNodeFilters.Port, portWithNodeFilters.NodeFilters)
		if len(portWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
			log.Infof("portmapping '%s' lacks a nodefilter, but there's more than one node: defaulting to %s", portWithNodeFilters.Port, DefaultTargetsNodefiltersPortMappings)
			portWithNodeFilters.NodeFilters = DefaultTargetsNodefiltersPortMappings
		}

		for _, f := range portWithNodeFilters.NodeFilters {
			if strings.HasPrefix(f, "loadbalancer") {
				log.Infof("portmapping '%s' targets the loadbalancer: defaulting to %s", portWithNodeFilters.Port, DefaultTargetsNodefiltersPortMappings)
				portWithNodeFilters.NodeFilters = DefaultTargetsNodefiltersPortMappings
				break
			}
		}

		filteredNodes, err := util.FilterNodesWithSuffix(nodeList, portWithNodeFilters.NodeFilters)
		if err != nil {
			return nil, err
		}

		nn := ""
		for _, n := range filteredNodes["proxy"] {
			nn = strings.Join([]string{nn, n.Name}, ",")
		}
		log.Debugf("Filtered nodes: %#v", nn)

		for suffix, nodes := range filteredNodes {
			portmappings, err := nat.ParsePortSpec(portWithNodeFilters.Port)
			if err != nil {
				return nil, fmt.Errorf("error parsing port spec '%s': %+v", portWithNodeFilters.Port, err)
			}

			if suffix == "proxy" || suffix == util.NodeFilterSuffixNone { // proxy is the default suffix for port mappings
				if newCluster.ServerLoadBalancer == nil {
					return nil, fmt.Errorf("port-mapping of type 'proxy' specified, but loadbalancer is disabled")
				}
				if err := addPortMappings(newCluster.ServerLoadBalancer.Node, portmappings); err != nil {
					return nil, err
				}
				for _, pm := range portmappings {
					if err := loadbalancerAddPortConfigs(newCluster.ServerLoadBalancer, pm, nodes); err != nil {
						return nil, err
					}
				}
			} else if suffix == "direct" {
				if len(nodes) > 1 {
					return nil, fmt.Errorf("error: cannot apply a direct port-mapping (%s) to more than one node", portmappings)
				}
				for _, node := range nodes {
					if err := addPortMappings(node, portmappings); err != nil {
						return nil, err
					}
				}
			} else if suffix != util.NodeFilterMapKeyAll {
				return nil, fmt.Errorf("error adding port mappings: unknown suffix %s", suffix)
			}
		}

	}

	// print generated loadbalancer config
	if log.GetLevel() >= log.DebugLevel {
		yamlized, err := yaml.Marshal(newCluster.ServerLoadBalancer.Config)
		if err != nil {
			log.Errorf("error printing loadbalancer config: %v", err)
		} else {
			log.Debugf("generated loadbalancer config:\n%s", string(yamlized))
		}
	}

	// -> K3S NODE LABELS
	for _, k3sNodeLabelWithNodeFilters := range simpleConfig.Options.K3sOptions.NodeLabels {
		if len(k3sNodeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
			return nil, fmt.Errorf("K3sNodeLabelmapping '%s' lacks a node filter, but there's more than one node", k3sNodeLabelWithNodeFilters.Label)
		}

		nodes, err := util.FilterNodes(nodeList, k3sNodeLabelWithNodeFilters.NodeFilters)
		if err != nil {
			return nil, err
		}

		for _, node := range nodes {
			if node.K3sNodeLabels == nil {
				node.K3sNodeLabels = make(map[string]string) // ensure that the map is initialized
			}
			k, v := util.SplitLabelKeyValue(k3sNodeLabelWithNodeFilters.Label)
			node.K3sNodeLabels[k] = v

		}
	}

	// -> RUNTIME LABELS
	for _, runtimeLabelWithNodeFilters := range simpleConfig.Options.Runtime.Labels {
		if len(runtimeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
			return nil, fmt.Errorf("RuntimeLabelmapping '%s' lacks a node filter, but there's more than one node", runtimeLabelWithNodeFilters.Label)
		}

		nodes, err := util.FilterNodes(nodeList, runtimeLabelWithNodeFilters.NodeFilters)
		if err != nil {
			return nil, err
		}

		for _, node := range nodes {
			if node.RuntimeLabels == nil {
				node.RuntimeLabels = make(map[string]string) // ensure that the map is initialized
			}
			k, v := util.SplitLabelKeyValue(runtimeLabelWithNodeFilters.Label)

			cliutil.ValidateRuntimeLabelKey(k)

			node.RuntimeLabels[k] = v
		}
	}

	// -> ENV
	for _, envVarWithNodeFilters := range simpleConfig.Env {
		if len(envVarWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
			return nil, fmt.Errorf("EnvVarMapping '%s' lacks a node filter, but there's more than one node", envVarWithNodeFilters.EnvVar)
		}

		nodes, err := util.FilterNodes(nodeList, envVarWithNodeFilters.NodeFilters)
		if err != nil {
			return nil, err
		}

		for _, node := range nodes {
			node.Env = append(node.Env, envVarWithNodeFilters.EnvVar)
		}
	}

	// -> ARGS
	for _, argWithNodeFilters := range simpleConfig.Options.K3sOptions.ExtraArgs {
		if len(argWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
			return nil, fmt.Errorf("K3sExtraArg '%s' lacks a node filter, but there's more than one node", argWithNodeFilters.Arg)
		}

		nodes, err := util.FilterNodes(nodeList, argWithNodeFilters.NodeFilters)
		if err != nil {
			return nil, err
		}

		for _, node := range nodes {
			node.Args = append(node.Args, argWithNodeFilters.Arg)
		}
	}

	/**************************
	 * Cluster Create Options *
	 **************************/

	clusterCreateOpts := k3d.ClusterCreateOpts{
		PrepDisableHostIPInjection: simpleConfig.Options.K3dOptions.PrepDisableHostIPInjection,
		DisableImageVolume:         simpleConfig.Options.K3dOptions.DisableImageVolume,
		WaitForServer:              simpleConfig.Options.K3dOptions.Wait,
		Timeout:                    simpleConfig.Options.K3dOptions.Timeout,
		DisableLoadBalancer:        simpleConfig.Options.K3dOptions.DisableLoadbalancer,
		GPURequest:                 simpleConfig.Options.Runtime.GPURequest,
		ServersMemory:              simpleConfig.Options.Runtime.ServersMemory,
		AgentsMemory:               simpleConfig.Options.Runtime.AgentsMemory,
		GlobalLabels:               map[string]string{}, // empty init
		GlobalEnv:                  []string{},          // empty init
	}

	// ensure, that we have the default object labels
	for k, v := range k3d.DefaultRuntimeLabels {
		clusterCreateOpts.GlobalLabels[k] = v
	}

	/*
	 * Registries
	 */
	if simpleConfig.Registries.Create {
		regPort, err := cliutil.ParsePortExposureSpec("random", k3d.DefaultRegistryPort)
		if err != nil {
			return nil, fmt.Errorf("Failed to get port for registry: %+v", err)
		}
		clusterCreateOpts.Registries.Create = &k3d.Registry{
			ClusterRef:   newCluster.Name,
			Host:         fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, newCluster.Name),
			Image:        fmt.Sprintf("%s:%s", k3d.DefaultRegistryImageRepo, k3d.DefaultRegistryImageTag),
			ExposureOpts: *regPort,
		}
	}

	for _, usereg := range simpleConfig.Registries.Use {
		reg, err := util.ParseRegistryRef(usereg)
		if err != nil {
			return nil, fmt.Errorf("Failed to parse use-registry string  '%s': %+v", usereg, err)
		}
		log.Tracef("Parsed registry reference: %+v", reg)
		clusterCreateOpts.Registries.Use = append(clusterCreateOpts.Registries.Use, reg)
	}

	if simpleConfig.Registries.Config != "" {
		var k3sRegistry *k3s.Registry

		if strings.Contains(simpleConfig.Registries.Config, "\n") { // CASE 1: embedded registries.yaml (multiline string)
			log.Debugf("Found multiline registries config embedded in SimpleConfig:\n%s", simpleConfig.Registries.Config)
			if err := yaml.Unmarshal([]byte(simpleConfig.Registries.Config), &k3sRegistry); err != nil {
				return nil, fmt.Errorf("Failed to read embedded registries config: %+v", err)
			}
		} else { // CASE 2: registries.yaml file referenced by path (single line)
			registryConfigFile, err := os.Open(simpleConfig.Registries.Config)
			if err != nil {
				return nil, fmt.Errorf("Failed to open registry config file at %s: %+v", simpleConfig.Registries.Config, err)
			}
			configBytes, err := ioutil.ReadAll(registryConfigFile)
			if err != nil {
				return nil, fmt.Errorf("Failed to read registry config file at %s: %+v", registryConfigFile.Name(), err)
			}

			if err := yaml.Unmarshal(configBytes, &k3sRegistry); err != nil {
				return nil, fmt.Errorf("Failed to read registry configuration: %+v", err)
			}
		}

		log.Tracef("Registry: read config from input:\n%+v", k3sRegistry)
		clusterCreateOpts.Registries.Config = k3sRegistry
	}

	/**********************
	 * Kubeconfig Options *
	 **********************/

	// Currently, the kubeconfig options for the cluster config are the same as for the simple config

	/******************************
	 * Create Full Cluster Config *
	 ******************************/

	clusterConfig := &conf.ClusterConfig{
		Cluster:           newCluster,
		ClusterCreateOpts: clusterCreateOpts,
		KubeconfigOpts:    simpleConfig.Options.KubeconfigOptions,
	}

	return clusterConfig, nil
}

func addPortMappings(node *k3d.Node, portmappings []nat.PortMapping) error {

	if node.Ports == nil {
		node.Ports = nat.PortMap{}
	}
	for _, pm := range portmappings {
		if _, exists := node.Ports[pm.Port]; exists {
			node.Ports[pm.Port] = append(node.Ports[pm.Port], pm.Binding)
		} else {
			node.Ports[pm.Port] = []nat.PortBinding{pm.Binding}
		}
	}
	return nil
}

func loadbalancerAddPortConfigs(loadbalancer *k3d.Loadbalancer, pm nat.PortMapping, nodes []*k3d.Node) error {
	portconfig := fmt.Sprintf("%s.%s", pm.Port.Port(), pm.Port.Proto())
	nodenames := []string{}
	for _, node := range nodes {
		if node.Role == k3d.LoadBalancerRole {
			return fmt.Errorf("error adding port config to loadbalancer: cannot add port config referencing the loadbalancer itself (loop)")
		}
		nodenames = append(nodenames, node.Name)
	}

	// entry for that port doesn't exist yet, so we simply create it with the list of node names
	if _, ok := loadbalancer.Config.Ports[portconfig]; !ok {
		loadbalancer.Config.Ports[portconfig] = nodenames
		return nil
	}

nodenameLoop:
	for _, nodename := range nodenames {
		for _, existingNames := range loadbalancer.Config.Ports[portconfig] {
			if nodename == existingNames {
				continue nodenameLoop
			}
			loadbalancer.Config.Ports[portconfig] = append(loadbalancer.Config.Ports[portconfig], nodename)
		}
	}

	return nil
}