[Refactoring/Preparation] use real port structs as prep for registries (#427)

Makefile

@@ -49,6 +49,7 @@ E2E_LOG_LEVEL ?= WARN
 E2E_SKIP ?=
 E2E_EXTRA ?=
 E2E_RUNNER_START_TIMEOUT ?= 10
+E2E_HELPER_IMAGE_TAG ?=
 
 ########## Go Build Options ##########
 # Build targets
@@ -168,7 +169,7 @@ test:
 
 e2e: build-docker-dind
 	@echo "Running e2e tests in k3d:$(K3D_IMAGE_TAG)"
-	LOG_LEVEL="$(E2E_LOG_LEVEL)" E2E_SKIP="$(E2E_SKIP)" E2E_EXTRA="$(E2E_EXTRA)" E2E_RUNNER_START_TIMEOUT=$(E2E_RUNNER_START_TIMEOUT) tests/dind.sh "${K3D_IMAGE_TAG}-dind"
+	LOG_LEVEL="$(E2E_LOG_LEVEL)" E2E_SKIP="$(E2E_SKIP)" E2E_EXTRA="$(E2E_EXTRA)" E2E_RUNNER_START_TIMEOUT=$(E2E_RUNNER_START_TIMEOUT) E2E_HELPER_IMAGE_TAG="$(E2E_HELPER_IMAGE_TAG)" tests/dind.sh "${K3D_IMAGE_TAG}-dind"
 
 ci-tests: fmt check e2e
 

cmd/cluster/clusterCreate.go

@@ -51,17 +51,12 @@ Every cluster will consist of one or more containers:
 
 // flags that go through some pre-processing before transforming them to config
 type preProcessedFlags struct {
-	APIPort string
-	Volumes []string
-	Ports   []string
-	Labels  []string
-	Env     []string
-}
-
-// registry
-type registryFlags struct {
-	Use    []string
-	Create bool
+	APIPort     string
+	Volumes     []string
+	Ports       []string
+	Labels      []string
+	Env         []string
+	RegistryUse []string
 }
 
 // NewCmdClusterCreate returns a new cobra command
@@ -70,7 +65,6 @@ func NewCmdClusterCreate() *cobra.Command {
 	cliConfig := &conf.SimpleConfig{}
 	var configFile string
 	ppFlags := &preProcessedFlags{}
-	regFlags := &registryFlags{}
 
 	// create new command
 	cmd := &cobra.Command{
@@ -213,11 +207,8 @@ func NewCmdClusterCreate() *cobra.Command {
 	}
 
 	/* Registry */
-	cmd.Flags().StringArrayVar(&regFlags.Use, "registry-use", nil, "Connect to one or more registries running locally")
-	if err := cmd.Flags().MarkHidden("registry-use"); err != nil {
-		log.Fatalln("Failed to mark flag `cluster create --registry-use` as hidden")
-	}
-	cmd.Flags().BoolVar(&cliConfig.Registries.Create, "registry-create", false, "Create a registry and connect it to the cluster")
+	cmd.Flags().StringArrayVar(&cliConfig.Registries.Use, "registry-use", nil, "Connect to one or more k3d-managed registries running locally")
+	cmd.Flags().BoolVar(&cliConfig.Registries.Create, "registry-create", false, "Create a k3d-managed registry and connect it to the cluster")
 
 	/* Multi Server Configuration */
 
@@ -265,11 +256,15 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, cliConfig *conf.Si
 
 	// -> API-PORT
 	// parse the port mapping
-	exposeAPI, err := cliutil.ParseExposePort(ppFlags.APIPort)
+	exposeAPI, err := cliutil.ParsePortExposureSpec(ppFlags.APIPort, k3d.DefaultAPIPort)
 	if err != nil {
 		log.Fatalln(err)
 	}
-	cliConfig.ExposeAPI = exposeAPI
+	cliConfig.ExposeAPI = conf.SimpleExposureOpts{
+		Host:     exposeAPI.Host,
+		HostIP:   exposeAPI.Binding.HostIP,
+		HostPort: exposeAPI.Binding.HostPort,
+	}
 
 	// -> VOLUMES
 	// volumeFilterMap will map volume mounts to applied node filters
@@ -384,5 +379,4 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, cliConfig *conf.Si
 	}
 
 	log.Tracef("EnvFilterMap: %+v", envFilterMap)
-
 }

cmd/registry/registryCreate.go

@@ -101,7 +101,7 @@ func parseCreateRegistryCmd(cmd *cobra.Command, args []string, flags *regCreateF
 	}
 
 	// --port
-	exposePort, err := cliutil.ParseExposePort(ppFlags.Port)
+	exposePort, err := cliutil.ParsePortExposureSpec(ppFlags.Port, k3d.DefaultRegistryPort)
 	if err != nil {
 		log.Errorln("Failed to parse registry port")
 		log.Fatalln(err)
@@ -113,5 +113,5 @@ func parseCreateRegistryCmd(cmd *cobra.Command, args []string, flags *regCreateF
 		registryName = fmt.Sprintf("%s-%s", k3d.DefaultObjectNamePrefix, args[0])
 	}
 
-	return &k3d.Registry{Host: registryName, Image: flags.Image, Port: k3d.MappedPort{InternalPort: k3d.DefaultRegistryPort, ExternalPort: exposePort}}, clusters
+	return &k3d.Registry{Host: registryName, Image: flags.Image, ExposureOpts: *exposePort}, clusters
 }

cmd/util/ports.go

@@ -24,60 +24,83 @@ package util
 import (
 	"fmt"
 	"net"
+	"regexp"
 	"strconv"
-	"strings"
 
+	"github.com/docker/go-connections/nat"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v4/pkg/util"
 	log "github.com/sirupsen/logrus"
 )
 
-// ParseExposePort parses/validates a string to create an exposePort struct from it
-func ParseExposePort(portString string) (k3d.ExposedPort, error) {
+var apiPortRegexp = regexp.MustCompile(`^(?P<hostref>(?P<hostip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?P<hostname>\S+):)?(?P<port>(\d{1,5}|random))$`)
 
-	var exposePort k3d.ExposedPort
+// ParsePortExposureSpec parses/validates a string to create an exposePort struct from it
+func ParsePortExposureSpec(exposedPortSpec, internalPort string) (*k3d.ExposureOpts, error) {
 
-	split := strings.Split(portString, ":")
-	if len(split) > 2 {
-		log.Errorln("Failed to parse API Port specification")
-		return exposePort, fmt.Errorf("api-port format error")
+	match := apiPortRegexp.FindStringSubmatch(exposedPortSpec)
+
+	if len(match) == 0 {
+		log.Errorln("Failed to parse Port Exposure specification")
+		return nil, fmt.Errorf("Port Exposure Spec format error: Must be [(HostIP|HostName):]HostPort")
+	}
+
+	submatches := util.MapSubexpNames(apiPortRegexp.SubexpNames(), match)
+
+	// no port specified (or not matched via regex)
+	if submatches["port"] == "" {
+		return nil, fmt.Errorf("Failed to find port in Port Exposure spec '%s'", exposedPortSpec)
 	}
 
-	if len(split) == 1 {
-		exposePort = k3d.ExposedPort{Port: split[0]}
-	} else {
-		// Make sure 'host' can be resolved to an IP address
-		addrs, err := net.LookupHost(split[0])
+	api := &k3d.ExposureOpts{}
+
+	// check if there's a host reference
+	if submatches["hostname"] != "" {
+		log.Tracef("Port Exposure: found hostname: %s", submatches["hostname"])
+		addrs, err := net.LookupHost(submatches["hostname"])
 		if err != nil {
-			return exposePort, err
+			return nil, fmt.Errorf("Failed to lookup host '%s' specified for Port Exposure: %+v", submatches["hostname"], err)
 		}
-		exposePort = k3d.ExposedPort{Host: split[0], HostIP: addrs[0], Port: split[1]}
+		api.Host = submatches["hostname"]
+		submatches["hostip"] = addrs[0] // set hostip to the resolved address
 	}
 
-	// Verify 'port' is an integer and within port ranges
-	if exposePort.Port == "" || exposePort.Port == "random" {
-		log.Debugf("API-Port Mapping didn't specify hostPort, choosing one randomly...")
+	realPortString := ""
+
+	if submatches["hostip"] == "" {
+		submatches["hostip"] = k3d.DefaultAPIHost
+	}
+
+	// start with the IP, if there is any
+	if submatches["hostip"] != "" {
+		realPortString += submatches["hostip"] + ":"
+	}
+
+	// port: get a free one if there's none defined or set to random
+	if submatches["port"] == "" || submatches["port"] == "random" {
+		log.Debugf("Port Exposure Mapping didn't specify hostPort, choosing one randomly...")
 		freePort, err := GetFreePort()
 		if err != nil || freePort == 0 {
-			log.Warnf("Failed to get random free port:\n%+v", err)
-			log.Warnf("Falling back to default port %s (may be blocked though)...", k3d.DefaultAPIPort)
-			exposePort.Port = k3d.DefaultAPIPort
+			log.Warnf("Failed to get random free port: %+v", err)
+			log.Warnf("Falling back to internal port %s (may be blocked though)...", internalPort)
+			submatches["port"] = internalPort
 		} else {
-			exposePort.Port = strconv.Itoa(freePort)
-			log.Debugf("Got free port for API: '%d'", freePort)
+			submatches["port"] = strconv.Itoa(freePort)
+			log.Debugf("Got free port for Port Exposure: '%d'", freePort)
 		}
 	}
-	p, err := strconv.Atoi(exposePort.Port)
+
+	realPortString += fmt.Sprintf("%s:%s/tcp", submatches["port"], internalPort)
+
+	portMapping, err := nat.ParsePortSpec(realPortString)
 	if err != nil {
-		log.Errorln("Failed to parse port mapping")
-		return exposePort, err
+		return nil, fmt.Errorf("Failed to parse port spec for Port Exposure '%s': %+v", realPortString, err)
 	}
 
-	if p < 0 || p > 65535 {
-		log.Errorln("Failed to parse API Port specification")
-		return exposePort, fmt.Errorf("Port value '%d' out of range", p)
-	}
+	api.Port = portMapping[0].Port // there can be only one due to our regexp
+	api.Binding = portMapping[0].Binding
 
-	return exposePort, nil
+	return api, nil
 
 }
 

pkg/client/cluster.go

@@ -33,6 +33,7 @@ import (
 
 	gort "runtime"
 
+	"github.com/docker/go-connections/nat"
 	"github.com/imdario/mergo"
 	"github.com/rancher/k3d/v4/pkg/actions"
 	config "github.com/rancher/k3d/v4/pkg/config/v1alpha1"
@@ -276,7 +277,7 @@ ClusterCreatOpts:
 	/*
 	 * Docker Machine Special Configuration
 	 */
-	if cluster.ExposeAPI.Host == k3d.DefaultAPIHost && runtime == k3drt.Docker {
+	if cluster.KubeAPI.Host == k3d.DefaultAPIHost && runtime == k3drt.Docker {
 		if gort.GOOS == "windows" || gort.GOOS == "darwin" {
 			log.Tracef("Running on %s: checking if it's using docker-machine", gort.GOOS)
 			machineIP, err := runtime.(docker.Docker).GetDockerMachineIP()
@@ -284,8 +285,8 @@ ClusterCreatOpts:
 				log.Warnf("Using docker-machine, but failed to get it's IP: %+v", err)
 			} else if machineIP != "" {
 				log.Infof("Using the docker-machine IP %s to connect to the Kubernetes API", machineIP)
-				cluster.ExposeAPI.Host = machineIP
-				cluster.ExposeAPI.HostIP = machineIP
+				cluster.KubeAPI.Host = machineIP
+				cluster.KubeAPI.Binding.HostIP = machineIP
 			} else {
 				log.Traceln("Not using docker-machine")
 			}
@@ -330,7 +331,7 @@ ClusterCreatOpts:
 		// node role specific settings
 		if node.Role == k3d.ServerRole {
 
-			node.ServerOpts.ExposeAPI = cluster.ExposeAPI
+			node.ServerOpts.KubeAPI = cluster.KubeAPI
 
 			// the cluster has an init server node, but its not this one, so connect it to the init node
 			if cluster.InitNode != nil && !node.ServerOpts.IsInit {
@@ -371,7 +372,7 @@ ClusterCreatOpts:
 
 		// in case the LoadBalancer was disabled, expose the API Port on the initializing server node
 		if clusterCreateOpts.DisableLoadBalancer {
-			cluster.InitNode.Ports = append(cluster.InitNode.Ports, fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort))
+			cluster.InitNode.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding}
 		}
 
 		if err := nodeSetup(cluster.InitNode, serverCount); err != nil {
@@ -390,7 +391,7 @@ ClusterCreatOpts:
 				continue
 			} else if serverCount == 0 && clusterCreateOpts.DisableLoadBalancer {
 				// if this is the first server node and the server loadbalancer is disabled, expose the API Port on this server node
-				node.Ports = append(node.Ports, fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort))
+				node.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding}
 			}
 
 			time.Sleep(1 * time.Second) // FIXME: arbitrary wait for one second to avoid race conditions of servers registering
@@ -431,34 +432,20 @@ ClusterCreatOpts:
 
 			// generate comma-separated list of extra ports to forward
 			ports := k3d.DefaultAPIPort
-			for _, portString := range cluster.ServerLoadBalancer.Ports {
-				split := strings.Split(portString, ":")
-				port := split[len(split)-1]
-				if strings.Contains(port, "-") {
-					split := strings.Split(port, "-")
-					start, err := strconv.Atoi(split[0])
-					if err != nil {
-						log.Errorf("Failed to parse port mapping for loadbalancer '%s'", port)
-						return err
-					}
-					end, err := strconv.Atoi(split[1])
-					if err != nil {
-						log.Errorf("Failed to parse port mapping for loadbalancer '%s'", port)
-						return err
-					}
-					for i := start; i <= end; i++ {
-						ports += "," + strconv.Itoa(i)
-					}
-				} else {
-					ports += "," + port
-				}
+			for exposedPort := range cluster.ServerLoadBalancer.Ports {
+				ports += "," + exposedPort.Port()
+			}
+
+			if cluster.ServerLoadBalancer.Ports == nil {
+				cluster.ServerLoadBalancer.Ports = nat.PortMap{}
 			}
+			cluster.ServerLoadBalancer.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding}
 
 			// Create LB as a modified node with loadbalancerRole
 			lbNode := &k3d.Node{
 				Name:  fmt.Sprintf("%s-%s-serverlb", k3d.DefaultObjectNamePrefix, cluster.Name),
 				Image: fmt.Sprintf("%s:%s", k3d.DefaultLBImageRepo, version.GetHelperImageVersion()),
-				Ports: append(cluster.ServerLoadBalancer.Ports, fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort)),
+				Ports: cluster.ServerLoadBalancer.Ports,
 				Env: []string{
 					fmt.Sprintf("SERVERS=%s", servers),
 					fmt.Sprintf("PORTS=%s", ports),

pkg/client/node.go

@@ -330,11 +330,11 @@ func patchServerSpec(node *k3d.Node) error {
 
 	// Add labels and TLS SAN for the exposed API
 	// FIXME: For now, the labels concerning the API on the server nodes are only being used for configuring the kubeconfig
-	node.Labels[k3d.LabelServerAPIHostIP] = node.ServerOpts.ExposeAPI.HostIP // TODO: maybe get docker machine IP here
-	node.Labels[k3d.LabelServerAPIHost] = node.ServerOpts.ExposeAPI.Host
-	node.Labels[k3d.LabelServerAPIPort] = node.ServerOpts.ExposeAPI.Port
+	node.Labels[k3d.LabelServerAPIHostIP] = node.ServerOpts.KubeAPI.Binding.HostIP // TODO: maybe get docker machine IP here
+	node.Labels[k3d.LabelServerAPIHost] = node.ServerOpts.KubeAPI.Host
+	node.Labels[k3d.LabelServerAPIPort] = node.ServerOpts.KubeAPI.Binding.HostPort
 
-	node.Args = append(node.Args, "--tls-san", node.ServerOpts.ExposeAPI.Host) // add TLS SAN for non default host name
+	node.Args = append(node.Args, "--tls-san", node.ServerOpts.KubeAPI.Host) // add TLS SAN for non default host name
 
 	return nil
 }

pkg/client/registry.go

@@ -25,6 +25,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/docker/go-connections/nat"
 	"github.com/rancher/k3d/v4/pkg/runtimes"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
 	"github.com/rancher/k3d/v4/pkg/types/k3s"
@@ -71,19 +72,19 @@ func RegistryCreate(ctx context.Context, runtime runtimes.Runtime, reg *k3d.Regi
 
 	// setup the node labels
 	registryNode.Labels = map[string]string{
-		k3d.LabelRole:           string(k3d.RegistryRole),
-		k3d.LabelRegistryHost:   reg.Port.ExternalPort.Host, // TODO: docker machine host?
-		k3d.LabelRegistryHostIP: reg.Port.ExternalPort.HostIP,
-		k3d.LabelRegistryPort:   reg.Port.ExternalPort.Port,
+		k3d.LabelRole:                 string(k3d.RegistryRole),
+		k3d.LabelRegistryHost:         reg.ExposureOpts.Host, // TODO: docker machine host?
+		k3d.LabelRegistryHostIP:       reg.ExposureOpts.Binding.HostIP,
+		k3d.LabelRegistryPortExternal: reg.ExposureOpts.Binding.HostPort,
+		k3d.LabelRegistryPortInternal: reg.ExposureOpts.Port.Port(),
 	}
 	for k, v := range k3d.DefaultObjectLabels {
 		registryNode.Labels[k] = v
 	}
 
 	// port
-	registryNode.Ports = []string{
-		fmt.Sprintf("%s:%s:%s/tcp", reg.Port.ExternalPort.HostIP, reg.Port.ExternalPort.Port, k3d.DefaultRegistryPort),
-	}
+	registryNode.Ports = nat.PortMap{}
+	registryNode.Ports[reg.ExposureOpts.Port] = []nat.PortBinding{reg.ExposureOpts.Binding}
 
 	// create the registry node
 	log.Infof("Creating node '%s'", registryNode.Name)
@@ -136,8 +137,8 @@ func RegistryGenerateK3sConfig(ctx context.Context, registries []*k3d.Registry)
 	regConf := &k3s.Registry{}
 
 	for _, reg := range registries {
-		internalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.Port.InternalPort)
-		externalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.Port.ExternalPort.Port)
+		internalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.ExposureOpts.Port.Port())
+		externalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.ExposureOpts.Binding.HostPort)
 
 		// init mirrors if nil
 		if regConf.Mirrors == nil {
@@ -159,3 +160,21 @@ func RegistryGenerateK3sConfig(ctx context.Context, registries []*k3d.Registry)
 
 	return regConf, nil
 }
+
+// RegistryGet gets a registry node by name and returns it as a registry object
+func RegistryGet(ctx context.Context, runtime runtimes.Runtime, name string) (*k3d.Registry, error) {
+	regNode, err := runtime.GetNode(ctx, &k3d.Node{
+		Name: name,
+		Role: k3d.RegistryRole,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("Failed to find registry '%s': %+v", name, err)
+	}
+
+	registry := &k3d.Registry{
+		Host: regNode.Name,
+	}
+	// TODO: finish RegistryGet
+	return registry, nil
+
+}

pkg/config/config_test.go

@@ -33,19 +33,20 @@ import (
 
 func TestReadSimpleConfig(t *testing.T) {
 
+	exposedAPI := conf.SimpleExposureOpts{}
+	exposedAPI.HostIP = "0.0.0.0"
+	exposedAPI.HostPort = "6443"
+
 	expectedConfig := conf.SimpleConfig{
 		TypeMeta: conf.TypeMeta{
 			APIVersion: "k3d.io/v1alpha1",
 			Kind:       "Simple",
 		},
-		Name:    "test",
-		Servers: 1,
-		Agents:  2,
-		ExposeAPI: k3d.ExposedPort{
-			HostIP: "0.0.0.0",
-			Port:   "6443",
-		},
-		Image: "rancher/k3s:latest",
+		Name:      "test",
+		Servers:   1,
+		Agents:    2,
+		ExposeAPI: exposedAPI,
+		Image:     "rancher/k3s:latest",
 		Volumes: []conf.VolumeWithNodeFilters{
 			{
 				Volume:      "/my/path:/some/path",

pkg/config/test_assets/config_test_simple.yaml

@@ -3,9 +3,9 @@ kind: Simple
 name: test
 servers: 1
 agents: 2
-exposeAPI:
+kubeAPI:
   hostIP: "0.0.0.0"
-  port: "6443"
+  hostPort: "6443"
 image: rancher/k3s:latest
 volumes:
   - volume: /my/path:/some/path

pkg/config/transform.go

@@ -26,6 +26,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/docker/go-connections/nat"
 	cliutil "github.com/rancher/k3d/v4/cmd/util" // TODO: move parseapiport to pkg
 	conf "github.com/rancher/k3d/v4/pkg/config/v1alpha1"
 	"github.com/rancher/k3d/v4/pkg/runtimes"
@@ -61,12 +62,21 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
 		simpleConfig.ExposeAPI.HostIP = k3d.DefaultAPIHost
 	}
 
+	kubeAPIExposureOpts := &k3d.ExposureOpts{
+		Host: simpleConfig.ExposeAPI.Host,
+	}
+	kubeAPIExposureOpts.Port = k3d.DefaultAPIPort
+	kubeAPIExposureOpts.Binding = nat.PortBinding{
+		HostIP:   simpleConfig.ExposeAPI.HostIP,
+		HostPort: simpleConfig.ExposeAPI.HostPort,
+	}
+
 	// FILL CLUSTER CONFIG
 	newCluster := k3d.Cluster{
-		Name:      simpleConfig.Name,
-		Network:   clusterNetwork,
-		Token:     simpleConfig.ClusterToken,
-		ExposeAPI: simpleConfig.ExposeAPI,
+		Name:    simpleConfig.Name,
+		Network: clusterNetwork,
+		Token:   simpleConfig.ClusterToken,
+		KubeAPI: kubeAPIExposureOpts,
 	}
 
 	// -> NODES
@@ -142,7 +152,20 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
 		}
 
 		for _, node := range nodes {
-			node.Ports = append(node.Ports, portWithNodeFilters.Port)
+			portmappings, err := nat.ParsePortSpec(portWithNodeFilters.Port)
+			if err != nil {
+				return nil, fmt.Errorf("Failed to parse port spec '%s': %+v", portWithNodeFilters.Port, err)
+			}
+			if node.Ports == nil {
+				node.Ports = nat.PortMap{}
+			}
+			for _, pm := range portmappings {
+				if _, exists := node.Ports[pm.Port]; exists {
+					node.Ports[pm.Port] = append(node.Ports[pm.Port], pm.Binding)
+				} else {
+					node.Ports[pm.Port] = []nat.PortBinding{pm.Binding}
+				}
+			}
 		}
 	}
 
@@ -200,18 +223,15 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
 	/*
 	 * Registries
 	 */
-	regPort, err := cliutil.ParseExposePort("random")
-	if err != nil {
-		return nil, fmt.Errorf("Failed to get port for registry: %+v", err)
-	}
 	if simpleConfig.Registries.Create {
+		regPort, err := cliutil.ParsePortExposureSpec("random", k3d.DefaultRegistryPort)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to get port for registry: %+v", err)
+		}
 		clusterCreateOpts.Registries.Create = &k3d.Registry{
-			Host:  fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, newCluster.Name),
-			Image: fmt.Sprintf("%s:%s", k3d.DefaultRegistryImageRepo, k3d.DefaultRegistryImageTag),
-			Port: k3d.MappedPort{
-				InternalPort: k3d.DefaultRegistryPort,
-				ExternalPort: regPort,
-			},
+			Host:         fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, newCluster.Name),
+			Image:        fmt.Sprintf("%s:%s", k3d.DefaultRegistryImageRepo, k3d.DefaultRegistryImageTag),
+			ExposureOpts: *regPort,
 		}
 	}
 

pkg/config/v1alpha1/types.go

@@ -116,7 +116,7 @@ type SimpleConfig struct {
 	Name         string                  `mapstructure:"name" yaml:"name" json:"name,omitempty"`
 	Servers      int                     `mapstructure:"servers" yaml:"servers" json:"servers,omitempty"` //nolint:lll    // default 1
 	Agents       int                     `mapstructure:"agents" yaml:"agents" json:"agents,omitempty"`    //nolint:lll    // default 0
-	ExposeAPI    k3d.ExposedPort         `mapstructure:"exposeAPI" yaml:"exposeAPI" json:"exposeAPI,omitempty"`
+	ExposeAPI    SimpleExposureOpts      `mapstructure:"kubeAPI" yaml:"kubeAPI" json:"kubeAPI,omitempty"`
 	Image        string                  `mapstructure:"image" yaml:"image" json:"image,omitempty"`
 	Network      string                  `mapstructure:"network" yaml:"network" json:"network,omitempty"`
 	ClusterToken string                  `mapstructure:"clusterToken" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated
@@ -126,11 +126,18 @@ type SimpleConfig struct {
 	Options      SimpleConfigOptions     `mapstructure:"options" yaml:"options" json:"options,omitempty"`
 	Env          []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"`
 	Registries   struct {
-		Use    []*k3d.Registry `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"`
-		Create bool            `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"`
+		Use    []string `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"`
+		Create bool     `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"`
 	} `mapstructure:"registries" yaml:"registries,omitempty" json:"registries,omitempty"`
 }
 
+// SimpleExposureOpts provides a simplified syntax compared to the original k3d.ExposureOpts
+type SimpleExposureOpts struct {
+	Host     string `mapstructure:"host" yaml:"host,omitempty" json:"host,omitempty"`
+	HostIP   string `mapstructure:"hostIP" yaml:"hostIP,omitempty" json:"hostIP,omitempty"`
+	HostPort string `mapstructure:"hostPort" yaml:"hostPort,omitempty" json:"hostPort,omitempty"`
+}
+
 // GetKind implements Config.GetKind
 func (c SimpleConfig) GetKind() string {
 	return "Cluster"

pkg/config/validate.go

@@ -57,7 +57,7 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config
 	}
 
 	// API-Port cannot be changed when using network=host
-	if config.Cluster.Network.Name == "host" && config.Cluster.ExposeAPI.Port != k3d.DefaultAPIPort {
+	if config.Cluster.Network.Name == "host" && config.Cluster.KubeAPI.Port.Port() != k3d.DefaultAPIPort {
 		// in hostNetwork mode, we're not going to map a hostport. Here it should always use 6443.
 		// Note that hostNetwork mode is super inflexible and since we don't change the backend port (on the container), it will only be one hostmode cluster allowed.
 		return fmt.Errorf("The API Port can not be changed when using 'host' network")
@@ -78,13 +78,6 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config
 				return err
 			}
 		}
-
-		// ports
-		for _, port := range node.Ports {
-			if err := util.ValidatePortMap(port); err != nil {
-				return err
-			}
-		}
 	}
 
 	return nil

pkg/runtimes/docker/translate.go

@@ -94,13 +94,14 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) {
 	// containerConfig.Volumes = map[string]struct{}{} // TODO: do we need this? We only used binds before
 
 	/* Ports */
-	exposedPorts, portBindings, err := nat.ParsePortSpecs(node.Ports)
-	if err != nil {
-		log.Errorf("Failed to parse port specs '%v'", node.Ports)
-		return nil, err
+	exposedPorts := nat.PortSet{}
+	for ep := range node.Ports {
+		if _, exists := exposedPorts[ep]; !exists {
+			exposedPorts[ep] = struct{}{}
+		}
 	}
 	containerConfig.ExposedPorts = exposedPorts
-	hostConfig.PortBindings = portBindings
+	hostConfig.PortBindings = node.Ports
 	/* Network */
 	networkingConfig.EndpointsConfig = map[string]*network.EndpointSettings{
 		node.Network: {},
@@ -135,14 +136,6 @@ func TranslateContainerToNode(cont *types.Container) (*k3d.Node, error) {
 // TranslateContainerDetailsToNode translates a docker containerJSON object into a k3d node representation
 func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d.Node, error) {
 
-	// translate portMap to string representation
-	ports := []string{}
-	for containerPort, portBindingList := range containerDetails.HostConfig.PortBindings {
-		for _, hostInfo := range portBindingList {
-			ports = append(ports, fmt.Sprintf("%s:%s:%s", hostInfo.HostIP, hostInfo.HostPort, containerPort))
-		}
-	}
-
 	// restart -> we only set 'unless-stopped' upon cluster creation
 	restart := false
 	if containerDetails.HostConfig.RestartPolicy.IsAlways() || containerDetails.HostConfig.RestartPolicy.IsUnlessStopped() {
@@ -159,13 +152,14 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d
 
 	// serverOpts
 	serverOpts := k3d.ServerOpts{IsInit: false}
+	serverOpts.KubeAPI = &k3d.ExposureOpts{}
 	for k, v := range containerDetails.Config.Labels {
 		if k == k3d.LabelServerAPIHostIP {
-			serverOpts.ExposeAPI.HostIP = v
+			serverOpts.KubeAPI.Binding.HostIP = v
 		} else if k == k3d.LabelServerAPIHost {
-			serverOpts.ExposeAPI.Host = v
+			serverOpts.KubeAPI.Host = v
 		} else if k == k3d.LabelServerAPIPort {
-			serverOpts.ExposeAPI.Port = v
+			serverOpts.KubeAPI.Binding.HostPort = v
 		}
 	}
 
@@ -199,7 +193,7 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d
 		Env:        env,
 		Cmd:        containerDetails.Config.Cmd,
 		Args:       []string{}, // empty, since Cmd already contains flags
-		Ports:      ports,
+		Ports:      containerDetails.HostConfig.PortBindings,
 		Restart:    restart,
 		Labels:     labels,
 		Network:    clusterNetwork,

pkg/runtimes/docker/translate_test.go

@@ -43,7 +43,14 @@ func TestTranslateNodeToContainer(t *testing.T) {
 		Env:     []string{"TEST_KEY_1=TEST_VAL_1"},
 		Cmd:     []string{"server", "--https-listen-port=6443"},
 		Args:    []string{"--some-boolflag"},
-		Ports:   []string{"0.0.0.0:6443:6443/tcp"},
+		Ports: nat.PortMap{
+			"6443/tcp": []nat.PortBinding{
+				{
+					HostIP:   "0.0.0.0",
+					HostPort: "6443",
+				},
+			},
+		},
 		Restart: true,
 		Labels:  map[string]string{k3d.LabelRole: string(k3d.ServerRole), "test_key_1": "test_val_1"},
 	}

pkg/types/types.go

@@ -25,6 +25,8 @@ import (
 	"context"
 	"fmt"
 	"time"
+
+	"github.com/docker/go-connections/nat"
 )
 
 // DefaultClusterName specifies the default name used for newly created clusters
@@ -106,19 +108,20 @@ var DefaultObjectLabels = map[string]string{
 
 // List of k3d technical label name
 const (
-	LabelClusterName     string = "k3d.cluster"
-	LabelClusterURL      string = "k3d.cluster.url"
-	LabelClusterToken    string = "k3d.cluster.token"
-	LabelImageVolume     string = "k3d.cluster.imageVolume"
-	LabelNetworkExternal string = "k3d.cluster.network.external"
-	LabelNetwork         string = "k3d.cluster.network"
-	LabelRole            string = "k3d.role"
-	LabelServerAPIPort   string = "k3d.server.api.port"
-	LabelServerAPIHost   string = "k3d.server.api.host"
-	LabelServerAPIHostIP string = "k3d.server.api.hostIP"
-	LabelRegistryHost    string = "k3d.registry.host"
-	LabelRegistryHostIP  string = "k3d.registry.hostIP"
-	LabelRegistryPort    string = "k3s.registry.port"
+	LabelClusterName          string = "k3d.cluster"
+	LabelClusterURL           string = "k3d.cluster.url"
+	LabelClusterToken         string = "k3d.cluster.token"
+	LabelImageVolume          string = "k3d.cluster.imageVolume"
+	LabelNetworkExternal      string = "k3d.cluster.network.external"
+	LabelNetwork              string = "k3d.cluster.network"
+	LabelRole                 string = "k3d.role"
+	LabelServerAPIPort        string = "k3d.server.api.port"
+	LabelServerAPIHost        string = "k3d.server.api.host"
+	LabelServerAPIHostIP      string = "k3d.server.api.hostIP"
+	LabelRegistryHost         string = "k3d.registry.host"
+	LabelRegistryHostIP       string = "k3d.registry.hostIP"
+	LabelRegistryPortExternal string = "k3s.registry.port.external"
+	LabelRegistryPortInternal string = "k3s.registry.port.internal"
 )
 
 // DefaultRoleCmds maps the node roles to their respective default commands
@@ -240,7 +243,7 @@ type Cluster struct {
 	Nodes              []*Node            `yaml:"nodes" json:"nodes,omitempty"`
 	InitNode           *Node              // init server node
 	ExternalDatastore  *ExternalDatastore `yaml:"externalDatastore,omitempty" json:"externalDatastore,omitempty"`
-	ExposeAPI          ExposedPort        `yaml:"exposeAPI" json:"exposeAPI,omitempty"`
+	KubeAPI            *ExposureOpts      `yaml:"kubeAPI" json:"kubeAPI,omitempty"`
 	ServerLoadBalancer *Node              `yaml:"serverLoadbalancer,omitempty" json:"serverLoadBalancer,omitempty"`
 	ImageVolume        string             `yaml:"imageVolume" json:"imageVolume,omitempty"`
 }
@@ -294,7 +297,7 @@ type Node struct {
 	Env        []string          `yaml:"env" json:"env,omitempty"`
 	Cmd        []string          // filled automatically based on role
 	Args       []string          `yaml:"extraArgs" json:"extraArgs,omitempty"`
-	Ports      []string          `yaml:"portMappings" json:"portMappings,omitempty"`
+	Ports      nat.PortMap       `yaml:"portMappings" json:"portMappings,omitempty"`
 	Restart    bool              `yaml:"restart" json:"restart,omitempty"`
 	Labels     map[string]string // filled automatically
 	Network    string            // filled automatically
@@ -307,8 +310,14 @@ type Node struct {
 
 // ServerOpts describes some additional server role specific opts
 type ServerOpts struct {
-	IsInit    bool        `yaml:"isInitializingServer" json:"isInitializingServer,omitempty"`
-	ExposeAPI ExposedPort // filled automatically
+	IsInit  bool          `yaml:"isInitializingServer" json:"isInitializingServer,omitempty"`
+	KubeAPI *ExposureOpts `yaml:"kubeAPI" json:"kubeAPI"`
+}
+
+// ExposureOpts describes settings that the user can set for accessing the Kubernetes API
+type ExposureOpts struct {
+	nat.PortMapping        // filled automatically (reference to normal portmapping)
+	Host            string `yaml:"host,omitempty" json:"host,omitempty"`
 }
 
 // ExternalDatastore describes an external datastore used for HA/multi-server clusters
@@ -320,19 +329,6 @@ type ExternalDatastore struct {
 	Network  string `yaml:"network" json:"network,omitempty"`
 }
 
-// MappedPort combines an internal port mapped to an exposed port
-type MappedPort struct {
-	InternalPort string      `yaml:"internal,omitempty" json:"internal,omitempty"`
-	ExternalPort ExposedPort `yaml:"expose,omitempty" json:"expose,omitempty"`
-}
-
-// ExposedPort describes a port exposed on the host system
-type ExposedPort struct {
-	Host   string `yaml:"host" json:"host,omitempty"`
-	HostIP string `yaml:"hostIP" json:"hostIP,omitempty"`
-	Port   string `yaml:"port" json:"port"`
-}
-
 // AgentOpts describes some additional agent role specific opts
 type AgentOpts struct{}
 
@@ -362,11 +358,12 @@ const (
 
 // Registry describes a k3d-managed registry
 type Registry struct {
-	ClusterRef string     // filled automatically -> if created with a cluster
-	Host       string     `yaml:"host" json:"host"`
-	Image      string     `yaml:"image,omitempty" json:"image,omitempty"`
-	Port       MappedPort `yaml:"port" json:"port"`
-	Options    struct {
+	ClusterRef   string       // filled automatically -> if created with a cluster
+	Protocol     string       `yaml:"protocol,omitempty" json:"protocol,omitempty"` // default: http
+	Host         string       `yaml:"host" json:"host"`
+	Image        string       `yaml:"image,omitempty" json:"image,omitempty"`
+	ExposureOpts ExposureOpts `yaml:"expose" json:"expose"`
+	Options      struct {
 		ConfigFile string `yaml:"configFile,omitempty" json:"configFile,omitempty"`
 		Proxy      struct {
 			RemoteURL string `yaml:"remoteURL" json:"remoteURL"`

pkg/util/ports.go

@@ -28,11 +28,6 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-// ValidatePortMap validates a port mapping
-func ValidatePortMap(portmap string) error {
-	return nil // TODO: ValidatePortMap: add validation of port mapping
-}
-
 // GetFreePort tries to fetch an open port from the OS-Kernel
 func GetFreePort() (int, error) {
 	tcpAddress, err := net.ResolveTCPAddr("tcp", "localhost:0")

tests/dind.sh

@@ -46,7 +46,13 @@ until docker inspect "$k3de2e" | jq ".[0].State.Running" && docker logs "$k3de2e
 done
 
 # build helper container images
-docker exec --workdir /src "$k3de2e" make build-helper-images
+if [ -z "$E2E_HELPER_IMAGE_TAG" ]; then
+  docker exec --workdir /src "$k3de2e" make build-helper-images
+  # execute tests
+  docker exec "$k3de2e" /src/tests/runner.sh
+else
+  # execute tests
+  docker exec -e "K3D_HELPER_IMAGE_TAG=$E2E_HELPER_IMAGE_TAG" "$k3de2e" /src/tests/runner.sh
+fi
+
 
-# execute tests
-docker exec "$k3de2e" /src/tests/runner.sh