From c44c576d694c56c85ea0dec4d2d136d4fa7e8f39 Mon Sep 17 00:00:00 2001 From: Thorsten Klein Date: Fri, 11 Dec 2020 16:36:10 +0100 Subject: [PATCH] [Refactoring/Preparation] use real port structs as prep for registries (#427) --- Makefile | 3 +- cmd/cluster/clusterCreate.go | 34 +++----- cmd/registry/registryCreate.go | 4 +- cmd/util/ports.go | 85 ++++++++++++------- pkg/client/cluster.go | 43 ++++------ pkg/client/node.go | 8 +- pkg/client/registry.go | 37 ++++++-- pkg/config/config_test.go | 17 ++-- .../test_assets/config_test_simple.yaml | 4 +- pkg/config/transform.go | 50 +++++++---- pkg/config/v1alpha1/types.go | 13 ++- pkg/config/validate.go | 9 +- pkg/runtimes/docker/translate.go | 28 +++--- pkg/runtimes/docker/translate_test.go | 9 +- pkg/types/types.go | 67 +++++++-------- pkg/util/ports.go | 5 -- tests/dind.sh | 12 ++- 17 files changed, 236 insertions(+), 192 deletions(-) diff --git a/Makefile b/Makefile index 1a5f72f1..222d8fd8 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,7 @@ E2E_LOG_LEVEL ?= WARN E2E_SKIP ?= E2E_EXTRA ?= E2E_RUNNER_START_TIMEOUT ?= 10 +E2E_HELPER_IMAGE_TAG ?= ########## Go Build Options ########## # Build targets @@ -168,7 +169,7 @@ test: e2e: build-docker-dind @echo "Running e2e tests in k3d:$(K3D_IMAGE_TAG)" - LOG_LEVEL="$(E2E_LOG_LEVEL)" E2E_SKIP="$(E2E_SKIP)" E2E_EXTRA="$(E2E_EXTRA)" E2E_RUNNER_START_TIMEOUT=$(E2E_RUNNER_START_TIMEOUT) tests/dind.sh "${K3D_IMAGE_TAG}-dind" + LOG_LEVEL="$(E2E_LOG_LEVEL)" E2E_SKIP="$(E2E_SKIP)" E2E_EXTRA="$(E2E_EXTRA)" E2E_RUNNER_START_TIMEOUT=$(E2E_RUNNER_START_TIMEOUT) E2E_HELPER_IMAGE_TAG="$(E2E_HELPER_IMAGE_TAG)" tests/dind.sh "${K3D_IMAGE_TAG}-dind" ci-tests: fmt check e2e diff --git a/cmd/cluster/clusterCreate.go b/cmd/cluster/clusterCreate.go index 17ed17a6..53165ddb 100644 --- a/cmd/cluster/clusterCreate.go +++ b/cmd/cluster/clusterCreate.go @@ -51,17 +51,12 @@ Every cluster will consist of one or more containers: // flags that go through some pre-processing before transforming them to config type preProcessedFlags struct { - APIPort string - Volumes []string - Ports []string - Labels []string - Env []string -} - -// registry -type registryFlags struct { - Use []string - Create bool + APIPort string + Volumes []string + Ports []string + Labels []string + Env []string + RegistryUse []string } // NewCmdClusterCreate returns a new cobra command @@ -70,7 +65,6 @@ func NewCmdClusterCreate() *cobra.Command { cliConfig := &conf.SimpleConfig{} var configFile string ppFlags := &preProcessedFlags{} - regFlags := ®istryFlags{} // create new command cmd := &cobra.Command{ @@ -213,11 +207,8 @@ func NewCmdClusterCreate() *cobra.Command { } /* Registry */ - cmd.Flags().StringArrayVar(®Flags.Use, "registry-use", nil, "Connect to one or more registries running locally") - if err := cmd.Flags().MarkHidden("registry-use"); err != nil { - log.Fatalln("Failed to mark flag `cluster create --registry-use` as hidden") - } - cmd.Flags().BoolVar(&cliConfig.Registries.Create, "registry-create", false, "Create a registry and connect it to the cluster") + cmd.Flags().StringArrayVar(&cliConfig.Registries.Use, "registry-use", nil, "Connect to one or more k3d-managed registries running locally") + cmd.Flags().BoolVar(&cliConfig.Registries.Create, "registry-create", false, "Create a k3d-managed registry and connect it to the cluster") /* Multi Server Configuration */ @@ -265,11 +256,15 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, cliConfig *conf.Si // -> API-PORT // parse the port mapping - exposeAPI, err := cliutil.ParseExposePort(ppFlags.APIPort) + exposeAPI, err := cliutil.ParsePortExposureSpec(ppFlags.APIPort, k3d.DefaultAPIPort) if err != nil { log.Fatalln(err) } - cliConfig.ExposeAPI = exposeAPI + cliConfig.ExposeAPI = conf.SimpleExposureOpts{ + Host: exposeAPI.Host, + HostIP: exposeAPI.Binding.HostIP, + HostPort: exposeAPI.Binding.HostPort, + } // -> VOLUMES // volumeFilterMap will map volume mounts to applied node filters @@ -384,5 +379,4 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, cliConfig *conf.Si } log.Tracef("EnvFilterMap: %+v", envFilterMap) - } diff --git a/cmd/registry/registryCreate.go b/cmd/registry/registryCreate.go index aacc7895..404969d8 100644 --- a/cmd/registry/registryCreate.go +++ b/cmd/registry/registryCreate.go @@ -101,7 +101,7 @@ func parseCreateRegistryCmd(cmd *cobra.Command, args []string, flags *regCreateF } // --port - exposePort, err := cliutil.ParseExposePort(ppFlags.Port) + exposePort, err := cliutil.ParsePortExposureSpec(ppFlags.Port, k3d.DefaultRegistryPort) if err != nil { log.Errorln("Failed to parse registry port") log.Fatalln(err) @@ -113,5 +113,5 @@ func parseCreateRegistryCmd(cmd *cobra.Command, args []string, flags *regCreateF registryName = fmt.Sprintf("%s-%s", k3d.DefaultObjectNamePrefix, args[0]) } - return &k3d.Registry{Host: registryName, Image: flags.Image, Port: k3d.MappedPort{InternalPort: k3d.DefaultRegistryPort, ExternalPort: exposePort}}, clusters + return &k3d.Registry{Host: registryName, Image: flags.Image, ExposureOpts: *exposePort}, clusters } diff --git a/cmd/util/ports.go b/cmd/util/ports.go index f89230c6..6e1f2f2a 100644 --- a/cmd/util/ports.go +++ b/cmd/util/ports.go @@ -24,60 +24,83 @@ package util import ( "fmt" "net" + "regexp" "strconv" - "strings" + "github.com/docker/go-connections/nat" k3d "github.com/rancher/k3d/v4/pkg/types" + "github.com/rancher/k3d/v4/pkg/util" log "github.com/sirupsen/logrus" ) -// ParseExposePort parses/validates a string to create an exposePort struct from it -func ParseExposePort(portString string) (k3d.ExposedPort, error) { +var apiPortRegexp = regexp.MustCompile(`^(?P(?P\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?P\S+):)?(?P(\d{1,5}|random))$`) - var exposePort k3d.ExposedPort +// ParsePortExposureSpec parses/validates a string to create an exposePort struct from it +func ParsePortExposureSpec(exposedPortSpec, internalPort string) (*k3d.ExposureOpts, error) { - split := strings.Split(portString, ":") - if len(split) > 2 { - log.Errorln("Failed to parse API Port specification") - return exposePort, fmt.Errorf("api-port format error") + match := apiPortRegexp.FindStringSubmatch(exposedPortSpec) + + if len(match) == 0 { + log.Errorln("Failed to parse Port Exposure specification") + return nil, fmt.Errorf("Port Exposure Spec format error: Must be [(HostIP|HostName):]HostPort") + } + + submatches := util.MapSubexpNames(apiPortRegexp.SubexpNames(), match) + + // no port specified (or not matched via regex) + if submatches["port"] == "" { + return nil, fmt.Errorf("Failed to find port in Port Exposure spec '%s'", exposedPortSpec) } - if len(split) == 1 { - exposePort = k3d.ExposedPort{Port: split[0]} - } else { - // Make sure 'host' can be resolved to an IP address - addrs, err := net.LookupHost(split[0]) + api := &k3d.ExposureOpts{} + + // check if there's a host reference + if submatches["hostname"] != "" { + log.Tracef("Port Exposure: found hostname: %s", submatches["hostname"]) + addrs, err := net.LookupHost(submatches["hostname"]) if err != nil { - return exposePort, err + return nil, fmt.Errorf("Failed to lookup host '%s' specified for Port Exposure: %+v", submatches["hostname"], err) } - exposePort = k3d.ExposedPort{Host: split[0], HostIP: addrs[0], Port: split[1]} + api.Host = submatches["hostname"] + submatches["hostip"] = addrs[0] // set hostip to the resolved address } - // Verify 'port' is an integer and within port ranges - if exposePort.Port == "" || exposePort.Port == "random" { - log.Debugf("API-Port Mapping didn't specify hostPort, choosing one randomly...") + realPortString := "" + + if submatches["hostip"] == "" { + submatches["hostip"] = k3d.DefaultAPIHost + } + + // start with the IP, if there is any + if submatches["hostip"] != "" { + realPortString += submatches["hostip"] + ":" + } + + // port: get a free one if there's none defined or set to random + if submatches["port"] == "" || submatches["port"] == "random" { + log.Debugf("Port Exposure Mapping didn't specify hostPort, choosing one randomly...") freePort, err := GetFreePort() if err != nil || freePort == 0 { - log.Warnf("Failed to get random free port:\n%+v", err) - log.Warnf("Falling back to default port %s (may be blocked though)...", k3d.DefaultAPIPort) - exposePort.Port = k3d.DefaultAPIPort + log.Warnf("Failed to get random free port: %+v", err) + log.Warnf("Falling back to internal port %s (may be blocked though)...", internalPort) + submatches["port"] = internalPort } else { - exposePort.Port = strconv.Itoa(freePort) - log.Debugf("Got free port for API: '%d'", freePort) + submatches["port"] = strconv.Itoa(freePort) + log.Debugf("Got free port for Port Exposure: '%d'", freePort) } } - p, err := strconv.Atoi(exposePort.Port) + + realPortString += fmt.Sprintf("%s:%s/tcp", submatches["port"], internalPort) + + portMapping, err := nat.ParsePortSpec(realPortString) if err != nil { - log.Errorln("Failed to parse port mapping") - return exposePort, err + return nil, fmt.Errorf("Failed to parse port spec for Port Exposure '%s': %+v", realPortString, err) } - if p < 0 || p > 65535 { - log.Errorln("Failed to parse API Port specification") - return exposePort, fmt.Errorf("Port value '%d' out of range", p) - } + api.Port = portMapping[0].Port // there can be only one due to our regexp + api.Binding = portMapping[0].Binding - return exposePort, nil + return api, nil } diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go index 0bd08b9f..c569b5ad 100644 --- a/pkg/client/cluster.go +++ b/pkg/client/cluster.go @@ -33,6 +33,7 @@ import ( gort "runtime" + "github.com/docker/go-connections/nat" "github.com/imdario/mergo" "github.com/rancher/k3d/v4/pkg/actions" config "github.com/rancher/k3d/v4/pkg/config/v1alpha1" @@ -276,7 +277,7 @@ ClusterCreatOpts: /* * Docker Machine Special Configuration */ - if cluster.ExposeAPI.Host == k3d.DefaultAPIHost && runtime == k3drt.Docker { + if cluster.KubeAPI.Host == k3d.DefaultAPIHost && runtime == k3drt.Docker { if gort.GOOS == "windows" || gort.GOOS == "darwin" { log.Tracef("Running on %s: checking if it's using docker-machine", gort.GOOS) machineIP, err := runtime.(docker.Docker).GetDockerMachineIP() @@ -284,8 +285,8 @@ ClusterCreatOpts: log.Warnf("Using docker-machine, but failed to get it's IP: %+v", err) } else if machineIP != "" { log.Infof("Using the docker-machine IP %s to connect to the Kubernetes API", machineIP) - cluster.ExposeAPI.Host = machineIP - cluster.ExposeAPI.HostIP = machineIP + cluster.KubeAPI.Host = machineIP + cluster.KubeAPI.Binding.HostIP = machineIP } else { log.Traceln("Not using docker-machine") } @@ -330,7 +331,7 @@ ClusterCreatOpts: // node role specific settings if node.Role == k3d.ServerRole { - node.ServerOpts.ExposeAPI = cluster.ExposeAPI + node.ServerOpts.KubeAPI = cluster.KubeAPI // the cluster has an init server node, but its not this one, so connect it to the init node if cluster.InitNode != nil && !node.ServerOpts.IsInit { @@ -371,7 +372,7 @@ ClusterCreatOpts: // in case the LoadBalancer was disabled, expose the API Port on the initializing server node if clusterCreateOpts.DisableLoadBalancer { - cluster.InitNode.Ports = append(cluster.InitNode.Ports, fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort)) + cluster.InitNode.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding} } if err := nodeSetup(cluster.InitNode, serverCount); err != nil { @@ -390,7 +391,7 @@ ClusterCreatOpts: continue } else if serverCount == 0 && clusterCreateOpts.DisableLoadBalancer { // if this is the first server node and the server loadbalancer is disabled, expose the API Port on this server node - node.Ports = append(node.Ports, fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort)) + node.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding} } time.Sleep(1 * time.Second) // FIXME: arbitrary wait for one second to avoid race conditions of servers registering @@ -431,34 +432,20 @@ ClusterCreatOpts: // generate comma-separated list of extra ports to forward ports := k3d.DefaultAPIPort - for _, portString := range cluster.ServerLoadBalancer.Ports { - split := strings.Split(portString, ":") - port := split[len(split)-1] - if strings.Contains(port, "-") { - split := strings.Split(port, "-") - start, err := strconv.Atoi(split[0]) - if err != nil { - log.Errorf("Failed to parse port mapping for loadbalancer '%s'", port) - return err - } - end, err := strconv.Atoi(split[1]) - if err != nil { - log.Errorf("Failed to parse port mapping for loadbalancer '%s'", port) - return err - } - for i := start; i <= end; i++ { - ports += "," + strconv.Itoa(i) - } - } else { - ports += "," + port - } + for exposedPort := range cluster.ServerLoadBalancer.Ports { + ports += "," + exposedPort.Port() + } + + if cluster.ServerLoadBalancer.Ports == nil { + cluster.ServerLoadBalancer.Ports = nat.PortMap{} } + cluster.ServerLoadBalancer.Ports[k3d.DefaultAPIPort] = []nat.PortBinding{cluster.KubeAPI.Binding} // Create LB as a modified node with loadbalancerRole lbNode := &k3d.Node{ Name: fmt.Sprintf("%s-%s-serverlb", k3d.DefaultObjectNamePrefix, cluster.Name), Image: fmt.Sprintf("%s:%s", k3d.DefaultLBImageRepo, version.GetHelperImageVersion()), - Ports: append(cluster.ServerLoadBalancer.Ports, fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort)), + Ports: cluster.ServerLoadBalancer.Ports, Env: []string{ fmt.Sprintf("SERVERS=%s", servers), fmt.Sprintf("PORTS=%s", ports), diff --git a/pkg/client/node.go b/pkg/client/node.go index 694692cc..101a3d45 100644 --- a/pkg/client/node.go +++ b/pkg/client/node.go @@ -330,11 +330,11 @@ func patchServerSpec(node *k3d.Node) error { // Add labels and TLS SAN for the exposed API // FIXME: For now, the labels concerning the API on the server nodes are only being used for configuring the kubeconfig - node.Labels[k3d.LabelServerAPIHostIP] = node.ServerOpts.ExposeAPI.HostIP // TODO: maybe get docker machine IP here - node.Labels[k3d.LabelServerAPIHost] = node.ServerOpts.ExposeAPI.Host - node.Labels[k3d.LabelServerAPIPort] = node.ServerOpts.ExposeAPI.Port + node.Labels[k3d.LabelServerAPIHostIP] = node.ServerOpts.KubeAPI.Binding.HostIP // TODO: maybe get docker machine IP here + node.Labels[k3d.LabelServerAPIHost] = node.ServerOpts.KubeAPI.Host + node.Labels[k3d.LabelServerAPIPort] = node.ServerOpts.KubeAPI.Binding.HostPort - node.Args = append(node.Args, "--tls-san", node.ServerOpts.ExposeAPI.Host) // add TLS SAN for non default host name + node.Args = append(node.Args, "--tls-san", node.ServerOpts.KubeAPI.Host) // add TLS SAN for non default host name return nil } diff --git a/pkg/client/registry.go b/pkg/client/registry.go index 329cab06..0c4e6af0 100644 --- a/pkg/client/registry.go +++ b/pkg/client/registry.go @@ -25,6 +25,7 @@ import ( "context" "fmt" + "github.com/docker/go-connections/nat" "github.com/rancher/k3d/v4/pkg/runtimes" k3d "github.com/rancher/k3d/v4/pkg/types" "github.com/rancher/k3d/v4/pkg/types/k3s" @@ -71,19 +72,19 @@ func RegistryCreate(ctx context.Context, runtime runtimes.Runtime, reg *k3d.Regi // setup the node labels registryNode.Labels = map[string]string{ - k3d.LabelRole: string(k3d.RegistryRole), - k3d.LabelRegistryHost: reg.Port.ExternalPort.Host, // TODO: docker machine host? - k3d.LabelRegistryHostIP: reg.Port.ExternalPort.HostIP, - k3d.LabelRegistryPort: reg.Port.ExternalPort.Port, + k3d.LabelRole: string(k3d.RegistryRole), + k3d.LabelRegistryHost: reg.ExposureOpts.Host, // TODO: docker machine host? + k3d.LabelRegistryHostIP: reg.ExposureOpts.Binding.HostIP, + k3d.LabelRegistryPortExternal: reg.ExposureOpts.Binding.HostPort, + k3d.LabelRegistryPortInternal: reg.ExposureOpts.Port.Port(), } for k, v := range k3d.DefaultObjectLabels { registryNode.Labels[k] = v } // port - registryNode.Ports = []string{ - fmt.Sprintf("%s:%s:%s/tcp", reg.Port.ExternalPort.HostIP, reg.Port.ExternalPort.Port, k3d.DefaultRegistryPort), - } + registryNode.Ports = nat.PortMap{} + registryNode.Ports[reg.ExposureOpts.Port] = []nat.PortBinding{reg.ExposureOpts.Binding} // create the registry node log.Infof("Creating node '%s'", registryNode.Name) @@ -136,8 +137,8 @@ func RegistryGenerateK3sConfig(ctx context.Context, registries []*k3d.Registry) regConf := &k3s.Registry{} for _, reg := range registries { - internalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.Port.InternalPort) - externalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.Port.ExternalPort.Port) + internalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.ExposureOpts.Port.Port()) + externalAddress := fmt.Sprintf("%s:%s", reg.Host, reg.ExposureOpts.Binding.HostPort) // init mirrors if nil if regConf.Mirrors == nil { @@ -159,3 +160,21 @@ func RegistryGenerateK3sConfig(ctx context.Context, registries []*k3d.Registry) return regConf, nil } + +// RegistryGet gets a registry node by name and returns it as a registry object +func RegistryGet(ctx context.Context, runtime runtimes.Runtime, name string) (*k3d.Registry, error) { + regNode, err := runtime.GetNode(ctx, &k3d.Node{ + Name: name, + Role: k3d.RegistryRole, + }) + if err != nil { + return nil, fmt.Errorf("Failed to find registry '%s': %+v", name, err) + } + + registry := &k3d.Registry{ + Host: regNode.Name, + } + // TODO: finish RegistryGet + return registry, nil + +} diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 615cc82a..1444ad37 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -33,19 +33,20 @@ import ( func TestReadSimpleConfig(t *testing.T) { + exposedAPI := conf.SimpleExposureOpts{} + exposedAPI.HostIP = "0.0.0.0" + exposedAPI.HostPort = "6443" + expectedConfig := conf.SimpleConfig{ TypeMeta: conf.TypeMeta{ APIVersion: "k3d.io/v1alpha1", Kind: "Simple", }, - Name: "test", - Servers: 1, - Agents: 2, - ExposeAPI: k3d.ExposedPort{ - HostIP: "0.0.0.0", - Port: "6443", - }, - Image: "rancher/k3s:latest", + Name: "test", + Servers: 1, + Agents: 2, + ExposeAPI: exposedAPI, + Image: "rancher/k3s:latest", Volumes: []conf.VolumeWithNodeFilters{ { Volume: "/my/path:/some/path", diff --git a/pkg/config/test_assets/config_test_simple.yaml b/pkg/config/test_assets/config_test_simple.yaml index e34db8e7..1842b3f6 100644 --- a/pkg/config/test_assets/config_test_simple.yaml +++ b/pkg/config/test_assets/config_test_simple.yaml @@ -3,9 +3,9 @@ kind: Simple name: test servers: 1 agents: 2 -exposeAPI: +kubeAPI: hostIP: "0.0.0.0" - port: "6443" + hostPort: "6443" image: rancher/k3s:latest volumes: - volume: /my/path:/some/path diff --git a/pkg/config/transform.go b/pkg/config/transform.go index 396c7dee..dd4c5db9 100644 --- a/pkg/config/transform.go +++ b/pkg/config/transform.go @@ -26,6 +26,7 @@ import ( "context" "fmt" + "github.com/docker/go-connections/nat" cliutil "github.com/rancher/k3d/v4/cmd/util" // TODO: move parseapiport to pkg conf "github.com/rancher/k3d/v4/pkg/config/v1alpha1" "github.com/rancher/k3d/v4/pkg/runtimes" @@ -61,12 +62,21 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim simpleConfig.ExposeAPI.HostIP = k3d.DefaultAPIHost } + kubeAPIExposureOpts := &k3d.ExposureOpts{ + Host: simpleConfig.ExposeAPI.Host, + } + kubeAPIExposureOpts.Port = k3d.DefaultAPIPort + kubeAPIExposureOpts.Binding = nat.PortBinding{ + HostIP: simpleConfig.ExposeAPI.HostIP, + HostPort: simpleConfig.ExposeAPI.HostPort, + } + // FILL CLUSTER CONFIG newCluster := k3d.Cluster{ - Name: simpleConfig.Name, - Network: clusterNetwork, - Token: simpleConfig.ClusterToken, - ExposeAPI: simpleConfig.ExposeAPI, + Name: simpleConfig.Name, + Network: clusterNetwork, + Token: simpleConfig.ClusterToken, + KubeAPI: kubeAPIExposureOpts, } // -> NODES @@ -142,7 +152,20 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim } for _, node := range nodes { - node.Ports = append(node.Ports, portWithNodeFilters.Port) + portmappings, err := nat.ParsePortSpec(portWithNodeFilters.Port) + if err != nil { + return nil, fmt.Errorf("Failed to parse port spec '%s': %+v", portWithNodeFilters.Port, err) + } + if node.Ports == nil { + node.Ports = nat.PortMap{} + } + for _, pm := range portmappings { + if _, exists := node.Ports[pm.Port]; exists { + node.Ports[pm.Port] = append(node.Ports[pm.Port], pm.Binding) + } else { + node.Ports[pm.Port] = []nat.PortBinding{pm.Binding} + } + } } } @@ -200,18 +223,15 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim /* * Registries */ - regPort, err := cliutil.ParseExposePort("random") - if err != nil { - return nil, fmt.Errorf("Failed to get port for registry: %+v", err) - } if simpleConfig.Registries.Create { + regPort, err := cliutil.ParsePortExposureSpec("random", k3d.DefaultRegistryPort) + if err != nil { + return nil, fmt.Errorf("Failed to get port for registry: %+v", err) + } clusterCreateOpts.Registries.Create = &k3d.Registry{ - Host: fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, newCluster.Name), - Image: fmt.Sprintf("%s:%s", k3d.DefaultRegistryImageRepo, k3d.DefaultRegistryImageTag), - Port: k3d.MappedPort{ - InternalPort: k3d.DefaultRegistryPort, - ExternalPort: regPort, - }, + Host: fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, newCluster.Name), + Image: fmt.Sprintf("%s:%s", k3d.DefaultRegistryImageRepo, k3d.DefaultRegistryImageTag), + ExposureOpts: *regPort, } } diff --git a/pkg/config/v1alpha1/types.go b/pkg/config/v1alpha1/types.go index 6e35b2b0..80c4158d 100644 --- a/pkg/config/v1alpha1/types.go +++ b/pkg/config/v1alpha1/types.go @@ -116,7 +116,7 @@ type SimpleConfig struct { Name string `mapstructure:"name" yaml:"name" json:"name,omitempty"` Servers int `mapstructure:"servers" yaml:"servers" json:"servers,omitempty"` //nolint:lll // default 1 Agents int `mapstructure:"agents" yaml:"agents" json:"agents,omitempty"` //nolint:lll // default 0 - ExposeAPI k3d.ExposedPort `mapstructure:"exposeAPI" yaml:"exposeAPI" json:"exposeAPI,omitempty"` + ExposeAPI SimpleExposureOpts `mapstructure:"kubeAPI" yaml:"kubeAPI" json:"kubeAPI,omitempty"` Image string `mapstructure:"image" yaml:"image" json:"image,omitempty"` Network string `mapstructure:"network" yaml:"network" json:"network,omitempty"` ClusterToken string `mapstructure:"clusterToken" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated @@ -126,11 +126,18 @@ type SimpleConfig struct { Options SimpleConfigOptions `mapstructure:"options" yaml:"options" json:"options,omitempty"` Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"` Registries struct { - Use []*k3d.Registry `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"` - Create bool `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"` + Use []string `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"` + Create bool `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"` } `mapstructure:"registries" yaml:"registries,omitempty" json:"registries,omitempty"` } +// SimpleExposureOpts provides a simplified syntax compared to the original k3d.ExposureOpts +type SimpleExposureOpts struct { + Host string `mapstructure:"host" yaml:"host,omitempty" json:"host,omitempty"` + HostIP string `mapstructure:"hostIP" yaml:"hostIP,omitempty" json:"hostIP,omitempty"` + HostPort string `mapstructure:"hostPort" yaml:"hostPort,omitempty" json:"hostPort,omitempty"` +} + // GetKind implements Config.GetKind func (c SimpleConfig) GetKind() string { return "Cluster" diff --git a/pkg/config/validate.go b/pkg/config/validate.go index 127fa03e..b7f1f926 100644 --- a/pkg/config/validate.go +++ b/pkg/config/validate.go @@ -57,7 +57,7 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config } // API-Port cannot be changed when using network=host - if config.Cluster.Network.Name == "host" && config.Cluster.ExposeAPI.Port != k3d.DefaultAPIPort { + if config.Cluster.Network.Name == "host" && config.Cluster.KubeAPI.Port.Port() != k3d.DefaultAPIPort { // in hostNetwork mode, we're not going to map a hostport. Here it should always use 6443. // Note that hostNetwork mode is super inflexible and since we don't change the backend port (on the container), it will only be one hostmode cluster allowed. return fmt.Errorf("The API Port can not be changed when using 'host' network") @@ -78,13 +78,6 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config return err } } - - // ports - for _, port := range node.Ports { - if err := util.ValidatePortMap(port); err != nil { - return err - } - } } return nil diff --git a/pkg/runtimes/docker/translate.go b/pkg/runtimes/docker/translate.go index 873419e1..8d8b842c 100644 --- a/pkg/runtimes/docker/translate.go +++ b/pkg/runtimes/docker/translate.go @@ -94,13 +94,14 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) { // containerConfig.Volumes = map[string]struct{}{} // TODO: do we need this? We only used binds before /* Ports */ - exposedPorts, portBindings, err := nat.ParsePortSpecs(node.Ports) - if err != nil { - log.Errorf("Failed to parse port specs '%v'", node.Ports) - return nil, err + exposedPorts := nat.PortSet{} + for ep := range node.Ports { + if _, exists := exposedPorts[ep]; !exists { + exposedPorts[ep] = struct{}{} + } } containerConfig.ExposedPorts = exposedPorts - hostConfig.PortBindings = portBindings + hostConfig.PortBindings = node.Ports /* Network */ networkingConfig.EndpointsConfig = map[string]*network.EndpointSettings{ node.Network: {}, @@ -135,14 +136,6 @@ func TranslateContainerToNode(cont *types.Container) (*k3d.Node, error) { // TranslateContainerDetailsToNode translates a docker containerJSON object into a k3d node representation func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d.Node, error) { - // translate portMap to string representation - ports := []string{} - for containerPort, portBindingList := range containerDetails.HostConfig.PortBindings { - for _, hostInfo := range portBindingList { - ports = append(ports, fmt.Sprintf("%s:%s:%s", hostInfo.HostIP, hostInfo.HostPort, containerPort)) - } - } - // restart -> we only set 'unless-stopped' upon cluster creation restart := false if containerDetails.HostConfig.RestartPolicy.IsAlways() || containerDetails.HostConfig.RestartPolicy.IsUnlessStopped() { @@ -159,13 +152,14 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d // serverOpts serverOpts := k3d.ServerOpts{IsInit: false} + serverOpts.KubeAPI = &k3d.ExposureOpts{} for k, v := range containerDetails.Config.Labels { if k == k3d.LabelServerAPIHostIP { - serverOpts.ExposeAPI.HostIP = v + serverOpts.KubeAPI.Binding.HostIP = v } else if k == k3d.LabelServerAPIHost { - serverOpts.ExposeAPI.Host = v + serverOpts.KubeAPI.Host = v } else if k == k3d.LabelServerAPIPort { - serverOpts.ExposeAPI.Port = v + serverOpts.KubeAPI.Binding.HostPort = v } } @@ -199,7 +193,7 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d Env: env, Cmd: containerDetails.Config.Cmd, Args: []string{}, // empty, since Cmd already contains flags - Ports: ports, + Ports: containerDetails.HostConfig.PortBindings, Restart: restart, Labels: labels, Network: clusterNetwork, diff --git a/pkg/runtimes/docker/translate_test.go b/pkg/runtimes/docker/translate_test.go index 6ca432e4..67794992 100644 --- a/pkg/runtimes/docker/translate_test.go +++ b/pkg/runtimes/docker/translate_test.go @@ -43,7 +43,14 @@ func TestTranslateNodeToContainer(t *testing.T) { Env: []string{"TEST_KEY_1=TEST_VAL_1"}, Cmd: []string{"server", "--https-listen-port=6443"}, Args: []string{"--some-boolflag"}, - Ports: []string{"0.0.0.0:6443:6443/tcp"}, + Ports: nat.PortMap{ + "6443/tcp": []nat.PortBinding{ + { + HostIP: "0.0.0.0", + HostPort: "6443", + }, + }, + }, Restart: true, Labels: map[string]string{k3d.LabelRole: string(k3d.ServerRole), "test_key_1": "test_val_1"}, } diff --git a/pkg/types/types.go b/pkg/types/types.go index f35f3172..9df61557 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -25,6 +25,8 @@ import ( "context" "fmt" "time" + + "github.com/docker/go-connections/nat" ) // DefaultClusterName specifies the default name used for newly created clusters @@ -106,19 +108,20 @@ var DefaultObjectLabels = map[string]string{ // List of k3d technical label name const ( - LabelClusterName string = "k3d.cluster" - LabelClusterURL string = "k3d.cluster.url" - LabelClusterToken string = "k3d.cluster.token" - LabelImageVolume string = "k3d.cluster.imageVolume" - LabelNetworkExternal string = "k3d.cluster.network.external" - LabelNetwork string = "k3d.cluster.network" - LabelRole string = "k3d.role" - LabelServerAPIPort string = "k3d.server.api.port" - LabelServerAPIHost string = "k3d.server.api.host" - LabelServerAPIHostIP string = "k3d.server.api.hostIP" - LabelRegistryHost string = "k3d.registry.host" - LabelRegistryHostIP string = "k3d.registry.hostIP" - LabelRegistryPort string = "k3s.registry.port" + LabelClusterName string = "k3d.cluster" + LabelClusterURL string = "k3d.cluster.url" + LabelClusterToken string = "k3d.cluster.token" + LabelImageVolume string = "k3d.cluster.imageVolume" + LabelNetworkExternal string = "k3d.cluster.network.external" + LabelNetwork string = "k3d.cluster.network" + LabelRole string = "k3d.role" + LabelServerAPIPort string = "k3d.server.api.port" + LabelServerAPIHost string = "k3d.server.api.host" + LabelServerAPIHostIP string = "k3d.server.api.hostIP" + LabelRegistryHost string = "k3d.registry.host" + LabelRegistryHostIP string = "k3d.registry.hostIP" + LabelRegistryPortExternal string = "k3s.registry.port.external" + LabelRegistryPortInternal string = "k3s.registry.port.internal" ) // DefaultRoleCmds maps the node roles to their respective default commands @@ -240,7 +243,7 @@ type Cluster struct { Nodes []*Node `yaml:"nodes" json:"nodes,omitempty"` InitNode *Node // init server node ExternalDatastore *ExternalDatastore `yaml:"externalDatastore,omitempty" json:"externalDatastore,omitempty"` - ExposeAPI ExposedPort `yaml:"exposeAPI" json:"exposeAPI,omitempty"` + KubeAPI *ExposureOpts `yaml:"kubeAPI" json:"kubeAPI,omitempty"` ServerLoadBalancer *Node `yaml:"serverLoadbalancer,omitempty" json:"serverLoadBalancer,omitempty"` ImageVolume string `yaml:"imageVolume" json:"imageVolume,omitempty"` } @@ -294,7 +297,7 @@ type Node struct { Env []string `yaml:"env" json:"env,omitempty"` Cmd []string // filled automatically based on role Args []string `yaml:"extraArgs" json:"extraArgs,omitempty"` - Ports []string `yaml:"portMappings" json:"portMappings,omitempty"` + Ports nat.PortMap `yaml:"portMappings" json:"portMappings,omitempty"` Restart bool `yaml:"restart" json:"restart,omitempty"` Labels map[string]string // filled automatically Network string // filled automatically @@ -307,8 +310,14 @@ type Node struct { // ServerOpts describes some additional server role specific opts type ServerOpts struct { - IsInit bool `yaml:"isInitializingServer" json:"isInitializingServer,omitempty"` - ExposeAPI ExposedPort // filled automatically + IsInit bool `yaml:"isInitializingServer" json:"isInitializingServer,omitempty"` + KubeAPI *ExposureOpts `yaml:"kubeAPI" json:"kubeAPI"` +} + +// ExposureOpts describes settings that the user can set for accessing the Kubernetes API +type ExposureOpts struct { + nat.PortMapping // filled automatically (reference to normal portmapping) + Host string `yaml:"host,omitempty" json:"host,omitempty"` } // ExternalDatastore describes an external datastore used for HA/multi-server clusters @@ -320,19 +329,6 @@ type ExternalDatastore struct { Network string `yaml:"network" json:"network,omitempty"` } -// MappedPort combines an internal port mapped to an exposed port -type MappedPort struct { - InternalPort string `yaml:"internal,omitempty" json:"internal,omitempty"` - ExternalPort ExposedPort `yaml:"expose,omitempty" json:"expose,omitempty"` -} - -// ExposedPort describes a port exposed on the host system -type ExposedPort struct { - Host string `yaml:"host" json:"host,omitempty"` - HostIP string `yaml:"hostIP" json:"hostIP,omitempty"` - Port string `yaml:"port" json:"port"` -} - // AgentOpts describes some additional agent role specific opts type AgentOpts struct{} @@ -362,11 +358,12 @@ const ( // Registry describes a k3d-managed registry type Registry struct { - ClusterRef string // filled automatically -> if created with a cluster - Host string `yaml:"host" json:"host"` - Image string `yaml:"image,omitempty" json:"image,omitempty"` - Port MappedPort `yaml:"port" json:"port"` - Options struct { + ClusterRef string // filled automatically -> if created with a cluster + Protocol string `yaml:"protocol,omitempty" json:"protocol,omitempty"` // default: http + Host string `yaml:"host" json:"host"` + Image string `yaml:"image,omitempty" json:"image,omitempty"` + ExposureOpts ExposureOpts `yaml:"expose" json:"expose"` + Options struct { ConfigFile string `yaml:"configFile,omitempty" json:"configFile,omitempty"` Proxy struct { RemoteURL string `yaml:"remoteURL" json:"remoteURL"` diff --git a/pkg/util/ports.go b/pkg/util/ports.go index 9aa8e468..1eaead82 100644 --- a/pkg/util/ports.go +++ b/pkg/util/ports.go @@ -28,11 +28,6 @@ import ( log "github.com/sirupsen/logrus" ) -// ValidatePortMap validates a port mapping -func ValidatePortMap(portmap string) error { - return nil // TODO: ValidatePortMap: add validation of port mapping -} - // GetFreePort tries to fetch an open port from the OS-Kernel func GetFreePort() (int, error) { tcpAddress, err := net.ResolveTCPAddr("tcp", "localhost:0") diff --git a/tests/dind.sh b/tests/dind.sh index 2a9a7f4d..fdce3f24 100755 --- a/tests/dind.sh +++ b/tests/dind.sh @@ -46,7 +46,13 @@ until docker inspect "$k3de2e" | jq ".[0].State.Running" && docker logs "$k3de2e done # build helper container images -docker exec --workdir /src "$k3de2e" make build-helper-images +if [ -z "$E2E_HELPER_IMAGE_TAG" ]; then + docker exec --workdir /src "$k3de2e" make build-helper-images + # execute tests + docker exec "$k3de2e" /src/tests/runner.sh +else + # execute tests + docker exec -e "K3D_HELPER_IMAGE_TAG=$E2E_HELPER_IMAGE_TAG" "$k3de2e" /src/tests/runner.sh +fi + -# execute tests -docker exec "$k3de2e" /src/tests/runner.sh