MasterLoadBalancer: add nodefilter & extra treatment

- masterlb is now an extra part of the cluster spec
- ports can now be attached to masterlb by using the 'loadbalancer'
nodefilter
- all ports exposed on the masterlb will be proxied to all master nodes
(on the same port)

cmd/create/createCluster.go

@@ -298,7 +298,7 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, createClusterOpts
 		}
 	}
 
-	log.Debugln(portFilterMap)
+	log.Debugf("PortFilterMap: %+v", portFilterMap)
 
 	/********************
 	 *									*
@@ -317,6 +317,13 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, createClusterOpts
 	// generate list of nodes
 	cluster.Nodes = []*k3d.Node{}
 
+	// MasterLoadBalancer
+	if !createClusterOpts.DisableLoadBalancer {
+		cluster.MasterLoadBalancer = &k3d.Node{
+			Role: k3d.LoadBalancerRole,
+		}
+	}
+
 	/****************
 	 * Master Nodes *
 	 ****************/
@@ -372,7 +379,7 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, createClusterOpts
 		if len(filters) == 0 && (masterCount+workerCount) > 1 {
 			log.Fatalf("Malformed portmapping '%s' lacks a node filter, but there is more than one node.", portmap)
 		}
-		nodes, err := cliutil.FilterNodes(cluster.Nodes, filters)
+		nodes, err := cliutil.FilterNodes(append(cluster.Nodes, cluster.MasterLoadBalancer), filters)
 		if err != nil {
 			log.Fatalln(err)
 		}

cmd/util/filter.go

@@ -34,7 +34,7 @@ import (
 )
 
 // Regexp pattern to match node filters
-var filterRegexp = regexp.MustCompile(`^(?P<group>master|worker|all)(?P<subsetSpec>\[(?P<subset>(?P<subsetList>(\d+,?)+)|(?P<subsetRange>\d*:\d*)|(?P<subsetWildcard>\*))\])?$`)
+var filterRegexp = regexp.MustCompile(`^(?P<group>master|worker|loadbalancer|all)(?P<subsetSpec>\[(?P<subset>(?P<subsetList>(\d+,?)+)|(?P<subsetRange>\d*:\d*)|(?P<subsetWildcard>\*))\])?$`)
 
 // SplitFiltersFromFlag separates a flag's value from the node filter, if there is one
 func SplitFiltersFromFlag(flag string) (string, []string, error) {
@@ -74,11 +74,14 @@ func FilterNodes(nodes []*k3d.Node, filters []string) ([]*k3d.Node, error) {
 	// map roles to subsets
 	masterNodes := []*k3d.Node{}
 	workerNodes := []*k3d.Node{}
+	var masterlb *k3d.Node
 	for _, node := range nodes {
 		if node.Role == k3d.MasterRole {
 			masterNodes = append(masterNodes, node)
 		} else if node.Role == k3d.WorkerRole {
 			workerNodes = append(workerNodes, node)
+		} else if node.Role == k3d.LoadBalancerRole {
+			masterlb = node
 		}
 	}
 
@@ -111,6 +114,9 @@ func FilterNodes(nodes []*k3d.Node, filters []string) ([]*k3d.Node, error) {
 			groupNodes = masterNodes
 		} else if submatches["group"] == string(k3d.WorkerRole) {
 			groupNodes = workerNodes
+		} else if submatches["group"] == string(k3d.LoadBalancerRole) {
+			filteredNodes = append(filteredNodes, masterlb)
+			return filteredNodes, nil // early exit if filtered group is the loadbalancer
 		}
 
 		/* Option 1) subset defined by list */

pkg/cluster/cluster.go

@@ -240,8 +240,10 @@ func CreateCluster(ctx context.Context, cluster *k3d.Cluster, runtime k3drt.Runt
 			suffix = workerCount
 			workerCount++
 		}
-		if err := nodeSetup(node, suffix); err != nil {
+		if node.Role == k3d.MasterRole || node.Role == k3d.WorkerRole {
-			return err
+			if err := nodeSetup(node, suffix); err != nil {
+				return err
+			}
 		}
 
 		// asynchronously wait for this master node to be ready (by checking the logs for a specific log mesage)
@@ -281,14 +283,21 @@ func CreateCluster(ctx context.Context, cluster *k3d.Cluster, runtime k3drt.Runt
 				}
 			}
 
+			// generate comma-separated list of extra ports to forward
+			ports := k3d.DefaultAPIPort
+			for _, portString := range cluster.MasterLoadBalancer.Ports {
+				split := strings.Split(portString, ":")
+				ports += "," + split[len(split)-1]
+			}
+
 			// Create LB as a modified node with loadbalancerRole
 			lbNode := &k3d.Node{
 				Name:  fmt.Sprintf("%s-%s-masterlb", k3d.DefaultObjectNamePrefix, cluster.Name),
 				Image: k3d.DefaultLBImage,
-				Ports: []string{fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort)},
+				Ports: append(cluster.MasterLoadBalancer.Ports, fmt.Sprintf("%s:%s:%s/tcp", cluster.ExposeAPI.Host, cluster.ExposeAPI.Port, k3d.DefaultAPIPort)),
 				Env: []string{
 					fmt.Sprintf("SERVERS=%s", servers),
-					fmt.Sprintf("PORT=%s", k3d.DefaultAPIPort),
+					fmt.Sprintf("PORTS=%s", ports),
 				},
 				Role:    k3d.LoadBalancerRole,
 				Labels:  k3d.DefaultObjectLabels, // TODO: createLoadBalancer: add more expressive labels

pkg/types/types.go

@@ -39,7 +39,7 @@ const DefaultClusterNameMaxLength = 32
 const DefaultK3sImageRepo = "docker.io/rancher/k3s"
 
 // DefaultLBImage defines the default cluster load balancer image
-const DefaultLBImage = "docker.io/iwilltry42/k3d-proxy:v0.0.1"
+const DefaultLBImage = "docker.io/iwilltry42/k3d-proxy:v0.0.2"
 
 // DefaultObjectNamePrefix defines the name prefix for every object created by k3d
 const DefaultObjectNamePrefix = "k3d"
@@ -114,15 +114,16 @@ type ClusterNetwork struct {
 
 // Cluster describes a k3d cluster
 type Cluster struct {
-	Name              string             `yaml:"name" json:"name,omitempty"`
+	Name               string             `yaml:"name" json:"name,omitempty"`
-	Network           ClusterNetwork     `yaml:"network" json:"network,omitempty"`
+	Network            ClusterNetwork     `yaml:"network" json:"network,omitempty"`
-	Secret            string             `yaml:"cluster_secret" json:"clusterSecret,omitempty"`
+	Secret             string             `yaml:"cluster_secret" json:"clusterSecret,omitempty"`
-	Nodes             []*Node            `yaml:"nodes" json:"nodes,omitempty"`
+	Nodes              []*Node            `yaml:"nodes" json:"nodes,omitempty"`
-	InitNode          *Node              // init master node
+	InitNode           *Node              // init master node
-	ExternalDatastore ExternalDatastore  `yaml:"external_datastore" json:"externalDatastore,omitempty"`
+	ExternalDatastore  ExternalDatastore  `yaml:"external_datastore" json:"externalDatastore,omitempty"`
-	CreateClusterOpts *CreateClusterOpts `yaml:"options" json:"options,omitempty"`
+	CreateClusterOpts  *CreateClusterOpts `yaml:"options" json:"options,omitempty"`
-	ExposeAPI         ExposeAPI          `yaml:"expose_api" json:"exposeAPI,omitempty"`
+	ExposeAPI          ExposeAPI          `yaml:"expose_api" json:"exposeAPI,omitempty"`
-	ImageVolume       string             `yaml:"image_volume" json:"imageVolume,omitempty"`
+	MasterLoadBalancer *Node              `yaml:"master_loadbalancer" json:"masterLoadBalancer,omitempty"`
+	ImageVolume        string             `yaml:"image_volume" json:"imageVolume,omitempty"`
 }
 
 // Node describes a k3d node

proxy/conf.d/nginx.toml

@@ -3,5 +3,5 @@ src = "nginx.tmpl"
 dest = "/etc/nginx/nginx.conf"
 keys = [
     "SERVERS",
-    "PORT",
+    "PORTS",
 ]

proxy/nginx-proxy

@@ -3,5 +3,10 @@
 # Run confd
 confd -onetime -backend env
 
+# Output Configuration
+echo "===== Initial nginx configuration ====="
+cat /etc/nginx/nginx.conf
+echo "======================================="
+
 # Start nginx
 nginx -g 'daemon off;'

proxy/templates/nginx.tmpl

@@ -1,3 +1,6 @@
+{{- $servers := split (getenv "SERVERS") "," -}}
+{{- $ports := split (getenv "PORTS") "," -}}
+
 error_log stderr notice;
 
 worker_processes auto;
@@ -8,16 +11,18 @@ events {
 }
 
 stream {
-  upstream kube_apiserver {
+  {{- range $port := $ports }}
-    {{ $servers := split (getenv "SERVERS") "," }}{{range $servers}}
+  upstream master_nodes_{{ $port }} {
-    server {{.}}:{{getenv "PORT"}} max_fails=1 fail_timeout=10s slow_start=10s;
+    {{- range $server := $servers }}
-    {{end}}
+    server {{ $server }}:{{ $port }} max_fails=1 fail_timeout=10s;
+    {{- end }}
   }
 
   server {
-    listen        {{getenv "PORT"}};
+    listen        {{ $port }};
-    proxy_pass    kube_apiserver;
+    proxy_pass    master_nodes_{{ $port }};
     proxy_timeout 30;
     proxy_connect_timeout 2s;
   }
+  {{- end }}
 }