From 220313524e28a942ad28932b9ed921638e9e66a1 Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Fri, 18 Jun 2021 12:57:27 +0200 Subject: [PATCH] generate new config file for lb and add helper functions to get images from env if needed --- pkg/client/cluster.go | 9 ++++- pkg/client/loadbalancer.go | 74 +++++++++++++++++++++----------------- pkg/types/images.go | 48 +++++++++++++++++++++++++ pkg/types/types.go | 12 +++++-- proxy/conf.d/nginx.toml | 3 +- proxy/templates/nginx.tmpl | 5 +-- proxy/test/portmap.yaml | 5 ++- 7 files changed, 114 insertions(+), 42 deletions(-) create mode 100644 pkg/types/images.go diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go index 6fdf2e7a..4905f692 100644 --- a/pkg/client/cluster.go +++ b/pkg/client/cluster.go @@ -507,9 +507,16 @@ ClusterCreatOpts: */ // *** ServerLoadBalancer *** if !clusterCreateOpts.DisableLoadBalancer { - if err := LoadbalancerCreate(ctx, runtime, cluster, &k3d.LoadbalancerCreateOpts{Labels: clusterCreateOpts.GlobalLabels}); err != nil { + node, nodeCreateOpts, err := LoadbalancerPrepare(ctx, runtime, cluster, &k3d.LoadbalancerCreateOpts{Labels: clusterCreateOpts.GlobalLabels}) + if err != nil { + return err + } + if err := NodeCreate(ctx, runtime, node, *nodeCreateOpts); err != nil { + log.Errorln("Failed to create loadbalancer") return err } + log.Debugf("Created loadbalancer '%s'", node.Name) + return err } return nil diff --git a/pkg/client/loadbalancer.go b/pkg/client/loadbalancer.go index f76440f3..db47c494 100644 --- a/pkg/client/loadbalancer.go +++ b/pkg/client/loadbalancer.go @@ -29,10 +29,10 @@ import ( "strings" "github.com/docker/go-connections/nat" + "github.com/rancher/k3d/v4/pkg/actions" "github.com/rancher/k3d/v4/pkg/runtimes" "github.com/rancher/k3d/v4/pkg/types" k3d "github.com/rancher/k3d/v4/pkg/types" - "github.com/rancher/k3d/v4/version" log "github.com/sirupsen/logrus" "sigs.k8s.io/yaml" ) @@ -112,28 +112,28 @@ func GetLoadbalancerConfig(ctx context.Context, runtime runtimes.Runtime, cluste return currentConfig, nil } -func LoadbalancerCreate(ctx context.Context, runtime runtimes.Runtime, cluster *types.Cluster, opts *k3d.LoadbalancerCreateOpts) error { - // Generate a comma-separated list of server/server names to pass to the LB container - servers := "" +func LoadbalancerPrepare(ctx context.Context, runtime runtimes.Runtime, cluster *types.Cluster, opts *k3d.LoadbalancerCreateOpts) (*k3d.Node, *k3d.NodeCreateOpts, error) { + + lbConfig := k3d.LoadbalancerConfig{ + Ports: map[string][]string{}, + Settings: k3d.LoadBalancerSettings{}, + } + + // get list of server nodes + servers := []string{} for _, node := range cluster.Nodes { if node.Role == k3d.ServerRole { - if servers == "" { - servers = node.Name - } else { - servers = fmt.Sprintf("%s,%s", servers, node.Name) - } + servers = append(servers, node.Name) } } - // generate comma-separated list of extra ports to forward - ports := []string{k3d.DefaultAPIPort} - var udp_ports []string + // Default API Port proxied to the server nodes + lbConfig.Ports[fmt.Sprintf("%s.tcp", k3d.DefaultAPIPort)] = servers + + // generate comma-separated list of extra ports to forward // TODO: no default targets? for exposedPort := range cluster.ServerLoadBalancer.Ports { - if exposedPort.Proto() == "udp" { - udp_ports = append(udp_ports, exposedPort.Port()) - continue - } - ports = append(ports, exposedPort.Port()) + // TODO: catch duplicates here? + lbConfig.Ports[fmt.Sprintf("%s.%s", exposedPort.Port(), exposedPort.Proto())] = servers } if cluster.ServerLoadBalancer.Ports == nil { @@ -143,28 +143,36 @@ func LoadbalancerCreate(ctx context.Context, runtime runtimes.Runtime, cluster * // Create LB as a modified node with loadbalancerRole lbNode := &k3d.Node{ - Name: fmt.Sprintf("%s-%s-serverlb", k3d.DefaultObjectNamePrefix, cluster.Name), - Image: fmt.Sprintf("%s:%s", k3d.DefaultLBImageRepo, version.GetHelperImageVersion()), - Ports: cluster.ServerLoadBalancer.Ports, - Env: []string{ - fmt.Sprintf("SERVERS=%s", servers), - fmt.Sprintf("PORTS=%s", strings.Join(ports, ",")), - fmt.Sprintf("WORKER_PROCESSES=%d", len(ports)), - }, + Name: fmt.Sprintf("%s-%s-serverlb", k3d.DefaultObjectNamePrefix, cluster.Name), + Image: k3d.GetLoadbalancerImage(), + Ports: cluster.ServerLoadBalancer.Ports, Role: k3d.LoadBalancerRole, RuntimeLabels: opts.Labels, // TODO: createLoadBalancer: add more expressive labels Networks: []string{cluster.Network.Name}, Restart: true, } - if len(udp_ports) > 0 { - lbNode.Env = append(lbNode.Env, fmt.Sprintf("UDP_PORTS=%s", strings.Join(udp_ports, ","))) - } cluster.Nodes = append(cluster.Nodes, lbNode) // append lbNode to list of cluster nodes, so it will be considered during rollback log.Infof("Creating LoadBalancer '%s'", lbNode.Name) - if err := NodeCreate(ctx, runtime, lbNode, k3d.NodeCreateOpts{}); err != nil { - log.Errorln("Failed to create loadbalancer") - return err + + // some additional nginx settings + lbConfig.Settings.WorkerProcesses = k3d.DefaultLoadbalancerWorkerProcesses + len(cluster.ServerLoadBalancer.Ports)*len(servers) + + // prepare to write config to lb container + configyaml, err := yaml.Marshal(lbConfig) + if err != nil { + return nil, nil, err } - log.Debugf("Created loadbalancer '%s'", lbNode.Name) - return nil + + writeLbConfigAction := k3d.NodeHook{ + Stage: k3d.LifecycleStagePreStart, + Action: actions.WriteFileAction{ + Runtime: runtime, + Dest: k3d.DefaultLoadbalancerConfigPath, + Mode: 0744, + Content: configyaml, + }, + } + + return lbNode, &k3d.NodeCreateOpts{NodeHooks: []k3d.NodeHook{writeLbConfigAction}}, nil + } diff --git a/pkg/types/images.go b/pkg/types/images.go new file mode 100644 index 00000000..713ab0a7 --- /dev/null +++ b/pkg/types/images.go @@ -0,0 +1,48 @@ +/* +Copyright © 2020-2021 The k3d Author(s) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package types + +import ( + "fmt" + "os" + + "github.com/rancher/k3d/v4/version" + log "github.com/sirupsen/logrus" +) + +func GetLoadbalancerImage() string { + if img := os.Getenv("K3D_IMAGE_LOADBALANCER"); img != "" { + log.Infof("Loadbalancer image set from env var $K3D_IMAGE_LOADBALANCER: %s", img) + return img + } + + return fmt.Sprintf("%s:%s", DefaultLBImageRepo, version.GetHelperImageVersion()) +} + +func GetToolsImage() string { + if img := os.Getenv("K3D_IMAGE_TOOLS"); img != "" { + log.Infof("Tools image set from env var $K3D_IMAGE_TOOLS: %s", img) + return img + } + + return fmt.Sprintf("%s:%s", DefaultToolsImageRepo, version.GetHelperImageVersion()) +} diff --git a/pkg/types/types.go b/pkg/types/types.go index 24fc41f4..141fe2c6 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -442,10 +442,18 @@ type RegistryExternal struct { * - k3d-k3s-default-agent-1 */ type LoadbalancerConfig struct { - Ports map[string][]string `yaml:"ports"` + Ports map[string][]string `yaml:"ports"` + Settings LoadBalancerSettings `yaml:"settings"` } -const DefaultLoadbalancerConfigPath = "/etc/confd/portmap.yaml" +type LoadBalancerSettings struct { + WorkerProcesses int `yaml:"workerProcesses"` +} + +const ( + DefaultLoadbalancerConfigPath = "/etc/confd/portmap.yaml" + DefaultLoadbalancerWorkerProcesses = 1024 +) type LoadbalancerCreateOpts struct { Labels map[string]string diff --git a/proxy/conf.d/nginx.toml b/proxy/conf.d/nginx.toml index 26decd79..96805409 100644 --- a/proxy/conf.d/nginx.toml +++ b/proxy/conf.d/nginx.toml @@ -2,5 +2,6 @@ src = "nginx.tmpl" dest = "/etc/nginx/nginx.conf" keys = [ - "ports" + "ports", + "settings" ] \ No newline at end of file diff --git a/proxy/templates/nginx.tmpl b/proxy/templates/nginx.tmpl index 963b61dd..fa8c3087 100644 --- a/proxy/templates/nginx.tmpl +++ b/proxy/templates/nginx.tmpl @@ -5,16 +5,13 @@ # ####### # ################################### -{{- $servers := split (getenv "SERVERS") "," -}} -{{- $ports := split (getenv "PORTS") "," -}} -{{- $udp_ports := split (getenv "UDP_PORTS") "," -}} error_log stderr notice; worker_processes auto; events { multi_accept on; use epoll; - worker_connections {{ add 1024 (len $ports) }}; + worker_connections {{ getv "/settings/workerProcesses" }}; } stream { diff --git a/proxy/test/portmap.yaml b/proxy/test/portmap.yaml index 02df30c3..d102645b 100644 --- a/proxy/test/portmap.yaml +++ b/proxy/test/portmap.yaml @@ -4,4 +4,7 @@ ports: - server-1 4321.udp: - agent-0 - - agent-1 \ No newline at end of file + - agent-1 + +settings: + workerProcesses: 1030 \ No newline at end of file