/ *
Copyright © 2020 - 2021 The k3d Author ( s )
Permission is hereby granted , free of charge , to any person obtaining a copy
of this software and associated documentation files ( the "Software" ) , to deal
in the Software without restriction , including without limitation the rights
to use , copy , modify , merge , publish , distribute , sublicense , and / or sell
copies of the Software , and to permit persons to whom the Software is
furnished to do so , subject to the following conditions :
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software .
THE SOFTWARE IS PROVIDED "AS IS" , WITHOUT WARRANTY OF ANY KIND , EXPRESS OR
IMPLIED , INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY ,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT . IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM , DAMAGES OR OTHER
LIABILITY , WHETHER IN AN ACTION OF CONTRACT , TORT OR OTHERWISE , ARISING FROM ,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE .
* /
package client
import (
"bytes"
"context"
_ "embed"
"errors"
"fmt"
"io"
"os"
"sort"
"strconv"
"strings"
"time"
gort "runtime"
"github.com/docker/go-connections/nat"
"github.com/imdario/mergo"
copystruct "github.com/mitchellh/copystructure"
"github.com/rancher/k3d/v5/pkg/actions"
config "github.com/rancher/k3d/v5/pkg/config/v1alpha3"
l "github.com/rancher/k3d/v5/pkg/logger"
k3drt "github.com/rancher/k3d/v5/pkg/runtimes"
"github.com/rancher/k3d/v5/pkg/runtimes/docker"
runtimeErr "github.com/rancher/k3d/v5/pkg/runtimes/errors"
"github.com/rancher/k3d/v5/pkg/types"
k3d "github.com/rancher/k3d/v5/pkg/types"
"github.com/rancher/k3d/v5/pkg/types/k3s"
"github.com/rancher/k3d/v5/pkg/util"
"github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
"gopkg.in/yaml.v2"
)
// ClusterRun orchestrates the steps of cluster creation, configuration and starting
func ClusterRun ( ctx context . Context , runtime k3drt . Runtime , clusterConfig * config . ClusterConfig ) error {
/ *
* Step 0 : ( Infrastructure ) Preparation
* /
if err := ClusterPrep ( ctx , runtime , clusterConfig ) ; err != nil {
return fmt . Errorf ( "Failed Cluster Preparation: %+v" , err )
}
// Create tools-node for later steps
go EnsureToolsNode ( ctx , runtime , & clusterConfig . Cluster )
/ *
* Step 1 : Create Containers
* /
if err := ClusterCreate ( ctx , runtime , & clusterConfig . Cluster , & clusterConfig . ClusterCreateOpts ) ; err != nil {
return fmt . Errorf ( "Failed Cluster Creation: %+v" , err )
}
/ *
* Step 2 : Pre - Start Configuration
* /
envInfo , err := GatherEnvironmentInfo ( ctx , runtime , & clusterConfig . Cluster )
if err != nil {
return fmt . Errorf ( "failed to gather environment information used for cluster creation: %w" , err )
}
/ *
* Step 3 : Start Containers
* /
if err := ClusterStart ( ctx , runtime , & clusterConfig . Cluster , k3d . ClusterStartOpts {
WaitForServer : clusterConfig . ClusterCreateOpts . WaitForServer ,
Timeout : clusterConfig . ClusterCreateOpts . Timeout , // TODO: here we should consider the time used so far
NodeHooks : clusterConfig . ClusterCreateOpts . NodeHooks ,
EnvironmentInfo : envInfo ,
Intent : k3d . IntentClusterCreate ,
} ) ; err != nil {
return fmt . Errorf ( "Failed Cluster Start: %+v" , err )
}
/ *
* Post - Start Configuration
* /
/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Additional Cluster Preparation *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
// create the registry hosting configmap
if len ( clusterConfig . ClusterCreateOpts . Registries . Use ) > 0 {
if err := prepCreateLocalRegistryHostingConfigMap ( ctx , runtime , & clusterConfig . Cluster ) ; err != nil {
l . Log ( ) . Warnf ( "Failed to create LocalRegistryHosting ConfigMap: %+v" , err )
}
}
return nil
}
// ClusterPrep takes care of the steps required before creating/starting the cluster containers
func ClusterPrep ( ctx context . Context , runtime k3drt . Runtime , clusterConfig * config . ClusterConfig ) error {
/ *
* Set up contexts
* Used for ( early ) termination ( across API boundaries )
* /
clusterPrepCtx := ctx
if clusterConfig . ClusterCreateOpts . Timeout > 0 * time . Second {
var cancelClusterPrepCtx context . CancelFunc
clusterPrepCtx , cancelClusterPrepCtx = context . WithTimeout ( ctx , clusterConfig . ClusterCreateOpts . Timeout )
defer cancelClusterPrepCtx ( )
}
/ *
* Step 0 : Pre - Pull Images
* /
// TODO: ClusterPrep: add image pre-pulling step
/ *
* Step 1 : Network
* /
if err := ClusterPrepNetwork ( clusterPrepCtx , runtime , & clusterConfig . Cluster , & clusterConfig . ClusterCreateOpts ) ; err != nil {
return fmt . Errorf ( "Failed Network Preparation: %+v" , err )
}
/ *
* Step 2 : Volume ( s )
* /
if ! clusterConfig . ClusterCreateOpts . DisableImageVolume {
if err := ClusterPrepImageVolume ( ctx , runtime , & clusterConfig . Cluster , & clusterConfig . ClusterCreateOpts ) ; err != nil {
return fmt . Errorf ( "Failed Image Volume Preparation: %+v" , err )
}
}
/ *
* Step 3 : Registries
* /
// Ensure referenced registries
for _ , reg := range clusterConfig . ClusterCreateOpts . Registries . Use {
l . Log ( ) . Debugf ( "Trying to find registry %s" , reg . Host )
regNode , err := runtime . GetNode ( ctx , & k3d . Node { Name : reg . Host } )
if err != nil {
return fmt . Errorf ( "Failed to find registry node '%s': %+v" , reg . Host , err )
}
regFromNode , err := RegistryFromNode ( regNode )
if err != nil {
return fmt . Errorf ( "failed to translate node to registry spec: %w" , err )
}
* reg = * regFromNode
}
// Create managed registry bound to this cluster
if clusterConfig . ClusterCreateOpts . Registries . Create != nil {
registryNode , err := RegistryCreate ( ctx , runtime , clusterConfig . ClusterCreateOpts . Registries . Create )
if err != nil {
return fmt . Errorf ( "Failed to create registry: %+v" , err )
}
clusterConfig . Cluster . Nodes = append ( clusterConfig . Cluster . Nodes , registryNode )
clusterConfig . ClusterCreateOpts . Registries . Use = append ( clusterConfig . ClusterCreateOpts . Registries . Use , clusterConfig . ClusterCreateOpts . Registries . Create )
}
// Use existing registries (including the new one, if created)
l . Log ( ) . Tracef ( "Using Registries: %+v" , clusterConfig . ClusterCreateOpts . Registries . Use )
var registryConfig * k3s . Registry
if len ( clusterConfig . ClusterCreateOpts . Registries . Use ) > 0 {
// ensure that all selected registries exist and connect them to the cluster network
for _ , externalReg := range clusterConfig . ClusterCreateOpts . Registries . Use {
regNode , err := runtime . GetNode ( ctx , & k3d . Node { Name : externalReg . Host } )
if err != nil {
return fmt . Errorf ( "Failed to find registry node '%s': %+v" , externalReg . Host , err )
}
if err := RegistryConnectNetworks ( ctx , runtime , regNode , [ ] string { clusterConfig . Cluster . Network . Name } ) ; err != nil {
return fmt . Errorf ( "Failed to connect registry node '%s' to cluster network: %+v" , regNode . Name , err )
}
}
// generate the registries.yaml
regConf , err := RegistryGenerateK3sConfig ( ctx , clusterConfig . ClusterCreateOpts . Registries . Use )
if err != nil {
return fmt . Errorf ( "Failed to generate registry config file for k3s: %+v" , err )
}
// generate the LocalRegistryHosting configmap
regCm , err := RegistryGenerateLocalRegistryHostingConfigMapYAML ( ctx , runtime , clusterConfig . ClusterCreateOpts . Registries . Use )
if err != nil {
return fmt . Errorf ( "Failed to generate LocalRegistryHosting configmap: %+v" , err )
}
l . Log ( ) . Tracef ( "Writing LocalRegistryHosting YAML:\n%s" , string ( regCm ) )
clusterConfig . ClusterCreateOpts . NodeHooks = append ( clusterConfig . ClusterCreateOpts . NodeHooks , k3d . NodeHook {
Stage : k3d . LifecycleStagePreStart ,
Action : actions . WriteFileAction {
Runtime : runtime ,
Content : regCm ,
Dest : k3d . DefaultLocalRegistryHostingConfigmapTempPath ,
Mode : 0644 ,
Description : "Write LocalRegistryHosting Configmap" ,
} ,
} )
registryConfig = regConf
}
// merge with pre-existing, referenced registries.yaml
if clusterConfig . ClusterCreateOpts . Registries . Config != nil {
if registryConfig != nil {
if err := RegistryMergeConfig ( ctx , registryConfig , clusterConfig . ClusterCreateOpts . Registries . Config ) ; err != nil {
return err
}
l . Log ( ) . Tracef ( "Merged registry config: %+v" , registryConfig )
} else {
registryConfig = clusterConfig . ClusterCreateOpts . Registries . Config
}
}
if registryConfig != nil {
regConfBytes , err := yaml . Marshal ( & registryConfig )
if err != nil {
return fmt . Errorf ( "Failed to marshal registry configuration: %+v" , err )
}
clusterConfig . ClusterCreateOpts . NodeHooks = append ( clusterConfig . ClusterCreateOpts . NodeHooks , k3d . NodeHook {
Stage : k3d . LifecycleStagePreStart ,
Action : actions . WriteFileAction {
Runtime : runtime ,
Content : regConfBytes ,
Dest : k3d . DefaultRegistriesFilePath ,
Mode : 0644 ,
Description : "Write Registry Configuration" ,
} ,
} )
}
return nil
}
// ClusterPrepNetwork creates a new cluster network, if needed or sets everything up to re-use an existing network
func ClusterPrepNetwork ( ctx context . Context , runtime k3drt . Runtime , cluster * k3d . Cluster , clusterCreateOpts * k3d . ClusterCreateOpts ) error {
l . Log ( ) . Infoln ( "Prep: Network" )
// error out if external cluster network should be used but no name was set
if cluster . Network . Name == "" && cluster . Network . External {
return fmt . Errorf ( "Failed to use external network because no name was specified" )
}
if cluster . Network . Name != "" && cluster . Network . External && ! cluster . Network . IPAM . IPPrefix . IsZero ( ) {
return fmt . Errorf ( "cannot specify subnet for exiting network" )
}
// generate cluster network name, if not set
if cluster . Network . Name == "" && ! cluster . Network . External {
cluster . Network . Name = fmt . Sprintf ( "%s-%s" , k3d . DefaultObjectNamePrefix , cluster . Name )
}
// handle hostnetwork
if cluster . Network . Name == "host" {
if len ( cluster . Nodes ) > 1 {
return fmt . Errorf ( "only one server node supported when using host network" )
}
}
// create cluster network or use an existing one
network , networkExists , err := runtime . CreateNetworkIfNotPresent ( ctx , & cluster . Network )
if err != nil {
return fmt . Errorf ( "failed to create cluster network: %w" , err )
}
cluster . Network = * network
clusterCreateOpts . GlobalLabels [ k3d . LabelNetworkID ] = network . ID
clusterCreateOpts . GlobalLabels [ k3d . LabelNetwork ] = cluster . Network . Name
clusterCreateOpts . GlobalLabels [ k3d . LabelNetworkIPRange ] = cluster . Network . IPAM . IPPrefix . String ( )
clusterCreateOpts . GlobalLabels [ k3d . LabelNetworkExternal ] = strconv . FormatBool ( cluster . Network . External )
if networkExists {
l . Log ( ) . Infof ( "Re-using existing network '%s' (%s)" , network . Name , network . ID )
clusterCreateOpts . GlobalLabels [ k3d . LabelNetworkExternal ] = "true" // if the network wasn't created, we say that it's managed externally (important for cluster deletion)
}
// just reserve some IPs for k3d (e.g. k3d-tools container), so we don't try to use them again
if cluster . Network . IPAM . Managed {
reservedIP , err := GetIP ( ctx , runtime , & cluster . Network )
if err != nil {
return fmt . Errorf ( "error reserving IP in new cluster network %s" , network . Name )
}
cluster . Network . IPAM . IPsUsed = append ( cluster . Network . IPAM . IPsUsed , reservedIP )
}
return nil
}
func ClusterPrepImageVolume ( ctx context . Context , runtime k3drt . Runtime , cluster * k3d . Cluster , clusterCreateOpts * k3d . ClusterCreateOpts ) error {
/ *
* Cluster - Wide volumes
* - image volume ( for importing images )
* /
imageVolumeName := fmt . Sprintf ( "%s-%s-images" , k3d . DefaultObjectNamePrefix , cluster . Name )
if err := runtime . CreateVolume ( ctx , imageVolumeName , map [ string ] string { k3d . LabelClusterName : cluster . Name } ) ; err != nil {
return fmt . Errorf ( "failed to create image volume '%s' for cluster '%s': %w" , imageVolumeName , cluster . Name , err )
}
clusterCreateOpts . GlobalLabels [ k3d . LabelImageVolume ] = imageVolumeName
cluster . ImageVolume = imageVolumeName
// attach volume to nodes
for _ , node := range cluster . Nodes {
node . Volumes = append ( node . Volumes , fmt . Sprintf ( "%s:%s" , imageVolumeName , k3d . DefaultImageVolumeMountPath ) )
}
return nil
}
// ClusterCreate creates a new cluster consisting of
// - some containerized k3s nodes
// - a docker network
func ClusterCreate ( ctx context . Context , runtime k3drt . Runtime , cluster * k3d . Cluster , clusterCreateOpts * k3d . ClusterCreateOpts ) error {
l . Log ( ) . Tracef ( `
== == = Creating Cluster == == =
Runtime :
% + v
Cluster :
% + v
ClusterCreatOpts :
% + v
== == == == == == == == == == == == == ==
` , runtime , cluster , clusterCreateOpts )
/ *
* Set up contexts
* Used for ( early ) termination ( across API boundaries )
* /
clusterCreateCtx := ctx
if clusterCreateOpts . Timeout > 0 * time . Second {
var cancelClusterCreateCtx context . CancelFunc
clusterCreateCtx , cancelClusterCreateCtx = context . WithTimeout ( ctx , clusterCreateOpts . Timeout )
defer cancelClusterCreateCtx ( )
}
/ *
* Docker Machine Special Configuration
* /
if cluster . KubeAPI . Host == k3d . DefaultAPIHost && runtime == k3drt . Docker {
if gort . GOOS == "windows" || gort . GOOS == "darwin" {
l . Log ( ) . Tracef ( "Running on %s: checking if it's using docker-machine" , gort . GOOS )
machineIP , err := runtime . ( docker . Docker ) . GetDockerMachineIP ( )
if err != nil {
l . Log ( ) . Warnf ( "Using docker-machine, but failed to get it's IP: %+v" , err )
} else if machineIP != "" {
l . Log ( ) . Infof ( "Using the docker-machine IP %s to connect to the Kubernetes API" , machineIP )
cluster . KubeAPI . Host = machineIP
cluster . KubeAPI . Binding . HostIP = machineIP
} else {
l . Log ( ) . Traceln ( "Not using docker-machine" )
}
}
}
/ *
* Cluster Token
* /
if cluster . Token == "" {
cluster . Token = GenerateClusterToken ( )
}
clusterCreateOpts . GlobalLabels [ k3d . LabelClusterToken ] = cluster . Token
/ *
* Nodes
* /
clusterCreateOpts . GlobalLabels [ k3d . LabelClusterName ] = cluster . Name
// agent defaults (per cluster)
// connection url is always the name of the first server node (index 0) // TODO: change this to the server loadbalancer
connectionURL := fmt . Sprintf ( "https://%s:%s" , GenerateNodeName ( cluster . Name , k3d . ServerRole , 0 ) , k3d . DefaultAPIPort )
clusterCreateOpts . GlobalLabels [ k3d . LabelClusterURL ] = connectionURL
clusterCreateOpts . GlobalEnv = append ( clusterCreateOpts . GlobalEnv , fmt . Sprintf ( "%s=%s" , k3s . EnvClusterToken , cluster . Token ) )
nodeSetup := func ( node * k3d . Node ) error {
// cluster specific settings
if node . RuntimeLabels == nil {
node . RuntimeLabels = make ( map [ string ] string ) // TODO: maybe create an init function?
}
// ensure global labels
for k , v := range clusterCreateOpts . GlobalLabels {
node . RuntimeLabels [ k ] = v
}
// ensure global env
node . Env = append ( node . Env , clusterCreateOpts . GlobalEnv ... )
// node role specific settings
if node . Role == k3d . ServerRole {
if cluster . Network . IPAM . Managed {
ip , err := GetIP ( ctx , runtime , & cluster . Network )
if err != nil {
return fmt . Errorf ( "failed to find free IP in network %s: %w" , cluster . Network . Name , err )
}
cluster . Network . IPAM . IPsUsed = append ( cluster . Network . IPAM . IPsUsed , ip ) // make sure that we're not reusing the same IP next time
node . IP . Static = true
node . IP . IP = ip
node . RuntimeLabels [ k3d . LabelNodeStaticIP ] = ip . String ( )
}
node . ServerOpts . KubeAPI = cluster . KubeAPI
// the cluster has an init server node, but its not this one, so connect it to the init node
if cluster . InitNode != nil && ! node . ServerOpts . IsInit {
node . Env = append ( node . Env , fmt . Sprintf ( "%s=%s" , k3s . EnvClusterConnectURL , connectionURL ) )
node . RuntimeLabels [ k3d . LabelServerIsInit ] = "false" // set label, that this server node is not the init server
}
} else if node . Role == k3d . AgentRole {
node . Env = append ( node . Env , fmt . Sprintf ( "%s=%s" , k3s . EnvClusterConnectURL , connectionURL ) )
}
node . Networks = [ ] string { cluster . Network . Name }
node . Restart = true
node . GPURequest = clusterCreateOpts . GPURequest
// create node
l . Log ( ) . Infof ( "Creating node '%s'" , node . Name )
if err := NodeCreate ( clusterCreateCtx , runtime , node , k3d . NodeCreateOpts { } ) ; err != nil {
return fmt . Errorf ( "failed to create node: %w" , err )
}
l . Log ( ) . Debugf ( "Created node '%s'" , node . Name )
return nil
}
// used for node suffices
serverCount := 0
// create init node first
if cluster . InitNode != nil {
l . Log ( ) . Infoln ( "Creating initializing server node" )
cluster . InitNode . Args = append ( cluster . InitNode . Args , "--cluster-init" )
if cluster . InitNode . RuntimeLabels == nil {
cluster . InitNode . RuntimeLabels = map [ string ] string { }
}
cluster . InitNode . RuntimeLabels [ k3d . LabelServerIsInit ] = "true" // set label, that this server node is the init server
// in case the LoadBalancer was disabled, expose the API Port on the initializing server node
if clusterCreateOpts . DisableLoadBalancer {
if cluster . InitNode . Ports == nil {
cluster . InitNode . Ports = nat . PortMap { }
}
cluster . InitNode . Ports [ k3d . DefaultAPIPort ] = [ ] nat . PortBinding { cluster . KubeAPI . Binding }
}
if err := nodeSetup ( cluster . InitNode ) ; err != nil {
return fmt . Errorf ( "failed init node setup: %w" , err )
}
serverCount ++
}
// create all other nodes, but skip the init node
for _ , node := range cluster . Nodes {
if node . Role == k3d . ServerRole {
// skip the init node here
if node == cluster . InitNode {
continue
} else if serverCount == 0 && clusterCreateOpts . DisableLoadBalancer {
// if this is the first server node and the server loadbalancer is disabled, expose the API Port on this server node
if node . Ports == nil {
node . Ports = nat . PortMap { }
}
node . Ports [ k3d . DefaultAPIPort ] = [ ] nat . PortBinding { cluster . KubeAPI . Binding }
}
time . Sleep ( 1 * time . Second ) // FIXME: arbitrary wait for one second to avoid race conditions of servers registering
serverCount ++
}
if node . Role == k3d . ServerRole || node . Role == k3d . AgentRole {
if err := nodeSetup ( node ) ; err != nil {
return fmt . Errorf ( "failed setup of server/agent node %s: %w" , node . Name , err )
}
}
}
// WARN, if there are exactly two server nodes: that means we're using etcd, but don't have fault tolerance
if serverCount == 2 {
l . Log ( ) . Warnln ( "You're creating 2 server nodes: Please consider creating at least 3 to achieve etcd quorum & fault tolerance" )
}
/ *
* Auxiliary Containers
* /
// *** ServerLoadBalancer ***
if ! clusterCreateOpts . DisableLoadBalancer {
if cluster . ServerLoadBalancer == nil {
l . Log ( ) . Infof ( "No loadbalancer specified, creating a default one..." )
lbNode , err := LoadbalancerPrepare ( ctx , runtime , cluster , & k3d . LoadbalancerCreateOpts { Labels : clusterCreateOpts . GlobalLabels } )
if err != nil {
return fmt . Errorf ( "failed to prepare loadbalancer: %w" , err )
}
cluster . Nodes = append ( cluster . Nodes , lbNode ) // append lbNode to list of cluster nodes, so it will be considered during rollback
}
if len ( cluster . ServerLoadBalancer . Config . Ports ) == 0 {
lbConfig , err := LoadbalancerGenerateConfig ( cluster )
if err != nil {
return fmt . Errorf ( "error generating loadbalancer config: %v" , err )
}
cluster . ServerLoadBalancer . Config = & lbConfig
}
cluster . ServerLoadBalancer . Node . RuntimeLabels = clusterCreateOpts . GlobalLabels
// prepare to write config to lb container
configyaml , err := yaml . Marshal ( cluster . ServerLoadBalancer . Config )
if err != nil {
return fmt . Errorf ( "failed to marshal loadbalancer config: %w" , err )
}
writeLbConfigAction := k3d . NodeHook {
Stage : k3d . LifecycleStagePreStart ,
Action : actions . WriteFileAction {
Runtime : runtime ,
Dest : k3d . DefaultLoadbalancerConfigPath ,
Mode : 0744 ,
Content : configyaml ,
Description : "Write Loadbalancer Configuration" ,
} ,
}
cluster . ServerLoadBalancer . Node . HookActions = append ( cluster . ServerLoadBalancer . Node . HookActions , writeLbConfigAction )
l . Log ( ) . Infof ( "Creating LoadBalancer '%s'" , cluster . ServerLoadBalancer . Node . Name )
if err := NodeCreate ( ctx , runtime , cluster . ServerLoadBalancer . Node , k3d . NodeCreateOpts { } ) ; err != nil {
return fmt . Errorf ( "error creating loadbalancer: %v" , err )
}
l . Log ( ) . Debugf ( "Created loadbalancer '%s'" , cluster . ServerLoadBalancer . Node . Name )
}
return nil
}
// ClusterDelete deletes an existing cluster
func ClusterDelete ( ctx context . Context , runtime k3drt . Runtime , cluster * k3d . Cluster , opts k3d . ClusterDeleteOpts ) error {
l . Log ( ) . Infof ( "Deleting cluster '%s'" , cluster . Name )
cluster , err := ClusterGet ( ctx , runtime , cluster )
if err != nil {
return fmt . Errorf ( "failed to get cluster: %w" , err )
}
l . Log ( ) . Debugf ( "Cluster Details: %+v" , cluster )
failed := 0
for _ , node := range cluster . Nodes {
// registry: only delete, if not connected to other networks
if node . Role == k3d . RegistryRole && ! opts . SkipRegistryCheck {
l . Log ( ) . Tracef ( "Registry Node has %d networks: %+v" , len ( node . Networks ) , node )
// check if node is connected to other networks, that are not
// - the cluster network
// - default docker networks
// -> if so, disconnect it from the cluster network and continue
connectedToOtherNet := false
for _ , net := range node . Networks {
if net == cluster . Network . Name {
continue
}
if net == "bridge" || net == "host" {
continue
}
l . Log ( ) . Tracef ( "net: %s" , net )
connectedToOtherNet = true
break
}
if connectedToOtherNet {
l . Log ( ) . Infof ( "Registry %s is also connected to other (non-default) networks (%+v), not deleting it..." , node . Name , node . Networks )
if err := runtime . DisconnectNodeFromNetwork ( ctx , node , cluster . Network . Name ) ; err != nil {
l . Log ( ) . Warnf ( "Failed to disconnect registry %s from cluster network %s" , node . Name , cluster . Network . Name )
}
continue
}
}
if err := NodeDelete ( ctx , runtime , node , k3d . NodeDeleteOpts { SkipLBUpdate : true } ) ; err != nil {
l . Log ( ) . Warningf ( "Failed to delete node '%s': Try to delete it manually" , node . Name )
failed ++
continue
}
}
// Delete the cluster network, if it was created for/by this cluster (and if it's not in use anymore)
if cluster . Network . Name != "" {
if ! cluster . Network . External {
l . Log ( ) . Infof ( "Deleting cluster network '%s'" , cluster . Network . Name )
if err := runtime . DeleteNetwork ( ctx , cluster . Network . Name ) ; err != nil {
if errors . Is ( err , runtimeErr . ErrRuntimeNetworkNotEmpty ) { // there are still containers connected to that network
connectedNodes , err := runtime . GetNodesInNetwork ( ctx , cluster . Network . Name ) // check, if there are any k3d nodes connected to the cluster
if err != nil {
l . Log ( ) . Warningf ( "Failed to check cluster network for connected nodes: %+v" , err )
}
if len ( connectedNodes ) > 0 { // there are still k3d-managed containers (aka nodes) connected to the network
connectedRegistryNodes := util . FilterNodesByRole ( connectedNodes , k3d . RegistryRole )
if len ( connectedRegistryNodes ) == len ( connectedNodes ) { // only registry node(s) left in the network
for _ , node := range connectedRegistryNodes {
l . Log ( ) . Debugf ( "Disconnecting registry node %s from the network..." , node . Name )
if err := runtime . DisconnectNodeFromNetwork ( ctx , node , cluster . Network . Name ) ; err != nil {
l . Log ( ) . Warnf ( "Failed to disconnect registry %s from network %s" , node . Name , cluster . Network . Name )
} else {
if err := runtime . DeleteNetwork ( ctx , cluster . Network . Name ) ; err != nil {
l . Log ( ) . Warningf ( "Failed to delete cluster network, even after disconnecting registry node(s): %+v" , err )
}
}
}
} else { // besides the registry node(s), there are still other nodes... maybe they still need a registry
l . Log ( ) . Debugf ( "There are some non-registry nodes left in the network" )
}
} else {
l . Log ( ) . Warningf ( "Failed to delete cluster network '%s' because it's still in use: is there another cluster using it?" , cluster . Network . Name )
}
} else {
l . Log ( ) . Warningf ( "Failed to delete cluster network '%s': '%+v'" , cluster . Network . Name , err )
}
}
} else if cluster . Network . External {
l . Log ( ) . Debugf ( "Skip deletion of cluster network '%s' because it's managed externally" , cluster . Network . Name )
}
}
// delete image volume
if cluster . ImageVolume != "" {
l . Log ( ) . Infof ( "Deleting image volume '%s'" , cluster . ImageVolume )
if err := runtime . DeleteVolume ( ctx , cluster . ImageVolume ) ; err != nil {
l . Log ( ) . Warningf ( "Failed to delete image volume '%s' of cluster '%s': Try to delete it manually" , cluster . ImageVolume , cluster . Name )
}
}
// return error if we failed to delete a node
if failed > 0 {
return fmt . Errorf ( "Failed to delete %d nodes: Try to delete them manually" , failed )
}
return nil
}
// ClusterList returns a list of all existing clusters
func ClusterList ( ctx context . Context , runtime k3drt . Runtime ) ( [ ] * k3d . Cluster , error ) {
l . Log ( ) . Traceln ( "Listing Clusters..." )
nodes , err := runtime . GetNodesByLabel ( ctx , k3d . DefaultRuntimeLabels )
if err != nil {
return nil , fmt . Errorf ( "runtime failed to list nodes: %w" , err )
}
l . Log ( ) . Debugf ( "Found %d nodes" , len ( nodes ) )
if l . Log ( ) . GetLevel ( ) == logrus . TraceLevel {
for _ , node := range nodes {
l . Log ( ) . Tracef ( "Found node %s of role %s" , node . Name , node . Role )
}
}
nodes = NodeFilterByRoles ( nodes , k3d . ClusterInternalNodeRoles , k3d . ClusterExternalNodeRoles )
l . Log ( ) . Tracef ( "Found %d cluster-internal nodes" , len ( nodes ) )
if l . Log ( ) . GetLevel ( ) == logrus . TraceLevel {
for _ , node := range nodes {
l . Log ( ) . Tracef ( "Found cluster-internal node %s of role %s belonging to cluster %s" , node . Name , node . Role , node . RuntimeLabels [ k3d . LabelClusterName ] )
}
}
clusters := [ ] * k3d . Cluster { }
// for each node, check, if we can add it to a cluster or add the cluster if it doesn't exist yet
for _ , node := range nodes {
clusterExists := false
for _ , cluster := range clusters {
if node . RuntimeLabels [ k3d . LabelClusterName ] == cluster . Name { // TODO: handle case, where this label doesn't exist
cluster . Nodes = append ( cluster . Nodes , node )
clusterExists = true
break
}
}
// cluster is not in the list yet, so we add it with the current node as its first member
if ! clusterExists {
clusters = append ( clusters , & k3d . Cluster {
Name : node . RuntimeLabels [ k3d . LabelClusterName ] ,
Nodes : [ ] * k3d . Node { node } ,
} )
}
}
// enrich cluster structs with label values
for _ , cluster := range clusters {
if err := populateClusterFieldsFromLabels ( cluster ) ; err != nil {
l . Log ( ) . Warnf ( "Failed to populate cluster fields from node label values for cluster '%s'" , cluster . Name )
l . Log ( ) . Warnln ( err )
}
}
l . Log ( ) . Debugf ( "Found %d clusters" , len ( clusters ) )
return clusters , nil
}
// populateClusterFieldsFromLabels inspects labels attached to nodes and translates them to struct fields
func populateClusterFieldsFromLabels ( cluster * k3d . Cluster ) error {
networkExternalSet := false
for _ , node := range cluster . Nodes {
// get the name of the cluster network
if cluster . Network . Name == "" {
if networkName , ok := node . RuntimeLabels [ k3d . LabelNetwork ] ; ok {
cluster . Network . Name = networkName
}
}
// check if the network is external
// since the struct value is a bool, initialized as false, we cannot check if it's unset
if ! cluster . Network . External && ! networkExternalSet {
if networkExternalString , ok := node . RuntimeLabels [ k3d . LabelNetworkExternal ] ; ok {
if networkExternal , err := strconv . ParseBool ( networkExternalString ) ; err == nil {
cluster . Network . External = networkExternal
networkExternalSet = true
}
}
}
// get image volume // TODO: enable external image volumes the same way we do it with networks
if cluster . ImageVolume == "" {
if imageVolumeName , ok := node . RuntimeLabels [ k3d . LabelImageVolume ] ; ok {
cluster . ImageVolume = imageVolumeName
}
}
// get k3s cluster's token
if cluster . Token == "" {
if token , ok := node . RuntimeLabels [ k3d . LabelClusterToken ] ; ok {
cluster . Token = token
}
}
}
return nil
}
var ClusterGetNoNodesFoundError = errors . New ( "No nodes found for given cluster" )
// ClusterGet returns an existing cluster with all fields and node lists populated
func ClusterGet ( ctx context . Context , runtime k3drt . Runtime , cluster * k3d . Cluster ) ( * k3d . Cluster , error ) {
// get nodes that belong to the selected cluster
nodes , err := runtime . GetNodesByLabel ( ctx , map [ string ] string { k3d . LabelClusterName : cluster . Name } )
if err != nil {
l . Log ( ) . Errorf ( "Failed to get nodes for cluster '%s': %v" , cluster . Name , err )
}
if len ( nodes ) == 0 {
return nil , ClusterGetNoNodesFoundError
}
// append nodes
for _ , node := range nodes {
// check if there's already a node in the struct
overwroteExisting := false
for _ , existingNode := range cluster . Nodes {
// overwrite existing node
if existingNode . Name == node . Name {
mergo . MergeWithOverwrite ( existingNode , node )
overwroteExisting = true
}
}
// no existing node overwritten: append new node
if ! overwroteExisting {
cluster . Nodes = append ( cluster . Nodes , node )
}
}
// Loadbalancer
if cluster . ServerLoadBalancer == nil {
for _ , node := range cluster . Nodes {
if node . Role == k3d . LoadBalancerRole {
cluster . ServerLoadBalancer = & k3d . Loadbalancer {
Node : node ,
}
}
}
if cluster . ServerLoadBalancer != nil && cluster . ServerLoadBalancer . Node != nil {
lbcfg , err := GetLoadbalancerConfig ( ctx , runtime , cluster )
if err != nil {
l . Log ( ) . Errorf ( "error getting loadbalancer config from %s: %v" , cluster . ServerLoadBalancer . Node . Name , err )
}
cluster . ServerLoadBalancer . Config = & lbcfg
}
}
if err := populateClusterFieldsFromLabels ( cluster ) ; err != nil {
l . Log ( ) . Warnf ( "Failed to populate cluster fields from node labels: %v" , err )
}
return cluster , nil
}
// GenerateClusterToken generates a random 20 character string
func GenerateClusterToken ( ) string {
return util . GenerateRandomString ( 20 )
}
func GenerateNodeName ( cluster string , role k3d . Role , suffix int ) string {
return fmt . Sprintf ( "%s-%s-%s-%d" , k3d . DefaultObjectNamePrefix , cluster , role , suffix )
}
// ClusterStart starts a whole cluster (i.e. all nodes of the cluster)
func ClusterStart ( ctx context . Context , runtime k3drt . Runtime , cluster * k3d . Cluster , clusterStartOpts types . ClusterStartOpts ) error {
l . Log ( ) . Infof ( "Starting cluster '%s'" , cluster . Name )
if clusterStartOpts . Intent == "" {
clusterStartOpts . Intent = k3d . IntentClusterStart
}
if clusterStartOpts . Timeout > 0 * time . Second {
var cancel context . CancelFunc
ctx , cancel = context . WithTimeout ( ctx , clusterStartOpts . Timeout )
defer cancel ( )
}
// sort the nodes into categories
var initNode * k3d . Node
var servers [ ] * k3d . Node
var agents [ ] * k3d . Node
var aux [ ] * k3d . Node
for _ , n := range cluster . Nodes {
if n . Role == k3d . ServerRole {
if n . ServerOpts . IsInit {
initNode = n
continue
}
servers = append ( servers , n )
} else if n . Role == k3d . AgentRole {
agents = append ( agents , n )
} else {
aux = append ( aux , n )
}
}
// sort list of servers for properly ordered sequential start
sort . Slice ( servers , func ( i , j int ) bool {
return servers [ i ] . Name < servers [ j ] . Name
} )
/ *
* Init Node
* /
if initNode != nil {
l . Log ( ) . Infoln ( "Starting the initializing server..." )
if err := NodeStart ( ctx , runtime , initNode , & k3d . NodeStartOpts {
Wait : true , // always wait for the init node
NodeHooks : clusterStartOpts . NodeHooks ,
ReadyLogMessage : types . GetReadyLogMessage ( initNode , clusterStartOpts . Intent ) , // initNode means, that we're using etcd -> this will need quorum, so "k3s is up and running" won't happen right now
EnvironmentInfo : clusterStartOpts . EnvironmentInfo ,
} ) ; err != nil {
return fmt . Errorf ( "Failed to start initializing server node: %+v" , err )
}
}
/ *
* Server Nodes
* /
l . Log ( ) . Infoln ( "Starting servers..." )
for _ , serverNode := range servers {
if err := NodeStart ( ctx , runtime , serverNode , & k3d . NodeStartOpts {
Wait : true ,
NodeHooks : append ( clusterStartOpts . NodeHooks , serverNode . HookActions ... ) ,
EnvironmentInfo : clusterStartOpts . EnvironmentInfo ,
} ) ; err != nil {
return fmt . Errorf ( "Failed to start server %s: %+v" , serverNode . Name , err )
}
}
/ *
* Agent Nodes
* /
agentWG , aCtx := errgroup . WithContext ( ctx )
l . Log ( ) . Infoln ( "Starting agents..." )
for _ , agentNode := range agents {
currentAgentNode := agentNode
agentWG . Go ( func ( ) error {
return NodeStart ( aCtx , runtime , currentAgentNode , & k3d . NodeStartOpts {
Wait : true ,
NodeHooks : clusterStartOpts . NodeHooks ,
EnvironmentInfo : clusterStartOpts . EnvironmentInfo ,
} )
} )
}
if err := agentWG . Wait ( ) ; err != nil {
return fmt . Errorf ( "Failed to add one or more agents: %w" , err )
}
/ *
* Auxiliary / Helper Nodes
* /
helperWG , hCtx := errgroup . WithContext ( ctx )
l . Log ( ) . Infoln ( "Starting helpers..." )
for _ , helperNode := range aux {
currentHelperNode := helperNode
helperWG . Go ( func ( ) error {
nodeStartOpts := & k3d . NodeStartOpts {
NodeHooks : currentHelperNode . HookActions ,
EnvironmentInfo : clusterStartOpts . EnvironmentInfo ,
}
if currentHelperNode . Role == k3d . LoadBalancerRole {
nodeStartOpts . Wait = true
}
return NodeStart ( hCtx , runtime , currentHelperNode , nodeStartOpts )
} )
}
if err := helperWG . Wait ( ) ; err != nil {
return fmt . Errorf ( "Failed to add one or more helper nodes: %w" , err )
}
/ *
* Additional Cluster Preparation ( post start )
* /
postStartErrgrp , postStartErrgrpCtx := errgroup . WithContext ( ctx )
/*** DNS ***/
// add host.k3d.internal record to /etc/hosts in all nodes
postStartErrgrp . Go ( func ( ) error {
return prepInjectHostIP ( postStartErrgrpCtx , runtime , cluster , & clusterStartOpts )
} )
postStartErrgrp . Go ( func ( ) error {
if cluster . Network . Name == "host" {
l . Log ( ) . Debugf ( "Not injecting host.k3d.internal into CoreDNS as clusternetwork is 'host'" )
return nil
}
hosts := fmt . Sprintf ( "%s %s\n" , clusterStartOpts . EnvironmentInfo . HostGateway . String ( ) , k3d . DefaultK3dInternalHostRecord )
net , err := runtime . GetNetwork ( ctx , & cluster . Network )
if err != nil {
return fmt . Errorf ( "failed to get cluster network %s to inject host records into CoreDNS: %w" , cluster . Network . Name , err )
}
for _ , member := range net . Members {
hosts += fmt . Sprintf ( "%s %s\n" , member . IP . String ( ) , member . Name )
}
l . Log ( ) . Infof ( "Injecting records for host.k3d.internal and for %d network members into CoreDNS configmap..." , len ( net . Members ) )
act := actions . RewriteFileAction {
Runtime : runtime ,
Path : "/var/lib/rancher/k3s/server/manifests/coredns.yaml" ,
Mode : 0744 ,
RewriteFunc : func ( input [ ] byte ) ( [ ] byte , error ) {
split , err := util . SplitYAML ( input )
if err != nil {
return nil , fmt . Errorf ( "error splitting yaml: %w" , err )
}
var outputBuf bytes . Buffer
outputEncoder := yaml . NewEncoder ( & outputBuf )
for _ , d := range split {
var doc map [ string ] interface { }
if err := yaml . Unmarshal ( d , & doc ) ; err != nil {
return nil , err
}
if kind , ok := doc [ "kind" ] ; ok {
if strings . ToLower ( kind . ( string ) ) == "configmap" {
configmapData := doc [ "data" ] . ( map [ interface { } ] interface { } )
configmapData [ "NodeHosts" ] = hosts
}
}
if err := outputEncoder . Encode ( doc ) ; err != nil {
return nil , err
}
}
outputEncoder . Close ( )
return outputBuf . Bytes ( ) , nil
} ,
}
// get the first server in the list and run action on it once it's ready for it
for _ , n := range cluster . Nodes {
if n . Role == k3d . ServerRole {
ts , err := time . Parse ( "2006-01-02T15:04:05.999999999Z" , n . State . Started )
if err != nil {
return err
}
if err := NodeWaitForLogMessage ( ctx , runtime , n , "Cluster dns configmap" , ts . Truncate ( time . Second ) ) ; err != nil {
return err
}
return act . Run ( ctx , n )
}
}
return nil
} )
if err := postStartErrgrp . Wait ( ) ; err != nil {
return fmt . Errorf ( "error during post-start cluster preparation: %w" , err )
}
return nil
}
// ClusterStop stops a whole cluster (i.e. all nodes of the cluster)
func ClusterStop ( ctx context . Context , runtime k3drt . Runtime , cluster * k3d . Cluster ) error {
l . Log ( ) . Infof ( "Stopping cluster '%s'" , cluster . Name )
failed := 0
for _ , node := range cluster . Nodes {
if err := runtime . StopNode ( ctx , node ) ; err != nil {
l . Log ( ) . Warningf ( "Failed to stop node '%s': Try to stop it manually" , node . Name )
failed ++
continue
}
}
if failed > 0 {
return fmt . Errorf ( "Failed to stop %d nodes: Try to stop them manually" , failed )
}
l . Log ( ) . Infof ( "Stopped cluster '%s'" , cluster . Name )
return nil
}
// SortClusters : in place sort cluster list by cluster name alphabetical order
func SortClusters ( clusters [ ] * k3d . Cluster ) [ ] * k3d . Cluster {
sort . Slice ( clusters , func ( i , j int ) bool {
return clusters [ i ] . Name < clusters [ j ] . Name
} )
return clusters
}
// corednsAddHost adds a host entry to the CoreDNS configmap if it doesn't exist (a host entry is a single line of the form "IP HOST")
func corednsAddHost ( ctx context . Context , runtime k3drt . Runtime , cluster * k3d . Cluster , ip string , name string ) error {
retries := 3
if v , ok := os . LookupEnv ( k3d . K3dEnvDebugCorednsRetries ) ; ok && v != "" {
l . Log ( ) . Debugf ( "Running with %s=%s" , k3d . K3dEnvDebugCorednsRetries , v )
if r , err := strconv . Atoi ( v ) ; err == nil {
retries = r
} else {
return fmt . Errorf ( "Invalid value set for env var %s (%s): %w" , k3d . K3dEnvDebugCorednsRetries , v , err )
}
}
// select any server node
var node * k3d . Node
for _ , n := range cluster . Nodes {
if n . Role == k3d . ServerRole {
node = n
}
}
hostsEntry := fmt . Sprintf ( "%s %s" , ip , name )
patchCmd := ` patch=$(kubectl get cm coredns -n kube-system --template=' {{ .data .NodeHosts }} ' | sed -n -E -e '/[0-9\.] { 4,12}\s ` + name + ` $/!p' -e '$a ` + hostsEntry + ` ' | tr '\n' '^' | busybox xargs -0 printf ' { "data": { "NodeHosts":"%s"}}'| sed -E 's%\^%\\n%g') && kubectl patch cm coredns -n kube-system -p="$patch" `
successInjectCoreDNSEntry := false
// try 3 (or K3D_DEBUG_COREDNS_RETRIES value) times, as e.g. on cluster startup it may take some time for the Configmap to be available and the server to be responsive
for i := 0 ; i < retries ; i ++ {
l . Log ( ) . Debugf ( "Running CoreDNS patch in node %s to add %s (try %d/%d)..." , node . Name , hostsEntry , i , retries )
logreader , err := runtime . ExecInNodeGetLogs ( ctx , node , [ ] string { "sh" , "-c" , patchCmd } )
if err == nil {
successInjectCoreDNSEntry = true
break
} else {
msg := fmt . Sprintf ( "(try %d/%d) error patching the CoreDNS ConfigMap to include entry '%s': %+v" , i , retries , hostsEntry , err )
if logreader != nil {
readlogs , err := io . ReadAll ( logreader )
if err != nil {
l . Log ( ) . Debugf ( "(try %d/%d) error reading the logs from failed CoreDNS patch exec process in node %s: %v" , i , retries , node . Name , err )
} else {
msg += fmt . Sprintf ( "\nLogs: %s" , string ( readlogs ) )
}
} else {
l . Log ( ) . Debugf ( "(try %d/%d) error reading the logs from failed CoreDNS patch exec process in node %s: no logreader returned for exec process" , i , retries , node . Name )
}
l . Log ( ) . Debugln ( msg )
time . Sleep ( 1 * time . Second )
}
}
if ! successInjectCoreDNSEntry {
return fmt . Errorf ( "failed to patch CoreDNS ConfigMap to include entry '%s' (%d tries, see debug logs)" , hostsEntry , retries )
}
l . Log ( ) . Debugf ( "Successfully patched CoreDNS Configmap with record '%s'" , hostsEntry )
return nil
}
// prepInjectHostIP adds /etc/hosts entry for host.k3d.internal, referring to the host system
func prepInjectHostIP ( ctx context . Context , runtime k3drt . Runtime , cluster * k3d . Cluster , clusterStartOpts * k3d . ClusterStartOpts ) error {
if cluster . Network . Name == "host" {
l . Log ( ) . Debugf ( "Not injecting host.k3d.internal into /etc/hosts as clusternetwork is 'host'" )
return nil
}
hostIP := clusterStartOpts . EnvironmentInfo . HostGateway
hostsEntry := fmt . Sprintf ( "%s %s" , hostIP . String ( ) , k3d . DefaultK3dInternalHostRecord )
l . Log ( ) . Infof ( "Injecting '%s' into /etc/hosts of all nodes..." , hostsEntry )
// entry in /etc/hosts
errgrp , errgrpctx := errgroup . WithContext ( ctx )
for _ , node := range cluster . Nodes {
n := node
errgrp . Go ( func ( ) error {
return runtime . ExecInNode ( errgrpctx , n , [ ] string { "sh" , "-c" , fmt . Sprintf ( "echo '%s' >> /etc/hosts" , hostsEntry ) } )
} )
}
if err := errgrp . Wait ( ) ; err != nil {
return fmt . Errorf ( "failed to add hosts entry %s: %w" , hostsEntry , err )
}
l . Log ( ) . Debugf ( "Successfully added host record \"%s\" to /etc/hosts in all nodes" , hostsEntry )
return nil
}
func prepCreateLocalRegistryHostingConfigMap ( ctx context . Context , runtime k3drt . Runtime , cluster * k3d . Cluster ) error {
success := false
for _ , node := range cluster . Nodes {
if node . Role == k3d . AgentRole || node . Role == k3d . ServerRole {
err := runtime . ExecInNode ( ctx , node , [ ] string { "sh" , "-c" , fmt . Sprintf ( "kubectl apply -f %s" , k3d . DefaultLocalRegistryHostingConfigmapTempPath ) } )
if err == nil {
success = true
break
} else {
l . Log ( ) . Debugf ( "Failed to create LocalRegistryHosting ConfigMap in node %s: %+v" , node . Name , err )
}
}
}
if success == false {
l . Log ( ) . Warnf ( "Failed to create LocalRegistryHosting ConfigMap" )
}
return nil
}
// ClusterEditChangesetSimple modifies an existing cluster with a given SimpleConfig changeset
func ClusterEditChangesetSimple ( ctx context . Context , runtime k3drt . Runtime , cluster * k3d . Cluster , changeset * config . SimpleConfig ) error {
// nodeCount := len(cluster.Nodes)
nodeList := cluster . Nodes
// === Ports ===
existingLB := cluster . ServerLoadBalancer
lbChangeset := & k3d . Loadbalancer { }
// copy existing loadbalancer
lbChangesetNode , err := CopyNode ( ctx , existingLB . Node , CopyNodeOpts { keepState : false } )
if err != nil {
return fmt . Errorf ( "error copying existing loadbalancer: %w" , err )
}
lbChangeset . Node = lbChangesetNode
// copy config from existing loadbalancer
lbChangesetConfig , err := copystruct . Copy ( existingLB . Config )
if err != nil {
return fmt . Errorf ( "error copying config from existing loadbalancer: %w" , err )
}
lbChangeset . Config = lbChangesetConfig . ( * k3d . LoadbalancerConfig )
// loop over ports
if len ( changeset . Ports ) > 0 {
// 1. ensure that there are only supported suffices in the node filters // TODO: overly complex right now, needs simplification
for _ , portWithNodeFilters := range changeset . Ports {
filteredNodes , err := util . FilterNodesWithSuffix ( nodeList , portWithNodeFilters . NodeFilters )
if err != nil {
return fmt . Errorf ( "failed to filter nodes: %w" , err )
}
for suffix := range filteredNodes {
switch suffix {
case "proxy" , util . NodeFilterSuffixNone , util . NodeFilterMapKeyAll :
continue
default :
return fmt . Errorf ( "error: 'cluster edit' does not (yet) support the '%s' opt/suffix for adding ports" , suffix )
}
}
}
// 2. transform
cluster . ServerLoadBalancer = lbChangeset // we're working with pointers, so let's point to the changeset here to not update the original that we keep as a reference
if err := TransformPorts ( ctx , runtime , cluster , changeset . Ports ) ; err != nil {
return fmt . Errorf ( "error transforming port config %s: %w" , changeset . Ports , err )
}
}
l . Log ( ) . Debugf ( "ORIGINAL:\n> Ports: %+v\n> Config: %+v\nCHANGESET:\n> Ports: %+v\n> Config: %+v" , existingLB . Node . Ports , existingLB . Config , lbChangeset . Node . Ports , lbChangeset . Config )
// prepare to write config to lb container
configyaml , err := yaml . Marshal ( lbChangeset . Config )
if err != nil {
return fmt . Errorf ( "failed to marshal loadbalancer config changeset: %w" , err )
}
writeLbConfigAction := k3d . NodeHook {
Stage : k3d . LifecycleStagePreStart ,
Action : actions . WriteFileAction {
Runtime : runtime ,
Dest : k3d . DefaultLoadbalancerConfigPath ,
Mode : 0744 ,
Content : configyaml ,
Description : "Write Loadbalancer Configuration" ,
} ,
}
if lbChangeset . Node . HookActions == nil {
lbChangeset . Node . HookActions = [ ] k3d . NodeHook { }
}
lbChangeset . Node . HookActions = append ( lbChangeset . Node . HookActions , writeLbConfigAction )
NodeReplace ( ctx , runtime , existingLB . Node , lbChangeset . Node )
return nil
}