From 185ffcd34f9a08af394e2e8564a0ca0e6523511e Mon Sep 17 00:00:00 2001 From: Thorsten Klein Date: Thu, 7 Jan 2021 11:32:30 +0100 Subject: [PATCH] [Enhancement] clusterDelete: proper node and network handling (#437) This comes with several fixes/improvements - only consider containers that have the default object label (app=k3d) - handle network deletion - check if there are other k3d containers connected - if there are only registries, disconnect them - if there are non-registry nodes, leave everything as it is - if there are any containers connected, that are not automatically disconnected, log a warning and continue --- cmd/cluster/clusterDelete.go | 1 + docs/usage/guides/registries.md | 2 +- pkg/client/cluster.go | 54 +++++++++++++++++++++++++++--- pkg/config/transform.go | 5 +++ pkg/runtimes/containerd/network.go | 5 +++ pkg/runtimes/containerd/node.go | 5 +++ pkg/runtimes/docker/network.go | 36 +++++++++++++++++++- pkg/runtimes/docker/node.go | 43 ++++++++++++++++++++++-- pkg/runtimes/docker/translate.go | 17 ++++++++++ pkg/runtimes/docker/volume.go | 5 +-- pkg/runtimes/errors/errors.go | 30 +++++++++++++++++ pkg/runtimes/runtime.go | 4 ++- pkg/util/filter.go | 11 ++++++ version/version.go | 2 +- 14 files changed, 206 insertions(+), 14 deletions(-) create mode 100644 pkg/runtimes/errors/errors.go diff --git a/cmd/cluster/clusterDelete.go b/cmd/cluster/clusterDelete.go index 952ea7e7..8a97161a 100644 --- a/cmd/cluster/clusterDelete.go +++ b/cmd/cluster/clusterDelete.go @@ -100,6 +100,7 @@ func parseDeleteClusterCmd(cmd *cobra.Command, args []string) []*k3d.Cluster { if all, err := cmd.Flags().GetBool("all"); err != nil { log.Fatalln(err) } else if all { + log.Infoln("Deleting all clusters...") clusters, err = client.ClusterList(cmd.Context(), runtimes.SelectedRuntime) if err != nil { log.Fatalln(err) diff --git a/docs/usage/guides/registries.md b/docs/usage/guides/registries.md index 53bf4847..66c336a1 100644 --- a/docs/usage/guides/registries.md +++ b/docs/usage/guides/registries.md @@ -67,7 +67,7 @@ Finally, we can create the cluster, mounting the CA file in the path we specifie ### Using k3d-managed registries -!!! info "Not ported yet" +!!! info "Just ported!" The k3d-managed registry is available again as of k3d v4.0.0 (January 2021) #### Create a dedicated registry together with your cluster diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go index 19318d0c..74426925 100644 --- a/pkg/client/cluster.go +++ b/pkg/client/cluster.go @@ -39,6 +39,7 @@ import ( config "github.com/rancher/k3d/v4/pkg/config/v1alpha1" k3drt "github.com/rancher/k3d/v4/pkg/runtimes" "github.com/rancher/k3d/v4/pkg/runtimes/docker" + runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors" "github.com/rancher/k3d/v4/pkg/types" k3d "github.com/rancher/k3d/v4/pkg/types" "github.com/rancher/k3d/v4/pkg/util" @@ -100,7 +101,6 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi // ClusterPrep takes care of the steps required before creating/starting the cluster containers func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *config.ClusterConfig) error { - /* * Set up contexts * Used for (early) termination (across API boundaries) @@ -139,6 +139,7 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf // Ensure referenced registries for _, reg := range clusterConfig.ClusterCreateOpts.Registries.Use { + log.Debugf("Trying to find registry %s", reg.Host) regNode, err := runtime.GetNode(ctx, &k3d.Node{Name: reg.Host}) if err != nil { return fmt.Errorf("Failed to find registry node '%s': %+v", reg.Host, err) @@ -243,7 +244,6 @@ func ClusterPrepImageVolume(ctx context.Context, runtime k3drt.Runtime, cluster * Cluster-Wide volumes * - image volume (for importing images) */ - imageVolumeName := fmt.Sprintf("%s-%s-images", k3d.DefaultObjectNamePrefix, cluster.Name) if err := runtime.CreateVolume(ctx, imageVolumeName, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil { log.Errorf("Failed to create image volume '%s' for cluster '%s'", imageVolumeName, cluster.Name) @@ -468,7 +468,7 @@ ClusterCreatOpts: fmt.Sprintf("WORKER_PROCESSES=%d", len(strings.Split(ports, ","))), }, Role: k3d.LoadBalancerRole, - Labels: k3d.DefaultObjectLabels, // TODO: createLoadBalancer: add more expressive labels + Labels: clusterCreateOpts.GlobalLabels, // TODO: createLoadBalancer: add more expressive labels Network: cluster.Network.Name, Restart: true, } @@ -491,6 +491,10 @@ ClusterCreatOpts: func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error { log.Infof("Deleting cluster '%s'", cluster.Name) + cluster, err := ClusterGet(ctx, runtime, cluster) + if err != nil { + return err + } log.Debugf("Cluster Details: %+v", cluster) failed := 0 @@ -507,8 +511,32 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus if !cluster.Network.External { log.Infof("Deleting cluster network '%s'", cluster.Network.Name) if err := runtime.DeleteNetwork(ctx, cluster.Network.Name); err != nil { - if strings.HasSuffix(err.Error(), "active endpoints") { - log.Warningf("Failed to delete cluster network '%s' because it's still in use: is there another cluster using it?", cluster.Network.Name) + if errors.Is(err, runtimeErr.ErrRuntimeNetworkNotEmpty) { // there are still containers connected to that network + + connectedNodes, err := runtime.GetNodesInNetwork(ctx, cluster.Network.Name) // check, if there are any k3d nodes connected to the cluster + if err != nil { + log.Warningf("Failed to check cluster network for connected nodes: %+v", err) + } + + if len(connectedNodes) > 0 { // there are still k3d-managed containers (aka nodes) connected to the network + connectedRegistryNodes := util.FilterNodesByRole(connectedNodes, k3d.RegistryRole) + if len(connectedRegistryNodes) == len(connectedNodes) { // only registry node(s) left in the network + for _, node := range connectedRegistryNodes { + log.Debugf("Disconnecting registry node %s from the network...", node.Name) + if err := runtime.DisconnectNodeFromNetwork(ctx, node, cluster.Network.Name); err != nil { + log.Warnf("Failed to disconnect registry %s from network %s", node.Name, cluster.Network.Name) + } else { + if err := runtime.DeleteNetwork(ctx, cluster.Network.Name); err != nil { + log.Warningf("Failed to delete cluster network, even after disconnecting registry node(s): %+v", err) + } + } + } + } else { // besides the registry node(s), there are still other nodes... maybe they still need a registry + log.Debugf("There are some non-registry nodes left in the network") + } + } else { + log.Warningf("Failed to delete cluster network '%s' because it's still in use: is there another cluster using it?", cluster.Network.Name) + } } else { log.Warningf("Failed to delete cluster network '%s': '%+v'", cluster.Network.Name, err) } @@ -535,14 +563,29 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus // ClusterList returns a list of all existing clusters func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, error) { + log.Traceln("Listing Clusters...") nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultObjectLabels) if err != nil { log.Errorln("Failed to get clusters") return nil, err } + log.Debugf("Found %d nodes", len(nodes)) + if log.GetLevel() == log.TraceLevel { + for _, node := range nodes { + log.Tracef("Found node %s of role %s", node.Name, node.Role) + } + } + nodes = NodeFilterByRoles(nodes, k3d.ClusterInternalNodeRoles, k3d.ClusterExternalNodeRoles) + log.Tracef("Found %d cluster-internal nodes", len(nodes)) + if log.GetLevel() == log.TraceLevel { + for _, node := range nodes { + log.Tracef("Found cluster-internal node %s of role %s belonging to cluster %s", node.Name, node.Role, node.Labels[k3d.LabelClusterName]) + } + } + clusters := []*k3d.Cluster{} // for each node, check, if we can add it to a cluster or add the cluster if it doesn't exist yet for _, node := range nodes { @@ -570,6 +613,7 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er log.Warnln(err) } } + log.Debugf("Found %d clusters", len(clusters)) return clusters, nil } diff --git a/pkg/config/transform.go b/pkg/config/transform.go index cbdf7ff1..8dbcc171 100644 --- a/pkg/config/transform.go +++ b/pkg/config/transform.go @@ -222,6 +222,11 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim GlobalEnv: []string{}, // empty init } + // ensure, that we have the default object labels + for k, v := range k3d.DefaultObjectLabels { + clusterCreateOpts.GlobalLabels[k] = v + } + /* * Registries */ diff --git a/pkg/runtimes/containerd/network.go b/pkg/runtimes/containerd/network.go index 2eccbb84..90d9ea43 100644 --- a/pkg/runtimes/containerd/network.go +++ b/pkg/runtimes/containerd/network.go @@ -41,3 +41,8 @@ func (d Containerd) DeleteNetwork(ctx context.Context, ID string) error { func (d Containerd) ConnectNodeToNetwork(ctx context.Context, node *k3d.Node, network string) error { return nil } + +// DisconnectNodeFromNetwork disconnects a node from a network (u don't say :O) +func (d Containerd) DisconnectNodeFromNetwork(ctx context.Context, node *k3d.Node, network string) error { + return nil +} diff --git a/pkg/runtimes/containerd/node.go b/pkg/runtimes/containerd/node.go index f2d9c9c0..5afec722 100644 --- a/pkg/runtimes/containerd/node.go +++ b/pkg/runtimes/containerd/node.go @@ -137,3 +137,8 @@ func (d Containerd) ExecInNode(ctx context.Context, node *k3d.Node, cmd []string func (d Containerd) ExecInNodeGetLogs(ctx context.Context, node *k3d.Node, cmd []string) (*bufio.Reader, error) { return nil, nil } + +// GetNodesInNetwork returns all the nodes connected to a given network +func (d Containerd) GetNodesInNetwork(ctx context.Context, network string) ([]*k3d.Node, error) { + return nil, nil +} diff --git a/pkg/runtimes/docker/network.go b/pkg/runtimes/docker/network.go index a32cc831..d7071b7e 100644 --- a/pkg/runtimes/docker/network.go +++ b/pkg/runtimes/docker/network.go @@ -25,12 +25,14 @@ import ( "context" "fmt" "net" + "strings" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" + runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors" k3d "github.com/rancher/k3d/v4/pkg/types" log "github.com/sirupsen/logrus" ) @@ -94,7 +96,13 @@ func (d Docker) DeleteNetwork(ctx context.Context, ID string) error { defer docker.Close() // (3) delete network - return docker.NetworkRemove(ctx, ID) + if err := docker.NetworkRemove(ctx, ID); err != nil { + if strings.HasSuffix(err.Error(), "active endpoints") { + return runtimeErr.ErrRuntimeNetworkNotEmpty + } + return err + } + return nil } // GetNetwork gets information about a network by its ID @@ -147,3 +155,29 @@ func (d Docker) ConnectNodeToNetwork(ctx context.Context, node *k3d.Node, networ // connect container to network return docker.NetworkConnect(ctx, networkResource.ID, container.ID, &network.EndpointSettings{}) } + +// DisconnectNodeFromNetwork disconnects a node from a network (u don't say :O) +func (d Docker) DisconnectNodeFromNetwork(ctx context.Context, node *k3d.Node, networkName string) error { + // get container + container, err := getNodeContainer(ctx, node) + if err != nil { + return err + } + + // get docker client + docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + log.Errorln("Failed to create docker client") + return err + } + defer docker.Close() + + // get network + networkResource, err := GetNetwork(ctx, networkName) + if err != nil { + log.Errorf("Failed to get network '%s'", networkName) + return err + } + + return docker.NetworkDisconnect(ctx, networkResource.ID, container.ID, true) +} diff --git a/pkg/runtimes/docker/node.go b/pkg/runtimes/docker/node.go index 6939d386..21cb66fd 100644 --- a/pkg/runtimes/docker/node.go +++ b/pkg/runtimes/docker/node.go @@ -25,6 +25,7 @@ package docker import ( "bufio" "context" + "errors" "fmt" "io" "io/ioutil" @@ -33,6 +34,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/client" + runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors" k3d "github.com/rancher/k3d/v4/pkg/types" log "github.com/sirupsen/logrus" ) @@ -59,6 +61,7 @@ func (d Docker) CreateNode(ctx context.Context, node *k3d.Node) error { // DeleteNode deletes a node func (d Docker) DeleteNode(ctx context.Context, nodeSpec *k3d.Node) error { + log.Debugf("Deleting node %s ...", nodeSpec.Name) return removeContainer(ctx, nodeSpec.Name) } @@ -218,7 +221,7 @@ func (d Docker) GetNode(ctx context.Context, node *k3d.Node) (*k3d.Node, error) node, err = TranslateContainerDetailsToNode(containerDetails) if err != nil { - log.Errorf("Failed to translate container details for node '%s' to node object", node.Name) + log.Errorf("Failed to translate container '%s' to node object", containerDetails.Name) return node, err } @@ -396,10 +399,44 @@ func executeInNode(ctx context.Context, node *k3d.Node, cmd []string) (*types.Hi if execInfo.ExitCode == 0 { // success log.Debugf("Exec process in node '%s' exited with '0'", node.Name) return &execConnection, nil - } else { // failed - return &execConnection, fmt.Errorf("Exec process in node '%s' failed with exit code '%d'", node.Name, execInfo.ExitCode) } + return &execConnection, fmt.Errorf("Exec process in node '%s' failed with exit code '%d'", node.Name, execInfo.ExitCode) + } +} + +// GetNodesInNetwork returns all the nodes connected to a given network +func (d Docker) GetNodesInNetwork(ctx context.Context, network string) ([]*k3d.Node, error) { + // create docker client + docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + log.Errorln("Failed to create docker client") + return nil, err + } + defer docker.Close() + net, err := GetNetwork(ctx, network) + if err != nil { + return nil, err + } + + connectedNodes := []*k3d.Node{} + + // loop over list of containers connected to this cluster and transform them into nodes internally + for cID := range net.Containers { + containerDetails, err := getContainerDetails(ctx, cID) + if err != nil { + return nil, err + } + node, err := TranslateContainerDetailsToNode(containerDetails) + if err != nil { + if errors.Is(err, runtimeErr.ErrRuntimeContainerUnknown) { + log.Tracef("GetNodesInNetwork: inspected non-k3d-managed container %s", containerDetails.Name) + continue + } + return nil, err + } + connectedNodes = append(connectedNodes, node) } + return connectedNodes, nil } diff --git a/pkg/runtimes/docker/translate.go b/pkg/runtimes/docker/translate.go index ed2a73f9..135f0f7c 100644 --- a/pkg/runtimes/docker/translate.go +++ b/pkg/runtimes/docker/translate.go @@ -31,6 +31,7 @@ import ( docker "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/network" "github.com/docker/go-connections/nat" + runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors" k3d "github.com/rancher/k3d/v4/pkg/types" log "github.com/sirupsen/logrus" @@ -135,6 +136,22 @@ func TranslateContainerToNode(cont *types.Container) (*k3d.Node, error) { // TranslateContainerDetailsToNode translates a docker containerJSON object into a k3d node representation func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d.Node, error) { + // first, make sure, that it's actually a k3d managed container by checking if it has all the default labels + for k, v := range k3d.DefaultObjectLabels { + log.Tracef("TranslateContainerDetailsToNode: Checking for default object label %s=%s", k, v) + found := false + for lk, lv := range containerDetails.Config.Labels { + if lk == k && lv == v { + found = true + break + } + } + if !found { + log.Debugf("Container %s is missing default label %s=%s in label set %+v", containerDetails.Name, k, v, containerDetails.Config.Labels) + return nil, runtimeErr.ErrRuntimeContainerUnknown + } + } + // restart -> we only set 'unless-stopped' upon cluster creation restart := false if containerDetails.HostConfig.RestartPolicy.IsAlways() || containerDetails.HostConfig.RestartPolicy.IsUnlessStopped() { diff --git a/pkg/runtimes/docker/volume.go b/pkg/runtimes/docker/volume.go index 7ec86854..95cd308f 100644 --- a/pkg/runtimes/docker/volume.go +++ b/pkg/runtimes/docker/volume.go @@ -45,11 +45,12 @@ func (d Docker) CreateVolume(ctx context.Context, name string, labels map[string // (1) create volume volumeCreateOptions := volume.VolumeCreateBody{ Name: name, - Labels: k3d.DefaultObjectLabels, + Labels: labels, Driver: "local", // TODO: allow setting driver + opts DriverOpts: map[string]string{}, } - for k, v := range labels { + + for k, v := range k3d.DefaultObjectLabels { volumeCreateOptions.Labels[k] = v } diff --git a/pkg/runtimes/errors/errors.go b/pkg/runtimes/errors/errors.go new file mode 100644 index 00000000..67e387b7 --- /dev/null +++ b/pkg/runtimes/errors/errors.go @@ -0,0 +1,30 @@ +/* +Copyright © 2020 The k3d Author(s) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package runtimes + +import "errors" + +// ErrRuntimeNetworkNotEmpty describes an error that occurs because a network still has containers connected to it (e.g. cannot be deleted) +var ErrRuntimeNetworkNotEmpty = errors.New("network not empty") + +// ErrRuntimeContainerUnknown describes the situation, where we're inspecting a container that's not obviously managed by k3d +var ErrRuntimeContainerUnknown = errors.New("container not managed by k3d: missing default label(s)") diff --git a/pkg/runtimes/runtime.go b/pkg/runtimes/runtime.go index e4ca0162..f807e236 100644 --- a/pkg/runtimes/runtime.go +++ b/pkg/runtimes/runtime.go @@ -56,6 +56,7 @@ type Runtime interface { GetNodesByLabel(context.Context, map[string]string) ([]*k3d.Node, error) GetNode(context.Context, *k3d.Node) (*k3d.Node, error) GetNodeStatus(context.Context, *k3d.Node) (bool, string, error) + GetNodesInNetwork(context.Context, string) ([]*k3d.Node, error) CreateNetworkIfNotPresent(context.Context, string) (string, bool, error) // @return NETWORK_NAME, EXISTS, ERROR GetKubeconfig(context.Context, *k3d.Node) (io.ReadCloser, error) DeleteNetwork(context.Context, string) error @@ -72,7 +73,8 @@ type Runtime interface { CopyToNode(context.Context, string, string, *k3d.Node) error // @param context, source, destination, node WriteToNode(context.Context, []byte, string, *k3d.Node) error // @param context, content, destination, node GetHostIP(context.Context, string) (net.IP, error) - ConnectNodeToNetwork(context.Context, *k3d.Node, string) error // @param context, node, network name + ConnectNodeToNetwork(context.Context, *k3d.Node, string) error // @param context, node, network name + DisconnectNodeFromNetwork(context.Context, *k3d.Node, string) error // @param context, node, network name } // GetRuntime checks, if a given name is represented by an implemented k3d runtime and returns it diff --git a/pkg/util/filter.go b/pkg/util/filter.go index fdc72d2a..db4fff54 100644 --- a/pkg/util/filter.go +++ b/pkg/util/filter.go @@ -174,3 +174,14 @@ func FilterNodes(nodes []*k3d.Node, filters []string) ([]*k3d.Node, error) { return filteredNodes, nil } + +// FilterNodesByRole returns a stripped list of nodes which do match the given role +func FilterNodesByRole(nodes []*k3d.Node, role k3d.Role) []*k3d.Node { + filteredNodes := []*k3d.Node{} + for _, node := range nodes { + if node.Role == role { + filteredNodes = append(filteredNodes, node) + } + } + return filteredNodes +} diff --git a/version/version.go b/version/version.go index e9fd57b5..8d78d27a 100644 --- a/version/version.go +++ b/version/version.go @@ -35,7 +35,7 @@ var Version string var HelperVersionOverride string // K3sVersion should contain the latest version tag of k3s (hardcoded at build time) -var K3sVersion = "rancher/k3s:v1.20.0-k3s2" +var K3sVersion = "v1.20.0-k3s2" // GetVersion returns the version for cli, it gets it from "git describe --tags" or returns "dev" when doing simple go build func GetVersion() string {