From 42ee62e552ee1dfbc19fd64400aae1b721f3fe01 Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Wed, 3 Jun 2020 14:55:50 +0200 Subject: [PATCH] createNode: add --wait and --timeout flags - new struct: createNodeOpts for wait and timeout values - new commands for creating/adding multiple nodes which then wait for all nodes to be up, if specified - tests/e2e: new test for adding a master node --- cmd/create/createNode.go | 14 +++++--- pkg/cluster/cluster.go | 4 +-- pkg/cluster/node.go | 64 +++++++++++++++++++++++++++++++++--- pkg/types/types.go | 14 +++++++- tests/test_full_lifecycle.sh | 13 +++----- 5 files changed, 88 insertions(+), 21 deletions(-) diff --git a/cmd/create/createNode.go b/cmd/create/createNode.go index 4c4d7eda..a47b2cef 100644 --- a/cmd/create/createNode.go +++ b/cmd/create/createNode.go @@ -23,6 +23,7 @@ package create import ( "fmt" + "time" "github.com/spf13/cobra" @@ -36,6 +37,8 @@ import ( // NewCmdCreateNode returns a new cobra command func NewCmdCreateNode() *cobra.Command { + createNodeOpts := k3d.CreateNodeOpts{} + // create new command cmd := &cobra.Command{ Use: "node NAME", @@ -44,11 +47,9 @@ func NewCmdCreateNode() *cobra.Command { Args: cobra.ExactArgs(1), // exactly one name accepted // TODO: if not specified, inherit from cluster that the node shall belong to, if that is specified Run: func(cmd *cobra.Command, args []string) { nodes, cluster := parseCreateNodeCmd(cmd, args) - for _, node := range nodes { - if err := k3dc.AddNodeToCluster(cmd.Context(), runtimes.SelectedRuntime, node, cluster); err != nil { - log.Errorf("Failed to add node '%s' to cluster '%s'", node.Name, cluster.Name) - log.Errorln(err) - } + if err := k3dc.AddNodesToCluster(cmd.Context(), runtimes.SelectedRuntime, nodes, cluster, createNodeOpts); err != nil { + log.Errorf("Failed to add nodes '%+v' to cluster '%s'", nodes, cluster.Name) + log.Errorln(err) } }, } @@ -63,6 +64,9 @@ func NewCmdCreateNode() *cobra.Command { cmd.Flags().StringP("image", "i", fmt.Sprintf("%s:%s", k3d.DefaultK3sImageRepo, version.GetK3sVersion(false)), "Specify k3s image used for the node(s)") + cmd.Flags().BoolVar(&createNodeOpts.Wait, "wait", false, "Wait for the node(s) to be ready before returning.") + cmd.Flags().DurationVar(&createNodeOpts.Timeout, "timeout", 0*time.Second, "Maximum waiting time for '--wait' before canceling/returning.") + // done return cmd } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index bba77a0b..9eb74736 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -155,7 +155,7 @@ func CreateCluster(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus // create node log.Infof("Creating node '%s'", node.Name) - if err := CreateNode(ctx, runtime, node); err != nil { + if err := CreateNode(ctx, runtime, node, k3d.CreateNodeOpts{}); err != nil { log.Errorln("Failed to create node") return err } @@ -301,7 +301,7 @@ func CreateCluster(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus } cluster.Nodes = append(cluster.Nodes, lbNode) // append lbNode to list of cluster nodes, so it will be considered during rollback log.Infof("Creating LoadBalancer '%s'", lbNode.Name) - if err := CreateNode(ctx, runtime, lbNode); err != nil { + if err := CreateNode(ctx, runtime, lbNode, k3d.CreateNodeOpts{}); err != nil { log.Errorln("Failed to create loadbalancer") return err } diff --git a/pkg/cluster/node.go b/pkg/cluster/node.go index 3303eeb6..1a03088e 100644 --- a/pkg/cluster/node.go +++ b/pkg/cluster/node.go @@ -33,10 +33,11 @@ import ( "github.com/rancher/k3d/pkg/runtimes" k3d "github.com/rancher/k3d/pkg/types" log "github.com/sirupsen/logrus" + "golang.org/x/sync/errgroup" ) // AddNodeToCluster adds a node to an existing cluster -func AddNodeToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, cluster *k3d.Cluster) error { +func AddNodeToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, cluster *k3d.Cluster, createNodeOpts k3d.CreateNodeOpts) error { cluster, err := GetCluster(ctx, runtime, cluster) if err != nil { log.Errorf("Failed to find specified cluster '%s'", cluster.Name) @@ -126,7 +127,7 @@ func AddNodeToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N } } - if err := CreateNode(ctx, runtime, node); err != nil { + if err := CreateNode(ctx, runtime, node, k3d.CreateNodeOpts{}); err != nil { return err } @@ -141,17 +142,70 @@ func AddNodeToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N return nil } +// AddNodesToCluster adds multiple nodes to a chosen cluster +func AddNodesToCluster(ctx context.Context, runtime runtimes.Runtime, nodes []*k3d.Node, cluster *k3d.Cluster, createNodeOpts k3d.CreateNodeOpts) error { + if createNodeOpts.Timeout > 0*time.Second { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, createNodeOpts.Timeout) + defer cancel() + } + + nodeWaitGroup, ctx := errgroup.WithContext(ctx) + for _, node := range nodes { + if err := AddNodeToCluster(ctx, runtime, node, cluster, k3d.CreateNodeOpts{}); err != nil { + return err + } + if createNodeOpts.Wait { + currentNode := node + nodeWaitGroup.Go(func() error { + log.Debugf("Starting to wait for node '%s'", currentNode.Name) + return WaitForNodeLogMessage(ctx, runtime, currentNode, k3d.ReadyLogMessageByRole[currentNode.Role], time.Time{}) + }) + } + } + if err := nodeWaitGroup.Wait(); err != nil { + log.Errorln("Failed to bring up all nodes in time. Check the logs:") + log.Errorf(">>> %+v", err) + return fmt.Errorf("Failed to add nodes") + } + + return nil +} + // CreateNodes creates a list of nodes -func CreateNodes(ctx context.Context, runtime runtimes.Runtime, nodes []*k3d.Node) { // TODO: pass `--atomic` flag, so we stop and return an error if any node creation fails? +func CreateNodes(ctx context.Context, runtime runtimes.Runtime, nodes []*k3d.Node, createNodeOpts k3d.CreateNodeOpts) error { // TODO: pass `--atomic` flag, so we stop and return an error if any node creation fails? + if createNodeOpts.Timeout > 0*time.Second { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, createNodeOpts.Timeout) + defer cancel() + } + + nodeWaitGroup, ctx := errgroup.WithContext(ctx) for _, node := range nodes { - if err := CreateNode(ctx, runtime, node); err != nil { + if err := CreateNode(ctx, runtime, node, k3d.CreateNodeOpts{}); err != nil { log.Error(err) } + if createNodeOpts.Wait { + currentNode := node + nodeWaitGroup.Go(func() error { + log.Debugf("Starting to wait for node '%s'", currentNode.Name) + return WaitForNodeLogMessage(ctx, runtime, currentNode, k3d.ReadyLogMessageByRole[currentNode.Role], time.Time{}) + }) + } } + + if err := nodeWaitGroup.Wait(); err != nil { + log.Errorln("Failed to bring up all nodes in time. Check the logs:") + log.Errorf(">>> %+v", err) + return fmt.Errorf("Failed to create nodes") + } + + return nil + } // CreateNode creates a new containerized k3s node -func CreateNode(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node) error { +func CreateNode(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, createNodeOpts k3d.CreateNodeOpts) error { log.Debugf("Creating node from spec\n%+v", node) /* diff --git a/pkg/types/types.go b/pkg/types/types.go index 13ae1e19..dd67d33e 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -47,7 +47,7 @@ const DefaultObjectNamePrefix = "k3d" // ReadyLogMessageMaster defines the log messages we wait for until a master node is considered ready var ReadyLogMessageByRole = map[Role]string{ MasterRole: "Wrote kubeconfig", - WorkerRole: "", + WorkerRole: "Successfully registered node", LoadBalancerRole: "start worker processes", } @@ -130,6 +130,18 @@ type StartClusterOpts struct { Timeout time.Duration } +// CreateNodeOpts describes a set of options one can set when creating a new node +type CreateNodeOpts struct { + Wait bool + Timeout time.Duration +} + +// StartNodeOpts describes a set of options one can set when (re-)starting a node +type StartNodeOpts struct { + Wait bool + Timeout time.Duration +} + // ClusterNetwork describes a network which a cluster is running in type ClusterNetwork struct { Name string `yaml:"name" json:"name,omitempty"` diff --git a/tests/test_full_lifecycle.sh b/tests/test_full_lifecycle.sh index 54b2d078..f3c5cf1a 100755 --- a/tests/test_full_lifecycle.sh +++ b/tests/test_full_lifecycle.sh @@ -39,14 +39,11 @@ info "Checking that we have 2 nodes online..." check_multi_node "$clustername" 2 || failed "failed to verify number of nodes" # 4. adding another worker node -# info "Adding one worker node..." -# LOG_LEVEL=debug $EXE create node "extra-worker" --cluster "$clustername" --role "worker" || failed "failed to add worker node" -# -# info "Waiting for a bit to give the new node enough time to boot and register..." -# sleep 10 -# -# info "Checking that we have 3 nodes available now..." -# check_multi_node "$clustername" 3 || failed "failed to verify number of nodes" +info "Adding one worker node..." +LOG_LEVEL=debug $EXE create node "extra-worker" --cluster "$clustername" --role "worker" --wait --timeout 360s || failed "failed to add worker node" + +info "Checking that we have 3 nodes available now..." +check_multi_node "$clustername" 3 || failed "failed to verify number of nodes" # 4. load an image into the cluster info "Loading an image into the cluster..."