diff --git a/cli/commands.go b/cli/commands.go index 0d148e17..10c55251 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -8,6 +8,7 @@ import ( "log" "os" "path" + "strconv" "strings" "time" @@ -40,6 +41,17 @@ func CreateCluster(c *cli.Context) error { return errors.New("Cannot use --timeout flag without --wait flag") } + // environment variables + env := []string{"K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml"} + if c.IsSet("env") || c.IsSet("e") { + env = append(env, c.StringSlice("env")...) + } + k3sClusterSecret := "" + if c.Int("workers") > 0 { + k3sClusterSecret = fmt.Sprintf("K3S_CLUSTER_SECRET=%s", GenerateRandomString(20)) + env = append(env, k3sClusterSecret) + } + // k3s server arguments k3sServerArgs := []string{"--https-listen-port", c.String("port")} if c.IsSet("server-arg") || c.IsSet("x") { @@ -53,7 +65,7 @@ func CreateCluster(c *cli.Context) error { fmt.Sprintf("docker.io/rancher/k3s:%s", c.String("version")), c.String("port"), k3sServerArgs, - []string{"K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml"}, + env, c.String("name"), strings.Split(c.String("volume"), ","), ) @@ -101,46 +113,73 @@ func CreateCluster(c *cli.Context) error { export KUBECONFIG="$(%s get-kubeconfig --name='%s')" kubectl cluster-info`, os.Args[0], c.String("name")) + + // worker nodes + if c.Int("workers") > 0 { + k3sWorkerArgs := []string{} + env := []string{k3sClusterSecret} + log.Printf("Booting %s workers for cluster %s", strconv.Itoa(c.Int("workers")), c.String("name")) + for i := 0; i < c.Int("workers"); i++ { + workerID, err := createWorker( + c.GlobalBool("verbose"), + fmt.Sprintf("docker.io/rancher/k3s:%s", c.String("version")), + k3sWorkerArgs, + env, + c.String("name"), + strings.Split(c.String("volume"), ","), + strconv.Itoa(i), + c.String("port"), + ) + if err != nil { + return fmt.Errorf("ERROR: failed to create worker node for cluster %s\n%+v", c.String("name"), err) + } + fmt.Printf("Created worker with ID %s\n", workerID) + } + } return nil } // DeleteCluster removes the cluster container and its cluster directory func DeleteCluster(c *cli.Context) error { - ctx := context.Background() - docker, err := client.NewEnvClient() - if err != nil { - return err - } - - clusterNames := []string{} // operate on one or all clusters + clusters := make(map[string]cluster) if !c.Bool("all") { - clusterNames = append(clusterNames, c.String("name")) + cluster, err := getCluster(c.String("name")) + if err != nil { + return err + } + clusters[c.String("name")] = cluster } else { - clusterList, err := getClusterNames() + clusterMap, err := getClusters() if err != nil { return fmt.Errorf("ERROR: `--all` specified, but no clusters were found\n%+v", err) } - clusterNames = append(clusterNames, clusterList...) + // copy clusterMap + for k, v := range clusterMap { + clusters[k] = v + } } // remove clusters one by one instead of appending all names to the docker command // this allows for more granular error handling and logging - for _, name := range clusterNames { - log.Printf("Removing cluster [%s]", name) - cluster, err := getCluster(name) - if err != nil { - log.Printf("WARNING: couldn't get docker info for %s", name) - continue - } - if err := docker.ContainerRemove(ctx, cluster.id, types.ContainerRemoveOptions{}); err != nil { - log.Printf("WARNING: couldn't delete cluster [%s], trying a force remove now.", cluster.name) - if err := docker.ContainerRemove(ctx, cluster.id, types.ContainerRemoveOptions{Force: true}); err != nil { - log.Printf("FAILURE: couldn't delete cluster container for [%s] -> %+v", cluster.name, err) + for _, cluster := range clusters { + log.Printf("Removing cluster [%s]", cluster.name) + if len(cluster.workers) > 0 { + log.Printf("...Removing %d workers\n", len(cluster.workers)) + for _, worker := range cluster.workers { + if err := removeContainer(worker.ID); err != nil { + log.Println(err) + continue + } } } + log.Println("...Removing server") deleteClusterDir(cluster.name) + if err := removeContainer(cluster.server.ID); err != nil { + return fmt.Errorf("ERROR: Couldn't remove server for cluster %s\n%+v", cluster.name, err) + } + log.Printf("SUCCESS: removed cluster [%s]", cluster.name) } @@ -178,7 +217,7 @@ func StopCluster(c *cli.Context) error { log.Printf("WARNING: couldn't get docker info for %s", name) continue } - if err := docker.ContainerStop(ctx, cluster.id, nil); err != nil { + if err := docker.ContainerStop(ctx, cluster.server.ID, nil); err != nil { fmt.Printf("WARNING: couldn't stop cluster %s\n%+v", cluster.name, err) continue } @@ -218,7 +257,7 @@ func StartCluster(c *cli.Context) error { log.Printf("WARNING: couldn't get docker info for %s", name) continue } - if err := docker.ContainerStart(ctx, cluster.id, types.ContainerStartOptions{}); err != nil { + if err := docker.ContainerStart(ctx, cluster.server.ID, types.ContainerStartOptions{}); err != nil { fmt.Printf("WARNING: couldn't start cluster %s\n%+v", cluster.name, err) continue } @@ -236,12 +275,12 @@ func ListClusters(c *cli.Context) error { // GetKubeConfig grabs the kubeconfig from the running cluster and prints the path to stdout func GetKubeConfig(c *cli.Context) error { - sourcePath := fmt.Sprintf("%s:/output/kubeconfig.yaml", c.String("name")) + sourcePath := fmt.Sprintf("k3d-%s-server:/output/kubeconfig.yaml", c.String("name")) destPath, _ := getClusterDir(c.String("name")) cmd := "docker" args := []string{"cp", sourcePath, destPath} if err := runCommand(c.GlobalBool("verbose"), cmd, args...); err != nil { - return fmt.Errorf("ERROR: Couldn't get kubeconfig for cluster [%s]\n%+v", fmt.Sprintf("%s-server", c.String("name")), err) + return fmt.Errorf("ERROR: Couldn't get kubeconfig for cluster [%s]\n%+v", fmt.Sprintf("k3d-%s-server", c.String("name")), err) } fmt.Printf("%s\n", path.Join(destPath, "kubeconfig.yaml")) return nil diff --git a/cli/config.go b/cli/config.go index 660fdbfd..95ca101f 100644 --- a/cli/config.go +++ b/cli/config.go @@ -11,18 +11,19 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/client" - dockerClient "github.com/docker/docker/client" "github.com/mitchellh/go-homedir" "github.com/olekukonko/tablewriter" ) type cluster struct { - name string - image string - status string - ports []string - id string + name string + image string + status string + serverPorts []string + server types.Container + workers []types.Container } // createDirIfNotExists checks for the existence of a directory and creates it along with all required parents if not. @@ -63,21 +64,27 @@ func getClusterDir(name string) (string, error) { // printClusters prints the names of existing clusters func printClusters(all bool) { - clusterNames, err := getClusterNames() + clusters, err := getClusters() if err != nil { - log.Fatalf("ERROR: Couldn't list clusters -> %+v", err) + log.Fatalf("ERROR: Couldn't list clusters\n%+v", err) } - if len(clusterNames) == 0 { + if len(clusters) == 0 { log.Printf("No clusters found!") return } table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"NAME", "IMAGE", "STATUS"}) - - for _, clusterName := range clusterNames { - cluster, _ := getCluster(clusterName) - clusterData := []string{cluster.name, cluster.image, cluster.status} + table.SetHeader([]string{"NAME", "IMAGE", "STATUS", "WORKERS"}) + + for _, cluster := range clusters { + workersRunning := 0 + for _, worker := range cluster.workers { + if worker.State == "running" { + workersRunning++ + } + } + workerData := fmt.Sprintf("%d/%d", workersRunning, len(cluster.workers)) + clusterData := []string{cluster.name, cluster.image, cluster.status, workerData} if cluster.status == "running" || all { table.Append(clusterData) } @@ -107,41 +114,56 @@ func getClusterNames() ([]string, error) { return clusters, nil } -// getCluster creates a cluster struct with populated information fields -func getCluster(name string) (cluster, error) { - cluster := cluster{ - name: name, - image: "UNKNOWN", - status: "UNKNOWN", - ports: []string{"UNKNOWN"}, - id: "UNKNOWN", - } +// getClusters uses the docker API to get existing clusters and compares that with the list of cluster directories +func getClusters() (map[string]cluster, error) { ctx := context.Background() - docker, err := dockerClient.NewEnvClient() + docker, err := client.NewEnvClient() if err != nil { - log.Printf("ERROR: couldn't create docker client -> %+v", err) - return cluster, err + return nil, fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) } filters := filters.NewArgs() filters.Add("label", "app=k3d") - filters.Add("label", fmt.Sprintf("cluster=%s", cluster.name)) filters.Add("label", "component=server") - containerList, err := docker.ContainerList(ctx, types.ContainerListOptions{ + k3dServers, err := docker.ContainerList(ctx, types.ContainerListOptions{ All: true, Filters: filters, }) if err != nil { - return cluster, fmt.Errorf("WARNING: couldn't get docker info for [%s] -> %+v", cluster.name, err) + return nil, fmt.Errorf("WARNING: couldn't list server containers\n%+v", err) } - container := containerList[0] - cluster.image = container.Image - cluster.status = container.State - for _, port := range container.Ports { - cluster.ports = append(cluster.ports, strconv.Itoa(int(port.PublicPort))) + + clusters := make(map[string]cluster) + for _, server := range k3dServers { + filters.Add("label", fmt.Sprintf("cluster=%s", server.Labels["cluster"])) + filters.Del("label", "component=server") + filters.Add("label", "component=worker") + workers, err := docker.ContainerList(ctx, types.ContainerListOptions{ + All: true, + Filters: filters, + }) + if err != nil { + return nil, fmt.Errorf("WARNING: couldn't list worker containers for cluster %s\n%+v", server.Labels["cluster"], err) + } + serverPorts := []string{} + for _, port := range server.Ports { + serverPorts = append(serverPorts, strconv.Itoa(int(port.PublicPort))) + } + clusters[server.Labels["cluster"]] = cluster{ + name: server.Labels["cluster"], + image: server.Image, + status: server.State, + serverPorts: serverPorts, + server: server, + workers: workers, + } } - cluster.id = container.ID + return clusters, nil +} - return cluster, nil +// getCluster creates a cluster struct with populated information fields +func getCluster(name string) (cluster, error) { + clusters, err := getClusters() + return clusters[name], err } diff --git a/cli/container.go b/cli/container.go index 3e28e4df..4bfd4f0f 100644 --- a/cli/container.go +++ b/cli/container.go @@ -27,7 +27,7 @@ func createServer(verbose bool, image string, port string, args []string, env [] return "", fmt.Errorf("ERROR: couldn't pull image %s\n%+v", image, err) } if verbose { - _, err := io.Copy(os.Stdout, reader) // TODO: only if verbose mode + _, err := io.Copy(os.Stdout, reader) if err != nil { log.Printf("WARNING: couldn't get docker output\n%+v", err) } @@ -39,10 +39,26 @@ func createServer(verbose bool, image string, port string, args []string, env [] containerLabels["created"] = time.Now().Format("2006-01-02 15:04:05") containerLabels["cluster"] = name - containerName := fmt.Sprintf("%s-server", name) + containerName := fmt.Sprintf("k3d-%s-server", name) containerPort := nat.Port(fmt.Sprintf("%s/tcp", port)) + hostConfig := &container.HostConfig{ + PortBindings: nat.PortMap{ + containerPort: []nat.PortBinding{ + { + HostIP: "0.0.0.0", + HostPort: port, + }, + }, + }, + Privileged: true, + } + + if len(volumes) > 0 && volumes[0] != "" { + hostConfig.Binds = volumes + } + resp, err := docker.ContainerCreate(ctx, &container.Config{ Image: image, Cmd: append([]string{"server"}, args...), @@ -51,18 +67,64 @@ func createServer(verbose bool, image string, port string, args []string, env [] }, Env: env, Labels: containerLabels, - }, &container.HostConfig{ - Binds: volumes, - PortBindings: nat.PortMap{ - containerPort: []nat.PortBinding{ - { - HostIP: "0.0.0.0", - HostPort: port, - }, - }, + }, hostConfig, nil, containerName) + if err != nil { + return "", fmt.Errorf("ERROR: couldn't create container %s\n%+v", containerName, err) + } + + if err := docker.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { + return "", fmt.Errorf("ERROR: couldn't start container %s\n%+v", containerName, err) + } + + return resp.ID, nil + +} + +func createWorker(verbose bool, image string, args []string, env []string, name string, volumes []string, postfix string, serverPort string) (string, error) { + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return "", fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) + } + + reader, err := docker.ImagePull(ctx, image, types.ImagePullOptions{}) + if err != nil { + return "", fmt.Errorf("ERROR: couldn't pull image %s\n%+v", image, err) + } + if verbose { + _, err := io.Copy(os.Stdout, reader) + if err != nil { + log.Printf("WARNING: couldn't get docker output\n%+v", err) + } + } + + containerLabels := make(map[string]string) + containerLabels["app"] = "k3d" + containerLabels["component"] = "worker" + containerLabels["created"] = time.Now().Format("2006-01-02 15:04:05") + containerLabels["cluster"] = name + + containerName := fmt.Sprintf("k3d-%s-worker-%s", name, postfix) + + env = append(env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", name, serverPort)) + + hostConfig := &container.HostConfig{ + Tmpfs: map[string]string{ + "/run": "", + "/var/run": "", }, Privileged: true, - }, nil, containerName) + } + + if len(volumes) > 0 && volumes[0] != "" { + hostConfig.Binds = volumes + } + + resp, err := docker.ContainerCreate(ctx, &container.Config{ + Image: image, + Env: env, + Labels: containerLabels, + }, hostConfig, nil, containerName) if err != nil { return "", fmt.Errorf("ERROR: couldn't create container %s\n%+v", containerName, err) } @@ -72,5 +134,19 @@ func createServer(verbose bool, image string, port string, args []string, env [] } return resp.ID, nil +} +func removeContainer(ID string) error { + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) + } + if err := docker.ContainerRemove(ctx, ID, types.ContainerRemoveOptions{}); err != nil { + log.Printf("WARNING: couldn't delete container [%s], trying a force remove now.", ID) + if err := docker.ContainerRemove(ctx, ID, types.ContainerRemoveOptions{Force: true}); err != nil { + return fmt.Errorf("FAILURE: couldn't delete container [%s] -> %+v", ID, err) + } + } + return nil } diff --git a/cli/util.go b/cli/util.go new file mode 100644 index 00000000..4033b4c8 --- /dev/null +++ b/cli/util.go @@ -0,0 +1,37 @@ +package run + +import ( + "math/rand" + "strings" + "time" +) + +const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + sb.WriteByte(letterBytes[idx]) + i-- + } + cache >>= letterIdxBits + remain-- + } + + return sb.String() +}