From 7eec5b061b3b244c19ae581b7f55779f9a2b7ded Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Mon, 15 Apr 2019 15:48:43 +0200 Subject: [PATCH 01/10] use docker sdk wherever possible --- cli/commands.go | 207 ++++++++++++++++++++++++--------------------- cli/config.go | 35 ++++++-- cli/container.go | 76 +++++++++++++++++ go.mod | 2 +- main.go | 14 ++- vendor/modules.txt | 16 ++-- 6 files changed, 236 insertions(+), 114 deletions(-) create mode 100644 cli/container.go diff --git a/cli/commands.go b/cli/commands.go index f37ae62e..0d148e17 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -1,80 +1,70 @@ package run import ( + "bytes" + "context" "errors" "fmt" "log" "os" - "os/exec" "path" "strings" "time" + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" "github.com/urfave/cli" ) // CheckTools checks if the installed tools work correctly func CheckTools(c *cli.Context) error { log.Print("Checking docker...") - cmd := "docker" - args := []string{"version"} - if err := runCommand(true, cmd, args...); err != nil { + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return err + } + ping, err := docker.Ping(ctx) + + if err != nil { return fmt.Errorf("ERROR: checking docker failed\n%+v", err) } - log.Println("SUCCESS: Checking docker succeeded") + log.Printf("SUCCESS: Checking docker succeeded (API: v%s)\n", ping.APIVersion) return nil } // CreateCluster creates a new single-node cluster container and initializes the cluster directory func CreateCluster(c *cli.Context) error { + if c.IsSet("timeout") && !c.IsSet("wait") { return errors.New("Cannot use --timeout flag without --wait flag") } - port := fmt.Sprintf("%s:%s", c.String("port"), c.String("port")) - image := fmt.Sprintf("rancher/k3s:%s", c.String("version")) - cmd := "docker" - - // default docker arguments - args := []string{ - "run", - "--name", c.String("name"), - "--publish", port, - "--privileged", - "--detach", - "--env", "K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml", - } - - // additional docker arguments - extraArgs := []string{} - if c.IsSet("env") || c.IsSet("e") { - for _, env := range c.StringSlice("env") { - extraArgs = append(extraArgs, "--env", env) - } - } - if c.IsSet("volume") { - extraArgs = append(extraArgs, "--volume", c.String("volume")) - } - if len(extraArgs) > 0 { - args = append(args, extraArgs...) - } - - // k3s version and options - args = append(args, - image, - "server", // cmd - "--https-listen-port", c.String("port"), //args - ) - - // additional k3s server arguments + // k3s server arguments + k3sServerArgs := []string{"--https-listen-port", c.String("port")} if c.IsSet("server-arg") || c.IsSet("x") { - args = append(args, c.StringSlice("server-arg")...) + k3sServerArgs = append(k3sServerArgs, c.StringSlice("server-arg")...) } // let's go log.Printf("Creating cluster [%s]", c.String("name")) - if err := runCommand(true, cmd, args...); err != nil { - return fmt.Errorf("ERROR: couldn't create cluster [%s]\n%+v", c.String("name"), err) + dockerID, err := createServer( + c.Bool("verbose"), + fmt.Sprintf("docker.io/rancher/k3s:%s", c.String("version")), + c.String("port"), + k3sServerArgs, + []string{"K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml"}, + c.String("name"), + strings.Split(c.String("volume"), ","), + ) + if err != nil { + log.Fatalf("ERROR: failed to create cluster\n%+v", err) + } + + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return err } // wait for k3s to be up and running if we want it @@ -88,17 +78,17 @@ func CreateCluster(c *cli.Context) error { } return errors.New("Cluster creation exceeded specified timeout") } - cmd := "docker" - args = []string{ - "logs", - c.String("name"), - } - prog := exec.Command(cmd, args...) - output, err := prog.CombinedOutput() + + out, err := docker.ContainerLogs(ctx, dockerID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true}) if err != nil { - return err + out.Close() + return fmt.Errorf("ERROR: couldn't get docker logs for %s\n%+v", c.String("name"), err) } - if strings.Contains(string(output), "Running kubelet") { + buf := new(bytes.Buffer) + nRead, _ := buf.ReadFrom(out) + out.Close() + output := buf.String() + if nRead > 0 && strings.Contains(string(output), "Running kubelet") { break } @@ -116,38 +106,42 @@ kubectl cluster-info`, os.Args[0], c.String("name")) // DeleteCluster removes the cluster container and its cluster directory func DeleteCluster(c *cli.Context) error { - cmd := "docker" - args := []string{"rm"} - clusters := []string{} + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return err + } + + clusterNames := []string{} // operate on one or all clusters if !c.Bool("all") { - clusters = append(clusters, c.String("name")) + clusterNames = append(clusterNames, c.String("name")) } else { clusterList, err := getClusterNames() if err != nil { return fmt.Errorf("ERROR: `--all` specified, but no clusters were found\n%+v", err) } - clusters = append(clusters, clusterList...) + clusterNames = append(clusterNames, clusterList...) } // remove clusters one by one instead of appending all names to the docker command // this allows for more granular error handling and logging - for _, cluster := range clusters { - log.Printf("Removing cluster [%s]", cluster) - args = append(args, cluster) - if err := runCommand(true, cmd, args...); err != nil { - log.Printf("WARNING: couldn't delete cluster [%s], trying a force remove now.", cluster) - args = args[:len(args)-1] // pop last element from list (name of cluster) - args = append(args, "-f", cluster) - if err := runCommand(true, cmd, args...); err != nil { - log.Printf("FAILURE: couldn't delete cluster [%s] -> %+v", cluster, err) + for _, name := range clusterNames { + log.Printf("Removing cluster [%s]", name) + cluster, err := getCluster(name) + if err != nil { + log.Printf("WARNING: couldn't get docker info for %s", name) + continue + } + if err := docker.ContainerRemove(ctx, cluster.id, types.ContainerRemoveOptions{}); err != nil { + log.Printf("WARNING: couldn't delete cluster [%s], trying a force remove now.", cluster.name) + if err := docker.ContainerRemove(ctx, cluster.id, types.ContainerRemoveOptions{Force: true}); err != nil { + log.Printf("FAILURE: couldn't delete cluster container for [%s] -> %+v", cluster.name, err) } - args = args[:len(args)-1] // pop last element from list (-f flag) } - deleteClusterDir(cluster) - log.Printf("SUCCESS: removed cluster [%s]", cluster) - args = args[:len(args)-1] // pop last element from list (name of last cluster) + deleteClusterDir(cluster.name) + log.Printf("SUCCESS: removed cluster [%s]", cluster.name) } return nil @@ -155,31 +149,40 @@ func DeleteCluster(c *cli.Context) error { // StopCluster stops a running cluster container (restartable) func StopCluster(c *cli.Context) error { - cmd := "docker" - args := []string{"stop"} - clusters := []string{} + + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return err + } + + clusterNames := []string{} // operate on one or all clusters if !c.Bool("all") { - clusters = append(clusters, c.String("name")) + clusterNames = append(clusterNames, c.String("name")) } else { clusterList, err := getClusterNames() if err != nil { return fmt.Errorf("ERROR: `--all` specified, but no clusters were found\n%+v", err) } - clusters = append(clusters, clusterList...) + clusterNames = append(clusterNames, clusterList...) } // stop clusters one by one instead of appending all names to the docker command // this allows for more granular error handling and logging - for _, cluster := range clusters { - log.Printf("Stopping cluster [%s]", cluster) - args = append(args, cluster) - if err := runCommand(true, cmd, args...); err != nil { - log.Printf("FAILURE: couldn't stop cluster [%s] -> %+v", cluster, err) + for _, name := range clusterNames { + log.Printf("Stopping cluster [%s]", name) + cluster, err := getCluster(name) + if err != nil { + log.Printf("WARNING: couldn't get docker info for %s", name) + continue + } + if err := docker.ContainerStop(ctx, cluster.id, nil); err != nil { + fmt.Printf("WARNING: couldn't stop cluster %s\n%+v", cluster.name, err) + continue } - log.Printf("SUCCESS: stopped cluster [%s]", cluster) - args = args[:len(args)-1] // pop last element from list (name of last cluster) + log.Printf("SUCCESS: stopped cluster [%s]", cluster.name) } return nil @@ -187,31 +190,39 @@ func StopCluster(c *cli.Context) error { // StartCluster starts a stopped cluster container func StartCluster(c *cli.Context) error { - cmd := "docker" - args := []string{"start"} - clusters := []string{} + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return err + } + + clusterNames := []string{} // operate on one or all clusters if !c.Bool("all") { - clusters = append(clusters, c.String("name")) + clusterNames = append(clusterNames, c.String("name")) } else { clusterList, err := getClusterNames() if err != nil { return fmt.Errorf("ERROR: `--all` specified, but no clusters were found\n%+v", err) } - clusters = append(clusters, clusterList...) + clusterNames = append(clusterNames, clusterList...) } - // start clusters one by one instead of appending all names to the docker command + // stop clusters one by one instead of appending all names to the docker command // this allows for more granular error handling and logging - for _, cluster := range clusters { - log.Printf("Starting cluster [%s]", cluster) - args = append(args, cluster) - if err := runCommand(true, cmd, args...); err != nil { - log.Printf("FAILURE: couldn't start cluster [%s] -> %+v", cluster, err) + for _, name := range clusterNames { + log.Printf("Starting cluster [%s]", name) + cluster, err := getCluster(name) + if err != nil { + log.Printf("WARNING: couldn't get docker info for %s", name) + continue + } + if err := docker.ContainerStart(ctx, cluster.id, types.ContainerStartOptions{}); err != nil { + fmt.Printf("WARNING: couldn't start cluster %s\n%+v", cluster.name, err) + continue } - log.Printf("SUCCESS: started cluster [%s]", cluster) - args = args[:len(args)-1] // pop last element from list (name of last cluster) + log.Printf("SUCCESS: started cluster [%s]", cluster.name) } return nil @@ -229,8 +240,8 @@ func GetKubeConfig(c *cli.Context) error { destPath, _ := getClusterDir(c.String("name")) cmd := "docker" args := []string{"cp", sourcePath, destPath} - if err := runCommand(false, cmd, args...); err != nil { - return fmt.Errorf("ERROR: Couldn't get kubeconfig for cluster [%s]\n%+v", c.String("name"), err) + if err := runCommand(c.GlobalBool("verbose"), cmd, args...); err != nil { + return fmt.Errorf("ERROR: Couldn't get kubeconfig for cluster [%s]\n%+v", fmt.Sprintf("%s-server", c.String("name")), err) } fmt.Printf("%s\n", path.Join(destPath, "kubeconfig.yaml")) return nil diff --git a/cli/config.go b/cli/config.go index 3b0c949e..660fdbfd 100644 --- a/cli/config.go +++ b/cli/config.go @@ -2,10 +2,15 @@ package run import ( "context" + "fmt" "io/ioutil" "log" "os" "path" + "strconv" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" dockerClient "github.com/docker/docker/client" "github.com/mitchellh/go-homedir" @@ -16,6 +21,8 @@ type cluster struct { name string image string status string + ports []string + id string } // createDirIfNotExists checks for the existence of a directory and creates it along with all required parents if not. @@ -106,19 +113,35 @@ func getCluster(name string) (cluster, error) { name: name, image: "UNKNOWN", status: "UNKNOWN", + ports: []string{"UNKNOWN"}, + id: "UNKNOWN", } - + ctx := context.Background() docker, err := dockerClient.NewEnvClient() if err != nil { log.Printf("ERROR: couldn't create docker client -> %+v", err) return cluster, err } - containerInfo, err := docker.ContainerInspect(context.Background(), cluster.name) + + filters := filters.NewArgs() + filters.Add("label", "app=k3d") + filters.Add("label", fmt.Sprintf("cluster=%s", cluster.name)) + filters.Add("label", "component=server") + + containerList, err := docker.ContainerList(ctx, types.ContainerListOptions{ + All: true, + Filters: filters, + }) if err != nil { - log.Printf("WARNING: couldn't get docker info for [%s] -> %+v", cluster.name, err) - } else { - cluster.image = containerInfo.Config.Image - cluster.status = containerInfo.ContainerJSONBase.State.Status + return cluster, fmt.Errorf("WARNING: couldn't get docker info for [%s] -> %+v", cluster.name, err) + } + container := containerList[0] + cluster.image = container.Image + cluster.status = container.State + for _, port := range container.Ports { + cluster.ports = append(cluster.ports, strconv.Itoa(int(port.PublicPort))) } + cluster.id = container.ID + return cluster, nil } diff --git a/cli/container.go b/cli/container.go new file mode 100644 index 00000000..3e28e4df --- /dev/null +++ b/cli/container.go @@ -0,0 +1,76 @@ +package run + +import ( + "context" + "fmt" + "io" + "log" + "os" + "time" + + "github.com/docker/go-connections/nat" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/client" +) + +func createServer(verbose bool, image string, port string, args []string, env []string, name string, volumes []string) (string, error) { + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return "", fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) + } + + reader, err := docker.ImagePull(ctx, image, types.ImagePullOptions{}) + if err != nil { + return "", fmt.Errorf("ERROR: couldn't pull image %s\n%+v", image, err) + } + if verbose { + _, err := io.Copy(os.Stdout, reader) // TODO: only if verbose mode + if err != nil { + log.Printf("WARNING: couldn't get docker output\n%+v", err) + } + } + + containerLabels := make(map[string]string) + containerLabels["app"] = "k3d" + containerLabels["component"] = "server" + containerLabels["created"] = time.Now().Format("2006-01-02 15:04:05") + containerLabels["cluster"] = name + + containerName := fmt.Sprintf("%s-server", name) + + containerPort := nat.Port(fmt.Sprintf("%s/tcp", port)) + + resp, err := docker.ContainerCreate(ctx, &container.Config{ + Image: image, + Cmd: append([]string{"server"}, args...), + ExposedPorts: nat.PortSet{ + containerPort: struct{}{}, + }, + Env: env, + Labels: containerLabels, + }, &container.HostConfig{ + Binds: volumes, + PortBindings: nat.PortMap{ + containerPort: []nat.PortBinding{ + { + HostIP: "0.0.0.0", + HostPort: port, + }, + }, + }, + Privileged: true, + }, nil, containerName) + if err != nil { + return "", fmt.Errorf("ERROR: couldn't create container %s\n%+v", containerName, err) + } + + if err := docker.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { + return "", fmt.Errorf("ERROR: couldn't start container %s\n%+v", containerName, err) + } + + return resp.ID, nil + +} diff --git a/go.mod b/go.mod index 4d3f9f05..43fc4441 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Microsoft/go-winio v0.4.12 // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v1.13.1 - github.com/docker/go-connections v0.4.0 // indirect + github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.3.3 // indirect github.com/mattn/go-runewidth v0.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 diff --git a/main.go b/main.go index bb66e12c..59841fbe 100644 --- a/main.go +++ b/main.go @@ -53,7 +53,7 @@ func main() { }, cli.StringFlag{ Name: "volume, v", - Usage: "Mount a volume into the cluster node (Docker notation: `source:destination`)", + Usage: "Mount one or more volumes into the cluster node (Docker notation: `source:destination[,source:destination]`)", }, cli.StringFlag{ Name: "version", @@ -82,6 +82,11 @@ func main() { Name: "env, e", Usage: "Pass an additional environment variable (new flag per variable)", }, + cli.IntFlag{ + Name: "workers", + Value: 0, + Usage: "Specify how many worker nodes you want to spawn", + }, }, Action: run.CreateCluster, }, @@ -169,6 +174,13 @@ func main() { }, } + app.Flags = []cli.Flag{ + cli.BoolFlag{ + Name: "verbose", + Usage: "Enable verbose output", + }, + } + // run the whole thing err := app.Run(os.Args) if err != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index 5dc1efe1..e46577e4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -4,26 +4,26 @@ github.com/Microsoft/go-winio github.com/docker/distribution/reference github.com/docker/distribution/digestset # github.com/docker/docker v1.13.1 -github.com/docker/docker/client github.com/docker/docker/api/types github.com/docker/docker/api/types/container -github.com/docker/docker/api/types/events github.com/docker/docker/api/types/filters +github.com/docker/docker/client +github.com/docker/docker/api/types/mount github.com/docker/docker/api/types/network -github.com/docker/docker/api/types/reference github.com/docker/docker/api/types/registry github.com/docker/docker/api/types/swarm -github.com/docker/docker/api/types/time +github.com/docker/docker/api/types/blkiodev +github.com/docker/docker/api/types/strslice github.com/docker/docker/api/types/versions +github.com/docker/docker/api/types/events +github.com/docker/docker/api/types/reference +github.com/docker/docker/api/types/time github.com/docker/docker/api/types/volume github.com/docker/docker/pkg/tlsconfig -github.com/docker/docker/api/types/mount -github.com/docker/docker/api/types/blkiodev -github.com/docker/docker/api/types/strslice # github.com/docker/go-connections v0.4.0 +github.com/docker/go-connections/nat github.com/docker/go-connections/sockets github.com/docker/go-connections/tlsconfig -github.com/docker/go-connections/nat # github.com/docker/go-units v0.3.3 github.com/docker/go-units # github.com/mattn/go-runewidth v0.0.4 From b0b08d21198b9b1bf3cfd0419c3c6699aa93746b Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Tue, 16 Apr 2019 11:12:05 +0200 Subject: [PATCH 02/10] bring back env vars, enable booting workers and base list output on docker data instead of dirs --- cli/commands.go | 91 ++++++++++++++++++++++++++++++------------ cli/config.go | 94 +++++++++++++++++++++++++++----------------- cli/container.go | 100 +++++++++++++++++++++++++++++++++++++++++------ cli/util.go | 37 ++++++++++++++++++ 4 files changed, 248 insertions(+), 74 deletions(-) create mode 100644 cli/util.go diff --git a/cli/commands.go b/cli/commands.go index 0d148e17..10c55251 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -8,6 +8,7 @@ import ( "log" "os" "path" + "strconv" "strings" "time" @@ -40,6 +41,17 @@ func CreateCluster(c *cli.Context) error { return errors.New("Cannot use --timeout flag without --wait flag") } + // environment variables + env := []string{"K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml"} + if c.IsSet("env") || c.IsSet("e") { + env = append(env, c.StringSlice("env")...) + } + k3sClusterSecret := "" + if c.Int("workers") > 0 { + k3sClusterSecret = fmt.Sprintf("K3S_CLUSTER_SECRET=%s", GenerateRandomString(20)) + env = append(env, k3sClusterSecret) + } + // k3s server arguments k3sServerArgs := []string{"--https-listen-port", c.String("port")} if c.IsSet("server-arg") || c.IsSet("x") { @@ -53,7 +65,7 @@ func CreateCluster(c *cli.Context) error { fmt.Sprintf("docker.io/rancher/k3s:%s", c.String("version")), c.String("port"), k3sServerArgs, - []string{"K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml"}, + env, c.String("name"), strings.Split(c.String("volume"), ","), ) @@ -101,46 +113,73 @@ func CreateCluster(c *cli.Context) error { export KUBECONFIG="$(%s get-kubeconfig --name='%s')" kubectl cluster-info`, os.Args[0], c.String("name")) + + // worker nodes + if c.Int("workers") > 0 { + k3sWorkerArgs := []string{} + env := []string{k3sClusterSecret} + log.Printf("Booting %s workers for cluster %s", strconv.Itoa(c.Int("workers")), c.String("name")) + for i := 0; i < c.Int("workers"); i++ { + workerID, err := createWorker( + c.GlobalBool("verbose"), + fmt.Sprintf("docker.io/rancher/k3s:%s", c.String("version")), + k3sWorkerArgs, + env, + c.String("name"), + strings.Split(c.String("volume"), ","), + strconv.Itoa(i), + c.String("port"), + ) + if err != nil { + return fmt.Errorf("ERROR: failed to create worker node for cluster %s\n%+v", c.String("name"), err) + } + fmt.Printf("Created worker with ID %s\n", workerID) + } + } return nil } // DeleteCluster removes the cluster container and its cluster directory func DeleteCluster(c *cli.Context) error { - ctx := context.Background() - docker, err := client.NewEnvClient() - if err != nil { - return err - } - - clusterNames := []string{} // operate on one or all clusters + clusters := make(map[string]cluster) if !c.Bool("all") { - clusterNames = append(clusterNames, c.String("name")) + cluster, err := getCluster(c.String("name")) + if err != nil { + return err + } + clusters[c.String("name")] = cluster } else { - clusterList, err := getClusterNames() + clusterMap, err := getClusters() if err != nil { return fmt.Errorf("ERROR: `--all` specified, but no clusters were found\n%+v", err) } - clusterNames = append(clusterNames, clusterList...) + // copy clusterMap + for k, v := range clusterMap { + clusters[k] = v + } } // remove clusters one by one instead of appending all names to the docker command // this allows for more granular error handling and logging - for _, name := range clusterNames { - log.Printf("Removing cluster [%s]", name) - cluster, err := getCluster(name) - if err != nil { - log.Printf("WARNING: couldn't get docker info for %s", name) - continue - } - if err := docker.ContainerRemove(ctx, cluster.id, types.ContainerRemoveOptions{}); err != nil { - log.Printf("WARNING: couldn't delete cluster [%s], trying a force remove now.", cluster.name) - if err := docker.ContainerRemove(ctx, cluster.id, types.ContainerRemoveOptions{Force: true}); err != nil { - log.Printf("FAILURE: couldn't delete cluster container for [%s] -> %+v", cluster.name, err) + for _, cluster := range clusters { + log.Printf("Removing cluster [%s]", cluster.name) + if len(cluster.workers) > 0 { + log.Printf("...Removing %d workers\n", len(cluster.workers)) + for _, worker := range cluster.workers { + if err := removeContainer(worker.ID); err != nil { + log.Println(err) + continue + } } } + log.Println("...Removing server") deleteClusterDir(cluster.name) + if err := removeContainer(cluster.server.ID); err != nil { + return fmt.Errorf("ERROR: Couldn't remove server for cluster %s\n%+v", cluster.name, err) + } + log.Printf("SUCCESS: removed cluster [%s]", cluster.name) } @@ -178,7 +217,7 @@ func StopCluster(c *cli.Context) error { log.Printf("WARNING: couldn't get docker info for %s", name) continue } - if err := docker.ContainerStop(ctx, cluster.id, nil); err != nil { + if err := docker.ContainerStop(ctx, cluster.server.ID, nil); err != nil { fmt.Printf("WARNING: couldn't stop cluster %s\n%+v", cluster.name, err) continue } @@ -218,7 +257,7 @@ func StartCluster(c *cli.Context) error { log.Printf("WARNING: couldn't get docker info for %s", name) continue } - if err := docker.ContainerStart(ctx, cluster.id, types.ContainerStartOptions{}); err != nil { + if err := docker.ContainerStart(ctx, cluster.server.ID, types.ContainerStartOptions{}); err != nil { fmt.Printf("WARNING: couldn't start cluster %s\n%+v", cluster.name, err) continue } @@ -236,12 +275,12 @@ func ListClusters(c *cli.Context) error { // GetKubeConfig grabs the kubeconfig from the running cluster and prints the path to stdout func GetKubeConfig(c *cli.Context) error { - sourcePath := fmt.Sprintf("%s:/output/kubeconfig.yaml", c.String("name")) + sourcePath := fmt.Sprintf("k3d-%s-server:/output/kubeconfig.yaml", c.String("name")) destPath, _ := getClusterDir(c.String("name")) cmd := "docker" args := []string{"cp", sourcePath, destPath} if err := runCommand(c.GlobalBool("verbose"), cmd, args...); err != nil { - return fmt.Errorf("ERROR: Couldn't get kubeconfig for cluster [%s]\n%+v", fmt.Sprintf("%s-server", c.String("name")), err) + return fmt.Errorf("ERROR: Couldn't get kubeconfig for cluster [%s]\n%+v", fmt.Sprintf("k3d-%s-server", c.String("name")), err) } fmt.Printf("%s\n", path.Join(destPath, "kubeconfig.yaml")) return nil diff --git a/cli/config.go b/cli/config.go index 660fdbfd..95ca101f 100644 --- a/cli/config.go +++ b/cli/config.go @@ -11,18 +11,19 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/client" - dockerClient "github.com/docker/docker/client" "github.com/mitchellh/go-homedir" "github.com/olekukonko/tablewriter" ) type cluster struct { - name string - image string - status string - ports []string - id string + name string + image string + status string + serverPorts []string + server types.Container + workers []types.Container } // createDirIfNotExists checks for the existence of a directory and creates it along with all required parents if not. @@ -63,21 +64,27 @@ func getClusterDir(name string) (string, error) { // printClusters prints the names of existing clusters func printClusters(all bool) { - clusterNames, err := getClusterNames() + clusters, err := getClusters() if err != nil { - log.Fatalf("ERROR: Couldn't list clusters -> %+v", err) + log.Fatalf("ERROR: Couldn't list clusters\n%+v", err) } - if len(clusterNames) == 0 { + if len(clusters) == 0 { log.Printf("No clusters found!") return } table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"NAME", "IMAGE", "STATUS"}) - - for _, clusterName := range clusterNames { - cluster, _ := getCluster(clusterName) - clusterData := []string{cluster.name, cluster.image, cluster.status} + table.SetHeader([]string{"NAME", "IMAGE", "STATUS", "WORKERS"}) + + for _, cluster := range clusters { + workersRunning := 0 + for _, worker := range cluster.workers { + if worker.State == "running" { + workersRunning++ + } + } + workerData := fmt.Sprintf("%d/%d", workersRunning, len(cluster.workers)) + clusterData := []string{cluster.name, cluster.image, cluster.status, workerData} if cluster.status == "running" || all { table.Append(clusterData) } @@ -107,41 +114,56 @@ func getClusterNames() ([]string, error) { return clusters, nil } -// getCluster creates a cluster struct with populated information fields -func getCluster(name string) (cluster, error) { - cluster := cluster{ - name: name, - image: "UNKNOWN", - status: "UNKNOWN", - ports: []string{"UNKNOWN"}, - id: "UNKNOWN", - } +// getClusters uses the docker API to get existing clusters and compares that with the list of cluster directories +func getClusters() (map[string]cluster, error) { ctx := context.Background() - docker, err := dockerClient.NewEnvClient() + docker, err := client.NewEnvClient() if err != nil { - log.Printf("ERROR: couldn't create docker client -> %+v", err) - return cluster, err + return nil, fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) } filters := filters.NewArgs() filters.Add("label", "app=k3d") - filters.Add("label", fmt.Sprintf("cluster=%s", cluster.name)) filters.Add("label", "component=server") - containerList, err := docker.ContainerList(ctx, types.ContainerListOptions{ + k3dServers, err := docker.ContainerList(ctx, types.ContainerListOptions{ All: true, Filters: filters, }) if err != nil { - return cluster, fmt.Errorf("WARNING: couldn't get docker info for [%s] -> %+v", cluster.name, err) + return nil, fmt.Errorf("WARNING: couldn't list server containers\n%+v", err) } - container := containerList[0] - cluster.image = container.Image - cluster.status = container.State - for _, port := range container.Ports { - cluster.ports = append(cluster.ports, strconv.Itoa(int(port.PublicPort))) + + clusters := make(map[string]cluster) + for _, server := range k3dServers { + filters.Add("label", fmt.Sprintf("cluster=%s", server.Labels["cluster"])) + filters.Del("label", "component=server") + filters.Add("label", "component=worker") + workers, err := docker.ContainerList(ctx, types.ContainerListOptions{ + All: true, + Filters: filters, + }) + if err != nil { + return nil, fmt.Errorf("WARNING: couldn't list worker containers for cluster %s\n%+v", server.Labels["cluster"], err) + } + serverPorts := []string{} + for _, port := range server.Ports { + serverPorts = append(serverPorts, strconv.Itoa(int(port.PublicPort))) + } + clusters[server.Labels["cluster"]] = cluster{ + name: server.Labels["cluster"], + image: server.Image, + status: server.State, + serverPorts: serverPorts, + server: server, + workers: workers, + } } - cluster.id = container.ID + return clusters, nil +} - return cluster, nil +// getCluster creates a cluster struct with populated information fields +func getCluster(name string) (cluster, error) { + clusters, err := getClusters() + return clusters[name], err } diff --git a/cli/container.go b/cli/container.go index 3e28e4df..4bfd4f0f 100644 --- a/cli/container.go +++ b/cli/container.go @@ -27,7 +27,7 @@ func createServer(verbose bool, image string, port string, args []string, env [] return "", fmt.Errorf("ERROR: couldn't pull image %s\n%+v", image, err) } if verbose { - _, err := io.Copy(os.Stdout, reader) // TODO: only if verbose mode + _, err := io.Copy(os.Stdout, reader) if err != nil { log.Printf("WARNING: couldn't get docker output\n%+v", err) } @@ -39,10 +39,26 @@ func createServer(verbose bool, image string, port string, args []string, env [] containerLabels["created"] = time.Now().Format("2006-01-02 15:04:05") containerLabels["cluster"] = name - containerName := fmt.Sprintf("%s-server", name) + containerName := fmt.Sprintf("k3d-%s-server", name) containerPort := nat.Port(fmt.Sprintf("%s/tcp", port)) + hostConfig := &container.HostConfig{ + PortBindings: nat.PortMap{ + containerPort: []nat.PortBinding{ + { + HostIP: "0.0.0.0", + HostPort: port, + }, + }, + }, + Privileged: true, + } + + if len(volumes) > 0 && volumes[0] != "" { + hostConfig.Binds = volumes + } + resp, err := docker.ContainerCreate(ctx, &container.Config{ Image: image, Cmd: append([]string{"server"}, args...), @@ -51,18 +67,64 @@ func createServer(verbose bool, image string, port string, args []string, env [] }, Env: env, Labels: containerLabels, - }, &container.HostConfig{ - Binds: volumes, - PortBindings: nat.PortMap{ - containerPort: []nat.PortBinding{ - { - HostIP: "0.0.0.0", - HostPort: port, - }, - }, + }, hostConfig, nil, containerName) + if err != nil { + return "", fmt.Errorf("ERROR: couldn't create container %s\n%+v", containerName, err) + } + + if err := docker.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { + return "", fmt.Errorf("ERROR: couldn't start container %s\n%+v", containerName, err) + } + + return resp.ID, nil + +} + +func createWorker(verbose bool, image string, args []string, env []string, name string, volumes []string, postfix string, serverPort string) (string, error) { + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return "", fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) + } + + reader, err := docker.ImagePull(ctx, image, types.ImagePullOptions{}) + if err != nil { + return "", fmt.Errorf("ERROR: couldn't pull image %s\n%+v", image, err) + } + if verbose { + _, err := io.Copy(os.Stdout, reader) + if err != nil { + log.Printf("WARNING: couldn't get docker output\n%+v", err) + } + } + + containerLabels := make(map[string]string) + containerLabels["app"] = "k3d" + containerLabels["component"] = "worker" + containerLabels["created"] = time.Now().Format("2006-01-02 15:04:05") + containerLabels["cluster"] = name + + containerName := fmt.Sprintf("k3d-%s-worker-%s", name, postfix) + + env = append(env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", name, serverPort)) + + hostConfig := &container.HostConfig{ + Tmpfs: map[string]string{ + "/run": "", + "/var/run": "", }, Privileged: true, - }, nil, containerName) + } + + if len(volumes) > 0 && volumes[0] != "" { + hostConfig.Binds = volumes + } + + resp, err := docker.ContainerCreate(ctx, &container.Config{ + Image: image, + Env: env, + Labels: containerLabels, + }, hostConfig, nil, containerName) if err != nil { return "", fmt.Errorf("ERROR: couldn't create container %s\n%+v", containerName, err) } @@ -72,5 +134,19 @@ func createServer(verbose bool, image string, port string, args []string, env [] } return resp.ID, nil +} +func removeContainer(ID string) error { + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) + } + if err := docker.ContainerRemove(ctx, ID, types.ContainerRemoveOptions{}); err != nil { + log.Printf("WARNING: couldn't delete container [%s], trying a force remove now.", ID) + if err := docker.ContainerRemove(ctx, ID, types.ContainerRemoveOptions{Force: true}); err != nil { + return fmt.Errorf("FAILURE: couldn't delete container [%s] -> %+v", ID, err) + } + } + return nil } diff --git a/cli/util.go b/cli/util.go new file mode 100644 index 00000000..4033b4c8 --- /dev/null +++ b/cli/util.go @@ -0,0 +1,37 @@ +package run + +import ( + "math/rand" + "strings" + "time" +) + +const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1<= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + sb.WriteByte(letterBytes[idx]) + i-- + } + cache >>= letterIdxBits + remain-- + } + + return sb.String() +} From a26aa65782434646296daa598e20be1e3fd027c7 Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Wed, 17 Apr 2019 09:45:42 +0200 Subject: [PATCH 03/10] make k3s v0.4.0 default version --- README.md | 7 +++---- main.go | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 869b4f41..fbc25868 100644 --- a/README.md +++ b/README.md @@ -46,8 +46,7 @@ Example Workflow: Create a new cluster and use it with `kubectl` ## TODO -- [ ] Use the docker client library instead of commands -- [ ] Test the docker version -- [ ] Improve cluster state management +- [x] Use the docker client library instead of commands +- [x] Improve cluster state management +- [x] Add install script - [ ] Use [sirupsen/logrus](https://github.com/sirupsen/logrus) for prettier logs -- [ ] Add install script \ No newline at end of file diff --git a/main.go b/main.go index 59841fbe..501d51de 100644 --- a/main.go +++ b/main.go @@ -57,7 +57,7 @@ func main() { }, cli.StringFlag{ Name: "version", - Value: "v0.3.0", + Value: "v0.4.0", Usage: "Choose the k3s image version", }, cli.IntFlag{ From 7d5964c3b979844202dbbf6f339e673c1bb330f3 Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Thu, 18 Apr 2019 14:42:41 +0200 Subject: [PATCH 04/10] fix image not being pulled if reader is not being used --- cli/commands.go | 2 +- cli/container.go | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index 10c55251..27f9f773 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -61,7 +61,7 @@ func CreateCluster(c *cli.Context) error { // let's go log.Printf("Creating cluster [%s]", c.String("name")) dockerID, err := createServer( - c.Bool("verbose"), + c.GlobalBool("verbose"), fmt.Sprintf("docker.io/rancher/k3s:%s", c.String("version")), c.String("port"), k3sServerArgs, diff --git a/cli/container.go b/cli/container.go index 4bfd4f0f..a4d15c19 100644 --- a/cli/container.go +++ b/cli/container.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "io/ioutil" "log" "os" "time" @@ -16,12 +17,12 @@ import ( ) func createServer(verbose bool, image string, port string, args []string, env []string, name string, volumes []string) (string, error) { + log.Printf("Creating server using %s...\n", image) ctx := context.Background() docker, err := client.NewEnvClient() if err != nil { return "", fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) } - reader, err := docker.ImagePull(ctx, image, types.ImagePullOptions{}) if err != nil { return "", fmt.Errorf("ERROR: couldn't pull image %s\n%+v", image, err) @@ -31,6 +32,11 @@ func createServer(verbose bool, image string, port string, args []string, env [] if err != nil { log.Printf("WARNING: couldn't get docker output\n%+v", err) } + } else { + _, err := io.Copy(ioutil.Discard, reader) + if err != nil { + log.Printf("WARNING: couldn't get docker output\n%+v", err) + } } containerLabels := make(map[string]string) From 92039edbb17c9ea74452e38a81b1dc3a5b3bdb8c Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Thu, 18 Apr 2019 15:10:28 +0200 Subject: [PATCH 05/10] fix clusters not being filtered correctly --- cli/config.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/cli/config.go b/cli/config.go index 95ca101f..d751229c 100644 --- a/cli/config.go +++ b/cli/config.go @@ -122,10 +122,12 @@ func getClusters() (map[string]cluster, error) { return nil, fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) } + // Prepare docker label filters filters := filters.NewArgs() filters.Add("label", "app=k3d") filters.Add("label", "component=server") + // get all servers created by k3d k3dServers, err := docker.ContainerList(ctx, types.ContainerListOptions{ All: true, Filters: filters, @@ -135,17 +137,25 @@ func getClusters() (map[string]cluster, error) { } clusters := make(map[string]cluster) + + // don't filter for servers but for workers now + filters.Del("label", "component=server") + filters.Add("label", "component=worker") + + // for all servers created by k3d, get workers and cluster information for _, server := range k3dServers { filters.Add("label", fmt.Sprintf("cluster=%s", server.Labels["cluster"])) - filters.Del("label", "component=server") - filters.Add("label", "component=worker") + + // get workers workers, err := docker.ContainerList(ctx, types.ContainerListOptions{ All: true, Filters: filters, }) if err != nil { - return nil, fmt.Errorf("WARNING: couldn't list worker containers for cluster %s\n%+v", server.Labels["cluster"], err) + log.Printf("WARNING: couldn't get worker containers for cluster %s\n%+v", server.Labels["cluster"], err) } + + // save cluster information serverPorts := []string{} for _, port := range server.Ports { serverPorts = append(serverPorts, strconv.Itoa(int(port.PublicPort))) @@ -158,6 +168,8 @@ func getClusters() (map[string]cluster, error) { server: server, workers: workers, } + // clear label filters before searching for next cluster + filters.Del("label", fmt.Sprintf("cluster=%s", server.Labels["cluster"])) } return clusters, nil } From 6c072fb6d93d60e114fbc7a4ff68968af9768ddf Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Thu, 18 Apr 2019 16:08:15 +0200 Subject: [PATCH 06/10] connect workers and server via own docker network --- cli/commands.go | 11 +++++++++ cli/container.go | 21 +++++++++++++++-- cli/network.go | 59 ++++++++++++++++++++++++++++++++++++++++++++++ vendor/modules.txt | 2 +- 4 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 cli/network.go diff --git a/cli/commands.go b/cli/commands.go index 27f9f773..0026506b 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -37,6 +37,13 @@ func CheckTools(c *cli.Context) error { // CreateCluster creates a new single-node cluster container and initializes the cluster directory func CreateCluster(c *cli.Context) error { + // create cluster network + networkID, err := createClusterNetwork(c.String("name")) + if err != nil { + return err + } + log.Printf("Created cluster network with ID %s", networkID) + if c.IsSet("timeout") && !c.IsSet("wait") { return errors.New("Cannot use --timeout flag without --wait flag") } @@ -180,6 +187,10 @@ func DeleteCluster(c *cli.Context) error { return fmt.Errorf("ERROR: Couldn't remove server for cluster %s\n%+v", cluster.name, err) } + if err := deleteClusterNetwork(cluster.name); err != nil { + log.Printf("WARNING: couldn't delete cluster network for cluster %s\n%+v", cluster.name, err) + } + log.Printf("SUCCESS: removed cluster [%s]", cluster.name) } diff --git a/cli/container.go b/cli/container.go index a4d15c19..af77579b 100644 --- a/cli/container.go +++ b/cli/container.go @@ -13,6 +13,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" "github.com/docker/docker/client" ) @@ -65,6 +66,14 @@ func createServer(verbose bool, image string, port string, args []string, env [] hostConfig.Binds = volumes } + networkingConfig := &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + name: &network.EndpointSettings{ + Aliases: []string{containerName}, + }, + }, + } + resp, err := docker.ContainerCreate(ctx, &container.Config{ Image: image, Cmd: append([]string{"server"}, args...), @@ -73,7 +82,7 @@ func createServer(verbose bool, image string, port string, args []string, env [] }, Env: env, Labels: containerLabels, - }, hostConfig, nil, containerName) + }, hostConfig, networkingConfig, containerName) if err != nil { return "", fmt.Errorf("ERROR: couldn't create container %s\n%+v", containerName, err) } @@ -126,11 +135,19 @@ func createWorker(verbose bool, image string, args []string, env []string, name hostConfig.Binds = volumes } + networkingConfig := &network.NetworkingConfig{ + EndpointsConfig: map[string]*network.EndpointSettings{ + name: &network.EndpointSettings{ + Aliases: []string{containerName}, + }, + }, + } + resp, err := docker.ContainerCreate(ctx, &container.Config{ Image: image, Env: env, Labels: containerLabels, - }, hostConfig, nil, containerName) + }, hostConfig, networkingConfig, containerName) if err != nil { return "", fmt.Errorf("ERROR: couldn't create container %s\n%+v", containerName, err) } diff --git a/cli/network.go b/cli/network.go new file mode 100644 index 00000000..148a0241 --- /dev/null +++ b/cli/network.go @@ -0,0 +1,59 @@ +package run + +import ( + "context" + "fmt" + "log" + + "github.com/docker/docker/api/types/filters" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" +) + +func createClusterNetwork(clusterName string) (string, error) { + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return "", fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) + } + + resp, err := docker.NetworkCreate(ctx, clusterName, types.NetworkCreate{ + Labels: map[string]string{ + "app": "k3d", + "cluster": clusterName, + }, + }) + if err != nil { + return "", fmt.Errorf("ERROR: couldn't create network\n%+v", err) + } + + return resp.ID, nil +} + +func deleteClusterNetwork(clusterName string) error { + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) + } + + filters := filters.NewArgs() + filters.Add("label", "app=k3d") + filters.Add("label", fmt.Sprintf("cluster=%s", clusterName)) + + networks, err := docker.NetworkList(ctx, types.NetworkListOptions{ + Filters: filters, + }) + if err != nil { + return fmt.Errorf("ERROR: couldn't find network for cluster %s\n%+v", clusterName, err) + } + + for _, network := range networks { + if err := docker.NetworkRemove(ctx, network.ID); err != nil { + log.Printf("WARNING: couldn't remove network for cluster %s\n%+v", clusterName, err) + continue + } + } + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index e46577e4..9a44086b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -7,9 +7,9 @@ github.com/docker/distribution/digestset github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/api/types/filters +github.com/docker/docker/api/types/network github.com/docker/docker/client github.com/docker/docker/api/types/mount -github.com/docker/docker/api/types/network github.com/docker/docker/api/types/registry github.com/docker/docker/api/types/swarm github.com/docker/docker/api/types/blkiodev From 9142529c62bf079070815c9c4dd0fadacc034ff4 Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Tue, 23 Apr 2019 10:51:02 +0200 Subject: [PATCH 07/10] get-kubeconfig using docker sdk --- cli/commands.go | 61 ++++++++++++++++++++++++++++++++++++++++++------- cli/run.go | 18 --------------- 2 files changed, 53 insertions(+), 26 deletions(-) delete mode 100644 cli/run.go diff --git a/cli/commands.go b/cli/commands.go index 0026506b..597da877 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -5,13 +5,15 @@ import ( "context" "errors" "fmt" + "io/ioutil" "log" "os" - "path" "strconv" "strings" "time" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types" "github.com/docker/docker/client" "github.com/urfave/cli" @@ -286,13 +288,56 @@ func ListClusters(c *cli.Context) error { // GetKubeConfig grabs the kubeconfig from the running cluster and prints the path to stdout func GetKubeConfig(c *cli.Context) error { - sourcePath := fmt.Sprintf("k3d-%s-server:/output/kubeconfig.yaml", c.String("name")) - destPath, _ := getClusterDir(c.String("name")) - cmd := "docker" - args := []string{"cp", sourcePath, destPath} - if err := runCommand(c.GlobalBool("verbose"), cmd, args...); err != nil { - return fmt.Errorf("ERROR: Couldn't get kubeconfig for cluster [%s]\n%+v", fmt.Sprintf("k3d-%s-server", c.String("name")), err) + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return err + } + + filters := filters.NewArgs() + filters.Add("label", "app=k3d") + filters.Add("label", fmt.Sprintf("cluster=%s", c.String("name"))) + filters.Add("label", "component=server") + server, err := docker.ContainerList(ctx, types.ContainerListOptions{ + Filters: filters, + }) + if err != nil { + return fmt.Errorf("Couldn't get server container for cluster %s\n%+v", c.String("name"), err) + } + + // get kubeconfig file from container and read contents + reader, _, err := docker.CopyFromContainer(ctx, server[0].ID, "/output/kubeconfig.yaml") + if err != nil { + return fmt.Errorf("ERROR: couldn't copy kubeconfig.yaml from server container %s\n%+v", server[0].ID, err) + } + defer reader.Close() + + readBytes, err := ioutil.ReadAll(reader) + if err != nil { + return fmt.Errorf("ERROR: couldn't read kubeconfig from container\n%+v", err) } - fmt.Printf("%s\n", path.Join(destPath, "kubeconfig.yaml")) + + // create destination kubeconfig file + clusterDir, err := getClusterDir(c.String("name")) + destPath := fmt.Sprintf("%s/kubeconfig.yaml", clusterDir) + if err != nil { + return err + } + + kubeconfigfile, err := os.Create(destPath) + if err != nil { + return fmt.Errorf("ERROR: couldn't create kubeconfig.yaml in %s\n%+v", clusterDir, err) + } + defer kubeconfigfile.Close() + + // write to file, skipping the first 512 bytes which contain file metadata and trimming any NULL characters + _, err = kubeconfigfile.Write(bytes.Trim(readBytes[512:], "\x00")) + if err != nil { + return fmt.Errorf("ERROR: couldn't write to kubeconfig.yaml\n%+v", err) + } + + // output kubeconfig file path to stdout + fmt.Println(destPath) + return nil } diff --git a/cli/run.go b/cli/run.go deleted file mode 100644 index 770770f4..00000000 --- a/cli/run.go +++ /dev/null @@ -1,18 +0,0 @@ -package run - -import ( - "log" - "os" - "os/exec" -) - -// runCommand accepts the name and args and runs the specified command -func runCommand(verbose bool, name string, args ...string) error { - if verbose { - log.Printf("Running command: %+v", append([]string{name}, args...)) - } - cmd := exec.Command(name, args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() -} From 6fdc1e4a705705c468ed4d3fd8e06b5be861e842 Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Tue, 23 Apr 2019 11:12:42 +0200 Subject: [PATCH 08/10] automatically get latest k3s version tag at build time --- Makefile | 5 ++++- main.go | 2 +- version/version.go | 8 ++++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6131ede4..b7b52c67 100644 --- a/Makefile +++ b/Makefile @@ -10,13 +10,16 @@ ifeq ($(GIT_TAG),) GIT_TAG := $(shell git describe --always) endif +# get latest k3s version +K3S_TAG := $(shell curl --silent "https://api.github.com/repos/rancher/k3s/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') + # Go options GO ?= go PKG := $(shell go mod vendor) TAGS := TESTS := . TESTFLAGS := -LDFLAGS := -w -s -X github.com/rancher/k3d/version.Version=${GIT_TAG} +LDFLAGS := -w -s -X github.com/rancher/k3d/version.Version=${GIT_TAG} -X github.com/rancher/k3d/version.K3sVersion=${K3S_TAG} GOFLAGS := BINDIR := $(CURDIR)/bin BINARIES := k3d diff --git a/main.go b/main.go index 501d51de..c72529d8 100644 --- a/main.go +++ b/main.go @@ -57,7 +57,7 @@ func main() { }, cli.StringFlag{ Name: "version", - Value: "v0.4.0", + Value: version.GetK3sVersion(), Usage: "Choose the k3s image version", }, cli.IntFlag{ diff --git a/version/version.go b/version/version.go index 8fc22c07..f3b784a4 100644 --- a/version/version.go +++ b/version/version.go @@ -3,6 +3,9 @@ package version // Version is the string that contains version var Version string +// K3sVersion contains the latest version tag of K3s +var K3sVersion string + // GetVersion returns the version for cli, it gets it from "git describe --tags" or returns "dev" when doing simple go build func GetVersion() string { if len(Version) == 0 { @@ -10,3 +13,8 @@ func GetVersion() string { } return Version } + +// GetK3sVersion returns the version string for K3s +func GetK3sVersion() string { + return K3sVersion +} From e9206a74b2afa1ca38430637b490e93c7ad834fa Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Tue, 23 Apr 2019 11:22:59 +0200 Subject: [PATCH 09/10] make start/stop functions start/stop workers as well --- cli/commands.go | 110 +++++++++++++++++++++++++++++------------------- cli/config.go | 23 ---------- 2 files changed, 67 insertions(+), 66 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index 597da877..173e786e 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -202,39 +202,50 @@ func DeleteCluster(c *cli.Context) error { // StopCluster stops a running cluster container (restartable) func StopCluster(c *cli.Context) error { - ctx := context.Background() - docker, err := client.NewEnvClient() - if err != nil { - return err - } - - clusterNames := []string{} - // operate on one or all clusters + clusters := make(map[string]cluster) if !c.Bool("all") { - clusterNames = append(clusterNames, c.String("name")) + cluster, err := getCluster(c.String("name")) + if err != nil { + return err + } + clusters[c.String("name")] = cluster } else { - clusterList, err := getClusterNames() + clusterMap, err := getClusters() if err != nil { return fmt.Errorf("ERROR: `--all` specified, but no clusters were found\n%+v", err) } - clusterNames = append(clusterNames, clusterList...) + // copy clusterMap + for k, v := range clusterMap { + clusters[k] = v + } } - // stop clusters one by one instead of appending all names to the docker command + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) + } + + // remove clusters one by one instead of appending all names to the docker command // this allows for more granular error handling and logging - for _, name := range clusterNames { - log.Printf("Stopping cluster [%s]", name) - cluster, err := getCluster(name) - if err != nil { - log.Printf("WARNING: couldn't get docker info for %s", name) - continue + for _, cluster := range clusters { + log.Printf("Stopping cluster [%s]", cluster.name) + if len(cluster.workers) > 0 { + log.Printf("...Stopping %d workers\n", len(cluster.workers)) + for _, worker := range cluster.workers { + if err := docker.ContainerStop(ctx, worker.ID, nil); err != nil { + log.Println(err) + continue + } + } } + log.Println("...Stopping server") if err := docker.ContainerStop(ctx, cluster.server.ID, nil); err != nil { - fmt.Printf("WARNING: couldn't stop cluster %s\n%+v", cluster.name, err) - continue + return fmt.Errorf("ERROR: Couldn't stop server for cluster %s\n%+v", cluster.name, err) } - log.Printf("SUCCESS: stopped cluster [%s]", cluster.name) + + log.Printf("SUCCESS: Stopped cluster [%s]", cluster.name) } return nil @@ -242,39 +253,52 @@ func StopCluster(c *cli.Context) error { // StartCluster starts a stopped cluster container func StartCluster(c *cli.Context) error { - ctx := context.Background() - docker, err := client.NewEnvClient() - if err != nil { - return err - } - - clusterNames := []string{} - // operate on one or all clusters + clusters := make(map[string]cluster) if !c.Bool("all") { - clusterNames = append(clusterNames, c.String("name")) + cluster, err := getCluster(c.String("name")) + if err != nil { + return err + } + clusters[c.String("name")] = cluster } else { - clusterList, err := getClusterNames() + clusterMap, err := getClusters() if err != nil { return fmt.Errorf("ERROR: `--all` specified, but no clusters were found\n%+v", err) } - clusterNames = append(clusterNames, clusterList...) + // copy clusterMap + for k, v := range clusterMap { + clusters[k] = v + } } - // stop clusters one by one instead of appending all names to the docker command + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) + } + + // remove clusters one by one instead of appending all names to the docker command // this allows for more granular error handling and logging - for _, name := range clusterNames { - log.Printf("Starting cluster [%s]", name) - cluster, err := getCluster(name) - if err != nil { - log.Printf("WARNING: couldn't get docker info for %s", name) - continue - } + for _, cluster := range clusters { + log.Printf("Starting cluster [%s]", cluster.name) + + log.Println("...Starting server") if err := docker.ContainerStart(ctx, cluster.server.ID, types.ContainerStartOptions{}); err != nil { - fmt.Printf("WARNING: couldn't start cluster %s\n%+v", cluster.name, err) - continue + return fmt.Errorf("ERROR: Couldn't start server for cluster %s\n%+v", cluster.name, err) } - log.Printf("SUCCESS: started cluster [%s]", cluster.name) + + if len(cluster.workers) > 0 { + log.Printf("...Starting %d workers\n", len(cluster.workers)) + for _, worker := range cluster.workers { + if err := docker.ContainerStart(ctx, worker.ID, types.ContainerStartOptions{}); err != nil { + log.Println(err) + continue + } + } + } + + log.Printf("SUCCESS: Started cluster [%s]", cluster.name) } return nil diff --git a/cli/config.go b/cli/config.go index d751229c..8656d0e3 100644 --- a/cli/config.go +++ b/cli/config.go @@ -3,7 +3,6 @@ package run import ( "context" "fmt" - "io/ioutil" "log" "os" "path" @@ -92,28 +91,6 @@ func printClusters(all bool) { table.Render() } -// getClusterNames returns a list of cluster names which are folder names in the config directory -func getClusterNames() ([]string, error) { - homeDir, err := homedir.Dir() - if err != nil { - log.Printf("ERROR: Couldn't get user's home directory") - return nil, err - } - configDir := path.Join(homeDir, ".config", "k3d") - files, err := ioutil.ReadDir(configDir) - if err != nil { - log.Printf("ERROR: Couldn't list files in [%s]", configDir) - return nil, err - } - clusters := []string{} - for _, file := range files { - if file.IsDir() { - clusters = append(clusters, file.Name()) - } - } - return clusters, nil -} - // getClusters uses the docker API to get existing clusters and compares that with the list of cluster directories func getClusters() (map[string]cluster, error) { ctx := context.Background() From 068172aa1afec6d4ab70c56f868f1106e7e84ec3 Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Tue, 23 Apr 2019 15:20:06 +0200 Subject: [PATCH 10/10] fix output order for logs --- cli/commands.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index 173e786e..d2c3906b 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -117,11 +117,6 @@ func CreateCluster(c *cli.Context) error { } createClusterDir(c.String("name")) - log.Printf("SUCCESS: created cluster [%s]", c.String("name")) - log.Printf(`You can now use the cluster with: - -export KUBECONFIG="$(%s get-kubeconfig --name='%s')" -kubectl cluster-info`, os.Args[0], c.String("name")) // worker nodes if c.Int("workers") > 0 { @@ -145,6 +140,13 @@ kubectl cluster-info`, os.Args[0], c.String("name")) fmt.Printf("Created worker with ID %s\n", workerID) } } + + log.Printf("SUCCESS: created cluster [%s]", c.String("name")) + log.Printf(`You can now use the cluster with: + +export KUBECONFIG="$(%s get-kubeconfig --name='%s')" +kubectl cluster-info`, os.Args[0], c.String("name")) + return nil }