From 1eff9740647991abee73c4920e8f23eff903beea Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Thu, 23 May 2019 23:37:53 -0700 Subject: [PATCH 1/9] Add getClusterKubeConfigPath() Minor refactor. Add getClusterKubeConfigOath() to make logic reusable for later changes. --- cli/cluster.go | 5 +++++ cli/commands.go | 5 ++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cli/cluster.go b/cli/cluster.go index b8093f76..b43a773b 100644 --- a/cli/cluster.go +++ b/cli/cluster.go @@ -85,6 +85,11 @@ func getClusterDir(name string) (string, error) { return path.Join(homeDir, ".config", "k3d", name), nil } +func getClusterKubeConfigPath(cluster string) (string, error) { + clusterDir, err := getClusterDir(cluster) + return path.Join(clusterDir, "kubeconfig.yaml"), err +} + // printClusters prints the names of existing clusters func printClusters() { clusters, err := getClusters(true, "") diff --git a/cli/commands.go b/cli/commands.go index 51604767..6b492fb2 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -381,15 +381,14 @@ func GetKubeConfig(c *cli.Context) error { } // create destination kubeconfig file - clusterDir, err := getClusterDir(c.String("name")) - destPath := fmt.Sprintf("%s/kubeconfig.yaml", clusterDir) + destPath, err := getClusterKubeConfigPath(c.String("name")) if err != nil { return err } kubeconfigfile, err := os.Create(destPath) if err != nil { - return fmt.Errorf("ERROR: couldn't create kubeconfig.yaml in %s\n%+v", clusterDir, err) + return fmt.Errorf("ERROR: couldn't create kubeconfig file %s\n%+v", destPath, err) } defer kubeconfigfile.Close() From cdcc5e1de62b652eb390634566bf6e38596aada9 Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Fri, 24 May 2019 00:05:19 -0700 Subject: [PATCH 2/9] Add createKubeconfigFile() Refactoring. Make createKubeconfigFile() a stand along function, so we don't have to recreate the file every time we look up the file name. --- cli/cluster.go | 58 +++++++++++++++++++++++++++++++++++++++++++++++ cli/commands.go | 60 +++++++++++-------------------------------------- 2 files changed, 71 insertions(+), 47 deletions(-) diff --git a/cli/cluster.go b/cli/cluster.go index b43a773b..75cbad98 100644 --- a/cli/cluster.go +++ b/cli/cluster.go @@ -1,8 +1,10 @@ package run import ( + "bytes" "context" "fmt" + "io/ioutil" "log" "os" "path" @@ -90,6 +92,62 @@ func getClusterKubeConfigPath(cluster string) (string, error) { return path.Join(clusterDir, "kubeconfig.yaml"), err } +func createKubeConfigFile(cluster string) error { + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return err + } + + filters := filters.NewArgs() + filters.Add("label", "app=k3d") + filters.Add("label", fmt.Sprintf("cluster=%s", cluster)) + filters.Add("label", "component=server") + server, err := docker.ContainerList(ctx, types.ContainerListOptions{ + Filters: filters, + }) + + if err != nil { + return fmt.Errorf("Failed to get server container for cluster %s\n%+v", cluster, err) + } + + if len(server) == 0 { + return fmt.Errorf("No server container for cluster %s", cluster) + } + + // get kubeconfig file from container and read contents + reader, _, err := docker.CopyFromContainer(ctx, server[0].ID, "/output/kubeconfig.yaml") + if err != nil { + return fmt.Errorf("ERROR: couldn't copy kubeconfig.yaml from server container %s\n%+v", server[0].ID, err) + } + defer reader.Close() + + readBytes, err := ioutil.ReadAll(reader) + if err != nil { + return fmt.Errorf("ERROR: couldn't read kubeconfig from container\n%+v", err) + } + + // create destination kubeconfig file + destPath, err := getClusterKubeConfigPath(cluster) + if err != nil { + return err + } + + kubeconfigfile, err := os.Create(destPath) + if err != nil { + return fmt.Errorf("ERROR: couldn't create kubeconfig file %s\n%+v", destPath, err) + } + defer kubeconfigfile.Close() + + // write to file, skipping the first 512 bytes which contain file metadata and trimming any NULL characters + _, err = kubeconfigfile.Write(bytes.Trim(readBytes[512:], "\x00")) + if err != nil { + return fmt.Errorf("ERROR: couldn't write to kubeconfig.yaml\n%+v", err) + } + + return nil +} + // printClusters prints the names of existing clusters func printClusters() { clusters, err := getClusters(true, "") diff --git a/cli/commands.go b/cli/commands.go index 6b492fb2..e8f85ebb 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -9,15 +9,12 @@ import ( "context" "errors" "fmt" - "io/ioutil" "log" "os" "strconv" "strings" "time" - "github.com/docker/docker/api/types/filters" - "github.com/docker/docker/api/types" "github.com/docker/docker/client" "github.com/urfave/cli" @@ -346,60 +343,29 @@ func ListClusters(c *cli.Context) error { // GetKubeConfig grabs the kubeconfig from the running cluster and prints the path to stdout func GetKubeConfig(c *cli.Context) error { - ctx := context.Background() - docker, err := client.NewEnvClient() - if err != nil { - return err - } - - filters := filters.NewArgs() - filters.Add("label", "app=k3d") - filters.Add("label", fmt.Sprintf("cluster=%s", c.String("name"))) - filters.Add("label", "component=server") - server, err := docker.ContainerList(ctx, types.ContainerListOptions{ - Filters: filters, - }) - - if err != nil { - return fmt.Errorf("Failed to get server container for cluster %s\n%+v", c.String("name"), err) - } - - if len(server) == 0 { - return fmt.Errorf("No server container for cluster %s", c.String("name")) - } - - // get kubeconfig file from container and read contents - reader, _, err := docker.CopyFromContainer(ctx, server[0].ID, "/output/kubeconfig.yaml") - if err != nil { - return fmt.Errorf("ERROR: couldn't copy kubeconfig.yaml from server container %s\n%+v", server[0].ID, err) - } - defer reader.Close() - - readBytes, err := ioutil.ReadAll(reader) - if err != nil { - return fmt.Errorf("ERROR: couldn't read kubeconfig from container\n%+v", err) - } - - // create destination kubeconfig file + cluster := c.String("name") destPath, err := getClusterKubeConfigPath(c.String("name")) if err != nil { return err } - kubeconfigfile, err := os.Create(destPath) - if err != nil { - return fmt.Errorf("ERROR: couldn't create kubeconfig file %s\n%+v", destPath, err) + if clusters, err := getClusters(false, cluster); err != nil || len(clusters) != 1 { + if err != nil { + return err + } + return fmt.Errorf("Cluster %s does not exist", cluster) } - defer kubeconfigfile.Close() - // write to file, skipping the first 512 bytes which contain file metadata and trimming any NULL characters - _, err = kubeconfigfile.Write(bytes.Trim(readBytes[512:], "\x00")) - if err != nil { - return fmt.Errorf("ERROR: couldn't write to kubeconfig.yaml\n%+v", err) + // If kubeconfig.yaml has not been created, generate it now. + if _, err := os.Stat(destPath); os.IsNotExist(err) { + if err = createKubeConfigFile(cluster); err != nil { + return err + } + } else { + return err } // output kubeconfig file path to stdout fmt.Println(destPath) - return nil } From 9f276a68f3a96dac24e02969ff9a86fca854e343 Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Fri, 24 May 2019 00:27:56 -0700 Subject: [PATCH 3/9] Add getKubeConfig() Move the logic of retrive per cluster kube config file into cli/cluster.go. Simplify the CLI handling code. --- cli/cluster.go | 25 +++++++++++++++++++++++++ cli/commands.go | 20 ++------------------ 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/cli/cluster.go b/cli/cluster.go index 75cbad98..2adeb8ae 100644 --- a/cli/cluster.go +++ b/cli/cluster.go @@ -148,6 +148,31 @@ func createKubeConfigFile(cluster string) error { return nil } +func getKubeConfig(cluster string) (string, error) { + kubeConfigPath, err := getClusterKubeConfigPath(cluster) + if err != nil { + return "", err + } + + if clusters, err := getClusters(false, cluster); err != nil || len(clusters) != 1 { + if err != nil { + return "", err + } + return "", fmt.Errorf("Cluster %s does not exist", cluster) + } + + // If kubeconfig.yaml has not been created, generate it now + if _, err := os.Stat(kubeConfigPath); os.IsNotExist(err) { + if err = createKubeConfigFile(cluster); err != nil { + return "", err + } + } else { + return "", err + } + + return kubeConfigPath, nil +} + // printClusters prints the names of existing clusters func printClusters() { clusters, err := getClusters(true, "") diff --git a/cli/commands.go b/cli/commands.go index e8f85ebb..8c5cdf4b 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -344,28 +344,12 @@ func ListClusters(c *cli.Context) error { // GetKubeConfig grabs the kubeconfig from the running cluster and prints the path to stdout func GetKubeConfig(c *cli.Context) error { cluster := c.String("name") - destPath, err := getClusterKubeConfigPath(c.String("name")) + kubeConfigPath, err := getKubeConfig(cluster) if err != nil { return err } - if clusters, err := getClusters(false, cluster); err != nil || len(clusters) != 1 { - if err != nil { - return err - } - return fmt.Errorf("Cluster %s does not exist", cluster) - } - - // If kubeconfig.yaml has not been created, generate it now. - if _, err := os.Stat(destPath); os.IsNotExist(err) { - if err = createKubeConfigFile(cluster); err != nil { - return err - } - } else { - return err - } - // output kubeconfig file path to stdout - fmt.Println(destPath) + fmt.Println(kubeConfigPath) return nil } From 4ef6710e22289e11c02dd260f95627b3c2382872 Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Thu, 23 May 2019 23:28:42 -0700 Subject: [PATCH 4/9] Add basic bashShell() function Add the basic frame work for supporting spawning a bash shell by cli command. With this change, we can spawn a bash shell in the context of a cluster $ k3d create -n my-cluster $ k3d bash -n my-cluster [my-cluster] $> // execute commands with KUBECONFIG already set up [my-cluster] $> kubectl get pods --- cli/cluster.go | 12 +++++++----- cli/commands.go | 4 ++++ cli/shell.go | 31 +++++++++++++++++++++++++++++++ main.go | 13 +++++++++++++ 4 files changed, 55 insertions(+), 5 deletions(-) create mode 100644 cli/shell.go diff --git a/cli/cluster.go b/cli/cluster.go index 2adeb8ae..68151fc2 100644 --- a/cli/cluster.go +++ b/cli/cluster.go @@ -161,13 +161,15 @@ func getKubeConfig(cluster string) (string, error) { return "", fmt.Errorf("Cluster %s does not exist", cluster) } - // If kubeconfig.yaml has not been created, generate it now - if _, err := os.Stat(kubeConfigPath); os.IsNotExist(err) { - if err = createKubeConfigFile(cluster); err != nil { + // If kubeconfi.yaml has not been created, generate it now + if _, err := os.Stat(kubeConfigPath); err != nil { + if os.IsNotExist(err) { + if err = createKubeConfigFile(cluster); err != nil { + return "", err + } + } else { return "", err } - } else { - return "", err } return kubeConfigPath, nil diff --git a/cli/commands.go b/cli/commands.go index 8c5cdf4b..9c76b1f0 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -353,3 +353,7 @@ func GetKubeConfig(c *cli.Context) error { fmt.Println(kubeConfigPath) return nil } + +func Bash(c *cli.Context) error { + return bashShell(c.String("name")) +} diff --git a/cli/shell.go b/cli/shell.go new file mode 100644 index 00000000..6dbc772b --- /dev/null +++ b/cli/shell.go @@ -0,0 +1,31 @@ +package run + +import ( + "fmt" + "os" + "os/exec" +) + +func bashShell(cluster string) error { + kubeConfigPath, err := getKubeConfig(cluster) + if err != nil { + return err + } + cmd := exec.Command("/bin/bash", "--noprofile", "--norc") + + // Set up stdio + cmd.Stdout = os.Stdout + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + + // Set up Promot + setPS1 := fmt.Sprintf("PS1=[%s}%s", cluster, os.Getenv("PS1")) + + // Set up KUBECONFIG + setKube := fmt.Sprintf("KUBECONFIG=%s", kubeConfigPath) + newEnv := append(os.Environ(), setPS1, setKube) + + cmd.Env = newEnv + + return cmd.Run() +} diff --git a/main.go b/main.go index 8231df84..62b3423b 100644 --- a/main.go +++ b/main.go @@ -45,6 +45,19 @@ func main() { Usage: "Check if docker is running", Action: run.CheckTools, }, + { + // bash starts a bash shell in the context of a runnign cluster + Name: "bash", + Usage: "Start a bash subshell for a cluster", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "name, n", + Value: defaultK3sClusterName, + Usage: "Set a name for the cluster", + }, + }, + Action: run.Bash, + }, { // create creates a new k3s cluster in docker containers Name: "create", From 5c6f2d7dc5ae9aad10d734f168d323709c592bcc Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Fri, 24 May 2019 10:50:12 -0700 Subject: [PATCH 5/9] Avoid hard coding the bash path OS distribution and user may choose to install bash in different path. This patch uses bash found by "$PATH" environment, rather than hard code the bash path. This patch also handle the case 'bash' are not found. Mostly likely due to bash not being supported by the platform, or it is not installed. --- cli/shell.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/shell.go b/cli/shell.go index 6dbc772b..a3bdaf37 100644 --- a/cli/shell.go +++ b/cli/shell.go @@ -11,7 +11,13 @@ func bashShell(cluster string) error { if err != nil { return err } - cmd := exec.Command("/bin/bash", "--noprofile", "--norc") + + bashPath, err := exec.LookPath("bash") + if err != nil { + return err + } + + cmd := exec.Command(bashPath, "--noprofile", "--norc") // Set up stdio cmd.Stdout = os.Stdout From 2971dd68450a18b6b649056a9cbd74dd5e8da1df Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Fri, 24 May 2019 11:36:50 -0700 Subject: [PATCH 6/9] Add the ability to execute commands directly with the bash subcommand In addition to provide an interactive shell, this patch adds the '--command' and '-c' options to allow user to issue a command in the context of a cluster. For example: $ k3d bash -c 'kubectl cluster-info' --- cli/commands.go | 2 +- cli/shell.go | 7 ++++++- main.go | 4 ++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index 9c76b1f0..aaa63f77 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -355,5 +355,5 @@ func GetKubeConfig(c *cli.Context) error { } func Bash(c *cli.Context) error { - return bashShell(c.String("name")) + return bashShell(c.String("name"), c.String("command")) } diff --git a/cli/shell.go b/cli/shell.go index a3bdaf37..5f75bf06 100644 --- a/cli/shell.go +++ b/cli/shell.go @@ -6,7 +6,7 @@ import ( "os/exec" ) -func bashShell(cluster string) error { +func bashShell(cluster string, command string) error { kubeConfigPath, err := getKubeConfig(cluster) if err != nil { return err @@ -19,6 +19,11 @@ func bashShell(cluster string) error { cmd := exec.Command(bashPath, "--noprofile", "--norc") + if len(command) > 0 { + cmd.Args = append(cmd.Args, "-c", command) + + } + // Set up stdio cmd.Stdout = os.Stdout cmd.Stdin = os.Stdin diff --git a/main.go b/main.go index 62b3423b..7938483d 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,10 @@ func main() { Value: defaultK3sClusterName, Usage: "Set a name for the cluster", }, + cli.StringFlag{ + Name: "command, c", + Usage: "Run a shell command in the context of the cluster", + }, }, Action: run.Bash, }, From 8aaf70f4bfa86b1082c241d191d694bf028c77ef Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Fri, 24 May 2019 12:02:39 -0700 Subject: [PATCH 7/9] Block recursive bash invocation In theory, we can execute 'k3d bash' again within an cluster shell. I can't think of any practical value for allowing this capability. On the contrary, this can lead to confusion to the user. This patch adds a simple mechanism to detect and block recursive bash invocation. --- cli/shell.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cli/shell.go b/cli/shell.go index 5f75bf06..7139a11e 100644 --- a/cli/shell.go +++ b/cli/shell.go @@ -12,6 +12,11 @@ func bashShell(cluster string, command string) error { return err } + subShell := os.ExpandEnv("$__K3D_CLUSTER__") + if len(subShell) > 0 { + return fmt.Errorf("Error: Already in subshell of cluster %s", subShell) + } + bashPath, err := exec.LookPath("bash") if err != nil { return err @@ -34,7 +39,11 @@ func bashShell(cluster string, command string) error { // Set up KUBECONFIG setKube := fmt.Sprintf("KUBECONFIG=%s", kubeConfigPath) - newEnv := append(os.Environ(), setPS1, setKube) + + // Declare subshell + subShell = fmt.Sprintf("__K3D_CLUSTER__=%s", cluster) + + newEnv := append(os.Environ(), setPS1, setKube, subShell) cmd.Env = newEnv From d63f7d4bf2831387708367a623c88becd532dac4 Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Mon, 27 May 2019 17:05:52 -0700 Subject: [PATCH 8/9] Rename bash command to shell @iwilltry42 suggested to group all shell support under and single subcommand -- shell. --- cli/commands.go | 2 +- main.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cli/commands.go b/cli/commands.go index aaa63f77..98875bff 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -354,6 +354,6 @@ func GetKubeConfig(c *cli.Context) error { return nil } -func Bash(c *cli.Context) error { +func Shell(c *cli.Context) error { return bashShell(c.String("name"), c.String("command")) } diff --git a/main.go b/main.go index 7938483d..8d59f319 100644 --- a/main.go +++ b/main.go @@ -46,9 +46,9 @@ func main() { Action: run.CheckTools, }, { - // bash starts a bash shell in the context of a runnign cluster - Name: "bash", - Usage: "Start a bash subshell for a cluster", + // shell starts a shell in the context of a running cluster + Name: "shell", + Usage: "Start a subshell for a cluster", Flags: []cli.Flag{ cli.StringFlag{ Name: "name, n", @@ -60,7 +60,7 @@ func main() { Usage: "Run a shell command in the context of the cluster", }, }, - Action: run.Bash, + Action: run.Shell, }, { // create creates a new k3s cluster in docker containers From abd9a984eb22e9a55adae9abea843ddd75cac2c7 Mon Sep 17 00:00:00 2001 From: Andy Zhou Date: Mon, 27 May 2019 17:14:20 -0700 Subject: [PATCH 9/9] Add --shell argument Add subshell argument --shell. Currently, support only bash, which is also the default value. --- cli/commands.go | 4 ++++ main.go | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/cli/commands.go b/cli/commands.go index 98875bff..a5f41c90 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -355,5 +355,9 @@ func GetKubeConfig(c *cli.Context) error { } func Shell(c *cli.Context) error { + if c.String("shell") != "bash" { + return fmt.Errorf("%s is not supported. Only bash is supported", c.String("shell")) + } + return bashShell(c.String("name"), c.String("command")) } diff --git a/main.go b/main.go index 8d59f319..51386f3d 100644 --- a/main.go +++ b/main.go @@ -59,6 +59,11 @@ func main() { Name: "command, c", Usage: "Run a shell command in the context of the cluster", }, + cli.StringFlag{ + Name: "shell, s", + Value: "bash", + Usage: "Sub shell type. Only bash is supported. (default bash)", + }, }, Action: run.Shell, },