diff --git a/cli/cluster.go b/cli/cluster.go index aec873a1..825e2815 100644 --- a/cli/cluster.go +++ b/cli/cluster.go @@ -14,8 +14,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/filters" "github.com/docker/docker/client" - - "github.com/mitchellh/go-homedir" + homedir "github.com/mitchellh/go-homedir" "github.com/olekukonko/tablewriter" ) @@ -68,6 +67,10 @@ func createClusterDir(name string) { if err := createDirIfNotExists(clusterPath); err != nil { log.Fatalf("ERROR: couldn't create cluster directory [%s] -> %+v", clusterPath, err) } + // create subdir for sharing container images + if err := createDirIfNotExists(clusterPath + "/images"); err != nil { + log.Fatalf("ERROR: couldn't create cluster sub-directory [%s] -> %+v", clusterPath+"/images", err) + } } // deleteClusterDir contrary to createClusterDir, this deletes the cluster directory under $HOME/.config/k3d/ diff --git a/cli/commands.go b/cli/commands.go index 8b69a566..15b9aa01 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -100,7 +100,7 @@ func CreateCluster(c *cli.Context) error { if c.IsSet("port") { log.Println("INFO: As of v2.0.0 --port will be used for arbitrary port mapping. Please use --api-port/-a instead for configuring the Api Port") } - apiPort, err := parseApiPort(c.String("api-port")) + apiPort, err := parseAPIPort(c.String("api-port")) if err != nil { return err } @@ -111,7 +111,7 @@ func CreateCluster(c *cli.Context) error { if apiPort.Host == "" { apiPort.Host, err = getDockerMachineIp() // IP address is the same as the host - apiPort.HostIp = apiPort.Host + apiPort.HostIP = apiPort.Host // In case of error, Log a warning message, and continue on. Since it more likely caused by a miss configured // DOCKER_MACHINE_NAME environment variable. if err != nil { @@ -135,9 +135,18 @@ func CreateCluster(c *cli.Context) error { log.Fatal(err) } + // create a docker volume for sharing image tarballs with the cluster + imageVolume, err := createImageVolume(c.String("name")) + log.Println("Created docker volume ", imageVolume.Name) + if err != nil { + return err + } + volumes := c.StringSlice("volume") + volumes = append(volumes, fmt.Sprintf("%s:/images", imageVolume.Name)) + clusterSpec := &ClusterSpec{ AgentArgs: []string{}, - ApiPort: *apiPort, + APIPort: *apiPort, AutoRestart: c.Bool("auto-restart"), ClusterName: c.String("name"), Env: env, @@ -146,11 +155,15 @@ func CreateCluster(c *cli.Context) error { PortAutoOffset: c.Int("port-auto-offset"), ServerArgs: k3sServerArgs, Verbose: c.GlobalBool("verbose"), - Volumes: c.StringSlice("volume"), + Volumes: volumes, } // create the server log.Printf("Creating cluster [%s]", c.String("name")) + + // create the directory where we will put the kubeconfig file by default (when running `k3d get-config`) + createClusterDir(c.String("name")) + dockerID, err := createServer(clusterSpec) if err != nil { deleteCluster() @@ -192,10 +205,6 @@ func CreateCluster(c *cli.Context) error { time.Sleep(1 * time.Second) } - // create the directory where we will put the kubeconfig file by default (when running `k3d get-config`) - // TODO: this can probably be moved to `k3d get-config` or be removed in a different approach - createClusterDir(c.String("name")) - // spin up the worker nodes // TODO: do this concurrently in different goroutines if c.Int("workers") > 0 { @@ -241,8 +250,8 @@ func DeleteCluster(c *cli.Context) error { } } } - log.Println("...Removing server") deleteClusterDir(cluster.name) + log.Println("...Removing server") if err := removeContainer(cluster.server.ID); err != nil { return fmt.Errorf("ERROR: Couldn't remove server for cluster %s\n%+v", cluster.name, err) } @@ -251,6 +260,11 @@ func DeleteCluster(c *cli.Context) error { log.Printf("WARNING: couldn't delete cluster network for cluster %s\n%+v", cluster.name, err) } + log.Println("...Removing docker image volume") + if err := deleteImageVolume(cluster.name); err != nil { + log.Printf("WARNING: couldn't delete image docker volume for cluster %s\n%+v", cluster.name, err) + } + log.Printf("SUCCESS: removed cluster [%s]", cluster.name) } @@ -362,3 +376,14 @@ func GetKubeConfig(c *cli.Context) error { func Shell(c *cli.Context) error { return subShell(c.String("name"), c.String("shell"), c.String("command")) } + +// ImportImage saves an image locally and imports it into the k3d containers +func ImportImage(c *cli.Context) error { + images := make([]string, 0) + if strings.Contains(c.Args().First(), ",") { + images = append(images, strings.Split(c.Args().First(), ",")...) + } else { + images = append(images, c.Args()...) + } + return importImage(c.String("name"), images, c.Bool("no-remove")) +} diff --git a/cli/container.go b/cli/container.go index db3a1c81..2eaa8900 100644 --- a/cli/container.go +++ b/cli/container.go @@ -22,7 +22,7 @@ import ( type ClusterSpec struct { AgentArgs []string - ApiPort apiPort + APIPort apiPort AutoRestart bool ClusterName string Env []string @@ -94,14 +94,14 @@ func createServer(spec *ClusterSpec) (string, error) { return "", err } - hostIp := "0.0.0.0" + hostIP := "0.0.0.0" containerLabels["apihost"] = "localhost" - if spec.ApiPort.Host != "" { - hostIp = spec.ApiPort.HostIp - containerLabels["apihost"] = spec.ApiPort.Host + if spec.APIPort.Host != "" { + hostIP = spec.APIPort.HostIP + containerLabels["apihost"] = spec.APIPort.Host } - apiPortSpec := fmt.Sprintf("%s:%s:%s/tcp", hostIp, spec.ApiPort.Port, spec.ApiPort.Port) + apiPortSpec := fmt.Sprintf("%s:%s:%s/tcp", hostIP, spec.APIPort.Port, spec.APIPort.Port) serverPorts = append(serverPorts, apiPortSpec) @@ -157,7 +157,7 @@ func createWorker(spec *ClusterSpec, postfix int) (string, error) { containerName := GetContainerName("worker", spec.ClusterName, postfix) - env := append(spec.Env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", spec.ClusterName, spec.ApiPort.Port)) + env := append(spec.Env, fmt.Sprintf("K3S_URL=https://k3d-%s-server:%s", spec.ClusterName, spec.APIPort.Port)) // ports to be assigned to the server belong to roles // all, server or @@ -230,7 +230,7 @@ func removeContainer(ID string) error { } if err := docker.ContainerRemove(ctx, ID, options); err != nil { - return fmt.Errorf("FAILURE: couldn't delete container [%s] -> %+v", ID, err) + return fmt.Errorf("ERROR: couldn't delete container [%s] -> %+v", ID, err) } return nil } diff --git a/cli/image.go b/cli/image.go new file mode 100644 index 00000000..5ab51446 --- /dev/null +++ b/cli/image.go @@ -0,0 +1,209 @@ +package run + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "strings" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/api/types/network" + "github.com/docker/docker/client" +) + +const ( + imageBasePathRemote = "/images" + k3dToolsImage = "docker.io/iwilltry42/k3d-tools:v0.0.1" +) + +func importImage(clusterName string, images []string, noRemove bool) error { + // get a docker client + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) + } + + // get cluster directory to temporarily save the image tarball there + imageVolume, err := getImageVolume(clusterName) + if err != nil { + return fmt.Errorf("ERROR: couldn't get image volume for cluster [%s]\n%+v", clusterName, err) + } + + //*** first, save the images using the local docker daemon + log.Printf("INFO: Saving images %s from local docker daemon...", images) + toolsContainerName := fmt.Sprintf("k3d-%s-tools", clusterName) + tarFileName := fmt.Sprintf("%s/k3d-%s-images-%s.tar", imageBasePathRemote, clusterName, time.Now().Format("20060102150405")) + + // create a tools container to get the tarball into the named volume + containerConfig := container.Config{ + Hostname: toolsContainerName, + Image: k3dToolsImage, + Labels: map[string]string{ + "app": "k3d", + "cluster": clusterName, + "component": "tools", + }, + Cmd: append([]string{"save-image", "-d", tarFileName}, images...), + AttachStdout: true, + AttachStderr: true, + } + hostConfig := container.HostConfig{ + Binds: []string{ + "/var/run/docker.sock:/var/run/docker.sock", + fmt.Sprintf("%s:%s:rw", imageVolume.Name, imageBasePathRemote), + }, + } + + toolsContainerID, err := startContainer(false, &containerConfig, &hostConfig, &network.NetworkingConfig{}, toolsContainerName) + if err != nil { + return err + } + + defer func() { + if err = docker.ContainerRemove(ctx, toolsContainerID, types.ContainerRemoveOptions{ + Force: true, + }); err != nil { + log.Println(fmt.Errorf("WARN: couldn't remove tools container\n%+v", err)) + } + }() + + // loop to wait for tools container to exit (failed or successfully saved images) + for { + cont, err := docker.ContainerInspect(ctx, toolsContainerID) + if err != nil { + return fmt.Errorf("ERROR: couldn't get helper container's exit code\n%+v", err) + } + if !cont.State.Running { // container finished... + if cont.State.ExitCode == 0 { // ...successfully + log.Println("INFO: saved images to shared docker volume") + break + } else if cont.State.ExitCode != 0 { // ...failed + errTxt := "ERROR: helper container failed to save images" + logReader, err := docker.ContainerLogs(ctx, toolsContainerID, types.ContainerLogsOptions{ + ShowStdout: true, + ShowStderr: true, + }) + if err != nil { + return fmt.Errorf("%s\n> couldn't get logs from helper container\n%+v", errTxt, err) + } + logs, err := ioutil.ReadAll(logReader) // let's show somw logs indicating what happened + if err != nil { + return fmt.Errorf("%s\n> couldn't get logs from helper container\n%+v", errTxt, err) + } + return fmt.Errorf("%s -> Logs from [%s]:\n>>>>>>\n%s\n<<<<<<", errTxt, toolsContainerName, string(logs)) + } + } + time.Sleep(time.Second / 2) // wait for half a second so we don't spam the docker API too much + } + + // Get the container IDs for all containers in the cluster + clusters, err := getClusters(false, clusterName) + if err != nil { + return fmt.Errorf("ERROR: couldn't get cluster by name [%s]\n%+v", clusterName, err) + } + containerList := []types.Container{clusters[clusterName].server} + containerList = append(containerList, clusters[clusterName].workers...) + + // *** second, import the images using ctr in the k3d nodes + + // create exec configuration + cmd := []string{"ctr", "image", "import", tarFileName} + execConfig := types.ExecConfig{ + AttachStderr: true, + AttachStdout: true, + Cmd: cmd, + Tty: true, + Detach: true, + } + + execAttachConfig := types.ExecConfig{ + Tty: true, + } + + execStartConfig := types.ExecStartCheck{ + Tty: true, + } + + // import in each node separately + // TODO: import concurrently using goroutines or find a way to share the image cache + for _, container := range containerList { + + containerName := container.Names[0][1:] // trimming the leading "/" from name + log.Printf("INFO: Importing images %s in container [%s]", images, containerName) + + // create exec configuration + execResponse, err := docker.ContainerExecCreate(ctx, container.ID, execConfig) + if err != nil { + return fmt.Errorf("ERROR: Failed to create exec command for container [%s]\n%+v", containerName, err) + } + + // attach to exec process in container + containerConnection, err := docker.ContainerExecAttach(ctx, execResponse.ID, execAttachConfig) + if err != nil { + return fmt.Errorf("ERROR: couldn't attach to container [%s]\n%+v", containerName, err) + } + defer containerConnection.Close() + + // start exec + err = docker.ContainerExecStart(ctx, execResponse.ID, execStartConfig) + if err != nil { + return fmt.Errorf("ERROR: couldn't execute command in container [%s]\n%+v", containerName, err) + } + + // get output from container + content, err := ioutil.ReadAll(containerConnection.Reader) + if err != nil { + return fmt.Errorf("ERROR: couldn't read output from container [%s]\n%+v", containerName, err) + } + + // example output "unpacking image........ ...done" + if !strings.Contains(string(content), "done") { + return fmt.Errorf("ERROR: seems like something went wrong using `ctr image import` in container [%s]. Full output below:\n%s", containerName, string(content)) + } + } + + log.Printf("INFO: Successfully imported images %s in all nodes of cluster [%s]", images, clusterName) + + // remove tarball from inside the server container + if !noRemove { + log.Println("INFO: Cleaning up tarball") + + execID, err := docker.ContainerExecCreate(ctx, clusters[clusterName].server.ID, types.ExecConfig{ + Cmd: []string{"rm", "-f", tarFileName}, + }) + if err != nil { + log.Printf("WARN: failed to delete tarball: couldn't create remove in container [%s]\n%+v", clusters[clusterName].server.ID, err) + } + err = docker.ContainerExecStart(ctx, execID.ID, types.ExecStartCheck{ + Detach: true, + }) + if err != nil { + log.Printf("WARN: couldn't start tarball deletion action\n%+v", err) + } + + for { + execInspect, err := docker.ContainerExecInspect(ctx, execID.ID) + if err != nil { + log.Printf("WARN: couldn't verify deletion of tarball\n%+v", err) + } + + if !execInspect.Running { + if execInspect.ExitCode == 0 { + log.Println("INFO: deleted tarball") + break + } else { + log.Println("WARN: failed to delete tarball") + break + } + } + } + } + + log.Println("INFO: ...Done") + + return nil +} diff --git a/cli/util.go b/cli/util.go index 51f1b18e..6608056b 100644 --- a/cli/util.go +++ b/cli/util.go @@ -11,7 +11,7 @@ import ( type apiPort struct { Host string - HostIp string + HostIP string Port string } @@ -90,7 +90,7 @@ func ValidateHostname(name string) error { return nil } -func parseApiPort(portSpec string) (*apiPort, error) { +func parseAPIPort(portSpec string) (*apiPort, error) { var port *apiPort split := strings.Split(portSpec, ":") if len(split) > 2 { @@ -105,7 +105,7 @@ func parseApiPort(portSpec string) (*apiPort, error) { if err != nil { return nil, err } - port = &apiPort{Host: split[0], HostIp: addrs[0], Port: split[1]} + port = &apiPort{Host: split[0], HostIP: addrs[0], Port: split[1]} } // Verify 'port' is an integer and within port ranges diff --git a/cli/volume.go b/cli/volume.go new file mode 100644 index 00000000..aa93a66e --- /dev/null +++ b/cli/volume.go @@ -0,0 +1,92 @@ +package run + +import ( + "context" + "fmt" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" + "github.com/docker/docker/api/types/volume" + "github.com/docker/docker/client" +) + +// createImageVolume will create a new docker volume used for storing image tarballs that can be loaded into the clusters +func createImageVolume(clusterName string) (types.Volume, error) { + + var vol types.Volume + + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return vol, fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) + } + + volName := fmt.Sprintf("k3d-%s-images", clusterName) + + volumeCreateOptions := volume.VolumesCreateBody{ + Name: volName, + Labels: map[string]string{ + "app": "k3d", + "cluster": clusterName, + }, + Driver: "local", //TODO: allow setting driver + opts + DriverOpts: map[string]string{}, + } + vol, err = docker.VolumeCreate(ctx, volumeCreateOptions) + if err != nil { + return vol, fmt.Errorf("ERROR: failed to create image volume [%s] for cluster [%s]\n%+v", volName, clusterName, err) + } + + return vol, nil +} + +// deleteImageVolume will delete the volume we created for sharing images with this cluster +func deleteImageVolume(clusterName string) error { + + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) + } + + volName := fmt.Sprintf("k3d-%s-images", clusterName) + + if err = docker.VolumeRemove(ctx, volName, true); err != nil { + return fmt.Errorf("ERROR: couldn't remove volume [%s] for cluster [%s]\n%+v", volName, clusterName, err) + } + + return nil +} + +// getImageVolume returns the docker volume object representing the imagevolume for the cluster +func getImageVolume(clusterName string) (types.Volume, error) { + var vol types.Volume + volName := fmt.Sprintf("k3d-%s-images", clusterName) + + ctx := context.Background() + docker, err := client.NewEnvClient() + if err != nil { + return vol, fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) + } + + filters := filters.NewArgs() + filters.Add("label", "app=k3d") + filters.Add("label", fmt.Sprintf("cluster=%s", clusterName)) + volumeList, err := docker.VolumeList(ctx, filters) + if err != nil { + return vol, fmt.Errorf("ERROR: couldn't get volumes for cluster [%s]\n%+v ", clusterName, err) + } + volFound := false + for _, volume := range volumeList.Volumes { + if volume.Name == volName { + vol = *volume + volFound = true + break + } + } + if !volFound { + return vol, fmt.Errorf("ERROR: didn't find volume [%s] in list of volumes returned for cluster [%s]", volName, clusterName) + } + + return vol, nil +} diff --git a/main.go b/main.go index 1f94ee60..5b9b24ff 100644 --- a/main.go +++ b/main.go @@ -214,6 +214,24 @@ func main() { }, Action: run.GetKubeConfig, }, + { + // get-kubeconfig grabs the kubeconfig from the cluster and prints the path to it + Name: "import-images", + Aliases: []string{"i"}, + Usage: "Import a comma- or space-separated list of container images from your local docker daemon into the cluster", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "name, n, cluster, c", + Value: defaultK3sClusterName, + Usage: "Name of the cluster", + }, + cli.BoolFlag{ + Name: "no-remove, no-rm, keep, k", + Usage: "Disable automatic removal of the tarball", + }, + }, + Action: run.ImportImage, + }, } // Global flags diff --git a/vendor/modules.txt b/vendor/modules.txt index 9a44086b..395bfcee 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -8,6 +8,7 @@ github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/api/types/filters github.com/docker/docker/api/types/network +github.com/docker/docker/api/types/volume github.com/docker/docker/client github.com/docker/docker/api/types/mount github.com/docker/docker/api/types/registry @@ -18,7 +19,6 @@ github.com/docker/docker/api/types/versions github.com/docker/docker/api/types/events github.com/docker/docker/api/types/reference github.com/docker/docker/api/types/time -github.com/docker/docker/api/types/volume github.com/docker/docker/pkg/tlsconfig # github.com/docker/go-connections v0.4.0 github.com/docker/go-connections/nat