mirror of https://github.com/k3d-io/k3d
Merge pull request #91 from rancher/feature/import-images-helper
[Feature] import images from docker daemon into k3d using a helper containerpull/94/head v1.3.0
Thorsten Klein
5 years ago
committed by
GitHub
commit
c5e5adb0e2
@ -0,0 +1,209 @@ |
||||
package run |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"log" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/docker/docker/api/types" |
||||
"github.com/docker/docker/api/types/container" |
||||
"github.com/docker/docker/api/types/network" |
||||
"github.com/docker/docker/client" |
||||
) |
||||
|
||||
const ( |
||||
imageBasePathRemote = "/images" |
||||
k3dToolsImage = "docker.io/iwilltry42/k3d-tools:v0.0.1" |
||||
) |
||||
|
||||
func importImage(clusterName string, images []string, noRemove bool) error { |
||||
// get a docker client
|
||||
ctx := context.Background() |
||||
docker, err := client.NewEnvClient() |
||||
if err != nil { |
||||
return fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) |
||||
} |
||||
|
||||
// get cluster directory to temporarily save the image tarball there
|
||||
imageVolume, err := getImageVolume(clusterName) |
||||
if err != nil { |
||||
return fmt.Errorf("ERROR: couldn't get image volume for cluster [%s]\n%+v", clusterName, err) |
||||
} |
||||
|
||||
//*** first, save the images using the local docker daemon
|
||||
log.Printf("INFO: Saving images %s from local docker daemon...", images) |
||||
toolsContainerName := fmt.Sprintf("k3d-%s-tools", clusterName) |
||||
tarFileName := fmt.Sprintf("%s/k3d-%s-images-%s.tar", imageBasePathRemote, clusterName, time.Now().Format("20060102150405")) |
||||
|
||||
// create a tools container to get the tarball into the named volume
|
||||
containerConfig := container.Config{ |
||||
Hostname: toolsContainerName, |
||||
Image: k3dToolsImage, |
||||
Labels: map[string]string{ |
||||
"app": "k3d", |
||||
"cluster": clusterName, |
||||
"component": "tools", |
||||
}, |
||||
Cmd: append([]string{"save-image", "-d", tarFileName}, images...), |
||||
AttachStdout: true, |
||||
AttachStderr: true, |
||||
} |
||||
hostConfig := container.HostConfig{ |
||||
Binds: []string{ |
||||
"/var/run/docker.sock:/var/run/docker.sock", |
||||
fmt.Sprintf("%s:%s:rw", imageVolume.Name, imageBasePathRemote), |
||||
}, |
||||
} |
||||
|
||||
toolsContainerID, err := startContainer(false, &containerConfig, &hostConfig, &network.NetworkingConfig{}, toolsContainerName) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
defer func() { |
||||
if err = docker.ContainerRemove(ctx, toolsContainerID, types.ContainerRemoveOptions{ |
||||
Force: true, |
||||
}); err != nil { |
||||
log.Println(fmt.Errorf("WARN: couldn't remove tools container\n%+v", err)) |
||||
} |
||||
}() |
||||
|
||||
// loop to wait for tools container to exit (failed or successfully saved images)
|
||||
for { |
||||
cont, err := docker.ContainerInspect(ctx, toolsContainerID) |
||||
if err != nil { |
||||
return fmt.Errorf("ERROR: couldn't get helper container's exit code\n%+v", err) |
||||
} |
||||
if !cont.State.Running { // container finished...
|
||||
if cont.State.ExitCode == 0 { // ...successfully
|
||||
log.Println("INFO: saved images to shared docker volume") |
||||
break |
||||
} else if cont.State.ExitCode != 0 { // ...failed
|
||||
errTxt := "ERROR: helper container failed to save images" |
||||
logReader, err := docker.ContainerLogs(ctx, toolsContainerID, types.ContainerLogsOptions{ |
||||
ShowStdout: true, |
||||
ShowStderr: true, |
||||
}) |
||||
if err != nil { |
||||
return fmt.Errorf("%s\n> couldn't get logs from helper container\n%+v", errTxt, err) |
||||
} |
||||
logs, err := ioutil.ReadAll(logReader) // let's show somw logs indicating what happened
|
||||
if err != nil { |
||||
return fmt.Errorf("%s\n> couldn't get logs from helper container\n%+v", errTxt, err) |
||||
} |
||||
return fmt.Errorf("%s -> Logs from [%s]:\n>>>>>>\n%s\n<<<<<<", errTxt, toolsContainerName, string(logs)) |
||||
} |
||||
} |
||||
time.Sleep(time.Second / 2) // wait for half a second so we don't spam the docker API too much
|
||||
} |
||||
|
||||
// Get the container IDs for all containers in the cluster
|
||||
clusters, err := getClusters(false, clusterName) |
||||
if err != nil { |
||||
return fmt.Errorf("ERROR: couldn't get cluster by name [%s]\n%+v", clusterName, err) |
||||
} |
||||
containerList := []types.Container{clusters[clusterName].server} |
||||
containerList = append(containerList, clusters[clusterName].workers...) |
||||
|
||||
// *** second, import the images using ctr in the k3d nodes
|
||||
|
||||
// create exec configuration
|
||||
cmd := []string{"ctr", "image", "import", tarFileName} |
||||
execConfig := types.ExecConfig{ |
||||
AttachStderr: true, |
||||
AttachStdout: true, |
||||
Cmd: cmd, |
||||
Tty: true, |
||||
Detach: true, |
||||
} |
||||
|
||||
execAttachConfig := types.ExecConfig{ |
||||
Tty: true, |
||||
} |
||||
|
||||
execStartConfig := types.ExecStartCheck{ |
||||
Tty: true, |
||||
} |
||||
|
||||
// import in each node separately
|
||||
// TODO: import concurrently using goroutines or find a way to share the image cache
|
||||
for _, container := range containerList { |
||||
|
||||
containerName := container.Names[0][1:] // trimming the leading "/" from name
|
||||
log.Printf("INFO: Importing images %s in container [%s]", images, containerName) |
||||
|
||||
// create exec configuration
|
||||
execResponse, err := docker.ContainerExecCreate(ctx, container.ID, execConfig) |
||||
if err != nil { |
||||
return fmt.Errorf("ERROR: Failed to create exec command for container [%s]\n%+v", containerName, err) |
||||
} |
||||
|
||||
// attach to exec process in container
|
||||
containerConnection, err := docker.ContainerExecAttach(ctx, execResponse.ID, execAttachConfig) |
||||
if err != nil { |
||||
return fmt.Errorf("ERROR: couldn't attach to container [%s]\n%+v", containerName, err) |
||||
} |
||||
defer containerConnection.Close() |
||||
|
||||
// start exec
|
||||
err = docker.ContainerExecStart(ctx, execResponse.ID, execStartConfig) |
||||
if err != nil { |
||||
return fmt.Errorf("ERROR: couldn't execute command in container [%s]\n%+v", containerName, err) |
||||
} |
||||
|
||||
// get output from container
|
||||
content, err := ioutil.ReadAll(containerConnection.Reader) |
||||
if err != nil { |
||||
return fmt.Errorf("ERROR: couldn't read output from container [%s]\n%+v", containerName, err) |
||||
} |
||||
|
||||
// example output "unpacking image........ ...done"
|
||||
if !strings.Contains(string(content), "done") { |
||||
return fmt.Errorf("ERROR: seems like something went wrong using `ctr image import` in container [%s]. Full output below:\n%s", containerName, string(content)) |
||||
} |
||||
} |
||||
|
||||
log.Printf("INFO: Successfully imported images %s in all nodes of cluster [%s]", images, clusterName) |
||||
|
||||
// remove tarball from inside the server container
|
||||
if !noRemove { |
||||
log.Println("INFO: Cleaning up tarball") |
||||
|
||||
execID, err := docker.ContainerExecCreate(ctx, clusters[clusterName].server.ID, types.ExecConfig{ |
||||
Cmd: []string{"rm", "-f", tarFileName}, |
||||
}) |
||||
if err != nil { |
||||
log.Printf("WARN: failed to delete tarball: couldn't create remove in container [%s]\n%+v", clusters[clusterName].server.ID, err) |
||||
} |
||||
err = docker.ContainerExecStart(ctx, execID.ID, types.ExecStartCheck{ |
||||
Detach: true, |
||||
}) |
||||
if err != nil { |
||||
log.Printf("WARN: couldn't start tarball deletion action\n%+v", err) |
||||
} |
||||
|
||||
for { |
||||
execInspect, err := docker.ContainerExecInspect(ctx, execID.ID) |
||||
if err != nil { |
||||
log.Printf("WARN: couldn't verify deletion of tarball\n%+v", err) |
||||
} |
||||
|
||||
if !execInspect.Running { |
||||
if execInspect.ExitCode == 0 { |
||||
log.Println("INFO: deleted tarball") |
||||
break |
||||
} else { |
||||
log.Println("WARN: failed to delete tarball") |
||||
break |
||||
} |
||||
} |
||||
} |
||||
} |
||||
|
||||
log.Println("INFO: ...Done") |
||||
|
||||
return nil |
||||
} |
||||
@ -0,0 +1,92 @@ |
||||
package run |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
|
||||
"github.com/docker/docker/api/types" |
||||
"github.com/docker/docker/api/types/filters" |
||||
"github.com/docker/docker/api/types/volume" |
||||
"github.com/docker/docker/client" |
||||
) |
||||
|
||||
// createImageVolume will create a new docker volume used for storing image tarballs that can be loaded into the clusters
|
||||
func createImageVolume(clusterName string) (types.Volume, error) { |
||||
|
||||
var vol types.Volume |
||||
|
||||
ctx := context.Background() |
||||
docker, err := client.NewEnvClient() |
||||
if err != nil { |
||||
return vol, fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) |
||||
} |
||||
|
||||
volName := fmt.Sprintf("k3d-%s-images", clusterName) |
||||
|
||||
volumeCreateOptions := volume.VolumesCreateBody{ |
||||
Name: volName, |
||||
Labels: map[string]string{ |
||||
"app": "k3d", |
||||
"cluster": clusterName, |
||||
}, |
||||
Driver: "local", //TODO: allow setting driver + opts
|
||||
DriverOpts: map[string]string{}, |
||||
} |
||||
vol, err = docker.VolumeCreate(ctx, volumeCreateOptions) |
||||
if err != nil { |
||||
return vol, fmt.Errorf("ERROR: failed to create image volume [%s] for cluster [%s]\n%+v", volName, clusterName, err) |
||||
} |
||||
|
||||
return vol, nil |
||||
} |
||||
|
||||
// deleteImageVolume will delete the volume we created for sharing images with this cluster
|
||||
func deleteImageVolume(clusterName string) error { |
||||
|
||||
ctx := context.Background() |
||||
docker, err := client.NewEnvClient() |
||||
if err != nil { |
||||
return fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) |
||||
} |
||||
|
||||
volName := fmt.Sprintf("k3d-%s-images", clusterName) |
||||
|
||||
if err = docker.VolumeRemove(ctx, volName, true); err != nil { |
||||
return fmt.Errorf("ERROR: couldn't remove volume [%s] for cluster [%s]\n%+v", volName, clusterName, err) |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// getImageVolume returns the docker volume object representing the imagevolume for the cluster
|
||||
func getImageVolume(clusterName string) (types.Volume, error) { |
||||
var vol types.Volume |
||||
volName := fmt.Sprintf("k3d-%s-images", clusterName) |
||||
|
||||
ctx := context.Background() |
||||
docker, err := client.NewEnvClient() |
||||
if err != nil { |
||||
return vol, fmt.Errorf("ERROR: couldn't create docker client\n%+v", err) |
||||
} |
||||
|
||||
filters := filters.NewArgs() |
||||
filters.Add("label", "app=k3d") |
||||
filters.Add("label", fmt.Sprintf("cluster=%s", clusterName)) |
||||
volumeList, err := docker.VolumeList(ctx, filters) |
||||
if err != nil { |
||||
return vol, fmt.Errorf("ERROR: couldn't get volumes for cluster [%s]\n%+v ", clusterName, err) |
||||
} |
||||
volFound := false |
||||
for _, volume := range volumeList.Volumes { |
||||
if volume.Name == volName { |
||||
vol = *volume |
||||
volFound = true |
||||
break |
||||
} |
||||
} |
||||
if !volFound { |
||||
return vol, fmt.Errorf("ERROR: didn't find volume [%s] in list of volumes returned for cluster [%s]", volName, clusterName) |
||||
} |
||||
|
||||
return vol, nil |
||||
} |
||||
Loading…
Reference in new issue