implement load images command

pull/227/head v3.0.0-dev-20191204-0
iwilltry42 5 years ago
parent e6d7726ffb
commit 11b937a910
  1. 9
      cmd/create/createCluster.go
  2. 22
      cmd/load/loadImage.go
  3. 64
      pkg/cluster/cluster.go
  4. 41
      pkg/cluster/tools.go
  5. 5
      pkg/runtimes/containerd/containerd.go
  6. 5
      pkg/runtimes/containerd/node.go
  7. 5
      pkg/runtimes/docker/docker.go
  8. 85
      pkg/runtimes/docker/node.go
  9. 3
      pkg/runtimes/runtime.go
  10. 129
      pkg/tools/tools.go
  11. 12
      pkg/types/types.go
  12. 9
      thoughts.md

@ -58,7 +58,7 @@ func NewCmdCreateCluster() *cobra.Command {
cmd.Flags().IntP("masters", "m", 1, "Specify how many masters you want to create")
cmd.Flags().IntP("workers", "w", 0, "Specify how many workers you want to create")
// cmd.Flags().String("config", "", "Specify a cluster configuration file") // TODO: to implement
cmd.Flags().String("image", k3d.DefaultK3sImageRepo, "Specify k3s image that you want to use for the nodes") // TODO: get image version
cmd.Flags().String("image", fmt.Sprintf("%s:%s", k3d.DefaultK3sImageRepo, "v1.0.0"), "Specify k3s image that you want to use for the nodes") // TODO: get image version
cmd.Flags().String("network", "", "Join an existing network")
cmd.Flags().String("secret", "", "Specify a cluster secret. By default, we generate one.")
cmd.Flags().StringArrayP("volume", "v", nil, "Mount volumes into the nodes (Format: `--volume [SOURCE:]DEST[@NODEFILTER[;NODEFILTER...]]`")
@ -130,10 +130,15 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string) (runtimes.Runtime,
}
// --network
network, err := cmd.Flags().GetString("network")
networkName, err := cmd.Flags().GetString("network")
if err != nil {
log.Fatalln(err)
}
network := k3d.ClusterNetwork{}
if networkName != "" {
network.Name = networkName
network.External = true
}
// --secret
secret, err := cmd.Flags().GetString("secret")

@ -25,6 +25,7 @@ import (
"github.com/spf13/cobra"
"github.com/rancher/k3d/pkg/runtimes"
"github.com/rancher/k3d/pkg/tools"
k3d "github.com/rancher/k3d/pkg/types"
log "github.com/sirupsen/logrus"
@ -40,8 +41,16 @@ func NewCmdLoadImage() *cobra.Command {
Long: `Load an image from docker into a k3d cluster.`,
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
runtime, images, clusters := parseLoadImageCmd(cmd, args)
runtime, images, clusters, keepTarball := parseLoadImageCmd(cmd, args)
log.Debugf("Load images [%+v] from runtime [%s] into clusters [%+v]", runtime, images, clusters)
for _, cluster := range clusters {
log.Debugf("Loading images into '%s'", cluster.Name)
if err := tools.LoadImagesIntoCluster(runtime, images, &cluster, keepTarball); err != nil {
log.Errorf("Failed to load images into cluster '%s'", cluster.Name)
log.Errorln(err)
}
}
log.Debugln("Finished loading images into clusters")
},
}
@ -49,6 +58,7 @@ func NewCmdLoadImage() *cobra.Command {
* Flags *
*********/
cmd.Flags().StringArrayP("cluster", "c", []string{k3d.DefaultClusterName}, "Select clusters to load the image to.")
cmd.Flags().BoolP("keep-tarball", "k", false, "Do not delete the tarball which contains the saved images from the shared volume")
/* Subcommands */
@ -57,7 +67,7 @@ func NewCmdLoadImage() *cobra.Command {
}
// parseLoadImageCmd parses the command input into variables required to create a cluster
func parseLoadImageCmd(cmd *cobra.Command, args []string) (runtimes.Runtime, []string, []k3d.Cluster) {
func parseLoadImageCmd(cmd *cobra.Command, args []string) (runtimes.Runtime, []string, []k3d.Cluster, bool) {
// --runtime
rt, err := cmd.Flags().GetString("runtime")
if err != nil {
@ -68,6 +78,12 @@ func parseLoadImageCmd(cmd *cobra.Command, args []string) (runtimes.Runtime, []s
log.Fatalln(err)
}
// --keep-tarball
keepTarball, err := cmd.Flags().GetBool("keep-tarball")
if err != nil {
log.Fatalln(err)
}
// --cluster
clusterNames, err := cmd.Flags().GetStringArray("cluster")
if err != nil {
@ -84,5 +100,5 @@ func parseLoadImageCmd(cmd *cobra.Command, args []string) (runtimes.Runtime, []s
log.Fatalln("No images specified!")
}
return runtime, images, clusters
return runtime, images, clusters, keepTarball
}

@ -24,6 +24,7 @@ package cluster
import (
"bytes"
"fmt"
"strconv"
"strings"
"time"
@ -42,21 +43,26 @@ func CreateCluster(cluster *k3d.Cluster, runtime k3drt.Runtime) error {
* Network
*/
// error out if external cluster network should be used but no name was set
if cluster.Network.Name == "" && cluster.Network.External {
return fmt.Errorf("Failed to use external network because no name was specified")
}
// generate cluster network name, if not set
if cluster.Network == "" {
cluster.Network = fmt.Sprintf("%s-%s", k3d.DefaultObjectNamePrefix, cluster.Name)
if cluster.Network.Name == "" && !cluster.Network.External {
cluster.Network.Name = fmt.Sprintf("%s-%s", k3d.DefaultObjectNamePrefix, cluster.Name)
}
// create cluster network or use an existing one
networkID, networkExists, err := runtime.CreateNetworkIfNotPresent(cluster.Network)
networkID, networkExists, err := runtime.CreateNetworkIfNotPresent(cluster.Network.Name)
if err != nil {
log.Errorln("Failed to create cluster network")
return err
}
cluster.Network = networkID
cluster.Network.Name = networkID
extraLabels := map[string]string{
"k3d.cluster.network": networkID,
"k3d.cluster.network.external": "false",
"k3d.cluster.network.external": strconv.FormatBool(cluster.Network.External),
}
if networkExists {
extraLabels["k3d.cluster.network.external"] = "true" // if the network wasn't created, we say that it's managed externally (important for cluster deletion)
@ -75,7 +81,7 @@ func CreateCluster(cluster *k3d.Cluster, runtime k3drt.Runtime) error {
* - image volume (for importing images)
*/
if !cluster.ClusterCreationOpts.DisableImageVolume {
imageVolumeName := fmt.Sprintf("%s-images", cluster.Name)
imageVolumeName := fmt.Sprintf("%s-%s-images", k3d.DefaultObjectNamePrefix, cluster.Name)
if err := runtime.CreateVolume(imageVolumeName, map[string]string{"k3d.cluster": cluster.Name}); err != nil {
log.Errorln("Failed to create image volume '%s' for cluster '%s'", imageVolumeName, cluster.Name)
return err
@ -120,7 +126,7 @@ func CreateCluster(cluster *k3d.Cluster, runtime k3drt.Runtime) error {
}
node.Name = generateNodeName(cluster.Name, node.Role, suffix)
node.Network = cluster.Network
node.Network = cluster.Network.Name
// create node
log.Infof("Creating node '%s'", node.Name)
@ -214,7 +220,7 @@ func DeleteCluster(cluster *k3d.Cluster, runtime k3drt.Runtime) error {
// Delete the cluster network, if it was created for/by this cluster (and if it's not in use anymore) // TODO: does this make sense or should we always try to delete it? (Will fail anyway, if it's still in use)
if network, ok := cluster.Nodes[0].Labels["k3d.cluster.network"]; ok {
if cluster.Nodes[0].Labels["k3d.cluster.network.external"] == "false" {
if !cluster.Network.External || cluster.Nodes[0].Labels["k3d.cluster.network.external"] == "false" {
log.Infof("Deleting cluster network '%s'", network)
if err := runtime.DeleteNetwork(network); err != nil {
if strings.HasSuffix(err.Error(), "active endpoints") {
@ -223,7 +229,7 @@ func DeleteCluster(cluster *k3d.Cluster, runtime k3drt.Runtime) error {
log.Warningf("Failed to delete cluster network '%s': '%+v'", network, err)
}
}
} else if cluster.Nodes[0].Labels["k3d.cluster.network.external"] == "true" {
} else if cluster.Network.External || cluster.Nodes[0].Labels["k3d.cluster.network.external"] == "true" {
log.Debugf("Skip deletion of cluster network '%s' because it's managed externally", network)
}
}
@ -270,9 +276,44 @@ func GetClusters(runtime k3drt.Runtime) ([]*k3d.Cluster, error) {
})
}
}
// enrich cluster structs with label values
for _, cluster := range clusters {
if err := populateClusterFieldsFromLabels(cluster); err != nil {
log.Warnf("Failed to populate cluster fields from node label values for cluster '%s'", cluster.Name)
log.Warnln(err)
}
}
return clusters, nil
}
// populateClusterFieldsFromLabels inspects labels attached to nodes and translates them to struct fields
func populateClusterFieldsFromLabels(cluster *k3d.Cluster) error {
networkExternalSet := false
for _, node := range cluster.Nodes {
// get the name of the cluster network
if cluster.Network.Name == "" {
if networkName, ok := node.Labels["k3d.cluster.network"]; ok {
cluster.Network.Name = networkName
}
}
// check if the network is external
// since the struct value is a bool, initialized as false, we cannot check if it's unset
if !cluster.Network.External && !networkExternalSet {
if networkExternalString, ok := node.Labels["k3d.cluster.network.external"]; ok {
if networkExternal, err := strconv.ParseBool(networkExternalString); err == nil {
cluster.Network.External = networkExternal
networkExternalSet = true
}
}
}
}
return nil
}
// GetCluster returns an existing cluster
func GetCluster(cluster *k3d.Cluster, runtime k3drt.Runtime) (*k3d.Cluster, error) {
// get nodes that belong to the selected cluster
@ -290,6 +331,11 @@ func GetCluster(cluster *k3d.Cluster, runtime k3drt.Runtime) (*k3d.Cluster, erro
cluster.Nodes = append(cluster.Nodes, node)
}
if err := populateClusterFieldsFromLabels(cluster); err != nil {
log.Warnf("Failed to populate cluster fields from node labels")
log.Warnln(err)
}
return cluster, nil
}

@ -1,41 +0,0 @@
/*
Copyright © 2019 Thorsten Klein <iwilltry42@gmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package cluster
import (
"github.com/rancher/k3d/pkg/runtimes"
k3d "github.com/rancher/k3d/pkg/types"
log "github.com/sirupsen/logrus"
)
// LoadImagesIntoCluster starts up a k3d tools container for the selected clusters and uses it to export
// images from the runtime to import them into the nodes of the selected cluster
func LoadImagesIntoCluster(runtime runtimes.Runtime, images []string, clusters *k3d.Cluster) {
}
// startToolsContainer will start a new k3d tools container and connect it to the network of the chosen cluster
func startToolsContainer(runtime runtimes.Runtime, cluster *k3d.Cluster) (string, error) {
log.Debugln("starttoolscontainer")
return "", nil
}

@ -23,3 +23,8 @@ THE SOFTWARE.
package containerd
type Containerd struct{}
// GetRuntimePath returns the path of the containerd socket
func (d Containerd) GetRuntimePath() string {
return "/run/containerd/containerd.sock"
}

@ -125,3 +125,8 @@ func (d Containerd) GetNode(node *k3d.Node) (*k3d.Node, error) {
func (d Containerd) GetNodeLogs(node *k3d.Node) (io.ReadCloser, error) {
return nil, nil
}
// ExecInNode execs a command inside a node
func (d Containerd) ExecInNode(node *k3d.Node, cmd []string) error {
return nil
}

@ -23,3 +23,8 @@ THE SOFTWARE.
package docker
type Docker struct{}
// GetRuntimePath returns the path of the docker socket
func (d Docker) GetRuntimePath() string {
return "/var/run/docker.sock"
}

@ -26,6 +26,8 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
@ -211,3 +213,86 @@ func (d Docker) GetNodeLogs(node *k3d.Node) (io.ReadCloser, error) {
return logreader, nil
}
// ExecInNode execs a command inside a node
func (d Docker) ExecInNode(node *k3d.Node, cmd []string) error {
log.Debugf("Exec cmds '%+v' in node '%s'", cmd, node.Name)
// get the container for the given node
container, err := getNodeContainer(node)
if err != nil {
return err
}
// create docker client
ctx := context.Background()
docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
log.Errorln("Failed to create docker client")
return err
}
// exec
exec, err := docker.ContainerExecCreate(ctx, container.ID, types.ExecConfig{
Privileged: true,
Tty: true,
AttachStderr: true,
AttachStdout: true,
Cmd: cmd,
})
if err != nil {
log.Errorf("Failed to create exec config for node '%s'", node.Name)
return err
}
execConnection, err := docker.ContainerExecAttach(ctx, exec.ID, types.ExecStartCheck{
Tty: true,
})
if err != nil {
log.Errorf("Failed to connect to exec process in node '%s'", node.Name)
return err
}
defer execConnection.Close()
if err := docker.ContainerExecStart(ctx, exec.ID, types.ExecStartCheck{Tty: true}); err != nil {
log.Errorf("Failed to start exec process in node '%s'", node.Name)
return err
}
for {
// get info about exec process inside container
execInfo, err := docker.ContainerExecInspect(ctx, exec.ID)
if err != nil {
log.Errorln("Failed to inspect exec process in node '%s'", node.Name)
return err
}
// if still running, continue loop
if execInfo.Running {
log.Debugf("Exec process '%+v' still running in node '%s'.. sleeping for 1 second...", cmd, node.Name)
time.Sleep(1 * time.Second)
continue
}
// check exitcode
if execInfo.ExitCode == 0 { // success
log.Debugf("Exec process in node '%s' exited with '0'", node.Name)
break
}
if execInfo.ExitCode != 0 { // failed
logs, err := ioutil.ReadAll(execConnection.Reader)
if err != nil {
log.Errorln("Failed to get logs from node '%s'", node.Name)
return err
}
return fmt.Errorf("Logs from failed access process:\n%s", string(logs))
}
}
return nil
}

@ -49,7 +49,8 @@ type Runtime interface {
StopNode(*k3d.Node) error
CreateVolume(string, map[string]string) error
DeleteVolume(string) error
// ExecContainer() error
GetRuntimePath() string // returns e.g. '/var/run/docker.sock' for a default docker setup
ExecInNode(*k3d.Node, []string) error
// DeleteContainer() error
GetNodeLogs(*k3d.Node) (io.ReadCloser, error)
}

@ -0,0 +1,129 @@
/*
Copyright © 2019 Thorsten Klein <iwilltry42@gmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package tools
import (
"fmt"
"sync"
"time"
k3dc "github.com/rancher/k3d/pkg/cluster"
"github.com/rancher/k3d/pkg/runtimes"
k3d "github.com/rancher/k3d/pkg/types"
log "github.com/sirupsen/logrus"
)
// LoadImagesIntoCluster starts up a k3d tools container for the selected cluster and uses it to export
// images from the runtime to import them into the nodes of the selected cluster
func LoadImagesIntoCluster(runtime runtimes.Runtime, images []string, cluster *k3d.Cluster, keepTarball bool) error {
cluster, err := k3dc.GetCluster(cluster, runtime)
if err != nil {
log.Errorf("Failed to get cluster '%s'", cluster.Name)
return err
}
if cluster.Network.Name == "" {
return fmt.Errorf("Failed to get network for cluster '%s'", cluster.Name)
}
if _, ok := cluster.Nodes[0].Labels["k3d.cluster.volumes.imagevolume"]; !ok { // TODO: add failover solution
return fmt.Errorf("Failed to find image volume for cluster '%s'", cluster.Name)
}
imageVolume := cluster.Nodes[0].Labels["k3d.cluster.volumes.imagevolume"]
// create tools node to export images
log.Infoln("Starting k3d-tools node...")
toolsNode, err := startToolsNode( // TODO: re-use existing container
runtime,
cluster,
cluster.Network.Name,
[]string{
fmt.Sprintf("%s:%s", imageVolume, k3d.DefaultImageVolumeMountPath),
fmt.Sprintf("%s:%s", runtime.GetRuntimePath(), runtime.GetRuntimePath()),
})
if err != nil {
log.Errorf("Failed to start tools container for cluster '%s'", cluster.Name)
}
// save image to tarfile in shared volume
log.Infoln("Saving images...")
tarName := fmt.Sprintf("%s/k3d-%s-images-%s.tar", k3d.DefaultImageVolumeMountPath, cluster.Name, time.Now().Format("20060102150405")) // FIXME: change
if err := runtime.ExecInNode(toolsNode, append([]string{"./k3d-tools", "save-image", "-d", tarName}, images...)); err != nil {
log.Errorf("Failed to save images in tools container for cluster '%s'", cluster.Name)
return err
}
// import image in each node
log.Infoln("Importing images into nodes...")
var importWaitgroup sync.WaitGroup
for _, node := range cluster.Nodes {
importWaitgroup.Add(1)
go func(node *k3d.Node, wg *sync.WaitGroup) {
log.Infof("Importing images into node '%s'...", node.Name)
if err := runtime.ExecInNode(node, []string{"ctr", "image", "import", tarName}); err != nil {
log.Errorf("Failed to import images in node '%s'", node.Name)
log.Errorln(err)
}
wg.Done()
}(node, &importWaitgroup)
}
importWaitgroup.Wait()
// remove tarball
if !keepTarball {
log.Infoln("Removing the tarball...")
if err := runtime.ExecInNode(cluster.Nodes[0], []string{"rm", "-f", tarName}); err != nil { // TODO: do this in tools node (requires rm)
log.Errorf("Failed to delete tarball '%s'", tarName)
log.Errorln(err)
}
}
// delete tools container
log.Infoln("Removing k3d-tools node...")
if err := runtime.DeleteNode(toolsNode); err != nil {
log.Errorln("Failed to delete tools node '%s': Try to delete it manually", toolsNode.Name)
}
log.Infoln("...Done")
return nil
}
// startToolsNode will start a new k3d tools container and connect it to the network of the chosen cluster
func startToolsNode(runtime runtimes.Runtime, cluster *k3d.Cluster, network string, volumes []string) (*k3d.Node, error) {
node := &k3d.Node{
Name: fmt.Sprintf("%s-%s-tools", k3d.DefaultObjectNamePrefix, cluster.Name),
Image: k3d.DefaultToolsContainerImage,
Role: k3d.NoRole,
Volumes: volumes,
Network: network,
Cmd: []string{},
Args: []string{"noop"},
}
if err := runtime.CreateNode(node); err != nil {
log.Errorf("Failed to create tools container for cluster '%s'", cluster.Name)
return node, err
}
return node, nil
}

@ -48,6 +48,7 @@ type Role string
const (
MasterRole Role = "master"
WorkerRole Role = "worker"
NoRole Role = "nope"
)
// DefaultK3dRoles defines the roles available for nodes
@ -72,6 +73,9 @@ var DefaultNodeEnv = []string{
"K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml",
}
// DefaultToolsContainerImage defines the default image used for the tools container
const DefaultToolsContainerImage = "docker.io/iwilltry42/k3d-tools:v0.0.2" // TODO: get version dynamically or at build time
// DefaultImageVolumeMountPath defines the mount path inside k3d nodes where we will mount the shared image volume by default
const DefaultImageVolumeMountPath = "/k3d/images"
@ -80,10 +84,16 @@ type ClusterCreationOpts struct {
DisableImageVolume bool
}
// ClusterNetwork describes a network which a cluster is running in
type ClusterNetwork struct {
Name string `yaml:"name" json:"name,omitempty"`
External bool `yaml:"external" json:"isExternal,omitempty"`
}
// Cluster describes a k3d cluster
type Cluster struct {
Name string `yaml:"name" json:"name,omitempty"`
Network string `yaml:"network" json:"network,omitempty"`
Network ClusterNetwork `yaml:"network" json:"network,omitempty"`
Secret string `yaml:"cluster_secret" json:"clusterSecret,omitempty"`
Nodes []*Node `yaml:"nodes" json:"nodes,omitempty"`
InitNode *Node // init master node

@ -186,8 +186,17 @@ Here's how k3d types should translate to a runtime type:
- e.g. `k3d tools import-images`
- let's you set tools container version
- `k3d tools --image k3d-tools:v2 import-images`
- add `k3d create --image-vol NAME` flag to re-use existing image volume
- will add `k3d.volumes.imagevolume.external: true` label to nodes
- should not be deleted with cluster
- possibly add `k3d create volume` and `k3d create network` to create external networks?
## extra commands
- `k3d prune` to prune all dangling resources
- nodes, volumes, networks
## use OCI
- [https://github.com/opencontainers/runtime-spec/blob/master/specs-go/config.go](https://github.com/opencontainers/runtime-spec/blob/master/specs-go/config.go)
- move node -> container translation out of runtime

Loading…
Cancel
Save