Merge pull request #453 from rancher/feature/managed-registry-with-registries-file

cmd/cluster/clusterCreate.go

@@ -26,6 +26,7 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
@@ -209,6 +210,7 @@ func NewCmdClusterCreate() *cobra.Command {
 	/* Registry */
 	cmd.Flags().StringArrayVar(&cliConfig.Registries.Use, "registry-use", nil, "Connect to one or more k3d-managed registries running locally")
 	cmd.Flags().BoolVar(&cliConfig.Registries.Create, "registry-create", false, "Create a k3d-managed registry and connect it to the cluster")
+	cmd.Flags().StringVar(&cliConfig.Registries.Config, "registry-config", "", "Specify path to an extra registries.yaml file")
 
 	/* Multi Server Configuration */
 
@@ -277,6 +279,10 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, cliConfig *conf.Si
 			log.Fatalln(err)
 		}
 
+		if strings.Contains(volume, k3d.DefaultRegistriesFilePath) && (cliConfig.Registries.Create || cliConfig.Registries.Config != "" || len(cliConfig.Registries.Use) != 0) {
+			log.Warnf("Seems like you're mounting a file at '%s' while also using a referenced registries config or k3d-managed registries: Your mounted file will probably be overwritten!", k3d.DefaultRegistriesFilePath)
+		}
+
 		// create new entry or append filter to existing entry
 		if _, exists := volumeFilterMap[volume]; exists {
 			volumeFilterMap[volume] = append(volumeFilterMap[volume], filters...)

docs/usage/guides/registries.md

@@ -2,8 +2,11 @@
 
 ## Registries configuration file
 
-You can add registries by specifying them in a `registries.yaml` and mounting them at creation time:
-`#!bash k3d cluster create mycluster --volume "/home/YOU/my-registries.yaml:/etc/rancher/k3s/registries.yaml"`.
+You can add registries by specifying them in a `registries.yaml` and referencing it at creation time:
+`#!bash k3d cluster create mycluster --registry-config "/home/YOU/my-registries.yaml"`.
+
+??? Tip "Pre v4.0.0 solution"
+    Before we added the `--registry-config` flag in k3d v4.0.0, you had to bind-mount the file to the correct location: `--volume "/home/YOU/my-registries.yaml:/etc/rancher/k3s/registries.yaml"`
 
 This file is a regular [k3s registries configuration file](https://rancher.com/docs/k3s/latest/en/installation/private-registry/), and looks like this:
 
@@ -21,6 +24,27 @@ Note well there is an important limitation: **this configuration file will only
 
 This file can also be used for providing additional information necessary for accessing some registries, like [authentication](#authenticated-registries) and [certificates](#secure-registries).
 
+### Registries Configuration File embedded in k3d's SimpleConfig
+
+If you're using a `SimpleConfig` file to configure your k3d cluster, you may as well embed the registries.yaml in there directly:
+
+```yaml
+apiVersion: k3d.io/v1alpha1
+kind: Simple
+name: test
+servers: 1
+agents: 2
+registries:
+  create: true
+  config: |
+    mirrors:
+      "my.company.registry":
+        endpoint:
+          - http://my.company.registry:5000
+```
+
+Here, the config for the k3d-managed registry, created by the `create: true` flag will be merged with the config specified under `config: |`.
+
 ### Authenticated registries
 
 When using authenticated registries, we can add the _username_ and _password_ in a

pkg/client/cluster.go

@@ -42,6 +42,7 @@ import (
 	runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors"
 	"github.com/rancher/k3d/v4/pkg/types"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v4/pkg/types/k3s"
 	"github.com/rancher/k3d/v4/pkg/util"
 	"github.com/rancher/k3d/v4/version"
 	log "github.com/sirupsen/logrus"
@@ -97,8 +98,10 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi
 	}
 
 	// create the registry hosting configmap
-	if err := prepCreateLocalRegistryHostingConfigMap(ctx, runtime, &clusterConfig.Cluster); err != nil {
-		log.Warnf("Failed to create LocalRegistryHosting ConfigMap: %+v", err)
+	if len(clusterConfig.ClusterCreateOpts.Registries.Use) > 0 {
+		if err := prepCreateLocalRegistryHostingConfigMap(ctx, runtime, &clusterConfig.Cluster); err != nil {
+			log.Warnf("Failed to create LocalRegistryHosting ConfigMap: %+v", err)
+		}
 	}
 
 	return nil
@@ -171,6 +174,8 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf
 	// Use existing registries (including the new one, if created)
 	log.Tracef("Using Registries: %+v", clusterConfig.ClusterCreateOpts.Registries.Use)
 
+	var registryConfig *k3s.Registry
+
 	if len(clusterConfig.ClusterCreateOpts.Registries.Use) > 0 {
 		// ensure that all selected registries exist and connect them to the cluster network
 		for _, externalReg := range clusterConfig.ClusterCreateOpts.Registries.Use {
@@ -188,34 +193,49 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf
 		if err != nil {
 			return fmt.Errorf("Failed to generate registry config file for k3s: %+v", err)
 		}
-		regConfBytes, err := yaml.Marshal(&regConf)
+
+		// generate the LocalRegistryHosting configmap
+		regCm, err := RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx, clusterConfig.ClusterCreateOpts.Registries.Use)
 		if err != nil {
-			return fmt.Errorf("Failed to marshal registry configuration: %+v", err)
+			return fmt.Errorf("Failed to generate LocalRegistryHosting configmap: %+v", err)
 		}
+		log.Tracef("Writing LocalRegistryHosting YAML:\n%s", string(regCm))
 		clusterConfig.ClusterCreateOpts.NodeHooks = append(clusterConfig.ClusterCreateOpts.NodeHooks, k3d.NodeHook{
 			Stage: k3d.LifecycleStagePreStart,
 			Action: actions.WriteFileAction{
 				Runtime: runtime,
-				Content: regConfBytes,
-				Dest:    k3d.DefaultRegistriesFilePath,
+				Content: regCm,
+				Dest:    "/tmp/reg.yaml",
 			},
 		})
 
-		// generate the LocalRegistryHosting configmap
-		regCm, err := RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx, clusterConfig.ClusterCreateOpts.Registries.Use)
+		registryConfig = regConf
+
+	}
+	// merge with pre-existing, referenced registries.yaml
+	if clusterConfig.ClusterCreateOpts.Registries.Config != nil {
+		if registryConfig != nil {
+			if err := RegistryMergeConfig(ctx, registryConfig, clusterConfig.ClusterCreateOpts.Registries.Config); err != nil {
+				return err
+			}
+			log.Tracef("Merged registry config: %+v", registryConfig)
+		} else {
+			registryConfig = clusterConfig.ClusterCreateOpts.Registries.Config
+		}
+	}
+	if registryConfig != nil {
+		regConfBytes, err := yaml.Marshal(&registryConfig)
 		if err != nil {
-			return fmt.Errorf("Failed to generate LocalRegistryHosting configmap: %+v", err)
+			return fmt.Errorf("Failed to marshal registry configuration: %+v", err)
 		}
-		log.Tracef("Writing LocalRegistryHosting YAML:\n%s", string(regCm))
 		clusterConfig.ClusterCreateOpts.NodeHooks = append(clusterConfig.ClusterCreateOpts.NodeHooks, k3d.NodeHook{
 			Stage: k3d.LifecycleStagePreStart,
 			Action: actions.WriteFileAction{
 				Runtime: runtime,
-				Content: regCm,
-				Dest:    "/tmp/reg.yaml",
+				Content: regConfBytes,
+				Dest:    k3d.DefaultRegistriesFilePath,
 			},
 		})
-
 	}
 
 	return nil

pkg/client/registry.go

@@ -26,6 +26,7 @@ import (
 	"fmt"
 
 	"github.com/docker/go-connections/nat"
+	"github.com/imdario/mergo"
 	"github.com/rancher/k3d/v4/pkg/runtimes"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
 	"github.com/rancher/k3d/v4/pkg/types/k3s"
@@ -305,3 +306,11 @@ func RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx context.Context, regi
 
 	return cmYaml, nil
 }
+
+// RegistryMergeConfig merges a source registry config into an existing dest registry cofnig
+func RegistryMergeConfig(ctx context.Context, dest, src *k3s.Registry) error {
+	if err := mergo.MergeWithOverwrite(dest, src); err != nil {
+		return fmt.Errorf("Failed to merge registry configs: %+v", err)
+	}
+	return nil
+}

pkg/config/transform.go

@@ -25,14 +25,19 @@ package config
 import (
 	"context"
 	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
 
 	"github.com/docker/go-connections/nat"
 	cliutil "github.com/rancher/k3d/v4/cmd/util" // TODO: move parseapiport to pkg
 	conf "github.com/rancher/k3d/v4/pkg/config/v1alpha1"
 	"github.com/rancher/k3d/v4/pkg/runtimes"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v4/pkg/types/k3s"
 	"github.com/rancher/k3d/v4/pkg/util"
 	"github.com/rancher/k3d/v4/version"
+	"gopkg.in/yaml.v2"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -251,6 +256,33 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
 		clusterCreateOpts.Registries.Use = append(clusterCreateOpts.Registries.Use, reg)
 	}
 
+	if simpleConfig.Registries.Config != "" {
+		var k3sRegistry *k3s.Registry
+
+		if strings.Contains(simpleConfig.Registries.Config, "\n") { // CASE 1: embedded registries.yaml (multiline string)
+			log.Debugf("Found multiline registries config embedded in SimpleConfig:\n%s", simpleConfig.Registries.Config)
+			if err := yaml.Unmarshal([]byte(simpleConfig.Registries.Config), &k3sRegistry); err != nil {
+				return nil, fmt.Errorf("Failed to read embedded registries config: %+v", err)
+			}
+		} else { // CASE 2: registries.yaml file referenced by path (single line)
+			registryConfigFile, err := os.Open(simpleConfig.Registries.Config)
+			if err != nil {
+				return nil, fmt.Errorf("Failed to open registry config file at %s: %+v", simpleConfig.Registries.Config, err)
+			}
+			configBytes, err := ioutil.ReadAll(registryConfigFile)
+			if err != nil {
+				return nil, fmt.Errorf("Failed to read registry config file at %s: %+v", registryConfigFile.Name(), err)
+			}
+
+			if err := yaml.Unmarshal(configBytes, &k3sRegistry); err != nil {
+				return nil, fmt.Errorf("Failed to read registry configuration: %+v", err)
+			}
+		}
+
+		log.Tracef("Registry: read config from input:\n%+v", k3sRegistry)
+		clusterCreateOpts.Registries.Config = k3sRegistry
+	}
+
 	/**********************
 	 * Kubeconfig Options *
 	 **********************/

pkg/config/v1alpha1/types.go

@@ -128,6 +128,7 @@ type SimpleConfig struct {
 	Registries   struct {
 		Use    []string `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"`
 		Create bool     `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"`
+		Config string   `mapstructure:"config" yaml:"config,omitempty" json:"config,omitempty"` // registries.yaml (k3s config for containerd registry override)
 	} `mapstructure:"registries" yaml:"registries,omitempty" json:"registries,omitempty"`
 }
 

pkg/types/types.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/docker/go-connections/nat"
+	"github.com/rancher/k3d/v4/pkg/types/k3s"
 )
 
 // DefaultClusterName specifies the default name used for newly created clusters
@@ -179,8 +180,9 @@ type ClusterCreateOpts struct {
 	GlobalLabels               map[string]string `yaml:"globalLabels,omitempty" json:"globalLabels,omitempty"`
 	GlobalEnv                  []string          `yaml:"globalEnv,omitempty" json:"globalEnv,omitempty"`
 	Registries                 struct {
-		Create *Registry   `yaml:"create,omitempty" json:"create,omitempty"`
-		Use    []*Registry `yaml:"use,omitempty" json:"use,omitempty"`
+		Create *Registry     `yaml:"create,omitempty" json:"create,omitempty"`
+		Use    []*Registry   `yaml:"use,omitempty" json:"use,omitempty"`
+		Config *k3s.Registry `yaml:"config,omitempty" json:"config,omitempty"` // registries.yaml (k3s config for containerd registry override)
 	} `yaml:"registries,omitempty" json:"registries,omitempty"`
 }
 

pkg/util/filter.go

@@ -74,8 +74,9 @@ func FilterNodes(nodes []*k3d.Node, filters []string) ([]*k3d.Node, error) {
 
 		// if one of the filters is 'all', we only return this and drop all others
 		if submatches["group"] == "all" {
-			// TODO: filterNodes: only log if really more than one is specified
-			log.Warnf("Node filter 'all' set, but more were specified in '%+v'", filters)
+			if len(filters) > 1 {
+				log.Warnf("Node filter 'all' set, but more were specified in '%+v'", filters)
+			}
 			return nodes, nil
 		}
 

tests/assets/config_test_simple.yaml

@@ -30,6 +30,11 @@ labels:
 registries:
   create: true
   use: []
+  config: |
+    mirrors:
+      "my.company.registry":
+        endpoint:
+          - http://my.company.registry:5000
 
 options:
   k3d:

tests/assets/test_registries.yaml

@@ -0,0 +1,4 @@
+mirrors:
+  "my.company.registry":
+    endpoint:
+      - http://my.company.registry:5000

tests/common.sh

@@ -82,7 +82,7 @@ check_clusters() {
 
 check_cluster_count() {
   expectedClusterCount=$1
-  actualClusterCount=$($EXE cluster list --no-headers | wc -l)
+  actualClusterCount=$(LOG_LEVEL=warn $EXE cluster list --no-headers | wc -l) # this must always have a loglevel of <= warn or it will fail
   if [[ $actualClusterCount != $expectedClusterCount ]]; then
     failed "incorrect number of clusters available: $actualClusterCount != $expectedClusterCount"
     return 1
@@ -168,7 +168,7 @@ wait_for_pod_exec() {
 exec_in_node() {
   # $1 = container/node name
   # $2 = command
-  docker exec "$1" "$2"
+  docker exec "$1" sh -c "$2"
 }
 
 docker_assert_container_label() {

tests/test_config_file.sh

@@ -47,6 +47,11 @@ docker_assert_container_label "k3d-$clustername-server-0" "foo=bar" || failed "E
 info "Ensuring, that we have a registry node present"
 $EXE node list "k3d-$clustername-registry" || failed "Expected k3d-$clustername-registry to be present"
 
+## merged registries.yaml
+info "Ensuring, that the registries.yaml file contains both registries"
+exec_in_node "k3d-$clustername-server-0" "cat /etc/rancher/k3s/registries.yaml" | grep -i "my.company.registry"  || failed "Expected 'my.company.registry' to be in the /etc/rancher/k3s/registries.yaml"
+exec_in_node "k3d-$clustername-server-0" "cat /etc/rancher/k3s/registries.yaml" | grep -i "k3d-$clustername-registry" || failed "Expected 'k3d-$clustername-registry' to be in the /etc/rancher/k3s/registries.yaml"
+
 # Cleanup
 
 info "Deleting cluster $clustername..."

tests/test_full_lifecycle.sh

@@ -24,7 +24,7 @@ info "Creating cluster $clustername..."
 $EXE cluster create "$clustername" --agents 1 --api-port 6443 --wait --timeout 360s $EXTRA_FLAG || failed "could not create cluster $clustername $EXTRA_TITLE"
 
 info "Sleeping for 5 seconds to give the cluster enough time to get ready..."
-sleep 5
+sleep 10
 
 # 1. check initial access to the cluster
 info "Checking that we have access to the cluster..."