diff --git a/cmd/cluster/clusterCreate.go b/cmd/cluster/clusterCreate.go index 53165ddb..adbef273 100644 --- a/cmd/cluster/clusterCreate.go +++ b/cmd/cluster/clusterCreate.go @@ -26,6 +26,7 @@ import ( "fmt" "os" "runtime" + "strings" "time" "github.com/spf13/cobra" @@ -209,6 +210,7 @@ func NewCmdClusterCreate() *cobra.Command { /* Registry */ cmd.Flags().StringArrayVar(&cliConfig.Registries.Use, "registry-use", nil, "Connect to one or more k3d-managed registries running locally") cmd.Flags().BoolVar(&cliConfig.Registries.Create, "registry-create", false, "Create a k3d-managed registry and connect it to the cluster") + cmd.Flags().StringVar(&cliConfig.Registries.Config, "registry-config", "", "Specify path to an extra registries.yaml file") /* Multi Server Configuration */ @@ -277,6 +279,10 @@ func parseCreateClusterCmd(cmd *cobra.Command, args []string, cliConfig *conf.Si log.Fatalln(err) } + if strings.Contains(volume, k3d.DefaultRegistriesFilePath) && (cliConfig.Registries.Create || cliConfig.Registries.Config != "" || len(cliConfig.Registries.Use) != 0) { + log.Warnf("Seems like you're mounting a file at '%s' while also using a referenced registries config or k3d-managed registries: Your mounted file will probably be overwritten!", k3d.DefaultRegistriesFilePath) + } + // create new entry or append filter to existing entry if _, exists := volumeFilterMap[volume]; exists { volumeFilterMap[volume] = append(volumeFilterMap[volume], filters...) diff --git a/docs/usage/guides/registries.md b/docs/usage/guides/registries.md index 66c336a1..22da3ede 100644 --- a/docs/usage/guides/registries.md +++ b/docs/usage/guides/registries.md @@ -2,8 +2,11 @@ ## Registries configuration file -You can add registries by specifying them in a `registries.yaml` and mounting them at creation time: -`#!bash k3d cluster create mycluster --volume "/home/YOU/my-registries.yaml:/etc/rancher/k3s/registries.yaml"`. +You can add registries by specifying them in a `registries.yaml` and referencing it at creation time: +`#!bash k3d cluster create mycluster --registry-config "/home/YOU/my-registries.yaml"`. + +??? Tip "Pre v4.0.0 solution" + Before we added the `--registry-config` flag in k3d v4.0.0, you had to bind-mount the file to the correct location: `--volume "/home/YOU/my-registries.yaml:/etc/rancher/k3s/registries.yaml"` This file is a regular [k3s registries configuration file](https://rancher.com/docs/k3s/latest/en/installation/private-registry/), and looks like this: @@ -21,6 +24,27 @@ Note well there is an important limitation: **this configuration file will only This file can also be used for providing additional information necessary for accessing some registries, like [authentication](#authenticated-registries) and [certificates](#secure-registries). +### Registries Configuration File embedded in k3d's SimpleConfig + +If you're using a `SimpleConfig` file to configure your k3d cluster, you may as well embed the registries.yaml in there directly: + +```yaml +apiVersion: k3d.io/v1alpha1 +kind: Simple +name: test +servers: 1 +agents: 2 +registries: + create: true + config: | + mirrors: + "my.company.registry": + endpoint: + - http://my.company.registry:5000 +``` + +Here, the config for the k3d-managed registry, created by the `create: true` flag will be merged with the config specified under `config: |`. + ### Authenticated registries When using authenticated registries, we can add the _username_ and _password_ in a diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go index 4ce4950d..6047070b 100644 --- a/pkg/client/cluster.go +++ b/pkg/client/cluster.go @@ -42,6 +42,7 @@ import ( runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors" "github.com/rancher/k3d/v4/pkg/types" k3d "github.com/rancher/k3d/v4/pkg/types" + "github.com/rancher/k3d/v4/pkg/types/k3s" "github.com/rancher/k3d/v4/pkg/util" "github.com/rancher/k3d/v4/version" log "github.com/sirupsen/logrus" @@ -97,8 +98,10 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi } // create the registry hosting configmap - if err := prepCreateLocalRegistryHostingConfigMap(ctx, runtime, &clusterConfig.Cluster); err != nil { - log.Warnf("Failed to create LocalRegistryHosting ConfigMap: %+v", err) + if len(clusterConfig.ClusterCreateOpts.Registries.Use) > 0 { + if err := prepCreateLocalRegistryHostingConfigMap(ctx, runtime, &clusterConfig.Cluster); err != nil { + log.Warnf("Failed to create LocalRegistryHosting ConfigMap: %+v", err) + } } return nil @@ -171,6 +174,8 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf // Use existing registries (including the new one, if created) log.Tracef("Using Registries: %+v", clusterConfig.ClusterCreateOpts.Registries.Use) + var registryConfig *k3s.Registry + if len(clusterConfig.ClusterCreateOpts.Registries.Use) > 0 { // ensure that all selected registries exist and connect them to the cluster network for _, externalReg := range clusterConfig.ClusterCreateOpts.Registries.Use { @@ -188,34 +193,49 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf if err != nil { return fmt.Errorf("Failed to generate registry config file for k3s: %+v", err) } - regConfBytes, err := yaml.Marshal(®Conf) + + // generate the LocalRegistryHosting configmap + regCm, err := RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx, clusterConfig.ClusterCreateOpts.Registries.Use) if err != nil { - return fmt.Errorf("Failed to marshal registry configuration: %+v", err) + return fmt.Errorf("Failed to generate LocalRegistryHosting configmap: %+v", err) } + log.Tracef("Writing LocalRegistryHosting YAML:\n%s", string(regCm)) clusterConfig.ClusterCreateOpts.NodeHooks = append(clusterConfig.ClusterCreateOpts.NodeHooks, k3d.NodeHook{ Stage: k3d.LifecycleStagePreStart, Action: actions.WriteFileAction{ Runtime: runtime, - Content: regConfBytes, - Dest: k3d.DefaultRegistriesFilePath, + Content: regCm, + Dest: "/tmp/reg.yaml", }, }) - // generate the LocalRegistryHosting configmap - regCm, err := RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx, clusterConfig.ClusterCreateOpts.Registries.Use) + registryConfig = regConf + + } + // merge with pre-existing, referenced registries.yaml + if clusterConfig.ClusterCreateOpts.Registries.Config != nil { + if registryConfig != nil { + if err := RegistryMergeConfig(ctx, registryConfig, clusterConfig.ClusterCreateOpts.Registries.Config); err != nil { + return err + } + log.Tracef("Merged registry config: %+v", registryConfig) + } else { + registryConfig = clusterConfig.ClusterCreateOpts.Registries.Config + } + } + if registryConfig != nil { + regConfBytes, err := yaml.Marshal(®istryConfig) if err != nil { - return fmt.Errorf("Failed to generate LocalRegistryHosting configmap: %+v", err) + return fmt.Errorf("Failed to marshal registry configuration: %+v", err) } - log.Tracef("Writing LocalRegistryHosting YAML:\n%s", string(regCm)) clusterConfig.ClusterCreateOpts.NodeHooks = append(clusterConfig.ClusterCreateOpts.NodeHooks, k3d.NodeHook{ Stage: k3d.LifecycleStagePreStart, Action: actions.WriteFileAction{ Runtime: runtime, - Content: regCm, - Dest: "/tmp/reg.yaml", + Content: regConfBytes, + Dest: k3d.DefaultRegistriesFilePath, }, }) - } return nil diff --git a/pkg/client/registry.go b/pkg/client/registry.go index 05a8f557..f14d79ca 100644 --- a/pkg/client/registry.go +++ b/pkg/client/registry.go @@ -26,6 +26,7 @@ import ( "fmt" "github.com/docker/go-connections/nat" + "github.com/imdario/mergo" "github.com/rancher/k3d/v4/pkg/runtimes" k3d "github.com/rancher/k3d/v4/pkg/types" "github.com/rancher/k3d/v4/pkg/types/k3s" @@ -305,3 +306,11 @@ func RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx context.Context, regi return cmYaml, nil } + +// RegistryMergeConfig merges a source registry config into an existing dest registry cofnig +func RegistryMergeConfig(ctx context.Context, dest, src *k3s.Registry) error { + if err := mergo.MergeWithOverwrite(dest, src); err != nil { + return fmt.Errorf("Failed to merge registry configs: %+v", err) + } + return nil +} diff --git a/pkg/config/transform.go b/pkg/config/transform.go index 8dbcc171..095ff5e0 100644 --- a/pkg/config/transform.go +++ b/pkg/config/transform.go @@ -25,14 +25,19 @@ package config import ( "context" "fmt" + "io/ioutil" + "os" + "strings" "github.com/docker/go-connections/nat" cliutil "github.com/rancher/k3d/v4/cmd/util" // TODO: move parseapiport to pkg conf "github.com/rancher/k3d/v4/pkg/config/v1alpha1" "github.com/rancher/k3d/v4/pkg/runtimes" k3d "github.com/rancher/k3d/v4/pkg/types" + "github.com/rancher/k3d/v4/pkg/types/k3s" "github.com/rancher/k3d/v4/pkg/util" "github.com/rancher/k3d/v4/version" + "gopkg.in/yaml.v2" log "github.com/sirupsen/logrus" ) @@ -251,6 +256,33 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim clusterCreateOpts.Registries.Use = append(clusterCreateOpts.Registries.Use, reg) } + if simpleConfig.Registries.Config != "" { + var k3sRegistry *k3s.Registry + + if strings.Contains(simpleConfig.Registries.Config, "\n") { // CASE 1: embedded registries.yaml (multiline string) + log.Debugf("Found multiline registries config embedded in SimpleConfig:\n%s", simpleConfig.Registries.Config) + if err := yaml.Unmarshal([]byte(simpleConfig.Registries.Config), &k3sRegistry); err != nil { + return nil, fmt.Errorf("Failed to read embedded registries config: %+v", err) + } + } else { // CASE 2: registries.yaml file referenced by path (single line) + registryConfigFile, err := os.Open(simpleConfig.Registries.Config) + if err != nil { + return nil, fmt.Errorf("Failed to open registry config file at %s: %+v", simpleConfig.Registries.Config, err) + } + configBytes, err := ioutil.ReadAll(registryConfigFile) + if err != nil { + return nil, fmt.Errorf("Failed to read registry config file at %s: %+v", registryConfigFile.Name(), err) + } + + if err := yaml.Unmarshal(configBytes, &k3sRegistry); err != nil { + return nil, fmt.Errorf("Failed to read registry configuration: %+v", err) + } + } + + log.Tracef("Registry: read config from input:\n%+v", k3sRegistry) + clusterCreateOpts.Registries.Config = k3sRegistry + } + /********************** * Kubeconfig Options * **********************/ diff --git a/pkg/config/v1alpha1/types.go b/pkg/config/v1alpha1/types.go index 80c4158d..a120fcb1 100644 --- a/pkg/config/v1alpha1/types.go +++ b/pkg/config/v1alpha1/types.go @@ -128,6 +128,7 @@ type SimpleConfig struct { Registries struct { Use []string `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"` Create bool `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"` + Config string `mapstructure:"config" yaml:"config,omitempty" json:"config,omitempty"` // registries.yaml (k3s config for containerd registry override) } `mapstructure:"registries" yaml:"registries,omitempty" json:"registries,omitempty"` } diff --git a/pkg/types/types.go b/pkg/types/types.go index ff6410f3..90a985ff 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -27,6 +27,7 @@ import ( "time" "github.com/docker/go-connections/nat" + "github.com/rancher/k3d/v4/pkg/types/k3s" ) // DefaultClusterName specifies the default name used for newly created clusters @@ -179,8 +180,9 @@ type ClusterCreateOpts struct { GlobalLabels map[string]string `yaml:"globalLabels,omitempty" json:"globalLabels,omitempty"` GlobalEnv []string `yaml:"globalEnv,omitempty" json:"globalEnv,omitempty"` Registries struct { - Create *Registry `yaml:"create,omitempty" json:"create,omitempty"` - Use []*Registry `yaml:"use,omitempty" json:"use,omitempty"` + Create *Registry `yaml:"create,omitempty" json:"create,omitempty"` + Use []*Registry `yaml:"use,omitempty" json:"use,omitempty"` + Config *k3s.Registry `yaml:"config,omitempty" json:"config,omitempty"` // registries.yaml (k3s config for containerd registry override) } `yaml:"registries,omitempty" json:"registries,omitempty"` } diff --git a/pkg/util/filter.go b/pkg/util/filter.go index 3416e5d1..262fbf73 100644 --- a/pkg/util/filter.go +++ b/pkg/util/filter.go @@ -74,8 +74,9 @@ func FilterNodes(nodes []*k3d.Node, filters []string) ([]*k3d.Node, error) { // if one of the filters is 'all', we only return this and drop all others if submatches["group"] == "all" { - // TODO: filterNodes: only log if really more than one is specified - log.Warnf("Node filter 'all' set, but more were specified in '%+v'", filters) + if len(filters) > 1 { + log.Warnf("Node filter 'all' set, but more were specified in '%+v'", filters) + } return nodes, nil } diff --git a/tests/assets/config_test_simple.yaml b/tests/assets/config_test_simple.yaml index 6f98dc1f..dc8351e8 100644 --- a/tests/assets/config_test_simple.yaml +++ b/tests/assets/config_test_simple.yaml @@ -30,6 +30,11 @@ labels: registries: create: true use: [] + config: | + mirrors: + "my.company.registry": + endpoint: + - http://my.company.registry:5000 options: k3d: diff --git a/tests/assets/test_registries.yaml b/tests/assets/test_registries.yaml new file mode 100644 index 00000000..c9fa033a --- /dev/null +++ b/tests/assets/test_registries.yaml @@ -0,0 +1,4 @@ +mirrors: + "my.company.registry": + endpoint: + - http://my.company.registry:5000 \ No newline at end of file diff --git a/tests/common.sh b/tests/common.sh index 37fb218b..fe5227c9 100755 --- a/tests/common.sh +++ b/tests/common.sh @@ -82,7 +82,7 @@ check_clusters() { check_cluster_count() { expectedClusterCount=$1 - actualClusterCount=$($EXE cluster list --no-headers | wc -l) + actualClusterCount=$(LOG_LEVEL=warn $EXE cluster list --no-headers | wc -l) # this must always have a loglevel of <= warn or it will fail if [[ $actualClusterCount != $expectedClusterCount ]]; then failed "incorrect number of clusters available: $actualClusterCount != $expectedClusterCount" return 1 @@ -168,7 +168,7 @@ wait_for_pod_exec() { exec_in_node() { # $1 = container/node name # $2 = command - docker exec "$1" "$2" + docker exec "$1" sh -c "$2" } docker_assert_container_label() { diff --git a/tests/test_config_file.sh b/tests/test_config_file.sh index ef036348..f39f74a3 100755 --- a/tests/test_config_file.sh +++ b/tests/test_config_file.sh @@ -47,6 +47,11 @@ docker_assert_container_label "k3d-$clustername-server-0" "foo=bar" || failed "E info "Ensuring, that we have a registry node present" $EXE node list "k3d-$clustername-registry" || failed "Expected k3d-$clustername-registry to be present" +## merged registries.yaml +info "Ensuring, that the registries.yaml file contains both registries" +exec_in_node "k3d-$clustername-server-0" "cat /etc/rancher/k3s/registries.yaml" | grep -i "my.company.registry" || failed "Expected 'my.company.registry' to be in the /etc/rancher/k3s/registries.yaml" +exec_in_node "k3d-$clustername-server-0" "cat /etc/rancher/k3s/registries.yaml" | grep -i "k3d-$clustername-registry" || failed "Expected 'k3d-$clustername-registry' to be in the /etc/rancher/k3s/registries.yaml" + # Cleanup info "Deleting cluster $clustername..." diff --git a/tests/test_full_lifecycle.sh b/tests/test_full_lifecycle.sh index f64bcf2d..d4fd8db6 100755 --- a/tests/test_full_lifecycle.sh +++ b/tests/test_full_lifecycle.sh @@ -24,7 +24,7 @@ info "Creating cluster $clustername..." $EXE cluster create "$clustername" --agents 1 --api-port 6443 --wait --timeout 360s $EXTRA_FLAG || failed "could not create cluster $clustername $EXTRA_TITLE" info "Sleeping for 5 seconds to give the cluster enough time to get ready..." -sleep 5 +sleep 10 # 1. check initial access to the cluster info "Checking that we have access to the cluster..."