[Enhancement] More powerful registry-create opt (#727)

- `--registry-create NAME[:HOST][:HOSTPORT]` changed from bool flag
- respective config added to config file
pull/767/head v5.0.0-rc.0
Thorsten Klein 3 years ago committed by GitHub
parent 149dfdb9ab
commit 7071129df8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 27
      cmd/cluster/clusterCreate.go
  2. 3
      docs/usage/configfile.md
  3. 13
      docs/usage/guides/registries.md
  4. 49
      pkg/config/config_test.go
  5. 10
      pkg/config/test_assets/config_test_registries.yaml
  6. 23
      pkg/config/transform.go
  7. 23
      pkg/config/v1alpha3/migrations.go
  8. 50
      pkg/config/v1alpha3/schema.json
  9. 41
      pkg/config/v1alpha3/types.go
  10. 2
      pkg/runtimes/docker/network.go
  11. 3
      tests/assets/config_test_simple.yaml
  12. 5
      tests/assets/config_test_simple_migration_v1alpha3.yaml
  13. 7
      tests/test_config_file.sh
  14. 2
      tests/test_config_with_overrides.sh
  15. 11
      tests/test_registry.sh

@ -265,6 +265,9 @@ func NewCmdClusterCreate() *cobra.Command {
cmd.Flags().StringArrayP("runtime-label", "", nil, "Add label to container runtime (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --runtime-label \"my.label@agent:0,1\" --runtime-label \"other.label=somevalue@server:0\"`")
_ = ppViper.BindPFlag("cli.runtime-labels", cmd.Flags().Lookup("runtime-label"))
cmd.Flags().String("registry-create", "", "Create a k3d-managed registry and connect it to the cluster (Format: `NAME[:HOST][:HOSTPORT]`\n - Example: `k3d cluster create --registry-create mycluster-registry:0.0.0.0:5432`")
_ = ppViper.BindPFlag("cli.registries.create", cmd.Flags().Lookup("registry-create"))
/* k3s */
cmd.Flags().StringArray("k3s-arg", nil, "Additional args passed to k3s command (Format: `ARG@NODEFILTER[;@NODEFILTER]`)\n - Example: `k3d cluster create --k3s-arg \"--disable=traefik@server:0\"")
_ = cfgViper.BindPFlag("cli.k3sargs", cmd.Flags().Lookup("k3s-arg"))
@ -334,9 +337,6 @@ func NewCmdClusterCreate() *cobra.Command {
cmd.Flags().StringArray("registry-use", nil, "Connect to one or more k3d-managed registries running locally")
_ = cfgViper.BindPFlag("registries.use", cmd.Flags().Lookup("registry-use"))
cmd.Flags().Bool("registry-create", false, "Create a k3d-managed registry and connect it to the cluster")
_ = cfgViper.BindPFlag("registries.create", cmd.Flags().Lookup("registry-create"))
cmd.Flags().String("registry-config", "", "Specify path to an extra registries.yaml file")
_ = cfgViper.BindPFlag("registries.config", cmd.Flags().Lookup("registry-config"))
if err := cmd.MarkFlagFilename("registry-config", "yaml", "yml"); err != nil {
@ -418,7 +418,7 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
l.Log().Fatalln(err)
}
if strings.Contains(volume, k3d.DefaultRegistriesFilePath) && (cfg.Registries.Create || cfg.Registries.Config != "" || len(cfg.Registries.Use) != 0) {
if strings.Contains(volume, k3d.DefaultRegistriesFilePath) && (cfg.Registries.Create != nil || cfg.Registries.Config != "" || len(cfg.Registries.Use) != 0) {
l.Log().Warnf("Seems like you're mounting a file at '%s' while also using a referenced registries config or k3d-managed registries: Your mounted file will probably be overwritten!", k3d.DefaultRegistriesFilePath)
}
@ -576,5 +576,24 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
})
}
// --registry-create
if ppViper.IsSet("cli.registries.create") {
flagvalue := ppViper.GetString("cli.registries.create")
fvSplit := strings.SplitN(flagvalue, ":", 2)
if cfg.Registries.Create == nil {
cfg.Registries.Create = &conf.SimpleConfigRegistryCreateConfig{}
}
cfg.Registries.Create.Name = fvSplit[0]
if len(fvSplit) > 1 {
exposeAPI, err = cliutil.ParsePortExposureSpec(fvSplit[1], "1234") // internal port is unused after all
if err != nil {
return cfg, fmt.Errorf("failed to registry port spec: %w", err)
}
cfg.Registries.Create.Host = exposeAPI.Host
cfg.Registries.Create.HostPort = exposeAPI.Binding.HostPort
}
}
return cfg, nil
}

@ -77,7 +77,8 @@ env:
nodeFilters:
- server:0
registries: # define how registries should be created or used
create: true # creates a default registry to be used with the cluster; same as `--registry-create`
create:
name: registry.localhost # creates a default registry to be used with the cluster; same as `--registry-create registry.localhost`
use:
- k3d-myotherregistry:5000 # some other k3d-managed registry; same as `--registry-use 'k3d-myotherregistry:5000'`
config: | # define contents of the `registries.yaml` file (or reference a file); same as `--registry-config /path/to/config.yaml`

@ -35,7 +35,8 @@ name: test
servers: 1
agents: 2
registries:
create: true
create:
name: myregistry
config: |
mirrors:
"my.company.registry":
@ -43,7 +44,7 @@ registries:
- http://my.company.registry:5000
```
Here, the config for the k3d-managed registry, created by the `create: true` flag will be merged with the config specified under `config: |`.
Here, the config for the k3d-managed registry, created by the `create: {...}` option will be merged with the config specified under `config: |`.
### Authenticated registries
@ -100,14 +101,14 @@ k3d cluster create \
#### Create a dedicated registry together with your cluster
1. `#!bash k3d cluster create mycluster --registry-create`: This creates your cluster `mycluster` together with a registry container called `k3d-mycluster-registry`
1. `#!bash k3d cluster create mycluster --registry-create mycluster-registry`: This creates your cluster `mycluster` together with a registry container called `mycluster-registry`
- k3d sets everything up in the cluster for containerd to be able to pull images from that registry (using the `registries.yaml` file)
- the port, which the registry is listening on will be mapped to a random port on your host system
2. Check the k3d command output or `#!bash docker ps -f name=k3d-mycluster-registry` to find the exposed port (let's use `12345` here)
3. Pull some image (optional) `#!bash docker pull alpine:latest`, re-tag it to reference your newly created registry `#!bash docker tag alpine:latest k3d-mycluster-registry:12345/testimage:local` and push it `#!bash docker push k3d-mycluster-registry:12345/testimage:local`
4. Use kubectl to create a new pod in your cluster using that image to see, if the cluster can pull from the new registry: `#!bash kubectl run --image k3d-mycluster-registry:12345/testimage:local testimage --command -- tail -f /dev/null` (creates a container that will not do anything but keep on running)
2. Check the k3d command output or `#!bash docker ps -f name=mycluster-registry` to find the exposed port (let's use `12345` here)
3. Pull some image (optional) `#!bash docker pull alpine:latest`, re-tag it to reference your newly created registry `#!bash docker tag alpine:latest mycluster-registry:12345/testimage:local` and push it `#!bash docker push mycluster-registry:12345/testimage:local`
4. Use kubectl to create a new pod in your cluster using that image to see, if the cluster can pull from the new registry: `#!bash kubectl run --image mycluster-registry:12345/testimage:local testimage --command -- tail -f /dev/null` (creates a container that will not do anything but keep on running)
#### Create a customized k3d-managed registry

@ -256,3 +256,52 @@ func TestReadUnknownConfig(t *testing.T) {
}
}
func TestReadSimpleConfigRegistries(t *testing.T) {
exposedAPI := conf.SimpleExposureOpts{}
exposedAPI.HostIP = "0.0.0.0"
exposedAPI.HostPort = "6443"
expectedConfig := conf.SimpleConfig{
TypeMeta: configtypes.TypeMeta{
APIVersion: "k3d.io/v1alpha3",
Kind: "Simple",
},
Name: "test",
Servers: 1,
Agents: 1,
Registries: conf.SimpleConfigRegistries{
Create: &conf.SimpleConfigRegistryCreateConfig{
Name: "registry.localhost",
Host: "0.0.0.0",
HostPort: "5001",
},
},
}
cfgFile := "./test_assets/config_test_registries.yaml"
config := viper.New()
config.SetConfigFile(cfgFile)
// try to read config into memory (viper map structure)
if err := config.ReadInConfig(); err != nil {
if _, ok := err.(viper.ConfigFileNotFoundError); ok {
t.Error(err)
}
// config file found but some other error happened
t.Error(err)
}
readConfig, err := FromViper(config)
if err != nil {
t.Error(err)
}
t.Logf("\n========== Read Config ==========\n%+v\n=================================\n", readConfig)
if diff := deep.Equal(readConfig, expectedConfig); diff != nil {
t.Errorf("Actual representation\n%+v\ndoes not match expected representation\n%+v\nDiff:\n%+v", readConfig, expectedConfig, diff)
}
}

@ -0,0 +1,10 @@
apiVersion: k3d.io/v1alpha3
kind: Simple
name: test
servers: 1
agents: 1
registries:
create:
name: registry.localhost
host: "0.0.0.0"
hostPort: "5001"

@ -284,14 +284,31 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
/*
* Registries
*/
if simpleConfig.Registries.Create {
regPort, err := cliutil.ParsePortExposureSpec("random", k3d.DefaultRegistryPort)
if simpleConfig.Registries.Create != nil {
epSpecHost := "0.0.0.0"
epSpecPort := "random"
if simpleConfig.Registries.Create.HostPort != "" {
epSpecPort = simpleConfig.Registries.Create.HostPort
}
if simpleConfig.Registries.Create.Host != "" {
epSpecHost = simpleConfig.Registries.Create.Host
}
regPort, err := cliutil.ParsePortExposureSpec(fmt.Sprintf("%s:%s", epSpecHost, epSpecPort), k3d.DefaultRegistryPort)
if err != nil {
return nil, fmt.Errorf("failed to get port for registry: %w", err)
}
regName := fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, newCluster.Name)
if simpleConfig.Registries.Create.Name != "" {
regName = simpleConfig.Registries.Create.Name
}
clusterCreateOpts.Registries.Create = &k3d.Registry{
ClusterRef: newCluster.Name,
Host: fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, newCluster.Name),
Host: regName,
Image: fmt.Sprintf("%s:%s", k3d.DefaultRegistryImageRepo, k3d.DefaultRegistryImageTag),
ExposureOpts: *regPort,
}

@ -24,10 +24,12 @@ package v1alpha3
import (
"encoding/json"
"fmt"
configtypes "github.com/rancher/k3d/v5/pkg/config/types"
"github.com/rancher/k3d/v5/pkg/config/v1alpha2"
l "github.com/rancher/k3d/v5/pkg/logger"
k3d "github.com/rancher/k3d/v5/pkg/types"
)
var Migrations = map[string]func(configtypes.Config) (configtypes.Config, error){
@ -43,9 +45,18 @@ func MigrateV1Alpha2(input configtypes.Config) (configtypes.Config, error) {
}
if input.GetKind() == "Simple" {
cfg := SimpleConfig{}
cfgIntermediate := SimpleConfigIntermediateV1alpha2{}
if err := json.Unmarshal(injson, &cfg); err != nil {
if err := json.Unmarshal(injson, &cfgIntermediate); err != nil {
return nil, err
}
intermediateJSON, err := json.Marshal(cfgIntermediate)
if err != nil {
return nil, err
}
cfg := SimpleConfig{}
if err := json.Unmarshal(intermediateJSON, &cfg); err != nil {
return nil, err
}
@ -78,6 +89,14 @@ func MigrateV1Alpha2(input configtypes.Config) (configtypes.Config, error) {
})
}
if input.(v1alpha2.SimpleConfig).Registries.Create {
cfg.Registries.Create = &SimpleConfigRegistryCreateConfig{
Name: fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, cfg.Name),
Host: "0.0.0.0",
HostPort: "random",
}
}
cfg.APIVersion = ApiVersion
l.Log().Debugf("Migrated config: %+v", cfg)

@ -240,7 +240,55 @@
}
},
"registries": {
"type": "object"
"type": "object",
"properties": {
"create": {
"type": "object",
"description": "Create a new container image registry alongside the cluster.",
"properties": {
"name": {
"type": "string",
"examples": [
"myregistry",
"registry.localhost"
]
},
"host": {
"type": "string",
"examples": [
"0.0.0.0",
"localhost",
"127.0.0.1"
],
"default": "0.0.0.0"
},
"hostPort": {
"type": "string",
"examples": [
"5000",
"2345"
],
"default": "random"
}
},
"additionalProperties": false
},
"use": {
"type": "array",
"description": "Connect another container image registry to the cluster.",
"items": {
"type": "string"
},
"examples": [
"otherregistry:5000"
]
},
"config": {
"type": "string",
"description": "Reference a K3s registry configuration file or at it's contents here."
},
"additionalProperties": false
}
}
},
"additionalProperties": false,

@ -81,6 +81,12 @@ type K3sArgWithNodeFilters struct {
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters" json:"nodeFilters,omitempty"`
}
type SimpleConfigRegistryCreateConfig struct {
Name string `mapstructure:"name" yaml:"name" json:"name"`
Host string `mapstructure:"host" yaml:"host" json:"host"`
HostPort string `mapstructure:"hostPort" yaml:"hostPort" json:"hostPort"`
}
// SimpleConfigOptionsKubeconfig describes the set of options referring to the kubeconfig during cluster creation.
type SimpleConfigOptionsKubeconfig struct {
UpdateDefaultKubeconfig bool `mapstructure:"updateDefaultKubeconfig" yaml:"updateDefaultKubeconfig" json:"updateDefaultKubeconfig,omitempty"` // default: true
@ -120,6 +126,18 @@ type SimpleConfigOptionsK3s struct {
NodeLabels []LabelWithNodeFilters `mapstructure:"nodeLabels" yaml:"nodeLabels"`
}
type SimpleConfigRegistries struct {
Use []string `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"`
Create *SimpleConfigRegistryCreateConfig `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"`
Config string `mapstructure:"config" yaml:"config,omitempty" json:"config,omitempty"` // registries.yaml (k3s config for containerd registry override)
}
type SimpleConfigRegistriesIntermediateV1alpha2 struct {
Use []string `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"`
// Field "Create" changed significantly, so it's dropped here
Config string `mapstructure:"config" yaml:"config,omitempty" json:"config,omitempty"` // registries.yaml (k3s config for containerd registry override)
}
// SimpleConfig describes the toplevel k3d configuration file.
type SimpleConfig struct {
config.TypeMeta `mapstructure:",squash" yaml:",inline"`
@ -135,11 +153,24 @@ type SimpleConfig struct {
Ports []PortWithNodeFilters `mapstructure:"ports" yaml:"ports" json:"ports,omitempty"`
Options SimpleConfigOptions `mapstructure:"options" yaml:"options" json:"options,omitempty"`
Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"`
Registries struct {
Use []string `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"`
Create bool `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"`
Config string `mapstructure:"config" yaml:"config,omitempty" json:"config,omitempty"` // registries.yaml (k3s config for containerd registry override)
} `mapstructure:"registries" yaml:"registries,omitempty" json:"registries,omitempty"`
Registries SimpleConfigRegistries `mapstructure:"registries" yaml:"registries,omitempty" json:"registries,omitempty"`
}
type SimpleConfigIntermediateV1alpha2 struct {
config.TypeMeta `mapstructure:",squash" yaml:",inline"`
Name string `mapstructure:"name" yaml:"name" json:"name,omitempty"`
Servers int `mapstructure:"servers" yaml:"servers" json:"servers,omitempty"` //nolint:lll // default 1
Agents int `mapstructure:"agents" yaml:"agents" json:"agents,omitempty"` //nolint:lll // default 0
ExposeAPI SimpleExposureOpts `mapstructure:"kubeAPI" yaml:"kubeAPI" json:"kubeAPI,omitempty"`
Image string `mapstructure:"image" yaml:"image" json:"image,omitempty"`
Network string `mapstructure:"network" yaml:"network" json:"network,omitempty"`
Subnet string `mapstructure:"subnet" yaml:"subnet" json:"subnet,omitempty"`
ClusterToken string `mapstructure:"token" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated
Volumes []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes" json:"volumes,omitempty"`
Ports []PortWithNodeFilters `mapstructure:"ports" yaml:"ports" json:"ports,omitempty"`
Options SimpleConfigOptions `mapstructure:"options" yaml:"options" json:"options,omitempty"`
Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"`
Registries SimpleConfigRegistriesIntermediateV1alpha2 `mapstructure:"registries" yaml:"registries,omitempty" json:"registries,omitempty"`
}
// SimpleExposureOpts provides a simplified syntax compared to the original k3d.ExposureOpts

@ -204,7 +204,7 @@ func (d Docker) CreateNetworkIfNotPresent(ctx context.Context, inNet *k3d.Cluste
return nil, false, fmt.Errorf("docker failed to inspect newly created network '%s': %w", newNet.ID, err)
}
l.Log().Infof("Created network '%s' (%s)", inNet.Name, networkDetails.ID)
l.Log().Infof("Created network '%s'", inNet.Name)
prefix, err := netaddr.ParseIPPrefix(networkDetails.IPAM.Config[0].Subnet)
if err != nil {
return nil, false, fmt.Errorf("failed to parse IP Prefix of newly created network '%s': %w", newNet.ID, err)

@ -23,7 +23,8 @@ env:
nodeFilters:
- all
registries:
create: true
create:
name: registry.localhost
use: []
config: |
mirrors:

@ -23,7 +23,10 @@ env:
nodeFilters:
- all
registries:
create: true
create:
name: k3d-test-registry
host: "0.0.0.0"
hostPort: random
use: []
config: |
mirrors:

@ -21,7 +21,7 @@ configfileoriginal="$CURR_DIR/assets/config_test_simple.yaml"
configfile="/tmp/config_test_simple-tmp_$(date -u +'%Y%m%dT%H%M%SZ').yaml"
clustername="configtest"
sed -E "s/name:.+/name: $clustername/g" < "$configfileoriginal" > "$configfile" # replace cluster name in config file so we can use it in this script without running into override issues
sed -E "s/^name:.+/name: $clustername/g" < "$configfileoriginal" > "$configfile" # replace cluster name in config file so we can use it in this script without running into override issues
highlight "[START] ConfigTest $EXTRA_TITLE"
@ -53,13 +53,14 @@ info "Ensuring that k3s node labels have been set as stated in the config"
k3s_assert_node_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on node k3d-$clustername-server-0"
## Registry Node
registryname="registry.localhost"
info "Ensuring, that we have a registry node present"
$EXE node list "k3d-$clustername-registry" || failed "Expected k3d-$clustername-registry to be present"
$EXE node list "$registryname" || failed "Expected registry node $registryname to be present"
## merged registries.yaml
info "Ensuring, that the registries.yaml file contains both registries"
exec_in_node "k3d-$clustername-server-0" "cat /etc/rancher/k3s/registries.yaml" | grep -qi "my.company.registry" || failed "Expected 'my.company.registry' to be in the /etc/rancher/k3s/registries.yaml"
exec_in_node "k3d-$clustername-server-0" "cat /etc/rancher/k3s/registries.yaml" | grep -qi "k3d-$clustername-registry" || failed "Expected 'k3d-$clustername-registry' to be in the /etc/rancher/k3s/registries.yaml"
exec_in_node "k3d-$clustername-server-0" "cat /etc/rancher/k3s/registries.yaml" | grep -qi "$registryname" || failed "Expected '$registryname' to be in the /etc/rancher/k3s/registries.yaml"
# Cleanup

@ -21,7 +21,7 @@ clustername="cfgoverridetest"
highlight "[START] Config With Override $EXTRA_TITLE"
info "Creating cluster $clustername..."
$EXE cluster create "$clustername" --config "$CURR_DIR/assets/config_test_simple.yaml" --servers 4 -v /tmp/test:/tmp/test@loadbalancer --registry-create=false --env "x=y@agent:1" $EXTRA_FLAG || failed "could not create cluster $clustername $EXTRA_TITLE"
$EXE cluster create "$clustername" --config "$CURR_DIR/assets/config_test_simple.yaml" --servers 4 -v /tmp/test:/tmp/test@loadbalancer --env "x=y@agent:1" $EXTRA_FLAG || failed "could not create cluster $clustername $EXTRA_TITLE"
info "Sleeping for 5 seconds to give the cluster enough time to get ready..."
sleep 5

@ -19,11 +19,12 @@ export CURRENT_STAGE="Test | registry | $K3S_IMAGE_TAG"
clustername="registrytest"
registryname="$clustername-registry"
highlight "[START] RegistryTest $EXTRA_TITLE"
info "Creating cluster $clustername..."
$EXE cluster create "$clustername" --agents 1 --api-port 6443 --wait --timeout 360s --registry-create $EXTRA_FLAG || failed "could not create cluster $clustername $EXTRA_TITLE"
$EXE cluster create "$clustername" --agents 1 --api-port 6443 --wait --timeout 360s --registry-create "$registryname" $EXTRA_FLAG || failed "could not create cluster $clustername $EXTRA_TITLE"
info "Sleeping for 5 seconds to give the cluster enough time to get ready..."
sleep 5
@ -41,14 +42,14 @@ kubectl get configmap -n kube-public local-registry-hosting -o go-template='{{in
# 3. load an image into the registry
info "Pushing an image to the registry..."
registryPort=$(docker inspect k3d-$clustername-registry | jq '.[0].NetworkSettings.Ports["5000/tcp"][0].HostPort' | sed -E 's/"//g')
registryPort=$(docker inspect $registryname | jq '.[0].NetworkSettings.Ports["5000/tcp"][0].HostPort' | sed -E 's/"//g')
docker pull alpine:latest > /dev/null
docker tag alpine:latest k3d-$clustername-registry:$registryPort/alpine:local > /dev/null
docker push k3d-$clustername-registry:$registryPort/alpine:local || failed "Failed to push image to managed registry"
docker tag alpine:latest "localhost:$registryPort/alpine:local" > /dev/null
docker push "localhost:$registryPort/alpine:local" || failed "Failed to push image to managed registry"
# 4. use imported image
info "Spawning a pod using the pushed image..."
kubectl run --image k3d-$clustername-registry:$registryPort/alpine:local testimage --command -- tail -f /dev/null
kubectl run --image "$registryname:$registryPort/alpine:local" testimage --command -- tail -f /dev/null
info "Waiting for a bit for the pod to start..."
sleep 5

Loading…
Cancel
Save