From c80976744851cecf5db72d4f343643fdfdb134e8 Mon Sep 17 00:00:00 2001 From: Thorsten Klein Date: Tue, 19 Jan 2021 08:49:42 +0100 Subject: [PATCH] [Fix] registry: adjustments on how the registry is perceived (#454) - LocalRegistryHostingConfigMap - use localhost or docker-machine as external host IP - fromContainerRuntime: user container-name + internal port - registries.yaml: set up mirrors for both external and internal address --- pkg/client/cluster.go | 6 +++--- pkg/client/registry.go | 33 ++++++++++++++++++++++++++++++++- pkg/client/registry_test.go | 3 ++- pkg/types/types.go | 2 ++ 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go index 6047070b..eb8ec7c7 100644 --- a/pkg/client/cluster.go +++ b/pkg/client/cluster.go @@ -195,7 +195,7 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf } // generate the LocalRegistryHosting configmap - regCm, err := RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx, clusterConfig.ClusterCreateOpts.Registries.Use) + regCm, err := RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx, runtime, clusterConfig.ClusterCreateOpts.Registries.Use) if err != nil { return fmt.Errorf("Failed to generate LocalRegistryHosting configmap: %+v", err) } @@ -205,7 +205,7 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf Action: actions.WriteFileAction{ Runtime: runtime, Content: regCm, - Dest: "/tmp/reg.yaml", + Dest: k3d.DefaultLocalRegistryHostingConfigmapTempPath, }, }) @@ -935,7 +935,7 @@ func prepCreateLocalRegistryHostingConfigMap(ctx context.Context, runtime k3drt. success := false for _, node := range cluster.Nodes { if node.Role == k3d.AgentRole || node.Role == k3d.ServerRole { - err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", "kubectl apply -f /tmp/reg.yaml"}) + err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", fmt.Sprintf("kubectl apply -f %s", k3d.DefaultLocalRegistryHostingConfigmapTempPath)}) if err == nil { success = true break diff --git a/pkg/client/registry.go b/pkg/client/registry.go index f14d79ca..6e2686cf 100644 --- a/pkg/client/registry.go +++ b/pkg/client/registry.go @@ -24,10 +24,12 @@ package client import ( "context" "fmt" + gort "runtime" "github.com/docker/go-connections/nat" "github.com/imdario/mergo" "github.com/rancher/k3d/v4/pkg/runtimes" + "github.com/rancher/k3d/v4/pkg/runtimes/docker" k3d "github.com/rancher/k3d/v4/pkg/types" "github.com/rancher/k3d/v4/pkg/types/k3s" "github.com/rancher/k3d/v4/pkg/types/k8s" @@ -182,6 +184,12 @@ func RegistryGenerateK3sConfig(ctx context.Context, registries []*k3d.Registry) }, } + regConf.Mirrors[internalAddress] = k3s.Mirror{ + Endpoints: []string{ + fmt.Sprintf("http://%s", internalAddress), + }, + } + if reg.Options.Proxy.RemoteURL != "" { regConf.Mirrors[reg.Options.Proxy.RemoteURL] = k3s.Mirror{ Endpoints: []string{fmt.Sprintf("http://%s", internalAddress)}, @@ -242,7 +250,7 @@ func RegistryFromNode(node *k3d.Node) (*k3d.Registry, error) { } // RegistryGenerateLocalRegistryHostingConfigMapYAML generates a ConfigMap used to advertise the registries in the cluster -func RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx context.Context, registries []*k3d.Registry) ([]byte, error) { +func RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx context.Context, runtime runtimes.Runtime, registries []*k3d.Registry) ([]byte, error) { type cmMetadata struct { Name string `yaml:"name"` @@ -269,11 +277,34 @@ func RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx context.Context, regi return nil, nil } + // if no host is set, fallback onto the HostIP used to bind the port host := registries[0].ExposureOpts.Host if host == "" { host = registries[0].ExposureOpts.Binding.HostIP } + // if the host is now 0.0.0.0, check if we can set it to the IP of the docker-machine, if it's used + if host == k3d.DefaultAPIHost && runtime == runtimes.Docker { + if gort.GOOS == "windows" || gort.GOOS == "darwin" { + log.Tracef("Running on %s: checking if it's using docker-machine", gort.GOOS) + machineIP, err := runtime.(docker.Docker).GetDockerMachineIP() + if err != nil { + log.Warnf("Using docker-machine, but failed to get it's IP for usage in LocalRegistryHosting Config Map: %+v", err) + } else if machineIP != "" { + log.Infof("Using the docker-machine IP %s in the LocalRegistryHosting Config Map", machineIP) + host = machineIP + } else { + log.Traceln("Not using docker-machine") + } + } + } + + // if host is still 0.0.0.0, use localhost instead + if host == k3d.DefaultAPIHost { + host = "localhost" // we prefer localhost over 0.0.0.0 + } + + // transform configmap data to YAML dat, err := yaml.Marshal( k8s.LocalRegistryHostingV1{ Host: fmt.Sprintf("%s:%s", host, registries[0].ExposureOpts.Binding.HostPort), diff --git a/pkg/client/registry_test.go b/pkg/client/registry_test.go index 59012719..c1538c73 100644 --- a/pkg/client/registry_test.go +++ b/pkg/client/registry_test.go @@ -27,6 +27,7 @@ import ( "testing" "github.com/docker/go-connections/nat" + "github.com/rancher/k3d/v4/pkg/runtimes" k3d "github.com/rancher/k3d/v4/pkg/types" ) @@ -54,7 +55,7 @@ data: regs := []*k3d.Registry{reg} - cm, err := RegistryGenerateLocalRegistryHostingConfigMapYAML(context.Background(), regs) + cm, err := RegistryGenerateLocalRegistryHostingConfigMapYAML(context.Background(), runtimes.Docker, regs) if err != nil { t.Error(err) } diff --git a/pkg/types/types.go b/pkg/types/types.go index 90a985ff..bb5a125b 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -358,6 +358,8 @@ const ( DefaultRegistriesFilePath = "/etc/rancher/k3s/registries.yaml" DefaultRegistryMountPath = "/var/lib/registry" DefaultDockerHubAddress = "registry-1.docker.io" + // Default temporary path for the LocalRegistryHosting configmap, from where it will be applied via kubectl + DefaultLocalRegistryHostingConfigmapTempPath = "/tmp/localRegistryHostingCM.yaml" ) // Registry describes a k3d-managed registry