mirror of https://github.com/k3d-io/k3d
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
432 lines
15 KiB
432 lines
15 KiB
/*
|
|
Copyright © 2020-2023 The k3d Author(s)
|
|
|
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
of this software and associated documentation files (the "Software"), to deal
|
|
in the Software without restriction, including without limitation the rights
|
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
copies of the Software, and to permit persons to whom the Software is
|
|
furnished to do so, subject to the following conditions:
|
|
|
|
The above copyright notice and this permission notice shall be included in
|
|
all copies or substantial portions of the Software.
|
|
|
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
THE SOFTWARE.
|
|
*/
|
|
|
|
package config
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/netip"
|
|
"os"
|
|
"strings"
|
|
|
|
wharfie "github.com/rancher/wharfie/pkg/registries"
|
|
|
|
"github.com/docker/go-connections/nat"
|
|
"sigs.k8s.io/yaml"
|
|
|
|
dockerunits "github.com/docker/go-units"
|
|
cliutil "github.com/k3d-io/k3d/v5/cmd/util" // TODO: move parseapiport to pkg
|
|
"github.com/k3d-io/k3d/v5/pkg/client"
|
|
conf "github.com/k3d-io/k3d/v5/pkg/config/v1alpha5"
|
|
l "github.com/k3d-io/k3d/v5/pkg/logger"
|
|
"github.com/k3d-io/k3d/v5/pkg/runtimes"
|
|
k3d "github.com/k3d-io/k3d/v5/pkg/types"
|
|
"github.com/k3d-io/k3d/v5/pkg/util"
|
|
"github.com/k3d-io/k3d/v5/version"
|
|
)
|
|
|
|
// TransformSimpleToClusterConfig transforms a simple configuration to a full-fledged cluster configuration
|
|
func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtime, simpleConfig conf.SimpleConfig, configFileName string) (*conf.ClusterConfig, error) {
|
|
// set default cluster name
|
|
if simpleConfig.Name == "" {
|
|
simpleConfig.Name = k3d.DefaultClusterName
|
|
}
|
|
|
|
/* Special cases for Image:
|
|
* - "latest" / "stable": get latest / stable channel image
|
|
* - starts with "+": get channel following the "+"
|
|
*/
|
|
if simpleConfig.Image == "latest" || simpleConfig.Image == "stable" || strings.HasPrefix(simpleConfig.Image, "+") {
|
|
searchChannel := strings.TrimPrefix(simpleConfig.Image, "+")
|
|
v, err := version.GetK3sVersion(searchChannel)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
l.Log().Debugf("Using fetched K3s version %s", v)
|
|
simpleConfig.Image = fmt.Sprintf("%s:%s", k3d.DefaultK3sImageRepo, v)
|
|
}
|
|
|
|
clusterNetwork := k3d.ClusterNetwork{}
|
|
if simpleConfig.Network != "" {
|
|
clusterNetwork.Name = simpleConfig.Network
|
|
clusterNetwork.External = true
|
|
} else {
|
|
clusterNetwork.Name = fmt.Sprintf("%s-%s", k3d.DefaultObjectNamePrefix, simpleConfig.Name)
|
|
clusterNetwork.External = false
|
|
}
|
|
|
|
if simpleConfig.Subnet != "" {
|
|
if simpleConfig.Subnet != "auto" {
|
|
subnet, err := netip.ParsePrefix(simpleConfig.Subnet)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid subnet '%s': %w", simpleConfig.Subnet, err)
|
|
}
|
|
clusterNetwork.IPAM.IPPrefix = subnet
|
|
}
|
|
clusterNetwork.IPAM.Managed = true
|
|
}
|
|
|
|
// -> API
|
|
if simpleConfig.ExposeAPI.HostIP == "" {
|
|
simpleConfig.ExposeAPI.HostIP = k3d.DefaultAPIHost
|
|
}
|
|
if simpleConfig.ExposeAPI.Host == "" {
|
|
simpleConfig.ExposeAPI.Host = simpleConfig.ExposeAPI.HostIP
|
|
}
|
|
|
|
kubeAPIExposureOpts := &k3d.ExposureOpts{
|
|
Host: simpleConfig.ExposeAPI.Host,
|
|
}
|
|
kubeAPIExposureOpts.Port = k3d.DefaultAPIPort
|
|
kubeAPIExposureOpts.Binding = nat.PortBinding{
|
|
HostIP: simpleConfig.ExposeAPI.HostIP,
|
|
HostPort: simpleConfig.ExposeAPI.HostPort,
|
|
}
|
|
|
|
// FILL CLUSTER CONFIG
|
|
newCluster := k3d.Cluster{
|
|
Name: simpleConfig.Name,
|
|
Network: clusterNetwork,
|
|
Token: simpleConfig.ClusterToken,
|
|
KubeAPI: kubeAPIExposureOpts,
|
|
}
|
|
|
|
// -> NODES
|
|
newCluster.Nodes = []*k3d.Node{}
|
|
|
|
if !simpleConfig.Options.K3dOptions.DisableLoadbalancer {
|
|
newCluster.ServerLoadBalancer = k3d.NewLoadbalancer()
|
|
lbCreateOpts := &k3d.LoadbalancerCreateOpts{}
|
|
if simpleConfig.Options.K3dOptions.Loadbalancer.ConfigOverrides != nil && len(simpleConfig.Options.K3dOptions.Loadbalancer.ConfigOverrides) > 0 {
|
|
lbCreateOpts.ConfigOverrides = simpleConfig.Options.K3dOptions.Loadbalancer.ConfigOverrides
|
|
}
|
|
var err error
|
|
newCluster.ServerLoadBalancer.Node, err = client.LoadbalancerPrepare(ctx, runtime, &newCluster, lbCreateOpts)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error preparing the loadbalancer: %w", err)
|
|
}
|
|
newCluster.Nodes = append(newCluster.Nodes, newCluster.ServerLoadBalancer.Node)
|
|
} else {
|
|
l.Log().Debugln("Disabling the load balancer")
|
|
}
|
|
|
|
/*************
|
|
* Add Nodes *
|
|
*************/
|
|
|
|
for i := 0; i < simpleConfig.Servers; i++ {
|
|
serverNode := k3d.Node{
|
|
Name: client.GenerateNodeName(newCluster.Name, k3d.ServerRole, i),
|
|
Role: k3d.ServerRole,
|
|
Image: simpleConfig.Image,
|
|
ServerOpts: k3d.ServerOpts{},
|
|
Memory: simpleConfig.Options.Runtime.ServersMemory,
|
|
HostPidMode: simpleConfig.Options.Runtime.HostPidMode,
|
|
}
|
|
|
|
// first server node will be init node if we have more than one server specified but no external datastore
|
|
if i == 0 && simpleConfig.Servers > 1 {
|
|
serverNode.ServerOpts.IsInit = true
|
|
newCluster.InitNode = &serverNode
|
|
}
|
|
|
|
newCluster.Nodes = append(newCluster.Nodes, &serverNode)
|
|
|
|
if !simpleConfig.Options.K3dOptions.DisableLoadbalancer {
|
|
newCluster.ServerLoadBalancer.Config.Ports[fmt.Sprintf("%s.tcp", k3d.DefaultAPIPort)] = append(newCluster.ServerLoadBalancer.Config.Ports[fmt.Sprintf("%s.tcp", k3d.DefaultAPIPort)], serverNode.Name)
|
|
}
|
|
}
|
|
|
|
for i := 0; i < simpleConfig.Agents; i++ {
|
|
agentNode := k3d.Node{
|
|
Name: client.GenerateNodeName(newCluster.Name, k3d.AgentRole, i),
|
|
Role: k3d.AgentRole,
|
|
Image: simpleConfig.Image,
|
|
Memory: simpleConfig.Options.Runtime.AgentsMemory,
|
|
HostPidMode: simpleConfig.Options.Runtime.HostPidMode,
|
|
}
|
|
newCluster.Nodes = append(newCluster.Nodes, &agentNode)
|
|
}
|
|
|
|
/****************************
|
|
* Extra Node Configuration *
|
|
****************************/
|
|
nodeCount := len(newCluster.Nodes)
|
|
nodeList := newCluster.Nodes
|
|
|
|
// -> VOLUMES
|
|
for _, volumeWithNodeFilters := range simpleConfig.Volumes {
|
|
nodes, err := util.FilterNodes(nodeList, volumeWithNodeFilters.NodeFilters)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to filter nodes for volume mapping '%s': %w", volumeWithNodeFilters.Volume, err)
|
|
}
|
|
|
|
for _, node := range nodes {
|
|
node.Volumes = append(node.Volumes, volumeWithNodeFilters.Volume)
|
|
}
|
|
}
|
|
|
|
// -> PORTS
|
|
if err := client.TransformPorts(ctx, runtime, &newCluster, simpleConfig.Ports); err != nil {
|
|
return nil, fmt.Errorf("failed to transform ports: %w", err)
|
|
}
|
|
|
|
// -> K3S NODE LABELS
|
|
for _, k3sNodeLabelWithNodeFilters := range simpleConfig.Options.K3sOptions.NodeLabels {
|
|
if len(k3sNodeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
|
|
return nil, fmt.Errorf("k3s node label mapping '%s' lacks a node filter, but there's more than one node", k3sNodeLabelWithNodeFilters.Label)
|
|
}
|
|
|
|
nodes, err := util.FilterNodes(nodeList, k3sNodeLabelWithNodeFilters.NodeFilters)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to filter nodes for k3s node label mapping '%s': %w", k3sNodeLabelWithNodeFilters.Label, err)
|
|
}
|
|
|
|
for _, node := range nodes {
|
|
if node.K3sNodeLabels == nil {
|
|
node.K3sNodeLabels = make(map[string]string) // ensure that the map is initialized
|
|
}
|
|
k, v := util.SplitLabelKeyValue(k3sNodeLabelWithNodeFilters.Label)
|
|
node.K3sNodeLabels[k] = v
|
|
}
|
|
}
|
|
|
|
// -> RUNTIME LABELS
|
|
for _, runtimeLabelWithNodeFilters := range simpleConfig.Options.Runtime.Labels {
|
|
if len(runtimeLabelWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
|
|
return nil, fmt.Errorf("RuntimeLabelmapping '%s' lacks a node filter, but there's more than one node", runtimeLabelWithNodeFilters.Label)
|
|
}
|
|
|
|
nodes, err := util.FilterNodes(nodeList, runtimeLabelWithNodeFilters.NodeFilters)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to filter nodes for runtime label mapping '%s': %w", runtimeLabelWithNodeFilters.Label, err)
|
|
}
|
|
|
|
for _, node := range nodes {
|
|
if node.RuntimeLabels == nil {
|
|
node.RuntimeLabels = make(map[string]string) // ensure that the map is initialized
|
|
}
|
|
k, v := util.SplitLabelKeyValue(runtimeLabelWithNodeFilters.Label)
|
|
|
|
cliutil.ValidateRuntimeLabelKey(k)
|
|
|
|
node.RuntimeLabels[k] = v
|
|
}
|
|
}
|
|
|
|
// -> RUNTIME ULIMITS
|
|
for index, runtimeUlimit := range simpleConfig.Options.Runtime.Ulimits {
|
|
for _, node := range nodeList {
|
|
if node.RuntimeUlimits == nil {
|
|
node.RuntimeUlimits = make([]*dockerunits.Ulimit, len(simpleConfig.Options.Runtime.Ulimits)) // ensure that the map is initialized
|
|
}
|
|
|
|
cliutil.ValidateRuntimeUlimitKey(runtimeUlimit.Name)
|
|
|
|
node.RuntimeUlimits[index] = &dockerunits.Ulimit{
|
|
Name: runtimeUlimit.Name,
|
|
Soft: runtimeUlimit.Soft,
|
|
Hard: runtimeUlimit.Hard,
|
|
}
|
|
}
|
|
}
|
|
|
|
// -> ENV
|
|
for _, envVarWithNodeFilters := range simpleConfig.Env {
|
|
if len(envVarWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
|
|
return nil, fmt.Errorf("EnvVarMapping '%s' lacks a node filter, but there's more than one node", envVarWithNodeFilters.EnvVar)
|
|
}
|
|
|
|
nodes, err := util.FilterNodes(nodeList, envVarWithNodeFilters.NodeFilters)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to filter nodes for environment variable config '%s': %w", envVarWithNodeFilters.EnvVar, err)
|
|
}
|
|
|
|
for _, node := range nodes {
|
|
node.Env = append(node.Env, envVarWithNodeFilters.EnvVar)
|
|
}
|
|
}
|
|
|
|
// -> ARGS
|
|
for _, argWithNodeFilters := range simpleConfig.Options.K3sOptions.ExtraArgs {
|
|
if len(argWithNodeFilters.NodeFilters) == 0 && nodeCount > 1 {
|
|
return nil, fmt.Errorf("K3sExtraArg '%s' lacks a node filter, but there's more than one node", argWithNodeFilters.Arg)
|
|
}
|
|
|
|
nodes, err := util.FilterNodes(nodeList, argWithNodeFilters.NodeFilters)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to filter nodes for k3s extra args config '%s': %w", argWithNodeFilters.Arg, err)
|
|
}
|
|
|
|
for _, node := range nodes {
|
|
node.Args = append(node.Args, argWithNodeFilters.Arg)
|
|
}
|
|
}
|
|
|
|
/**************************
|
|
* Cluster Create Options *
|
|
**************************/
|
|
|
|
clusterCreateOpts := k3d.ClusterCreateOpts{
|
|
DisableImageVolume: simpleConfig.Options.K3dOptions.DisableImageVolume,
|
|
WaitForServer: simpleConfig.Options.K3dOptions.Wait,
|
|
Timeout: simpleConfig.Options.K3dOptions.Timeout,
|
|
DisableLoadBalancer: simpleConfig.Options.K3dOptions.DisableLoadbalancer,
|
|
GPURequest: simpleConfig.Options.Runtime.GPURequest,
|
|
ServersMemory: simpleConfig.Options.Runtime.ServersMemory,
|
|
AgentsMemory: simpleConfig.Options.Runtime.AgentsMemory,
|
|
HostAliases: simpleConfig.HostAliases,
|
|
GlobalLabels: map[string]string{}, // empty init
|
|
GlobalEnv: []string{}, // empty init
|
|
}
|
|
|
|
// ensure, that we have the default object labels
|
|
for k, v := range k3d.DefaultRuntimeLabels {
|
|
clusterCreateOpts.GlobalLabels[k] = v
|
|
}
|
|
|
|
/*
|
|
* Registries
|
|
*/
|
|
if simpleConfig.Registries.Create != nil {
|
|
epSpecHost := "0.0.0.0"
|
|
epSpecPort := "random"
|
|
|
|
if simpleConfig.Registries.Create.HostPort != "" {
|
|
epSpecPort = simpleConfig.Registries.Create.HostPort
|
|
}
|
|
if simpleConfig.Registries.Create.Host != "" {
|
|
epSpecHost = simpleConfig.Registries.Create.Host
|
|
}
|
|
|
|
regPort, err := cliutil.ParsePortExposureSpec(fmt.Sprintf("%s:%s", epSpecHost, epSpecPort), k3d.DefaultRegistryPort)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to get port for registry: %w", err)
|
|
}
|
|
|
|
regName := fmt.Sprintf("%s-%s-registry", k3d.DefaultObjectNamePrefix, newCluster.Name)
|
|
if simpleConfig.Registries.Create.Name != "" {
|
|
regName = simpleConfig.Registries.Create.Name
|
|
}
|
|
|
|
image := fmt.Sprintf("%s:%s", k3d.DefaultRegistryImageRepo, k3d.DefaultRegistryImageTag)
|
|
if simpleConfig.Registries.Create.Image != "" {
|
|
image = simpleConfig.Registries.Create.Image
|
|
}
|
|
|
|
clusterCreateOpts.Registries.Create = &k3d.Registry{
|
|
ClusterRef: newCluster.Name,
|
|
Host: regName,
|
|
Image: image,
|
|
ExposureOpts: *regPort,
|
|
Volumes: simpleConfig.Registries.Create.Volumes,
|
|
Options: k3d.RegistryOptions{
|
|
Proxy: simpleConfig.Registries.Create.Proxy,
|
|
},
|
|
}
|
|
}
|
|
|
|
for _, usereg := range simpleConfig.Registries.Use {
|
|
reg, err := util.ParseRegistryRef(usereg)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to parse use-registry string '%s': %w", usereg, err)
|
|
}
|
|
l.Log().Tracef("Parsed registry reference: %+v", reg)
|
|
clusterCreateOpts.Registries.Use = append(clusterCreateOpts.Registries.Use, reg)
|
|
}
|
|
|
|
if simpleConfig.Registries.Config != "" {
|
|
var k3sRegistry *wharfie.Registry
|
|
|
|
if strings.Contains(simpleConfig.Registries.Config, "\n") { // CASE 1: embedded registries.yaml (multiline string)
|
|
l.Log().Debugf("Found multiline registries config embedded in SimpleConfig:\n%s", simpleConfig.Registries.Config)
|
|
if err := yaml.Unmarshal([]byte(simpleConfig.Registries.Config), &k3sRegistry); err != nil {
|
|
return nil, fmt.Errorf("failed to read embedded registries config: %w", err)
|
|
}
|
|
} else { // CASE 2: registries.yaml file referenced by path (single line)
|
|
registryConfigFile, err := os.Open(simpleConfig.Registries.Config)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to open registry config file at %s: %w", simpleConfig.Registries.Config, err)
|
|
}
|
|
configBytes, err := io.ReadAll(registryConfigFile)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read registry config file at %s: %w", registryConfigFile.Name(), err)
|
|
}
|
|
|
|
if err := yaml.Unmarshal(configBytes, &k3sRegistry); err != nil {
|
|
return nil, fmt.Errorf("failed to read registry configuration: %w", err)
|
|
}
|
|
}
|
|
|
|
l.Log().Tracef("Registry: read config from input:\n%+v", k3sRegistry)
|
|
clusterCreateOpts.Registries.Config = k3sRegistry
|
|
}
|
|
|
|
/*
|
|
* Files
|
|
*/
|
|
|
|
for _, fileWithNodeFilters := range simpleConfig.Files {
|
|
nodes, err := util.FilterNodes(nodeList, fileWithNodeFilters.NodeFilters)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to filter nodes for file copying '%s': %w", fileWithNodeFilters, err)
|
|
}
|
|
|
|
content, err := util.ReadFileSource(configFileName, fileWithNodeFilters.Source)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("failed to read source content: %w", err)
|
|
}
|
|
|
|
destination, err := util.ResolveFileDestination(fileWithNodeFilters.Destination)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("destination path is not correct: %w", err)
|
|
}
|
|
|
|
for _, node := range nodes {
|
|
node.Files = append(node.Files, k3d.File{
|
|
Content: content,
|
|
Destination: destination,
|
|
Description: fileWithNodeFilters.Description,
|
|
})
|
|
}
|
|
}
|
|
|
|
/**********************
|
|
* Kubeconfig Options *
|
|
**********************/
|
|
|
|
// Currently, the kubeconfig options for the cluster config are the same as for the simple config
|
|
|
|
/******************************
|
|
* Create Full Cluster Config *
|
|
******************************/
|
|
|
|
clusterConfig := &conf.ClusterConfig{
|
|
Cluster: newCluster,
|
|
ClusterCreateOpts: clusterCreateOpts,
|
|
KubeconfigOpts: simpleConfig.Options.KubeconfigOptions,
|
|
}
|
|
|
|
return clusterConfig, nil
|
|
}
|
|
|