/ *
Copyright © 2020 - 2021 The k3d Author ( s )
Permission is hereby granted , free of charge , to any person obtaining a copy
of this software and associated documentation files ( the "Software" ) , to deal
in the Software without restriction , including without limitation the rights
to use , copy , modify , merge , publish , distribute , sublicense , and / or sell
copies of the Software , and to permit persons to whom the Software is
furnished to do so , subject to the following conditions :
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software .
THE SOFTWARE IS PROVIDED "AS IS" , WITHOUT WARRANTY OF ANY KIND , EXPRESS OR
IMPLIED , INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY ,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT . IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM , DAMAGES OR OTHER
LIABILITY , WHETHER IN AN ACTION OF CONTRACT , TORT OR OTHERWISE , ARISING FROM ,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE .
* /
package types
import (
"context"
"fmt"
"time"
"github.com/docker/go-connections/nat"
"github.com/rancher/k3d/v4/pkg/types/k3s"
"github.com/rancher/k3d/v4/version"
"inet.af/netaddr"
)
// DefaultClusterName specifies the default name used for newly created clusters
const DefaultClusterName = "k3s-default"
// DefaultClusterNameMaxLength specifies the maximal length of a passed in cluster name
// This restriction allows us to construct an name consisting of
// <DefaultObjectNamePrefix[3]>-<ClusterName>-<TypeSuffix[5-10]>-<Counter[1-3]>
// ... and still stay within the 64 character limit (e.g. of docker)
const DefaultClusterNameMaxLength = 32
// DefaultK3sImageRepo specifies the default image repository for the used k3s image
const DefaultK3sImageRepo = "docker.io/rancher/k3s"
// DefaultLBImageRepo defines the default cluster load balancer image
const DefaultLBImageRepo = "docker.io/rancher/k3d-proxy"
// DefaultToolsImageRepo defines the default image used for the tools container
const DefaultToolsImageRepo = "docker.io/rancher/k3d-tools"
// DefaultRegistryImageRepo defines the default image used for the k3d-managed registry
const DefaultRegistryImageRepo = "docker.io/library/registry"
// DefaultRegistryImageTag defines the default image tag used for the k3d-managed registry
const DefaultRegistryImageTag = "2"
// DefaultObjectNamePrefix defines the name prefix for every object created by k3d
const DefaultObjectNamePrefix = "k3d"
// ReadyLogMessageByRole defines the log messages we wait for until a server node is considered ready
var ReadyLogMessageByRole = map [ Role ] string {
ServerRole : "k3s is up and running" ,
AgentRole : "Successfully registered node" ,
LoadBalancerRole : "start worker processes" ,
RegistryRole : "listening on" ,
}
// NodeWaitForLogMessageRestartWarnTime is the time after which to warn about a restarting container
const NodeWaitForLogMessageRestartWarnTime = 2 * time . Minute
// NodeStatusRestarting defines the status string that signals the node container is restarting
const NodeStatusRestarting = "restarting"
// Role defines a k3d node role
type Role string
// existing k3d node roles
const (
ServerRole Role = "server"
AgentRole Role = "agent"
NoRole Role = "noRole"
LoadBalancerRole Role = "loadbalancer"
RegistryRole Role = "registry"
)
// NodeRoles defines the roles available for nodes
var NodeRoles = map [ string ] Role {
string ( ServerRole ) : ServerRole ,
string ( AgentRole ) : AgentRole ,
string ( LoadBalancerRole ) : LoadBalancerRole ,
string ( RegistryRole ) : RegistryRole ,
}
// ClusterInternalNodeRoles is a list of roles for nodes that belong to a cluster
var ClusterInternalNodeRoles = [ ] Role {
ServerRole ,
AgentRole ,
LoadBalancerRole ,
}
// ClusterExternalNodeRoles is a list of roles for nodes that do not belong to a specific cluster
var ClusterExternalNodeRoles = [ ] Role {
RegistryRole ,
}
// DefaultRuntimeLabels specifies a set of labels that will be attached to k3d runtime objects by default
var DefaultRuntimeLabels = map [ string ] string {
"app" : "k3d" ,
}
// DefaultRuntimeLabelsVar specifies a set of labels that will be attached to k3d runtime objects by default but are not static (e.g. across k3d versions)
var DefaultRuntimeLabelsVar = map [ string ] string {
"k3d.version" : version . GetVersion ( ) ,
}
// List of k3d technical label name
const (
LabelClusterName string = "k3d.cluster"
LabelClusterURL string = "k3d.cluster.url"
LabelClusterToken string = "k3d.cluster.token"
LabelImageVolume string = "k3d.cluster.imageVolume"
LabelNetworkExternal string = "k3d.cluster.network.external"
LabelNetwork string = "k3d.cluster.network"
LabelNetworkID string = "k3d.cluster.network.id"
LabelNetworkIPRange string = "k3d.cluster.network.iprange"
LabelRole string = "k3d.role"
LabelServerAPIPort string = "k3d.server.api.port"
LabelServerAPIHost string = "k3d.server.api.host"
LabelServerAPIHostIP string = "k3d.server.api.hostIP"
LabelServerIsInit string = "k3d.server.init"
LabelRegistryHost string = "k3d.registry.host"
LabelRegistryHostIP string = "k3d.registry.hostIP"
LabelRegistryPortExternal string = "k3s.registry.port.external"
LabelRegistryPortInternal string = "k3s.registry.port.internal"
LabelNodeStaticIP string = "k3d.node.staticIP"
)
// DefaultRoleCmds maps the node roles to their respective default commands
var DefaultRoleCmds = map [ Role ] [ ] string {
ServerRole : { "server" } ,
AgentRole : { "agent" } ,
}
// DefaultTmpfsMounts specifies tmpfs mounts that are required for all k3d nodes
var DefaultTmpfsMounts = [ ] string {
"/run" ,
"/var/run" ,
}
// DefaultNodeEnv defines some default environment variables that should be set on every node
var DefaultNodeEnv = [ ] string {
"K3S_KUBECONFIG_OUTPUT=/output/kubeconfig.yaml" ,
}
// DefaultK3dInternalHostRecord defines the default /etc/hosts entry for the k3d host
const DefaultK3dInternalHostRecord = "host.k3d.internal"
// DefaultImageVolumeMountPath defines the mount path inside k3d nodes where we will mount the shared image volume by default
const DefaultImageVolumeMountPath = "/k3d/images"
// DefaultConfigDirName defines the name of the config directory (where we'll e.g. put the kubeconfigs)
const DefaultConfigDirName = ".k3d" // should end up in $HOME/
// DefaultKubeconfigPrefix defines the default prefix for kubeconfig files
const DefaultKubeconfigPrefix = DefaultObjectNamePrefix + "-kubeconfig"
// DefaultAPIPort defines the default Kubernetes API Port
const DefaultAPIPort = "6443"
// DefaultAPIHost defines the default host (IP) for the Kubernetes API
const DefaultAPIHost = "0.0.0.0"
// DoNotCopyServerFlags defines a list of commands/args that shouldn't be copied from an existing node when adding a similar node to a cluster
var DoNotCopyServerFlags = [ ] string {
"--cluster-init" ,
}
// ClusterCreateOpts describe a set of options one can set when creating a cluster
type ClusterCreateOpts struct {
PrepDisableHostIPInjection bool ` yaml:"prepDisableHostIPInjection" json:"prepDisableHostIPInjection,omitempty" `
DisableImageVolume bool ` yaml:"disableImageVolume" json:"disableImageVolume,omitempty" `
WaitForServer bool ` yaml:"waitForServer" json:"waitForServer,omitempty" `
Timeout time . Duration ` yaml:"timeout" json:"timeout,omitempty" `
DisableLoadBalancer bool ` yaml:"disableLoadbalancer" json:"disableLoadbalancer,omitempty" `
GPURequest string ` yaml:"gpuRequest" json:"gpuRequest,omitempty" `
ServersMemory string ` yaml:"serversMemory" json:"serversMemory,omitempty" `
AgentsMemory string ` yaml:"agentsMemory" json:"agentsMemory,omitempty" `
NodeHooks [ ] NodeHook ` yaml:"nodeHooks,omitempty" json:"nodeHooks,omitempty" `
GlobalLabels map [ string ] string ` yaml:"globalLabels,omitempty" json:"globalLabels,omitempty" `
GlobalEnv [ ] string ` yaml:"globalEnv,omitempty" json:"globalEnv,omitempty" `
Registries struct {
Create * Registry ` yaml:"create,omitempty" json:"create,omitempty" `
Use [ ] * Registry ` yaml:"use,omitempty" json:"use,omitempty" `
Config * k3s . Registry ` yaml:"config,omitempty" json:"config,omitempty" ` // registries.yaml (k3s config for containerd registry override)
} ` yaml:"registries,omitempty" json:"registries,omitempty" `
}
// NodeHook is an action that is bound to a specifc stage of a node lifecycle
type NodeHook struct {
Stage LifecycleStage ` yaml:"stage,omitempty" json:"stage,omitempty" `
Action NodeHookAction ` yaml:"action,omitempty" json:"action,omitempty" `
}
// LifecycleStage defines descriptors for specific stages in the lifecycle of a node or cluster object
type LifecycleStage string
// all defined lifecyclestages
const (
LifecycleStagePreStart LifecycleStage = "preStart"
LifecycleStagePostStart LifecycleStage = "postStart"
)
// ClusterStartOpts describe a set of options one can set when (re-)starting a cluster
type ClusterStartOpts struct {
WaitForServer bool
Timeout time . Duration
NodeHooks [ ] NodeHook ` yaml:"nodeHooks,omitempty" json:"nodeHooks,omitempty" `
}
// ClusterDeleteOpts describe a set of options one can set when deleting a cluster
type ClusterDeleteOpts struct {
SkipRegistryCheck bool // skip checking if this is a registry (and act accordingly)
}
// NodeCreateOpts describes a set of options one can set when creating a new node
type NodeCreateOpts struct {
Wait bool
Timeout time . Duration
NodeHooks [ ] NodeHook ` yaml:"nodeHooks,omitempty" json:"nodeHooks,omitempty" `
}
// NodeStartOpts describes a set of options one can set when (re-)starting a node
type NodeStartOpts struct {
Wait bool
Timeout time . Duration
NodeHooks [ ] NodeHook ` yaml:"nodeHooks,omitempty" json:"nodeHooks,omitempty" `
ReadyLogMessage string
}
// NodeDeleteOpts describes a set of options one can set when deleting a node
type NodeDeleteOpts struct {
SkipLBUpdate bool // skip updating the loadbalancer
}
// NodeHookAction is an interface to implement actions that should trigger at specific points of the node lifecycle
type NodeHookAction interface {
Run ( ctx context . Context , node * Node ) error
}
// ImageImportOpts describes a set of options one can set for loading image(s) into cluster(s)
type ImageImportOpts struct {
KeepTar bool
KeepToolsNode bool
}
type IPAM struct {
IPPrefix netaddr . IPPrefix ` yaml:"ipPrefix" json:"ipPrefix,omitempty" `
IPsUsed [ ] netaddr . IP ` yaml:"ipsUsed" json:"ipsUsed,omitempty" `
Managed bool // IPAM is done by k3d
}
// ClusterNetwork describes a network which a cluster is running in
type ClusterNetwork struct {
Name string ` yaml:"name" json:"name,omitempty" `
ID string ` yaml:"id" json:"id" ` // may be the same as name, but e.g. docker only differentiates by random ID, not by name
External bool ` yaml:"external" json:"isExternal,omitempty" `
IPAM IPAM ` yaml:"ipam" json:"ipam,omitempty" `
}
// Cluster describes a k3d cluster
type Cluster struct {
Name string ` yaml:"name" json:"name,omitempty" `
Network ClusterNetwork ` yaml:"network" json:"network,omitempty" `
Token string ` yaml:"clusterToken" json:"clusterToken,omitempty" `
Nodes [ ] * Node ` yaml:"nodes" json:"nodes,omitempty" `
InitNode * Node // init server node
ExternalDatastore * ExternalDatastore ` yaml:"externalDatastore,omitempty" json:"externalDatastore,omitempty" `
KubeAPI * ExposureOpts ` yaml:"kubeAPI" json:"kubeAPI,omitempty" `
ServerLoadBalancer * Node ` yaml:"serverLoadbalancer,omitempty" json:"serverLoadBalancer,omitempty" `
ImageVolume string ` yaml:"imageVolume" json:"imageVolume,omitempty" `
}
// ServerCountRunning returns the number of server nodes running in the cluster and the total number
func ( c * Cluster ) ServerCountRunning ( ) ( int , int ) {
serverCount := 0
serversRunning := 0
for _ , node := range c . Nodes {
if node . Role == ServerRole {
serverCount ++
if node . State . Running {
serversRunning ++
}
}
}
return serverCount , serversRunning
}
// AgentCountRunning returns the number of agent nodes running in the cluster and the total number
func ( c * Cluster ) AgentCountRunning ( ) ( int , int ) {
agentCount := 0
agentsRunning := 0
for _ , node := range c . Nodes {
if node . Role == AgentRole {
agentCount ++
if node . State . Running {
agentsRunning ++
}
}
}
return agentCount , agentsRunning
}
// HasLoadBalancer returns true if cluster has a loadbalancer node
func ( c * Cluster ) HasLoadBalancer ( ) bool {
for _ , node := range c . Nodes {
if node . Role == LoadBalancerRole {
return true
}
}
return false
}
type NodeIP struct {
IP netaddr . IP
Static bool
}
// Node describes a k3d node
type Node struct {
Name string ` yaml:"name" json:"name,omitempty" `
Role Role ` yaml:"role" json:"role,omitempty" `
Image string ` yaml:"image" json:"image,omitempty" `
Volumes [ ] string ` yaml:"volumes" json:"volumes,omitempty" `
Env [ ] string ` yaml:"env" json:"env,omitempty" `
Cmd [ ] string // filled automatically based on role
Args [ ] string ` yaml:"extraArgs" json:"extraArgs,omitempty" `
Ports nat . PortMap ` yaml:"portMappings" json:"portMappings,omitempty" `
Restart bool ` yaml:"restart" json:"restart,omitempty" `
Created string ` yaml:"created" json:"created,omitempty" `
RuntimeLabels map [ string ] string ` yaml:"runtimeLabels" json:"runtimeLabels,omitempty" `
K3sNodeLabels map [ string ] string ` yaml:"k3sNodeLabels" json:"k3sNodeLabels,omitempty" `
Networks [ ] string // filled automatically
ExtraHosts [ ] string // filled automatically
ServerOpts ServerOpts ` yaml:"serverOpts" json:"serverOpts,omitempty" `
AgentOpts AgentOpts ` yaml:"agentOpts" json:"agentOpts,omitempty" `
GPURequest string // filled automatically
Memory string // filled automatically
State NodeState // filled automatically
IP NodeIP // filled automatically
}
// ServerOpts describes some additional server role specific opts
type ServerOpts struct {
IsInit bool ` yaml:"isInitializingServer" json:"isInitializingServer,omitempty" `
KubeAPI * ExposureOpts ` yaml:"kubeAPI" json:"kubeAPI" `
}
// ExposureOpts describes settings that the user can set for accessing the Kubernetes API
type ExposureOpts struct {
nat . PortMapping // filled automatically (reference to normal portmapping)
Host string ` yaml:"host,omitempty" json:"host,omitempty" `
}
// ExternalDatastore describes an external datastore used for HA/multi-server clusters
type ExternalDatastore struct {
Endpoint string ` yaml:"endpoint" json:"endpoint,omitempty" `
CAFile string ` yaml:"caFile" json:"caFile,omitempty" `
CertFile string ` yaml:"certFile" json:"certFile,omitempty" `
KeyFile string ` yaml:"keyFile" json:"keyFile,omitempty" `
Network string ` yaml:"network" json:"network,omitempty" `
}
// AgentOpts describes some additional agent role specific opts
type AgentOpts struct { }
// GetDefaultObjectName prefixes the passed name with the default prefix
func GetDefaultObjectName ( name string ) string {
return fmt . Sprintf ( "%s-%s" , DefaultObjectNamePrefix , name )
}
// NodeState describes the current state of a node
type NodeState struct {
Running bool
Status string
Started string
}
/ *
* Registry
* /
// Registry Defaults
const (
DefaultRegistryPort = "5000"
DefaultRegistryName = DefaultObjectNamePrefix + "-registry"
DefaultRegistriesFilePath = "/etc/rancher/k3s/registries.yaml"
DefaultRegistryMountPath = "/var/lib/registry"
DefaultDockerHubAddress = "registry-1.docker.io"
// Default temporary path for the LocalRegistryHosting configmap, from where it will be applied via kubectl
DefaultLocalRegistryHostingConfigmapTempPath = "/tmp/localRegistryHostingCM.yaml"
)
// Registry describes a k3d-managed registry
type Registry struct {
ClusterRef string // filled automatically -> if created with a cluster
Protocol string ` yaml:"protocol,omitempty" json:"protocol,omitempty" ` // default: http
Host string ` yaml:"host" json:"host" `
Image string ` yaml:"image,omitempty" json:"image,omitempty" `
ExposureOpts ExposureOpts ` yaml:"expose" json:"expose" `
Options struct {
ConfigFile string ` yaml:"configFile,omitempty" json:"configFile,omitempty" `
Proxy struct {
RemoteURL string ` yaml:"remoteURL" json:"remoteURL" `
Username string ` yaml:"username,omitempty" json:"username,omitempty" `
Password string ` yaml:"password,omitempty" json:"password,omitempty" `
} ` yaml:"proxy,omitempty" json:"proxy,omitempty" `
} ` yaml:"options,omitempty" json:"options,omitempty" `
}
// RegistryExternal describes a minimal spec for an "external" registry
// "external" meaning, that it's unrelated to the current cluster
// e.g. used for the --registry-use flag registry reference
type RegistryExternal struct {
Protocol string ` yaml:"protocol,omitempty" json:"protocol,omitempty" ` // default: http
Host string ` yaml:"host" json:"host" `
Port string ` yaml:"port" json:"port" `
}
/ *
* Loadbalancer
* /
/ * LoadbalancerConfig defines the coarse file structure to configure the k3d - proxy
* Example :
* ports :
* 1234. tcp :
* - k3d - k3s - default - server - 0
* - k3d - k3s - default - server - 1
* 4321. udp :
* - k3d - k3s - default - agent - 0
* - k3d - k3s - default - agent - 1
* /
type LoadbalancerConfig struct {
Ports map [ string ] [ ] string ` yaml:"ports" `
}
const DefaultLoadbalancerConfigPath = "/etc/confd/portmap.yaml"
type LoadbalancerCreateOpts struct {
Labels map [ string ] string
}