/ *
Copyright © 2020 The k3d Author ( s )
Permission is hereby granted , free of charge , to any person obtaining a copy
of this software and associated documentation files ( the "Software" ) , to deal
in the Software without restriction , including without limitation the rights
to use , copy , modify , merge , publish , distribute , sublicense , and / or sell
copies of the Software , and to permit persons to whom the Software is
furnished to do so , subject to the following conditions :
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software .
THE SOFTWARE IS PROVIDED "AS IS" , WITHOUT WARRANTY OF ANY KIND , EXPRESS OR
IMPLIED , INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY ,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT . IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM , DAMAGES OR OTHER
LIABILITY , WHETHER IN AN ACTION OF CONTRACT , TORT OR OTHERWISE , ARISING FROM ,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE .
* /
package cluster
import (
"fmt"
"os"
"runtime"
"time"
"github.com/spf13/cobra"
cliutil "github.com/rancher/k3d/v3/cmd/util"
"github.com/rancher/k3d/v3/pkg/cluster"
k3dCluster "github.com/rancher/k3d/v3/pkg/cluster"
"github.com/rancher/k3d/v3/pkg/runtimes"
k3d "github.com/rancher/k3d/v3/pkg/types"
"github.com/rancher/k3d/v3/version"
log "github.com/sirupsen/logrus"
)
const clusterCreateDescription = `
Create a new k3s cluster with containerized nodes ( k3s in docker ) .
Every cluster will consist of one or more containers :
- 1 ( or more ) server node container ( k3s )
- ( optionally ) 1 loadbalancer container as the entrypoint to the cluster ( nginx )
- ( optionally ) 1 ( or more ) agent node containers ( k3s )
`
// NewCmdClusterCreate returns a new cobra command
func NewCmdClusterCreate ( ) * cobra . Command {
createClusterOpts := & k3d . ClusterCreateOpts { }
var noRollback bool
var updateDefaultKubeconfig , updateCurrentContext bool
// create new command
cmd := & cobra . Command {
Use : "create NAME" ,
Short : "Create a new cluster" ,
Long : clusterCreateDescription ,
Args : cobra . RangeArgs ( 0 , 1 ) , // exactly one cluster name can be set (default: k3d.DefaultClusterName)
Run : func ( cmd * cobra . Command , args [ ] string ) {
// parse args and flags
cluster := parseCreateClusterCmd ( cmd , args , createClusterOpts )
// check if a cluster with that name exists already
if _ , err := k3dCluster . ClusterGet ( cmd . Context ( ) , runtimes . SelectedRuntime , cluster ) ; err == nil {
log . Fatalf ( "Failed to create cluster '%s' because a cluster with that name already exists" , cluster . Name )
}
if ! updateDefaultKubeconfig && updateCurrentContext {
log . Infoln ( "--update-default-kubeconfig=false --> sets --switch-context=false" )
updateCurrentContext = false
}
// create cluster
if updateDefaultKubeconfig {
log . Debugln ( "'--update-default-kubeconfig set: enabling wait-for-server" )
cluster . CreateClusterOpts . WaitForServer = true
}
if err := k3dCluster . ClusterCreate ( cmd . Context ( ) , runtimes . SelectedRuntime , cluster ) ; err != nil {
log . Errorln ( err )
if noRollback {
log . Fatalln ( "Cluster creation FAILED, rollback deactivated." )
}
// rollback if creation failed
log . Errorln ( "Failed to create cluster >>> Rolling Back" )
if err := k3dCluster . ClusterDelete ( cmd . Context ( ) , runtimes . SelectedRuntime , cluster ) ; err != nil {
log . Errorln ( err )
log . Fatalln ( "Cluster creation FAILED, also FAILED to rollback changes!" )
}
log . Fatalln ( "Cluster creation FAILED, all changes have been rolled back!" )
}
log . Infof ( "Cluster '%s' created successfully!" , cluster . Name )
if updateDefaultKubeconfig {
log . Debugf ( "Updating default kubeconfig with a new context for cluster %s" , cluster . Name )
if _ , err := k3dCluster . KubeconfigGetWrite ( cmd . Context ( ) , runtimes . SelectedRuntime , cluster , "" , & k3dCluster . WriteKubeConfigOptions { UpdateExisting : true , OverwriteExisting : false , UpdateCurrentContext : updateCurrentContext } ) ; err != nil {
log . Warningln ( err )
}
}
// print information on how to use the cluster with kubectl
log . Infoln ( "You can now use it like this:" )
if updateDefaultKubeconfig && ! updateCurrentContext {
fmt . Printf ( "kubectl config use-context %s\n" , fmt . Sprintf ( "%s-%s" , k3d . DefaultObjectNamePrefix , cluster . Name ) )
} else if ! updateCurrentContext {
if runtime . GOOS == "windows" {
fmt . Printf ( "$env:KUBECONFIG=(%s kubeconfig write %s)\n" , os . Args [ 0 ] , cluster . Name )
} else {
fmt . Printf ( "export KUBECONFIG=$(%s kubeconfig write %s)\n" , os . Args [ 0 ] , cluster . Name )
}
}
fmt . Println ( "kubectl cluster-info" )
} ,
}
/ * * * * * * * * *
* Flags *
* * * * * * * * * /
cmd . Flags ( ) . String ( "api-port" , "random" , "Specify the Kubernetes API server port exposed on the LoadBalancer (Format: `[HOST:]HOSTPORT`)\n - Example: `k3d cluster create --servers 3 --api-port 0.0.0.0:6550`" )
cmd . Flags ( ) . IntP ( "servers" , "s" , 1 , "Specify how many servers you want to create" )
cmd . Flags ( ) . IntP ( "agents" , "a" , 0 , "Specify how many agents you want to create" )
cmd . Flags ( ) . StringP ( "image" , "i" , fmt . Sprintf ( "%s:%s" , k3d . DefaultK3sImageRepo , version . GetK3sVersion ( false ) ) , "Specify k3s image that you want to use for the nodes" )
cmd . Flags ( ) . String ( "network" , "" , "Join an existing network" )
cmd . Flags ( ) . String ( "token" , "" , "Specify a cluster token. By default, we generate one." )
cmd . Flags ( ) . StringArrayP ( "volume" , "v" , nil , "Mount volumes into the nodes (Format: `[SOURCE:]DEST[@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -v \"/my/path@agent[0,1]\" -v \"/tmp/test:/tmp/other@server[0]\"`" )
cmd . Flags ( ) . StringArrayP ( "port" , "p" , nil , "Map ports from the node containers to the host (Format: `[HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]`)\n - Example: `k3d cluster create --agents 2 -p \"8080:80@agent[0]\" -p \"8081@agent[1]\"`" )
cmd . Flags ( ) . BoolVar ( & createClusterOpts . WaitForServer , "wait" , true , "Wait for the server(s) to be ready before returning. Use '--timeout DURATION' to not wait forever." )
cmd . Flags ( ) . DurationVar ( & createClusterOpts . Timeout , "timeout" , 0 * time . Second , "Rollback changes if cluster couldn't be created in specified duration." )
cmd . Flags ( ) . BoolVar ( & updateDefaultKubeconfig , "update-default-kubeconfig" , true , "Directly update the default kubeconfig with the new cluster's context" )
cmd . Flags ( ) . BoolVar ( & updateCurrentContext , "switch-context" , true , "Directly switch the default kubeconfig's current-context to the new cluster's context (requires --update-default-kubeconfig)" )
cmd . Flags ( ) . BoolVar ( & createClusterOpts . DisableLoadBalancer , "no-lb" , false , "Disable the creation of a LoadBalancer in front of the server nodes" )
cmd . Flags ( ) . BoolVar ( & noRollback , "no-rollback" , false , "Disable the automatic rollback actions, if anything goes wrong" )
cmd . Flags ( ) . BoolVar ( & createClusterOpts . PrepDisableHostIPInjection , "no-hostip" , false , "Disable the automatic injection of the Host IP as 'host.k3d.internal' into the containers and CoreDNS" )
/* Image Importing */
cmd . Flags ( ) . BoolVar ( & createClusterOpts . DisableImageVolume , "no-image-volume" , false , "Disable the creation of a volume for importing images" )
/* Multi Server Configuration */
// multi-server - datastore
// TODO: implement multi-server setups with external data store
// cmd.Flags().String("datastore-endpoint", "", "[WIP] Specify external datastore endpoint (e.g. for multi server clusters)")
/ *
cmd . Flags ( ) . String ( "datastore-network" , "" , "Specify container network where we can find the datastore-endpoint (add a connection)" )
// TODO: set default paths and hint, that one should simply mount the files using --volume flag
cmd . Flags ( ) . String ( "datastore-cafile" , "" , "Specify external datastore's TLS Certificate Authority (CA) file" )
cmd . Flags ( ) . String ( "datastore-certfile" , "" , "Specify external datastore's TLS certificate file'" )
cmd . Flags ( ) . String ( "datastore-keyfile" , "" , "Specify external datastore's TLS key file'" )
* /
/* k3s */
cmd . Flags ( ) . StringArrayVar ( & createClusterOpts . K3sServerArgs , "k3s-server-arg" , nil , "Additional args passed to the `k3s server` command on server nodes (new flag per arg)" )
cmd . Flags ( ) . StringArrayVar ( & createClusterOpts . K3sAgentArgs , "k3s-agent-arg" , nil , "Additional args passed to the `k3s agent` command on agent nodes (new flag per arg)" )
/* Subcommands */
// done
return cmd
}
// parseCreateClusterCmd parses the command input into variables required to create a cluster
func parseCreateClusterCmd ( cmd * cobra . Command , args [ ] string , createClusterOpts * k3d . ClusterCreateOpts ) * k3d . Cluster {
/ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Parse and validate arguments *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * /
clustername := k3d . DefaultClusterName
if len ( args ) != 0 {
clustername = args [ 0 ]
}
if err := cluster . CheckName ( clustername ) ; err != nil {
log . Fatal ( err )
}
/ * * * * * * * * * * * * * * * * * * * * * * * * * * * *
* Parse and validate flags *
* * * * * * * * * * * * * * * * * * * * * * * * * * * * /
// --image
image , err := cmd . Flags ( ) . GetString ( "image" )
if err != nil {
log . Errorln ( "No image specified" )
log . Fatalln ( err )
}
if image == "latest" {
image = version . GetK3sVersion ( true )
}
// --servers
serverCount , err := cmd . Flags ( ) . GetInt ( "servers" )
if err != nil {
log . Fatalln ( err )
}
// --agents
agentCount , err := cmd . Flags ( ) . GetInt ( "agents" )
if err != nil {
log . Fatalln ( err )
}
// --network
networkName , err := cmd . Flags ( ) . GetString ( "network" )
if err != nil {
log . Fatalln ( err )
}
network := k3d . ClusterNetwork { }
if networkName != "" {
network . Name = networkName
network . External = true
}
if networkName == "host" && ( serverCount + agentCount ) > 1 {
log . Fatalln ( "Can only run a single node in hostnetwork mode" )
}
// --token
token , err := cmd . Flags ( ) . GetString ( "token" )
if err != nil {
log . Fatalln ( err )
}
// --timeout
if cmd . Flags ( ) . Changed ( "timeout" ) && createClusterOpts . Timeout <= 0 * time . Second {
log . Fatalln ( "--timeout DURATION must be >= 1s" )
}
// --api-port
apiPort , err := cmd . Flags ( ) . GetString ( "api-port" )
if err != nil {
log . Fatalln ( err )
}
// parse the port mapping
exposeAPI , err := cliutil . ParseAPIPort ( apiPort )
if err != nil {
log . Fatalln ( err )
}
if exposeAPI . Host == "" {
exposeAPI . Host = k3d . DefaultAPIHost
}
if exposeAPI . HostIP == "" {
exposeAPI . HostIP = k3d . DefaultAPIHost
}
if networkName == "host" {
// in hostNetwork mode, we're not going to map a hostport. Here it should always use 6443.
// Note that hostNetwork mode is super inflexible and since we don't change the backend port (on the container), it will only be one hostmode cluster allowed.
exposeAPI . Port = k3d . DefaultAPIPort
}
// --volume
volumeFlags , err := cmd . Flags ( ) . GetStringArray ( "volume" )
if err != nil {
log . Fatalln ( err )
}
// volumeFilterMap will map volume mounts to applied node filters
volumeFilterMap := make ( map [ string ] [ ] string , 1 )
for _ , volumeFlag := range volumeFlags {
// split node filter from the specified volume
volume , filters , err := cliutil . SplitFiltersFromFlag ( volumeFlag )
if err != nil {
log . Fatalln ( err )
}
// validate the specified volume mount and return it in SRC:DEST format
volume , err = cliutil . ValidateVolumeMount ( runtimes . SelectedRuntime , volume )
if err != nil {
log . Fatalln ( err )
}
// create new entry or append filter to existing entry
if _ , exists := volumeFilterMap [ volume ] ; exists {
volumeFilterMap [ volume ] = append ( volumeFilterMap [ volume ] , filters ... )
} else {
volumeFilterMap [ volume ] = filters
}
}
// --port
portFlags , err := cmd . Flags ( ) . GetStringArray ( "port" )
if err != nil {
log . Fatalln ( err )
}
portFilterMap := make ( map [ string ] [ ] string , 1 )
for _ , portFlag := range portFlags {
// split node filter from the specified volume
portmap , filters , err := cliutil . SplitFiltersFromFlag ( portFlag )
if err != nil {
log . Fatalln ( err )
}
if len ( filters ) > 1 {
log . Fatalln ( "Can only apply a Portmap to one node" )
}
// the same portmapping can't be applied to multiple nodes
// validate the specified volume mount and return it in SRC:DEST format
portmap , err = cliutil . ValidatePortMap ( portmap )
if err != nil {
log . Fatalln ( err )
}
// create new entry or append filter to existing entry
if _ , exists := portFilterMap [ portmap ] ; exists {
log . Fatalln ( "Same Portmapping can not be used for multiple nodes" )
} else {
portFilterMap [ portmap ] = filters
}
}
log . Tracef ( "PortFilterMap: %+v" , portFilterMap )
/ * * * * * * * * * * * * * * * * * * * *
* *
* generate cluster *
* *
* * * * * * * * * * * * * * * * * * * * /
cluster := & k3d . Cluster {
Name : clustername ,
Network : network ,
Token : token ,
CreateClusterOpts : createClusterOpts ,
ExposeAPI : exposeAPI ,
}
// generate list of nodes
cluster . Nodes = [ ] * k3d . Node { }
// ServerLoadBalancer
if ! createClusterOpts . DisableLoadBalancer {
cluster . ServerLoadBalancer = & k3d . Node {
Role : k3d . LoadBalancerRole ,
}
}
/ * * * * * * * * * * * * * * * *
* Server Nodes *
* * * * * * * * * * * * * * * * /
for i := 0 ; i < serverCount ; i ++ {
node := k3d . Node {
Role : k3d . ServerRole ,
Image : image ,
Args : createClusterOpts . K3sServerArgs ,
ServerOpts : k3d . ServerOpts { } ,
}
// TODO: by default, we don't expose an API port: should we change that?
// -> if we want to change that, simply add the exposeAPI struct here
// first server node will be init node if we have more than one server specified but no external datastore
if i == 0 && serverCount > 1 {
node . ServerOpts . IsInit = true
cluster . InitNode = & node
}
// append node to list
cluster . Nodes = append ( cluster . Nodes , & node )
}
/ * * * * * * * * * * * * * * * *
* Agent Nodes *
* * * * * * * * * * * * * * * * /
for i := 0 ; i < agentCount ; i ++ {
node := k3d . Node {
Role : k3d . AgentRole ,
Image : image ,
Args : createClusterOpts . K3sAgentArgs ,
}
cluster . Nodes = append ( cluster . Nodes , & node )
}
// append volumes
for volume , filters := range volumeFilterMap {
nodes , err := cliutil . FilterNodes ( cluster . Nodes , filters )
if err != nil {
log . Fatalln ( err )
}
for _ , node := range nodes {
node . Volumes = append ( node . Volumes , volume )
}
}
// append ports
nodeCount := serverCount + agentCount
nodeList := cluster . Nodes
if ! createClusterOpts . DisableLoadBalancer {
nodeCount ++
nodeList = append ( nodeList , cluster . ServerLoadBalancer )
}
for portmap , filters := range portFilterMap {
if len ( filters ) == 0 && ( nodeCount ) > 1 {
log . Fatalf ( "Malformed portmapping '%s' lacks a node filter, but there is more than one node (including the loadbalancer, if there is any)." , portmap )
}
nodes , err := cliutil . FilterNodes ( nodeList , filters )
if err != nil {
log . Fatalln ( err )
}
for _ , node := range nodes {
node . Ports = append ( node . Ports , portmap )
}
}
/ * * * * * * * * * * * * * * * * * * * * * *
* Utility Containers *
* * * * * * * * * * * * * * * * * * * * * * /
// ...
return cluster
}