- Check the documentation, help text and tutorials for more details
- Communicate managed registry using the LocalRegistryHostingV1 spec from [KEP-1755](https://github.com/kubernetes/enhancements/blob/0d69f7cea6fbe73a7d70fab569c6898f5ccb7be0/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry/README.md)
- interesting especially for tools that reload images, like Tilt or Skaffold
- Config File Support
- Put all your CLI-Arguments/Flags into a more readable config file and re-use it everywhere (keep it in your repo)
- Note: this is not always a 1:1 matching in naming/syntax/semantics
- `k3d cluster create --config myconfig.yaml`
```yaml
apiVersion: k3d.io/v1alpha1
kind: Simple
name: mycluster
servers: 3
agents: 2
ports:
- port: 8080:80
nodeFilters:
- loadbalancer
```
- Check out our test cases in [pkg/config/test_assets/](./pkg/config/test_assets/) for more config file examples
- [WIP] Support for Lifecycle Hooks
- Run any executable at specific stages during the cluster and node lifecycles
- e.g. we modify the `registries.yaml` in the `preStart` stage of nodes
- Guides will follow
#### Misc
- Now building with Go 1.15
- same for the k3d-tools code
- updated dependencies (including Docker v20.10)
- tests/e2e: add E2E_INCLUDE and rename E2E_SKIP to E2E_EXCLUDE
- use [Homebrew](https://brew.sh): `brew install k3d` (Homebrew is available for MacOS and Linux)
- use [Homebrew](https://brew.sh): `brew install k3d` (Homebrew is available for MacOS and Linux)
- Formula can be found in [homebrew/homebrew-core](https://github.com/Homebrew/homebrew-core/blob/master/Formula/k3d.rb) and is mirrored to [homebrew/linuxbrew-core](https://github.com/Homebrew/linuxbrew-core/blob/master/Formula/k3d.rb)
- Formula can be found in [homebrew/homebrew-core](https://github.com/Homebrew/homebrew-core/blob/master/Formula/k3d.rb) and is mirrored to [homebrew/linuxbrew-core](https://github.com/Homebrew/linuxbrew-core/blob/master/Formula/k3d.rb)
@ -67,7 +68,7 @@ or...
## Build
## Build
1. Clone this repo, e.g. via `git clone git@github.com:rancher/k3d.git` or `go get github.com/rancher/k3d/v3@main`
1. Clone this repo, e.g. via `git clone git@github.com:rancher/k3d.git` or `go get github.com/rancher/k3d/v4@main`
2. Inside the repo run
2. Inside the repo run
- 'make install-tools' to make sure required go packages are installed
- 'make install-tools' to make sure required go packages are installed
3. Inside the repo run one of the following commands
3. Inside the repo run one of the following commands
@ -82,7 +83,7 @@ Check out what you can do via `k3d help` or check the docs @ [k3d.io](https://k3
Example Workflow: Create a new cluster and use it with `kubectl`
Example Workflow: Create a new cluster and use it with `kubectl`
1. `k3d cluster create CLUSTER_NAME` to create a new single-node cluster (= 1 container running k3s + 1 loadbalancer container)
1. `k3d cluster create CLUSTER_NAME` to create a new single-node cluster (= 1 container running k3s + 1 loadbalancer container)
2. `k3d kubeconfig merge CLUSTER_NAME --switch-context` to update your default kubeconfig and switch the current-context to the new one
2. [Optional, included in cluster create] `k3d kubeconfig merge CLUSTER_NAME --kubeconfig-switch-context` to update your default kubeconfig and switch the current-context to the new one
3. execute some commands like `kubectl get pods --all-namespaces`
3. execute some commands like `kubectl get pods --all-namespaces`
4. `k3d cluster delete CLUSTER_NAME` to delete the default cluster
4. `k3d cluster delete CLUSTER_NAME` to delete the default cluster
@ -98,7 +99,8 @@ This repository is based on [@zeerorg](https://github.com/zeerorg/)'s [zeerorg/k
## Related Projects
## Related Projects
- [k3x](https://github.com/inercia/k3x): a graphics interface (for Linux) to k3d.
- [k3x](https://github.com/inercia/k3x): GUI (Linux) to k3d
- [vscode-k3d](https://github.com/inercia/vscode-k3d): vscode plugin for k3d
cmd.Flags().String("api-port","random","Specify the Kubernetes API server port exposed on the LoadBalancer (Format: `[HOST:]HOSTPORT`)\n - Example: `k3d cluster create --servers 3 --api-port 0.0.0.0:6550`")
cmd.Flags().StringVar(&ppFlags.APIPort,"api-port","random","Specify the Kubernetes API server port exposed on the LoadBalancer (Format: `[HOST:]HOSTPORT`)\n - Example: `k3d cluster create --servers 3 --api-port 0.0.0.0:6550`")
cmd.Flags().IntP("servers","s",1,"Specify how many servers you want to create")
cmd.Flags().IntVarP(&cliConfig.Servers,"servers","s",1,"Specify how many servers you want to create")
cmd.Flags().IntP("agents","a",0,"Specify how many agents you want to create")
cmd.Flags().IntVarP(&cliConfig.Agents,"agents","a",0,"Specify how many agents you want to create")
cmd.Flags().StringP("image","i",fmt.Sprintf("%s:%s",k3d.DefaultK3sImageRepo,version.GetK3sVersion(false)),"Specify k3s image that you want to use for the nodes")
cmd.Flags().StringVarP(&cliConfig.Image,"image","i",fmt.Sprintf("%s:%s",k3d.DefaultK3sImageRepo,version.GetK3sVersion(false)),"Specify k3s image that you want to use for the nodes")
cmd.Flags().String("network","","Join an existing network")
cmd.Flags().StringVar(&cliConfig.Network,"network","","Join an existing network")
cmd.Flags().String("token","","Specify a cluster token. By default, we generate one.")
cmd.Flags().StringVar(&cliConfig.ClusterToken,"token","","Specify a cluster token. By default, we generate one.")
cmd.Flags().StringArrayP("volume","v",nil,"Mount volumes into the nodes (Format: `[SOURCE:]DEST[@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -v \"/my/path@agent[0,1]\" -v \"/tmp/test:/tmp/other@server[0]\"`")
cmd.Flags().StringArrayVarP(&ppFlags.Volumes,"volume","v",nil,"Mount volumes into the nodes (Format: `[SOURCE:]DEST[@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -v /my/path@agent[0,1] -v /tmp/test:/tmp/other@server[0]`")
cmd.Flags().StringArrayP("port","p",nil,"Map ports from the node containers to the host (Format: `[HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]`)\n - Example: `k3d cluster create --agents 2 -p \"8080:80@agent[0]\" -p \"8081@agent[1]\"`")
cmd.Flags().StringArrayVarP(&ppFlags.Ports,"port","p",nil,"Map ports from the node containers to the host (Format: `[HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]`)\n - Example: `k3d cluster create --agents 2 -p 8080:80@agent[0] -p 8081@agent[1]`")
cmd.Flags().BoolVar(&createClusterOpts.WaitForServer,"wait",true,"Wait for the server(s) to be ready before returning. Use '--timeout DURATION' to not wait forever.")
cmd.Flags().BoolVar(&cliConfig.Options.K3dOptions.Wait,"wait",true,"Wait for the server(s) to be ready before returning. Use '--timeout DURATION' to not wait forever.")
cmd.Flags().DurationVar(&createClusterOpts.Timeout,"timeout",0*time.Second,"Rollback changes if cluster couldn't be created in specified duration.")
cmd.Flags().DurationVar(&cliConfig.Options.K3dOptions.Timeout,"timeout",0*time.Second,"Rollback changes if cluster couldn't be created in specified duration.")
cmd.Flags().BoolVar(&updateDefaultKubeconfig,"update-default-kubeconfig",true,"Directly update the default kubeconfig with the new cluster's context")
cmd.Flags().BoolVar(&cliConfig.Options.KubeconfigOptions.UpdateDefaultKubeconfig,"kubeconfig-update-default",true,"Directly update the default kubeconfig with the new cluster's context")
cmd.Flags().BoolVar(&updateCurrentContext,"switch-context",true,"Directly switch the default kubeconfig's current-context to the new cluster's context (requires --update-default-kubeconfig)")
cmd.Flags().BoolVar(&cliConfig.Options.KubeconfigOptions.SwitchCurrentContext,"kubeconfig-switch-context",true,"Directly switch the default kubeconfig's current-context to the new cluster's context (requires --kubeconfig-update-default)")
cmd.Flags().BoolVar(&createClusterOpts.DisableLoadBalancer,"no-lb",false,"Disable the creation of a LoadBalancer in front of the server nodes")
cmd.Flags().BoolVar(&cliConfig.Options.K3dOptions.DisableLoadbalancer,"no-lb",false,"Disable the creation of a LoadBalancer in front of the server nodes")
cmd.Flags().BoolVar(&noRollback,"no-rollback",false,"Disable the automatic rollback actions, if anything goes wrong")
cmd.Flags().BoolVar(&cliConfig.Options.K3dOptions.NoRollback,"no-rollback",false,"Disable the automatic rollback actions, if anything goes wrong")
cmd.Flags().BoolVar(&createClusterOpts.PrepDisableHostIPInjection,"no-hostip",false,"Disable the automatic injection of the Host IP as 'host.k3d.internal' into the containers and CoreDNS")
cmd.Flags().BoolVar(&cliConfig.Options.K3dOptions.PrepDisableHostIPInjection,"no-hostip",false,"Disable the automatic injection of the Host IP as 'host.k3d.internal' into the containers and CoreDNS")
cmd.Flags().StringVar(&createClusterOpts.GPURequest,"gpus","","GPU devices to add to the cluster node containers ('all' to pass all GPUs) [From docker]")
cmd.Flags().StringVar(&cliConfig.Options.Runtime.GPURequest,"gpus","","GPU devices to add to the cluster node containers ('all' to pass all GPUs) [From docker]")
cmd.Flags().StringArrayVar(&createClusterOpts.K3sServerArgs,"k3s-server-arg",nil,"Additional args passed to the `k3s server` command on server nodes (new flag per arg)")
cmd.Flags().StringArrayVar(&cliConfig.Options.K3sOptions.ExtraServerArgs,"k3s-server-arg",nil,"Additional args passed to the `k3s server` command on server nodes (new flag per arg)")
cmd.Flags().StringArrayVar(&createClusterOpts.K3sAgentArgs,"k3s-agent-arg",nil,"Additional args passed to the `k3s agent` command on agent nodes (new flag per arg)")
cmd.Flags().StringArrayVar(&cliConfig.Options.K3sOptions.ExtraAgentArgs,"k3s-agent-arg",nil,"Additional args passed to the `k3s agent` command on agent nodes (new flag per arg)")
// in hostNetwork mode, we're not going to map a hostport. Here it should always use 6443.
// Note that hostNetwork mode is super inflexible and since we don't change the backend port (on the container), it will only be one hostmode cluster allowed.
log.Fatalln("Failed to mark flag --output as filename")
log.Fatalln("Failed to mark flag --output as filename")
}
}
cmd.Flags().BoolVarP(&mergeKubeconfigFlags.targetDefault,"merge-default-kubeconfig","d",false,fmt.Sprintf("Merge into the default kubeconfig ($KUBECONFIG or %s)",clientcmd.RecommendedHomeFile))
cmd.Flags().BoolVarP(&mergeKubeconfigFlags.targetDefault,"kubeconfig-merge-default","d",false,fmt.Sprintf("Merge into the default kubeconfig ($KUBECONFIG or %s)",clientcmd.RecommendedHomeFile))
cmd.Flags().BoolVarP(&writeKubeConfigOptions.UpdateExisting,"update","u",true,"Update conflicting fields in existing kubeconfig")
cmd.Flags().BoolVarP(&writeKubeConfigOptions.UpdateExisting,"update","u",true,"Update conflicting fields in existing kubeconfig")
cmd.Flags().BoolVarP(&writeKubeConfigOptions.UpdateCurrentContext,"switch-context","s",true,"Switch to new context")
cmd.Flags().BoolVarP(&writeKubeConfigOptions.UpdateCurrentContext,"kubeconfig-switch-context","s",true,"Switch to new context")
cmd.Flags().BoolVar(&writeKubeConfigOptions.OverwriteExisting,"overwrite",false,"[Careful!] Overwrite existing file, ignoring its contents")
cmd.Flags().BoolVar(&writeKubeConfigOptions.OverwriteExisting,"overwrite",false,"[Careful!] Overwrite existing file, ignoring its contents")
cmd.Flags().BoolVarP(&mergeKubeconfigFlags.all,"all","a",false,"Get kubeconfigs from all existing clusters")
cmd.Flags().BoolVarP(&mergeKubeconfigFlags.all,"all","a",false,"Get kubeconfigs from all existing clusters")
log.Fatalln("Failed to hide --cluster flag on registry create command")
}
cmd.Flags().StringVarP(&flags.Image,"image","i",fmt.Sprintf("%s:%s",k3d.DefaultRegistryImageRepo,k3d.DefaultRegistryImageTag),"Specify image used for the registry")
cmd.Flags().StringVarP(&ppFlags.Port,"port","p","random","Select which port the registry should be listening on on your machine (localhost) (Format: `[HOST:]HOSTPORT`)\n - Example: `k3d registry create --port 0.0.0.0:5111`")
// done
returncmd
}
// parseCreateRegistryCmd parses the command input into variables required to create a registry
- use [Homebrew](https://brew.sh): `#!bash brew install k3d` (Homebrew is available for MacOS and Linux)
- use [Homebrew](https://brew.sh): `#!bash brew install k3d` (Homebrew is available for MacOS and Linux)
- Formula can be found in [homebrew/homebrew-core](https://github.com/Homebrew/homebrew-core/blob/master/Formula/k3d.rb) and is mirrored to [homebrew/linuxbrew-core](https://github.com/Homebrew/linuxbrew-core/blob/master/Formula/k3d.rb)
- Formula can be found in [homebrew/homebrew-core](https://github.com/Homebrew/homebrew-core/blob/master/Formula/k3d.rb) and is mirrored to [homebrew/linuxbrew-core](https://github.com/Homebrew/linuxbrew-core/blob/master/Formula/k3d.rb)
@ -61,7 +61,7 @@ k3d cluster create mycluster
Get the new cluster's connection details merged into your default kubeconfig (usually specified using the `KUBECONFIG` environment variable or the default path `#!bash $HOME/.kube/config`) and directly switch to the new context:
Get the new cluster's connection details merged into your default kubeconfig (usually specified using the `KUBECONFIG` environment variable or the default path `#!bash $HOME/.kube/config`) and directly switch to the new context:
completion [bash | zsh | (psh | powershell)] # generate completion scripts for common shells
cluster [CLUSTERNAME] # default cluster name is 'k3s-default'
cluster [CLUSTERNAME] # default cluster name is 'k3s-default'
create
create
--api-port # specify the port on which the cluster will be accessible (e.g. via kubectl)
-a, --agents # specify how many agent nodes you want to create (integer, default: 0)
-i, --image # specify which k3s image should be used for the nodes
--api-port # specify the port on which the cluster will be accessible (format '[HOST:]HOSTPORT', default: random)
--k3s-agent-arg # add additional arguments to the k3s agent (see https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/#k3s-agent-cli-help)
-c, --config # use a config file (format 'PATH')
--k3s-server-arg # add additional arguments to the k3s server (see https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/#k3s-server-cli-help)
-e, --env # add environment variables to the nodes (quoted string, format: 'KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]', use flag multiple times)
-s, --servers # specify how many server nodes you want to create
--gpus # [from docker CLI] add GPU devices to the node containers (string, e.g. 'all')
--network # specify a network you want to connect to
-i, --image # specify which k3s image should be used for the nodes (string, default: 'docker.io/rancher/k3s:v1.20.0-k3s2', tag changes per build)
--no-hostip # disable the automatic injection of the Host IP as 'host.k3d.internal' into the containers and CoreDN
--k3s-agent-arg # add additional arguments to the k3s agent (quoted string, use flag multiple times) (see https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/#k3s-agent-cli-help)
--no-image-volume # disable the creation of a volume for storing images (used for the 'k3d load image' command)
--k3s-server-arg # add additional arguments to the k3s server (quoted string, use flag multiple times) (see https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/#k3s-server-cli-help)
--no-lb # disable the creation of a LoadBalancer in front of the server nodes
--kubeconfig-switch-context # (implies --kubeconfig-update-default) automatically sets the current-context of your default kubeconfig to the new cluster's context (default: true)
--no-rollback # disable the automatic rollback actions, if anything goes wrong
--kubeconfig-update-default # enable the automated update of the default kubeconfig with the details of the newly created cluster (also sets '--wait=true') (default: true)
-p, --port # add some more port mappings
-l, --label # add (docker) labels to the node containers (format: 'KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]', use flag multiple times)
--token # specify a cluster token (default: auto-generated)
--network # specify an existing (docker) network you want to connect to (string)
--timeout # specify a timeout, after which the cluster creation will be interrupted and changes rolled back
--no-hostip # disable the automatic injection of the Host IP as 'host.k3d.internal' into the containers and CoreDNS (default: false)
--update-default-kubeconfig # enable the automated update of the default kubeconfig with the details of the newly created cluster (also sets '--wait=true')
--no-image-volume # disable the creation of a volume for storing images (used for the 'k3d image import' command) (default: false)
--switch-context # (implies --update-default-kubeconfig) automatically sets the current-context of your default kubeconfig to the new cluster's context
--no-lb # disable the creation of a load balancer in front of the server nodes (default: false)
-v, --volume # specify additional bind-mounts
--no-rollback # disable the automatic rollback actions, if anything goes wrong (default: false)
--wait # enable waiting for all server nodes to be ready before returning
-p, --port # add some more port mappings (format: '[HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]', use flag multiple times)
-a, --agents # specify how many agent nodes you want to create
--registry-create # create a new (docker) registry dedicated for this cluster (default: false)
-e, --env # add environment variables to the node containers
--registry-use # use an existing local (docker) registry with this cluster (string, use multiple times)
-s, --servers # specify how many server nodes you want to create (integer, default: 1)
--token # specify a cluster token (string, default: auto-generated)
--timeout # specify a timeout, after which the cluster creation will be interrupted and changes rolled back (duration, e.g. '10s')
-v, --volume # specify additional bind-mounts (format: '[SOURCE:]DEST[@NODEFILTER[;NODEFILTER...]]', use flag multiple times)
--wait # enable waiting for all server nodes to be ready before returning (default: true)
start CLUSTERNAME # start a (stopped) cluster
start CLUSTERNAME # start a (stopped) cluster
-a, --all # start all clusters
-a, --all # start all clusters (default: false)
--wait # wait for all servers and server-loadbalancer to be up before returning
--wait # wait for all servers and server-loadbalancer to be up before returning (default: true)
--timeout # maximum waiting time for '--wait' before canceling/returning
--timeout # maximum waiting time for '--wait' before canceling/returning (duration, e.g. '10s')
stop CLUSTERNAME # stop a cluster
stop CLUSTERNAME # stop a cluster
-a, --all # stop all clusters
-a, --all # stop all clusters (default: false)
delete CLUSTERNAME # delete an existing cluster
delete CLUSTERNAME # delete an existing cluster
-a, --all # delete all existing clusters
-a, --all # delete all existing clusters (default: false)
list [CLUSTERNAME [CLUSTERNAME ...]]
list [CLUSTERNAME [CLUSTERNAME ...]]
--no-headers # do not print headers
--no-headers # do not print headers (default: false)
--token # show column with cluster tokens
--token # show column with cluster tokens (default: false)
-o, --output # format the output (format: 'json|yaml')
completion [bash | zsh | fish | (psh | powershell)] # generate completion scripts for common shells
config
init # write a default k3d config (as a starting point)
-f, --force # force overwrite target file (default: false)
-o, --output # file to write to (string, default "k3d-default.yaml")
help [COMMAND] # show help text for any command
image
import [IMAGE | ARCHIVE [IMAGE | ARCHIVE ...]] # Load one or more images from the local runtime environment or tar-archives into k3d clusters
-c, --cluster # clusters to load the image into (string, use flag multiple times, default: k3s-default)
-k, --keep-tarball # do not delete the image tarball from the shared volume after completion (default: false)
kubeconfig
get (CLUSTERNAME [CLUSTERNAME ...] | --all) # get kubeconfig from cluster(s) and write it to stdout
-a, --all # get kubeconfigs from all clusters (default: false)
merge | write (CLUSTERNAME [CLUSTERNAME ...] | --all) # get kubeconfig from cluster(s) and merge it/them into a (kubeconfig-)file
-a, --all # get kubeconfigs from all clusters (default: false)
-s, --kubeconfig-switch-context # switch current-context in kubeconfig to the new context (default: true)
-d, --kubeconfig-merge-default # update the default kubeconfig (usually $KUBECONFIG or $HOME/.kube/config)
-o, --output # specify the output file where the kubeconfig should be written to (string)
create NODENAME # Create new nodes (and add them to existing clusters)
create NODENAME # Create new nodes (and add them to existing clusters)
-c, --cluster # specify the cluster that the node shall connect to
-c, --cluster # specify the cluster that the node shall connect to (string, default: k3s-default)
-i, --image # specify which k3s image should be used for the node(s)
-i, --image # specify which k3s image should be used for the node(s) (string, default: 'docker.io/rancher/k3s:v1.20.0-k3s2', tag changes per build)
--replicas # specify how many replicas you want to create with this spec
--replicas # specify how many replicas you want to create with this spec (integer, default: 1)
--role # specify the node role
--role # specify the node role (string, format: 'agent|server', default: agent)
--wait # wait for the node to be up and running before returning
--timeout # specify a timeout duration, after which the node creation will be interrupted, if not done yet (duration, e.g. '10s')
--timeout # specify a timeout duration, after which the node creation will be interrupted, if not done yet
--wait # wait for the node to be up and running before returning (default: true)
start NODENAME # start a (stopped) node
start NODENAME # start a (stopped) node
stop NODENAME # stop a node
stop NODENAME # stop a node
delete NODENAME # delete an existing node
delete NODENAME # delete an existing node
-a, --all # delete all existing nodes
-a, --all # delete all existing nodes (default: false)
list NODENAME
list NODENAME
--no-headers # do not print headers
--no-headers # do not print headers (default: false)
kubeconfig
registry
get (CLUSTERNAME [CLUSTERNAME ...] | --all) # get kubeconfig from cluster(s) and write it to stdout
create REGISTRYNAME
-a, --all # get kubeconfigs from all clusters
-i, --image # specify image used for the registry (string, default: "docker.io/library/registry:2")
merge | write (CLUSTERNAME [CLUSTERNAME ...] | --all) # get kubeconfig from cluster(s) and merge it/them into into a file in $HOME/.k3d (or whatever you specify via the flags)
-p, --port # select host port to map to (format: '[HOST:]HOSTPORT', default: 'random')
-a, --all # get kubeconfigs from all clusters
delete REGISTRYNAME
--output # specify the output file where the kubeconfig should be written to
-a, --all # delete all existing registries (default: false)
If you want to run CUDA workloads on the K3S container you need to customize the container.
If you want to run CUDA workloads on the K3S container you need to customize the container.
CUDA workloads require the NVIDIA Container Runtime, so containerd needs to be configured to use this runtime.
CUDA workloads require the NVIDIA Container Runtime, so containerd needs to be configured to use this runtime.
The K3S container itself also needs to run with this runtime. If you are using Docker you can install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html).
The K3S container itself also needs to run with this runtime. If you are using Docker you can install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html).
## Building a customized K3S image
## Building a customized K3S image
To get the NVIDIA container runtime in the K3S image you need to build your own K3S image. The native K3S image is based on Alpine but the NVIDIA container runtime is not supported on Alpine yet. To get around this we need to build the image with a supported base image.
To get the NVIDIA container runtime in the K3S image you need to build your own K3S image. The native K3S image is based on Alpine but the NVIDIA container runtime is not supported on Alpine yet. To get around this we need to build the image with a supported base image.
### Adapt the Dockerfile
### Adapt the Dockerfile
@ -48,13 +50,16 @@ ENV PATH="$PATH:/bin/aux"
ENTRYPOINT ["/bin/k3s"]
ENTRYPOINT ["/bin/k3s"]
CMD ["agent"]
CMD ["agent"]
```
```
This [Dockerfile](cuda/Dockerfile) is based on the [K3S Dockerfile](https://github.com/rancher/k3s/blob/master/package/Dockerfile).
This [Dockerfile](cuda/Dockerfile) is based on the [K3S Dockerfile](https://github.com/rancher/k3s/blob/master/package/Dockerfile).
The following changes are applied:
The following changes are applied:
1. Change the base images to Ubuntu 18.04 so the NVIDIA Container Runtime can be installed
1. Change the base images to Ubuntu 18.04 so the NVIDIA Container Runtime can be installed
2. Add a custom containerd `config.toml` template to add the NVIDIA Container Runtime. This replaces the default `runc` runtime
2. Add a custom containerd `config.toml` template to add the NVIDIA Container Runtime. This replaces the default `runc` runtime
3. Add a manifest for the NVIDIA driver plugin for Kubernetes
3. Add a manifest for the NVIDIA driver plugin for Kubernetes
### Configure containerd
### Configure containerd
We need to configure containerd to use the NVIDIA Container Runtime. We need to customize the config.toml that is used at startup. K3S provides a way to do this using a [config.toml.tmpl](cuda/config.toml.tmpl) file. More information can be found on the [K3S site](https://rancher.com/docs/k3s/latest/en/advanced/#configuring-containerd).
We need to configure containerd to use the NVIDIA Container Runtime. We need to customize the config.toml that is used at startup. K3S provides a way to do this using a [config.toml.tmpl](cuda/config.toml.tmpl) file. More information can be found on the [K3S site](https://rancher.com/docs/k3s/latest/en/advanced/#configuring-containerd).
```go
```go
@ -116,7 +121,9 @@ We need to configure containerd to use the NVIDIA Container Runtime. We need to
```
```
### The NVIDIA device plugin
### The NVIDIA device plugin
To enable NVIDIA GPU support on Kubernetes you also need to install the [NVIDIA device plugin](https://github.com/NVIDIA/k8s-device-plugin). The device plugin is a daemonset and allows you to automatically:
To enable NVIDIA GPU support on Kubernetes you also need to install the [NVIDIA device plugin](https://github.com/NVIDIA/k8s-device-plugin). The device plugin is a deamonset and allows you to automatically:
* Expose the number of GPUs on each nodes of your cluster
* Expose the number of GPUs on each nodes of your cluster
* Keep track of the health of your GPUs
* Keep track of the health of your GPUs
* Run GPU enabled containers in your Kubernetes cluster.
* Run GPU enabled containers in your Kubernetes cluster.
@ -166,9 +173,11 @@ spec:
```
```
### Build the K3S image
### Build the K3S image
To build the custom image we need to build K3S because we need the generated output.
To build the custom image we need to build K3S because we need the generated output.
Put the following files in a directory:
Put the following files in a directory:
* [Dockerfile](cuda/Dockerfile)
* [Dockerfile](cuda/Dockerfile)
* [config.toml.tmpl](cuda/config.toml.tmpl)
* [config.toml.tmpl](cuda/config.toml.tmpl)
* [gpu.yaml](cuda/gpu.yaml)
* [gpu.yaml](cuda/gpu.yaml)
@ -176,6 +185,7 @@ Put the following files in a directory:
k3d cluster create --no-lb --image k3s-gpu:v1.18.10-k3s1 --gpus all
k3d cluster create --no-lb --image k3s-gpu:v1.18.10-k3s1 --gpus all
```
```
Deploy a [test pod](cuda/cuda-vector-add.yaml):
Deploy a [test pod](cuda/cuda-vector-add.yaml):
```
```bash
kubectl apply -f cuda-vector-add.yaml
kubectl apply -f cuda-vector-add.yaml
kubectl logs cuda-vector-add
kubectl logs cuda-vector-add
```
```
## Known issues
## Known issues
* This approach does not work on WSL2 yet. The NVIDIA driver plugin and container runtime rely on the NVIDIA Management Library (NVML) which is not yet supported. See the [CUDA on WSL User Guide](https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations).
* This approach does not work on WSL2 yet. The NVIDIA driver plugin and container runtime rely on the NVIDIA Management Library (NVML) which is not yet supported. See the [CUDA on WSL User Guide](https://docs.nvidia.com/cuda/wsl-user-guide/index.html#known-limitations).
## Acknowledgements:
## Acknowledgements
Most of the information in this article was obtained from various sources:
Most of the information in this article was obtained from various sources:
* [Add NVIDIA GPU support to k3s with containerd](https://dev.to/mweibel/add-nvidia-gpu-support-to-k3s-with-containerd-4j17)
* [Add NVIDIA GPU support to k3s with containerd](https://dev.to/mweibel/add-nvidia-gpu-support-to-k3s-with-containerd-4j17)
In this example, we will deploy a simple nginx webserver deployment and make it accessible via ingress.
In this example, we will deploy a simple nginx webserver deployment and make it accessible via ingress.
Therefore, we have to create the cluster in a way, that the internal port 80 (where the `traefik` ingress controller is listening on) is exposed on the host system.
Therefore, we have to create the cluster in a way, that the internal port 80 (where the `traefik` ingress controller is listening on) is exposed on the host system.
@ -16,7 +16,7 @@ Therefore, we have to create the cluster in a way, that the internal port 80 (wh
- the `loadbalancer` nodefilter matches only the `serverlb` that's deployed in front of a cluster's server nodes
- the `loadbalancer` nodefilter matches only the `serverlb` that's deployed in front of a cluster's server nodes
- all ports exposed on the `serverlb` will be proxied to the same ports on all server nodes in the cluster
- all ports exposed on the `serverlb` will be proxied to the same ports on all server nodes in the cluster
2. Get the kubeconfig file
2. Get the kubeconfig file (redundant, as `k3d cluster create` already merges it into your default kubeconfig file)
@ -65,6 +65,7 @@ Therefore, we have to create the cluster in a way, that the internal port 80 (wh
- **Note**: Kubernetes' default NodePort range is [`30000-32767`](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport)
- **Note**: Kubernetes' default NodePort range is [`30000-32767`](https://kubernetes.io/docs/concepts/services-networking/service/#nodeport)
- **Note**: You may as well expose the whole NodePort range from the very beginning, e.g. via `k3d cluster create mycluster --agents 3 -p "30000-32767:30000-32767@server[0]"` (See [this video from @portainer](https://www.youtube.com/watch?v=5HaU6338lAk))
- **Note**: You may as well expose the whole NodePort range from the very beginning, e.g. via `k3d cluster create mycluster --agents 3 -p "30000-32767:30000-32767@server[0]"` (See [this video from @portainer](https://www.youtube.com/watch?v=5HaU6338lAk))
- **Warning**: Docker creates iptable entries and a new proxy process per port-mapping, so this may take a very long time or even freeze your system!
@ -65,12 +65,27 @@ Finally, we can create the cluster, mounting the CA file in the path we specifie
## Using a local registry
## Using a local registry
### Using the k3d registry
### Using k3d-managed registries
!!! info "Not ported yet"
!!! info "Just ported!"
The k3d-managed registry has not yet been ported from v1.x to v3.x
The k3d-managed registry is available again as of k3d v4.0.0 (January 2021)
### Using your own local registry
#### Create a dedicated registry together with your cluster
1. `#!bash k3d cluster create mycluster --registry-create`: This creates your cluster `mycluster` together with a registry container called `k3d-mycluster-registry`
- k3d sets everything up in the cluster for containerd to be able to pull images from that registry (using the `registries.yaml` file)
- the port, which the registry is listening on will be mapped to a random port on your host system
2. Check the k3d command output or `#!bash docker ps -f name=k3d-mycluster-registry` to find the exposed port (let's use `12345` here)
3. Pull some image (optional) `#!bash docker pull alpine:latest`, re-tag it to reference your newly created registry `#!bash docker tag alpine:latest k3d-mycluster-registry:12345/testimage:local` and push it `#!bash docker push k3d-mycluster-registry:12345/testimage:local`
4. Use kubectl to create a new pod in your cluster using that image to see, if the cluster can pull from the new registry: `#!bash kubectl run --image k3d-mycluster-registry:12345/testimage:local testimage --command -- tail -f /dev/null` (creates a container that will not do anything but keep on running)
#### Create a customized k3d-managed registry
1. `#!bash k3d registry create myregistry.localhost --port 5111` creates a new registry called `myregistry.localhost` (could be used with automatic resolution of `*.localhost`, see next section)
2. `#!bash k3d cluster create newcluster --registry-use k3d-myregistry.localhost:5111` (make sure you use the `k3d-` prefix here) creates a new cluster set up to us that registry
3. continue with step 3 and 4 from the last section for testing
### Using your own (not k3d-managed) local registry
You can start your own local registry it with some `docker` commands, like:
You can start your own local registry it with some `docker` commands, like:
@ -103,14 +118,14 @@ Once again, this will only work with k3s >= v0.10.0 (see the some sections below
You should test that you can
You should test that you can
* push to your registry from your local development machine.
- push to your registry from your local development machine.
* use images from that registry in `Deployments` in your k3d cluster.
- use images from that registry in `Deployments` in your k3d cluster.
We will verify these two things for a local registry (located at `registry.localhost:5000`) running in your development machine. Things would be basically the same for checking an external registry, but some additional configuration could be necessary in your local machine when using an authenticated or secure registry (please refer to Docker's documentation for this).
We will verify these two things for a local registry (located at `registry.localhost:5000`) running in your development machine. Things would be basically the same for checking an external registry, but some additional configuration could be necessary in your local machine when using an authenticated or secure registry (please refer to Docker's documentation for this).
First, we can download some image (like `nginx`) and push it to our local registry with:
First, we can download some image (like `nginx`) and push it to our local registry with:
```shell script
```bash
docker pull nginx:latest
docker pull nginx:latest
docker tag nginx:latest registry.localhost:5000/nginx:latest
docker tag nginx:latest registry.localhost:5000/nginx:latest
@ -30,7 +30,7 @@ To get a kubeconfig set up for you to connect to a k3d cluster, you can go diffe
!!! info "Switching the current context"
!!! info "Switching the current context"
None of the above options switch the current-context by default.
None of the above options switch the current-context by default.
This is intended to be least intrusive, since the current-context has a global effect.
This is intended to be least intrusive, since the current-context has a global effect.
You can switch the current-context directly with the `kubeconfig merge` command by adding the `--switch-context` flag.
You can switch the current-context directly with the `kubeconfig merge` command by adding the `--kubeconfig-switch-context` flag.
## Removing cluster details from the kubeconfig
## Removing cluster details from the kubeconfig
@ -40,6 +40,6 @@ It will also delete the respective kubeconfig file in `$HOME/.k3d/` if it exists
## Handling multiple clusters
## Handling multiple clusters
`k3d kubeconfig merge` let's you specify one or more clusters via arguments _or_ all via `--all`.
`k3d kubeconfig merge` let's you specify one or more clusters via arguments _or_ all via `--all`.
All kubeconfigs will then be merged into a single file if `--merge-default-kubeconfig` or `--output` is specified.
All kubeconfigs will then be merged into a single file if `--kubeconfig-merge-default` or `--output` is specified.
If none of those two flags was specified, a new file will be created per cluster and the merged path (e.g. `$HOME/.k3d/kubeconfig-cluster1.yaml:$HOME/.k3d/cluster2.yaml`) will be returned.
If none of those two flags was specified, a new file will be created per cluster and the merged path (e.g. `$HOME/.k3d/kubeconfig-cluster1.yaml:$HOME/.k3d/cluster2.yaml`) will be returned.
Note, that with multiple cluster specified, the `--switch-context` flag will change the current context to the cluster which was last in the list.
Note, that with multiple cluster specified, the `--kubeconfig-switch-context` flag will change the current context to the cluster which was last in the list.
extraLabels[k3d.LabelNetworkExternal]="true"// if the network wasn't created, we say that it's managed externally (important for cluster deletion)
clusterCreateOpts.GlobalLabels[k3d.LabelNetworkExternal]="true"// if the network wasn't created, we say that it's managed externally (important for cluster deletion)
returnfmt.Errorf("Node %s is restarting, early exit to avoid crash loop",node.Name)
log.Warnf("Node '%s' is restarting for more than a minute now. Possibly it will recover soon (e.g. when it's waiting to join). Consider using a creation timeout to avoid waiting forever in a Restart Loop.",node.Name)
}
}
time.Sleep(500*time.Millisecond)// wait for half a second to avoid overloading docker (error `socket: too many open files`)
time.Sleep(500*time.Millisecond)// wait for half a second to avoid overloading docker (error `socket: too many open files`)
returnnil,fmt.Errorf("Failed to parse registry spec from node %+v: 0 or multiple ports defined, where one is expected",node)
}
forport,bindings:=rangenode.Ports{
registry.ExposureOpts.Port=port
// we expect 0 or 1 binding for that port
iflen(bindings)>1{
returnnil,fmt.Errorf("Failed to parse registry spec from node %+v: Multiple bindings '%+v' specified for port '%s' where one is expected",node,bindings,port)
}
for_,binding:=rangebindings{
registry.ExposureOpts.Binding=binding
}
}
log.Tracef("Got registry %+v from node %+v",registry,node)
returnregistry,nil
}
// RegistryGenerateLocalRegistryHostingConfigMapYAML generates a ConfigMap used to advertise the registries in the cluster
t.Errorf("Computed configmap\n-> Actual: %s\n does not match expected YAML\n-> Expected: %s",strings.TrimSpace(string(cm)),strings.TrimSpace(expectedYAMLString))
// in hostNetwork mode, we're not going to map a hostport. Here it should always use 6443.
// Note that hostNetwork mode is super inflexible and since we don't change the backend port (on the container), it will only be one hostmode cluster allowed.
returnfmt.Errorf("The API Port can not be changed when using 'host' network")
}
// validate nodes one by one
for_,node:=rangeconfig.Cluster.Nodes{
// node names have to be valid hostnames // TODO: validate hostnames once we generate them before this step
/*iferr:=k3dc.CheckName(node.Name);err!=nil{
returnerr
}*/
// volumes have to be either an existing path on the host or a named runtime volume