[Feature] Volume Shortcuts and k3d-managed volumes (#916)

- volume-mount destination shortcuts, e.g. `k3s-storage`
- non-existing named volumes starting with `k3d-` created/handled by k3d
- removed unused volume validation logic in `cmd/util`
pull/917/head^2
Thorsten Klein 3 years ago committed by GitHub
parent 280de52b01
commit 5a00a39323
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 21
      CHANGELOG.md
  2. 107
      cmd/util/volumes.go
  3. 19
      pkg/client/cluster.go
  4. 17
      pkg/config/process.go
  5. 12
      pkg/config/process_test.go
  6. 2
      pkg/config/validate.go
  7. 38
      pkg/runtimes/docker/volume.go
  8. 5
      pkg/runtimes/errors/errors.go
  9. 1
      pkg/runtimes/runtime.go
  10. 79
      pkg/runtimes/util/volumes.go
  11. 40
      pkg/types/k3s/paths.go
  12. 1
      pkg/types/types.go

@ -1,5 +1,26 @@
# Changelog
## v5.3.0
**Note:** Now trying to follow a standard scheme defined by <https://keepachangelog.com/en/1.0.0/>
### Added
- Volumemount-Shortcuts (#916)
- Use some destination shortcuts with the `--volume/-v` flag that k3d automatically expands
- `k3s-storage` -> `/var/lib/rancher/k3s/storage`
- `k3s-manifests` -> `/var/lib/rancher/k3s/server/manifests`
- `k3s-manifests-custom` -> `/var/lib/rancher/k3s/server/manifests/custom` (not K3s default: this is just some sub-directory inside the auto-deploy manifests directory which will also be parsed)
- `k3s-containerd` -> `/var/lib/rancher/k3s/agent/etc/containerd/config.toml` (use with caution, K3s generates this file!)
- `k3s-containerd-tmpl` -> `/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl` (used by K3s to generate the real config above)
- `k3s-registry-config` -> `/etc/rancher/k3s/registries.yaml` (or just use `--registry-config`)
- k3d-managed volumes (#916)
- non-existing named volumes starting with a `k3d-` prefix will now be created and managed by `k3d`
### Removed
- unused volume validation functionality in `cmd/util`, does not affect the CLI (#916)
## v5.2.2
### Fixes

@ -1,107 +0,0 @@
/*
Copyright © 2020-2021 The k3d Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package util
import (
"fmt"
"os"
rt "runtime"
"strings"
"github.com/rancher/k3d/v5/pkg/runtimes"
l "github.com/rancher/k3d/v5/pkg/logger"
)
// ValidateVolumeMount checks, if the source of volume mounts exists and if the destination is an absolute path
// - SRC: source directory/file -> tests: must exist
// - DEST: source directory/file -> tests: must be absolute path
// ValidateVolumeMount checks, if the source of volume mounts exists and if the destination is an absolute path
// - SRC: source directory/file -> tests: must exist
// - DEST: source directory/file -> tests: must be absolute path
func ValidateVolumeMount(runtime runtimes.Runtime, volumeMount string) (string, error) {
src := ""
dest := ""
// validate 'SRC[:DEST]' substring
split := strings.Split(volumeMount, ":")
// a volume mapping can have 3 parts seperated by a ':' followed by a node filter
// [SOURCE:]DEST[:OPT[,OPT]][@NODEFILTER[;NODEFILTER...]]
// On Windows the source path needs to be an absolute path which means the path starts with
// a drive designator and will also have a ':' in it. So for Windows the maxParts is increased by one.
maxParts := 3
if rt.GOOS == "windows" {
maxParts++
}
if len(split) < 1 {
return "", fmt.Errorf("No volume/path specified")
}
if len(split) > maxParts {
return "", fmt.Errorf("Invalid volume mount '%s': maximal %d ':' allowed", volumeMount, maxParts-1)
}
// we only have SRC specified -> DEST = SRC
// On windows the first part of the SRC is the drive letter, so we need to concat the first and second parts to get the path.
if len(split) == 1 {
src = split[0]
dest = src
} else if rt.GOOS == "windows" {
src = split[0] + ":" + split[1]
dest = split[2]
} else {
src = split[0]
dest = split[1]
}
// verify that the source exists
if src != "" {
// a) named volume
isNamedVolume := true
if err := verifyNamedVolume(runtime, src); err != nil {
isNamedVolume = false
}
if !isNamedVolume {
if _, err := os.Stat(src); err != nil {
l.Log().Warnf("Failed to stat file/directory/named volume that you're trying to mount: '%s' in '%s' -> Please make sure it exists", src, volumeMount)
}
}
}
// verify that the destination is an absolute path
if !strings.HasPrefix(dest, "/") {
return "", fmt.Errorf("Volume mount destination doesn't appear to be an absolute path: '%s' in '%s'", dest, volumeMount)
}
return volumeMount, nil
}
// verifyNamedVolume checks whether a named volume exists in the runtime
func verifyNamedVolume(runtime runtimes.Runtime, volumeName string) error {
volumeName, err := runtime.GetVolume(volumeName)
if err != nil {
return fmt.Errorf("Failed to verify named volume: %w", err)
}
if volumeName == "" {
return fmt.Errorf("Failed to find named volume '%s'", volumeName)
}
return nil
}

@ -309,9 +309,11 @@ func ClusterPrepImageVolume(ctx context.Context, runtime k3drt.Runtime, cluster
if err := runtime.CreateVolume(ctx, imageVolumeName, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil {
return fmt.Errorf("failed to create image volume '%s' for cluster '%s': %w", imageVolumeName, cluster.Name, err)
}
l.Log().Infof("Created image volume %s", imageVolumeName)
clusterCreateOpts.GlobalLabels[k3d.LabelImageVolume] = imageVolumeName
cluster.ImageVolume = imageVolumeName
cluster.Volumes = append(cluster.Volumes, imageVolumeName)
// attach volume to nodes
for _, node := range cluster.Nodes {
@ -640,11 +642,12 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
}
}
// delete image volume
if cluster.ImageVolume != "" {
l.Log().Infof("Deleting image volume '%s'", cluster.ImageVolume)
if err := runtime.DeleteVolume(ctx, cluster.ImageVolume); err != nil {
l.Log().Warningf("Failed to delete image volume '%s' of cluster '%s': Try to delete it manually", cluster.ImageVolume, cluster.Name)
// delete managed volumes attached to this cluster
l.Log().Infof("Deleting %d attached volumes...", len(cluster.Volumes))
for _, vol := range cluster.Volumes {
l.Log().Debugf("Deleting volume %s...", vol)
if err := runtime.DeleteVolume(ctx, vol); err != nil {
l.Log().Warningf("Failed to delete volume '%s' of cluster '%s': %v -> Try to delete it manually", cluster.ImageVolume, err, cluster.Name)
}
}
@ -806,6 +809,12 @@ func ClusterGet(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster
}
}
vols, err := runtime.GetVolumesByLabel(ctx, map[string]string{types.LabelClusterName: cluster.Name})
if err != nil {
return nil, err
}
cluster.Volumes = append(cluster.Volumes, vols...)
if err := populateClusterFieldsFromLabels(cluster); err != nil {
l.Log().Warnf("Failed to populate cluster fields from node labels: %v", err)
}

@ -23,9 +23,13 @@ THE SOFTWARE.
package config
import (
"strings"
conf "github.com/rancher/k3d/v5/pkg/config/v1alpha3"
l "github.com/rancher/k3d/v5/pkg/logger"
runtimeutil "github.com/rancher/k3d/v5/pkg/runtimes/util"
k3d "github.com/rancher/k3d/v5/pkg/types"
"github.com/rancher/k3d/v5/pkg/types/k3s"
)
// ProcessSimpleConfig applies processing to the simple config, sanitizing it and doing some modifications
@ -56,5 +60,18 @@ func ProcessClusterConfig(clusterConfig conf.ClusterConfig) (*conf.ClusterConfig
clusterConfig.ClusterCreateOpts.DisableLoadBalancer = true
}
for _, node := range clusterConfig.Cluster.Nodes {
for vIndex, volume := range node.Volumes {
_, dest, err := runtimeutil.ReadVolumeMount(volume)
if err != nil {
return nil, err
}
if path, ok := k3s.K3sPathShortcuts[dest]; ok {
l.Log().Tracef("[node: %s] expanding volume shortcut %s to %s", node.Name, dest, path)
node.Volumes[vIndex] = strings.Replace(volume, dest, path, 1)
}
}
}
return &clusterConfig, nil
}

@ -24,10 +24,12 @@ package config
import (
"context"
"strings"
"testing"
conf "github.com/rancher/k3d/v5/pkg/config/v1alpha3"
"github.com/rancher/k3d/v5/pkg/runtimes"
"github.com/rancher/k3d/v5/pkg/types/k3s"
"github.com/spf13/viper"
"gotest.tools/assert"
)
@ -51,11 +53,20 @@ func TestProcessClusterConfig(t *testing.T) {
t.Error(err)
}
// append some volume to test K3s volume shortcut expansion
clusterCfg.Cluster.Nodes[0].Volumes = append(clusterCfg.Cluster.Nodes[0].Volumes, "/tmp/testexpansion:k3s-storage:rw")
t.Logf("\n========== Process Cluster Config (non-host network) ==========\n%+v\n=================================\n", cfg)
clusterCfg, err = ProcessClusterConfig(*clusterCfg)
assert.Assert(t, clusterCfg.ClusterCreateOpts.DisableLoadBalancer == false, "The load balancer should be enabled")
for _, v := range clusterCfg.Cluster.Nodes[0].Volumes {
if strings.HasPrefix(v, "/tmp/testexpansion") {
assert.Assert(t, strings.Contains(v, k3s.K3sPathStorage), "volume path shortcut expansion of k3s-storage didn't work")
}
}
t.Logf("\n===== Resulting Cluster Config (non-host network) =====\n%+v\n===============\n", clusterCfg)
t.Logf("\n========== Process Cluster Config (host network) ==========\n%+v\n=================================\n", cfg)
@ -65,5 +76,6 @@ func TestProcessClusterConfig(t *testing.T) {
assert.Assert(t, clusterCfg.ClusterCreateOpts.DisableLoadBalancer == true, "The load balancer should be disabled")
t.Logf("\n===== Resulting Cluster Config (host network) =====\n%+v\n===============\n", clusterCfg)
t.Logf("\n===== First Node in Resulting Cluster Config (host network) =====\n%+v\n===============\n", clusterCfg.Cluster.Nodes[0])
}

@ -82,7 +82,7 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config
// volumes have to be either an existing path on the host or a named runtime volume
for _, volume := range node.Volumes {
if err := runtimeutil.ValidateVolumeMount(runtime, volume); err != nil {
if err := runtimeutil.ValidateVolumeMount(ctx, runtime, volume, &config.Cluster); err != nil {
return fmt.Errorf("failed to validate volume mount '%s': %w", volume, err)
}
}

@ -27,7 +27,7 @@ import (
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/volume"
l "github.com/rancher/k3d/v5/pkg/logger"
runtimeErrors "github.com/rancher/k3d/v5/pkg/runtimes/errors"
k3d "github.com/rancher/k3d/v5/pkg/types"
)
@ -55,11 +55,10 @@ func (d Docker) CreateVolume(ctx context.Context, name string, labels map[string
volumeCreateOptions.Labels[k] = v
}
vol, err := docker.VolumeCreate(ctx, volumeCreateOptions)
_, err = docker.VolumeCreate(ctx, volumeCreateOptions)
if err != nil {
return fmt.Errorf("failed to create volume '%s': %w", name, err)
}
l.Log().Infof("Created volume '%s'", vol.Name)
return nil
}
@ -110,9 +109,40 @@ func (d Docker) GetVolume(name string) (string, error) {
return "", fmt.Errorf("docker failed to list volumes: %w", err)
}
if len(volumeList.Volumes) < 1 {
return "", fmt.Errorf("failed to find named volume '%s'", name)
return "", fmt.Errorf("failed to find named volume '%s': %w", name, runtimeErrors.ErrRuntimeVolumeNotExists)
}
return volumeList.Volumes[0].Name, nil
}
func (d Docker) GetVolumesByLabel(ctx context.Context, labels map[string]string) ([]string, error) {
var volumes []string
// (0) create new docker client
docker, err := GetDockerClient()
if err != nil {
return volumes, fmt.Errorf("failed to get docker client: %w", err)
}
defer docker.Close()
// (1) list containers which have the default k3d labels attached
filters := filters.NewArgs()
for k, v := range k3d.DefaultRuntimeLabels {
filters.Add("label", fmt.Sprintf("%s=%s", k, v))
}
for k, v := range labels {
filters.Add("label", fmt.Sprintf("%s=%s", k, v))
}
volumeList, err := docker.VolumeList(ctx, filters)
if err != nil {
return volumes, fmt.Errorf("docker failed to list volumes: %w", err)
}
for _, v := range volumeList.Volumes {
volumes = append(volumes, v.Name)
}
return volumes, nil
}

@ -37,3 +37,8 @@ var (
// Container Filesystem Errors
var ErrRuntimeFileNotFound = errors.New("file not found")
// Runtime Volume Errors
var (
ErrRuntimeVolumeNotExists = errors.New("volume does not exist")
)

@ -65,6 +65,7 @@ type Runtime interface {
CreateVolume(context.Context, string, map[string]string) error
DeleteVolume(context.Context, string) error
GetVolume(string) (string, error)
GetVolumesByLabel(context.Context, map[string]string) ([]string, error) // @param context, labels - @return volumes, error
GetImageStream(context.Context, []string) (io.ReadCloser, error)
GetRuntimePath() string // returns e.g. '/var/run/docker.sock' for a default docker setup
ExecInNode(context.Context, *k3d.Node, []string) error

@ -22,12 +22,16 @@ THE SOFTWARE.
package util
import (
"context"
"errors"
"fmt"
"os"
rt "runtime"
"strings"
"github.com/rancher/k3d/v5/pkg/runtimes"
runtimeErrors "github.com/rancher/k3d/v5/pkg/runtimes/errors"
k3d "github.com/rancher/k3d/v5/pkg/types"
l "github.com/rancher/k3d/v5/pkg/logger"
)
@ -35,7 +39,49 @@ import (
// ValidateVolumeMount checks, if the source of volume mounts exists and if the destination is an absolute path
// - SRC: source directory/file -> tests: must exist
// - DEST: source directory/file -> tests: must be absolute path
func ValidateVolumeMount(runtime runtimes.Runtime, volumeMount string) error {
func ValidateVolumeMount(ctx context.Context, runtime runtimes.Runtime, volumeMount string, cluster *k3d.Cluster) error {
src, dest, err := ReadVolumeMount(volumeMount)
if err != nil {
return err
}
// verify that the source exists
if src != "" {
// directory/file: path containing / or \ (not allowed in named volumes)
if strings.ContainsAny(src, "/\\") {
if _, err := os.Stat(src); err != nil {
l.Log().Warnf("failed to stat file/directory '%s' volume mount '%s': please make sure it exists", src, volumeMount)
}
} else {
err := verifyNamedVolume(runtime, src)
if err != nil {
l.Log().Traceln(err)
if errors.Is(err, runtimeErrors.ErrRuntimeVolumeNotExists) {
if strings.HasPrefix(src, "k3d-") {
if err := runtime.CreateVolume(ctx, src, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil {
return fmt.Errorf("failed to create named volume '%s': %v", src, err)
}
cluster.Volumes = append(cluster.Volumes, src)
l.Log().Infof("Created named volume '%s'", src)
} else {
l.Log().Infof("No named volume '%s' found. The runtime will create it automatically.", src)
}
} else {
l.Log().Warnf("failed to get named volume: %v", err)
}
}
}
}
// verify that the destination is an absolute path
if !strings.HasPrefix(dest, "/") {
return fmt.Errorf("volume mount destination doesn't appear to be an absolute path: '%s' in '%s'", dest, volumeMount)
}
return nil
}
func ReadVolumeMount(volumeMount string) (string, string, error) {
src := ""
dest := ""
@ -50,10 +96,10 @@ func ValidateVolumeMount(runtime runtimes.Runtime, volumeMount string) error {
maxParts++
}
if len(split) < 1 {
return fmt.Errorf("No volume/path specified")
return src, dest, fmt.Errorf("no volume/path specified")
}
if len(split) > maxParts {
return fmt.Errorf("Invalid volume mount '%s': maximal %d ':' allowed", volumeMount, maxParts-1)
return src, dest, fmt.Errorf("invalid volume mount '%s': maximal %d ':' allowed", volumeMount, maxParts-1)
}
// we only have SRC specified -> DEST = SRC
@ -68,37 +114,14 @@ func ValidateVolumeMount(runtime runtimes.Runtime, volumeMount string) error {
src = split[0]
dest = split[1]
}
// verify that the source exists
if src != "" {
// a) named volume
isNamedVolume := true
if err := verifyNamedVolume(runtime, src); err != nil {
isNamedVolume = false
}
if !isNamedVolume {
if _, err := os.Stat(src); err != nil {
l.Log().Warnf("Failed to stat file/directory/named volume that you're trying to mount: '%s' in '%s' -> Please make sure it exists", src, volumeMount)
}
}
}
// verify that the destination is an absolute path
if !strings.HasPrefix(dest, "/") {
return fmt.Errorf("Volume mount destination doesn't appear to be an absolute path: '%s' in '%s'", dest, volumeMount)
}
return nil
return src, dest, nil
}
// verifyNamedVolume checks whether a named volume exists in the runtime
func verifyNamedVolume(runtime runtimes.Runtime, volumeName string) error {
foundVolName, err := runtime.GetVolume(volumeName)
_, err := runtime.GetVolume(volumeName)
if err != nil {
return fmt.Errorf("runtime failed to get volume '%s': %w", volumeName, err)
}
if foundVolName == "" {
return fmt.Errorf("failed to find named volume '%s'", volumeName)
}
return nil
}

@ -0,0 +1,40 @@
/*
Copyright © 2020-2021 The k3d Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package k3s
const (
K3sPathStorage = "/var/lib/rancher/k3s/storage"
K3sPathManifests = "/var/lib/rancher/k3s/server/manifests"
K3sPathManifestsCustom = "/var/lib/rancher/k3s/server/manifests/custom" // custom subfolder
K3sPathContainerdConfig = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml"
K3sPathContainerdConfigTmpl = "/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl"
K3sPathRegistryConfig = "/etc/rancher/k3s/registries.yaml"
)
var K3sPathShortcuts = map[string]string{
"k3s-storage": K3sPathStorage,
"k3s-manifests": K3sPathManifests,
"k3s-manifests-custom": K3sPathManifestsCustom,
"k3s-containerd": K3sPathContainerdConfig,
"k3s-containerd-tmpl": K3sPathContainerdConfigTmpl,
"k3s-registry-config": K3sPathRegistryConfig,
}

@ -234,6 +234,7 @@ type Cluster struct {
KubeAPI *ExposureOpts `yaml:"kubeAPI" json:"kubeAPI,omitempty"`
ServerLoadBalancer *Loadbalancer `yaml:"serverLoadbalancer,omitempty" json:"serverLoadBalancer,omitempty"`
ImageVolume string `yaml:"imageVolume" json:"imageVolume,omitempty"`
Volumes []string `yaml:"volumes,omitempty" json:"volumes,omitempty"` // k3d-managed volumes attached to this cluster
}
// ServerCountRunning returns the number of server nodes running in the cluster and the total number

Loading…
Cancel
Save