From e495fe83a8fb2104689e1b4a6963efc3f64e60b0 Mon Sep 17 00:00:00 2001 From: Konrad Malik Date: Mon, 29 Mar 2021 17:52:15 +0200 Subject: [PATCH] [FEATURE] Memory Limits (#494, @konradmalik) --- cmd/cluster/clusterCreate.go | 9 ++- cmd/node/nodeCreate.go | 13 ++++ go.mod | 1 + pkg/client/node.go | 46 ++++++++++++ pkg/config/transform.go | 11 ++- pkg/config/v1alpha2/schema.json | 6 ++ pkg/config/v1alpha2/types.go | 4 +- pkg/config/validate.go | 16 ++++ pkg/runtimes/docker/container.go | 59 +++++++++++++++ pkg/runtimes/docker/translate.go | 19 +++++ pkg/types/types.go | 3 + pkg/util/infofaker.go | 124 +++++++++++++++++++++++++++++++ vendor/modules.txt | 1 + 13 files changed, 307 insertions(+), 5 deletions(-) create mode 100644 pkg/util/infofaker.go diff --git a/cmd/cluster/clusterCreate.go b/cmd/cluster/clusterCreate.go index 70b263c7..e9efc4d5 100644 --- a/cmd/cluster/clusterCreate.go +++ b/cmd/cluster/clusterCreate.go @@ -24,12 +24,13 @@ package cluster import ( "fmt" - "github.com/docker/go-connections/nat" "os" "runtime" "strings" "time" + "github.com/docker/go-connections/nat" + "github.com/spf13/cobra" "github.com/spf13/viper" "gopkg.in/yaml.v2" @@ -316,6 +317,12 @@ func NewCmdClusterCreate() *cobra.Command { cmd.Flags().String("gpus", "", "GPU devices to add to the cluster node containers ('all' to pass all GPUs) [From docker]") _ = cfgViper.BindPFlag("options.runtime.gpurequest", cmd.Flags().Lookup("gpus")) + cmd.Flags().String("servers-memory", "", "Memory limit imposed on the server nodes [From docker]") + _ = cfgViper.BindPFlag("options.runtime.serversmemory", cmd.Flags().Lookup("servers-memory")) + + cmd.Flags().String("agents-memory", "", "Memory limit imposed on the agents nodes [From docker]") + _ = cfgViper.BindPFlag("options.runtime.agentsmemory", cmd.Flags().Lookup("agents-memory")) + /* Image Importing */ cmd.Flags().Bool("no-image-volume", false, "Disable the creation of a volume for importing images") _ = cfgViper.BindPFlag("options.k3d.disableimagevolume", cmd.Flags().Lookup("no-image-volume")) diff --git a/cmd/node/nodeCreate.go b/cmd/node/nodeCreate.go index bf96277f..c2d86281 100644 --- a/cmd/node/nodeCreate.go +++ b/cmd/node/nodeCreate.go @@ -27,6 +27,7 @@ import ( "github.com/spf13/cobra" + dockerunits "github.com/docker/go-units" "github.com/rancher/k3d/v4/cmd/util" k3dc "github.com/rancher/k3d/v4/pkg/client" "github.com/rancher/k3d/v4/pkg/runtimes" @@ -67,6 +68,7 @@ func NewCmdNodeCreate() *cobra.Command { } cmd.Flags().StringP("image", "i", fmt.Sprintf("%s:%s", k3d.DefaultK3sImageRepo, version.GetK3sVersion(false)), "Specify k3s image used for the node(s)") + cmd.Flags().String("memory", "", "Memory limit imposed on the node [From docker]") cmd.Flags().BoolVar(&createNodeOpts.Wait, "wait", false, "Wait for the node(s) to be ready before returning.") cmd.Flags().DurationVar(&createNodeOpts.Timeout, "timeout", 0*time.Second, "Maximum waiting time for '--wait' before canceling/returning.") @@ -112,6 +114,16 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl Name: clusterName, } + // --memory + memory, err := cmd.Flags().GetString("memory") + if err != nil { + log.Errorln("No memory specified") + log.Fatalln(err) + } + if _, err := dockerunits.RAMInBytes(memory); memory != "" && err != nil { + log.Errorf("Provided memory limit value is invalid") + } + // generate list of nodes nodes := []*k3d.Node{} for i := 0; i < replicas; i++ { @@ -123,6 +135,7 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl k3d.LabelRole: roleStr, }, Restart: true, + Memory: memory, } nodes = append(nodes, node) } diff --git a/go.mod b/go.mod index 8a802858..7d947da6 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v20.10.5+incompatible github.com/docker/go-connections v0.4.0 + github.com/docker/go-units v0.4.0 github.com/go-test/deep v1.0.4 github.com/gogo/protobuf v1.3.2 // indirect github.com/heroku/docker-registry-client v0.0.0-20190909225348-afc9e1acc3d5 diff --git a/pkg/client/node.go b/pkg/client/node.go index 9fb2a546..7b4fb1ac 100644 --- a/pkg/client/node.go +++ b/pkg/client/node.go @@ -26,13 +26,17 @@ import ( "bytes" "context" "fmt" + "os" "reflect" "strings" "time" + dockerunits "github.com/docker/go-units" "github.com/imdario/mergo" "github.com/rancher/k3d/v4/pkg/runtimes" + "github.com/rancher/k3d/v4/pkg/runtimes/docker" k3d "github.com/rancher/k3d/v4/pkg/types" + "github.com/rancher/k3d/v4/pkg/util" log "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" ) @@ -329,6 +333,37 @@ func NodeCreate(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, c } } + // memory limits + if node.Memory != "" { + if runtime != runtimes.Docker { + log.Warn("ignoring specified memory limits as runtime is not Docker") + } else { + memory, err := dockerunits.RAMInBytes(node.Memory) + if err != nil { + return fmt.Errorf("Invalid memory limit format: %+v", err) + } + // mount fake meminfo as readonly + fakemempath, err := util.MakeFakeMeminfo(memory, node.Name) + if err != nil { + return fmt.Errorf("Failed to create fake meminfo: %+v", err) + } + node.Volumes = append(node.Volumes, fmt.Sprintf("%s:%s:ro", fakemempath, util.MemInfoPath)) + // mount empty edac folder, but only if it exists + exists, err := docker.CheckIfDirectoryExists(ctx, node.Image, util.EdacFolderPath) + if err != nil { + return fmt.Errorf("Failed to check for the existence of edac folder: %+v", err) + } + if exists { + log.Debugln("Found edac folder") + fakeedacpath, err := util.MakeFakeEdac(node.Name) + if err != nil { + return fmt.Errorf("Failed to create fake edac: %+v", err) + } + node.Volumes = append(node.Volumes, fmt.Sprintf("%s:%s:ro", fakeedacpath, util.EdacFolderPath)) + } + } + } + /* * CREATION */ @@ -346,6 +381,17 @@ func NodeDelete(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, o log.Error(err) } + // delete fake folder created for limits + if node.Memory != "" { + log.Debug("Cleaning fake files folder from k3d config dir for this node...") + filepath, err := util.GetNodeFakerDirOrCreate(node.Name) + err = os.RemoveAll(filepath) + if err != nil { + // this err prob should not be fatal, just log it + log.Errorf("Could not remove fake files folder for node %s: %+v", node.Name, err) + } + } + // update the server loadbalancer if !opts.SkipLBUpdate && (node.Role == k3d.ServerRole || node.Role == k3d.AgentRole) { cluster, err := ClusterGet(ctx, runtime, &k3d.Cluster{Name: node.Labels[k3d.LabelClusterName]}) diff --git a/pkg/config/transform.go b/pkg/config/transform.go index 26951eaa..e39d50f1 100644 --- a/pkg/config/transform.go +++ b/pkg/config/transform.go @@ -107,6 +107,7 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim Image: simpleConfig.Image, Args: simpleConfig.Options.K3sOptions.ExtraServerArgs, ServerOpts: k3d.ServerOpts{}, + Memory: simpleConfig.Options.Runtime.ServersMemory, } // first server node will be init node if we have more than one server specified but no external datastore @@ -120,9 +121,10 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim for i := 0; i < simpleConfig.Agents; i++ { agentNode := k3d.Node{ - Role: k3d.AgentRole, - Image: simpleConfig.Image, - Args: simpleConfig.Options.K3sOptions.ExtraAgentArgs, + Role: k3d.AgentRole, + Image: simpleConfig.Image, + Args: simpleConfig.Options.K3sOptions.ExtraAgentArgs, + Memory: simpleConfig.Options.Runtime.AgentsMemory, } newCluster.Nodes = append(newCluster.Nodes, &agentNode) } @@ -226,6 +228,9 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim DisableLoadBalancer: simpleConfig.Options.K3dOptions.DisableLoadbalancer, K3sServerArgs: simpleConfig.Options.K3sOptions.ExtraServerArgs, K3sAgentArgs: simpleConfig.Options.K3sOptions.ExtraAgentArgs, + GPURequest: simpleConfig.Options.Runtime.GPURequest, + ServersMemory: simpleConfig.Options.Runtime.ServersMemory, + AgentsMemory: simpleConfig.Options.Runtime.AgentsMemory, GlobalLabels: map[string]string{}, // empty init GlobalEnv: []string{}, // empty init } diff --git a/pkg/config/v1alpha2/schema.json b/pkg/config/v1alpha2/schema.json index 0209b867..01a168c6 100644 --- a/pkg/config/v1alpha2/schema.json +++ b/pkg/config/v1alpha2/schema.json @@ -172,6 +172,12 @@ "properties": { "gpuRequest": { "type": "string" + }, + "serversMemory": { + "type": "string" + }, + "agentsMemory": { + "type": "string" } } } diff --git a/pkg/config/v1alpha2/types.go b/pkg/config/v1alpha2/types.go index 9309aa3a..02d74d47 100644 --- a/pkg/config/v1alpha2/types.go +++ b/pkg/config/v1alpha2/types.go @@ -97,7 +97,9 @@ type SimpleConfigOptions struct { } type SimpleConfigOptionsRuntime struct { - GPURequest string `mapstructure:"gpuRequest" yaml:"gpuRequest"` + GPURequest string `mapstructure:"gpuRequest" yaml:"gpuRequest"` + ServersMemory string `mapstructure:"serversMemory" yaml:"serversMemory"` + AgentsMemory string `mapstructure:"agentsMemory" yaml:"agentsMemory"` } type SimpleConfigOptionsK3d struct { diff --git a/pkg/config/validate.go b/pkg/config/validate.go index 86a23712..13cb924c 100644 --- a/pkg/config/validate.go +++ b/pkg/config/validate.go @@ -34,6 +34,7 @@ import ( "fmt" + dockerunits "github.com/docker/go-units" log "github.com/sirupsen/logrus" ) @@ -63,6 +64,21 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config return fmt.Errorf("The API Port can not be changed when using 'host' network") } + // memory limits must have proper format + // if empty we don't care about errors in parsing + if config.ClusterCreateOpts.ServersMemory != "" { + if _, err := dockerunits.RAMInBytes(config.ClusterCreateOpts.ServersMemory); err != nil { + return fmt.Errorf("Provided servers memory limit value is invalid") + } + + } + + if config.ClusterCreateOpts.AgentsMemory != "" { + if _, err := dockerunits.RAMInBytes(config.ClusterCreateOpts.AgentsMemory); err != nil { + return fmt.Errorf("Provided agents memory limit value is invalid") + } + } + // validate nodes one by one for _, node := range config.Cluster.Nodes { diff --git a/pkg/runtimes/docker/container.go b/pkg/runtimes/docker/container.go index fb28c973..aab1b178 100644 --- a/pkg/runtimes/docker/container.go +++ b/pkg/runtimes/docker/container.go @@ -180,3 +180,62 @@ func getNodeContainer(ctx context.Context, node *k3d.Node) (*types.Container, er return &containers[0], nil } + +// executes an arbitrary command in a container while returning its exit code. +// useful to check something in docker env +func executeCheckInContainer(ctx context.Context, image string, cmd []string) (int64, error) { + docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation()) + if err != nil { + log.Errorln("Failed to create docker client") + return -1, err + } + defer docker.Close() + + if err = pullImage(ctx, docker, image); err != nil { + return -1, err + } + + resp, err := docker.ContainerCreate(ctx, &container.Config{ + Image: image, + Cmd: cmd, + Tty: false, + Entrypoint: []string{}, + }, nil, nil, nil, "") + if err != nil { + log.Errorf("Failed to create container from image %s with cmd %s", image, cmd) + return -1, err + } + + if err = startContainer(ctx, resp.ID); err != nil { + return -1, err + } + + exitCode := -1 + statusCh, errCh := docker.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning) + select { + case err := <-errCh: + if err != nil { + log.Errorf("Error while waiting for container %s to exit", resp.ID) + return -1, err + } + case status := <-statusCh: + exitCode = int(status.StatusCode) + } + + if err = removeContainer(ctx, resp.ID); err != nil { + return -1, err + } + + return int64(exitCode), nil +} + +// CheckIfDirectoryExists checks for the existence of a given path inside the docker environment +func CheckIfDirectoryExists(ctx context.Context, image string, dir string) (bool, error) { + log.Tracef("checking if dir %s exists in docker environment...", dir) + shellCmd := fmt.Sprintf("[ -d \"%s\" ] && exit 0 || exit 1", dir) + cmd := []string{"sh", "-c", shellCmd} + exitCode, err := executeCheckInContainer(ctx, image, cmd) + log.Tracef("check dir container returned %d exist code", exitCode) + return exitCode == 0, err + +} diff --git a/pkg/runtimes/docker/translate.go b/pkg/runtimes/docker/translate.go index dd3d2a5d..78bc8bf6 100644 --- a/pkg/runtimes/docker/translate.go +++ b/pkg/runtimes/docker/translate.go @@ -36,6 +36,7 @@ import ( log "github.com/sirupsen/logrus" dockercliopts "github.com/docker/cli/opts" + dockerunits "github.com/docker/go-units" ) // TranslateNodeToContainer translates a k3d node specification to a docker container representation @@ -85,6 +86,16 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) { hostConfig.DeviceRequests = gpuopts.Value() } + // memory limits + // fake meminfo is mounted to hostConfig.Binds + if node.Memory != "" { + memory, err := dockerunits.RAMInBytes(node.Memory) + if err != nil { + return nil, fmt.Errorf("Failed to set memory limit: %+v", err) + } + hostConfig.Memory = memory + } + /* They have to run in privileged mode */ // TODO: can we replace this by a reduced set of capabilities? hostConfig.Privileged = true @@ -233,6 +244,13 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d Status: containerDetails.ContainerJSONBase.State.Status, } + // memory limit + memoryStr := dockerunits.HumanSize(float64(containerDetails.HostConfig.Memory)) + // no-limit is returned as 0B, filter this out + if memoryStr == "0B" { + memoryStr = "" + } + node := &k3d.Node{ Name: strings.TrimPrefix(containerDetails.Name, "/"), // container name with leading '/' cut off Role: k3d.NodeRoles[containerDetails.Config.Labels[k3d.LabelRole]], @@ -249,6 +267,7 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d ServerOpts: serverOpts, AgentOpts: k3d.AgentOpts{}, State: nodeState, + Memory: memoryStr, } return node, nil } diff --git a/pkg/types/types.go b/pkg/types/types.go index ba088985..9f7342d8 100644 --- a/pkg/types/types.go +++ b/pkg/types/types.go @@ -184,6 +184,8 @@ type ClusterCreateOpts struct { K3sServerArgs []string `yaml:"k3sServerArgs" json:"k3sServerArgs,omitempty"` K3sAgentArgs []string `yaml:"k3sAgentArgs" json:"k3sAgentArgs,omitempty"` GPURequest string `yaml:"gpuRequest" json:"gpuRequest,omitempty"` + ServersMemory string `yaml:"serversMemory" json:"serversMemory,omitempty"` + AgentsMemory string `yaml:"agentsMemory" json:"agentsMemory,omitempty"` NodeHooks []NodeHook `yaml:"nodeHooks,omitempty" json:"nodeHooks,omitempty"` GlobalLabels map[string]string `yaml:"globalLabels,omitempty" json:"globalLabels,omitempty"` GlobalEnv []string `yaml:"globalEnv,omitempty" json:"globalEnv,omitempty"` @@ -328,6 +330,7 @@ type Node struct { ServerOpts ServerOpts `yaml:"serverOpts" json:"serverOpts,omitempty"` AgentOpts AgentOpts `yaml:"agentOpts" json:"agentOpts,omitempty"` GPURequest string // filled automatically + Memory string // filled automatically State NodeState // filled automatically } diff --git a/pkg/util/infofaker.go b/pkg/util/infofaker.go new file mode 100644 index 00000000..035ec653 --- /dev/null +++ b/pkg/util/infofaker.go @@ -0,0 +1,124 @@ +/* +Copyright © 2020 The k3d Author(s) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package util + +import ( + "fmt" + "os" + "path" + "strings" + + dockerunits "github.com/docker/go-units" + log "github.com/sirupsen/logrus" +) + +const ( + EdacFolderPath = "/sys/devices/system/edac" + MemInfoPath = "/proc/meminfo" +) + +// creates a mininal fake meminfo with fields required by cadvisor (see machine.go in cadvisor) +func meminfoContent(totalKB int64) string { + var lines = []string{ + fmt.Sprintf("MemTotal: %d kB", totalKB), + // this may be configurable later + "SwapTotal: 0 kB", + } + return strings.Join(lines, "\n") +} + +// GetNodeFakerDirOrCreate creates or gets a hidden folder in k3d home dir +// to keep container(node)-specific fake files in it +func GetNodeFakerDirOrCreate(name string) (string, error) { + // this folder needs to be kept across reboots, keep it in ~/.k3d + configdir, err := GetConfigDirOrCreate() + if err != nil { + return "", err + } + fakeDir := path.Join(configdir, fmt.Sprintf(".%s", name)) + + // create directories if necessary + if err := createDirIfNotExists(fakeDir); err != nil { + log.Errorf("Failed to create fake files path '%s'", fakeDir) + return "", err + } + + return fakeDir, nil + +} + +// GetFakeMeminfoPathForName returns a path to (existent or not) fake meminfo file for a given node/container name +func GetFakeMeminfoPathForName(nodeName string) (string, error) { + return fakeInfoPathForName("meminfo", nodeName) +} + +// MakeFakeMeminfo creates a fake meminfo file to be mounted and provide a specific RAM capacity. +// This file is created on a per specific container/node basis, uniqueName must ensure that. +// Returns a path to the file +func MakeFakeMeminfo(memoryBytes int64, nodeName string) (string, error) { + fakeMeminfoPath, err := GetFakeMeminfoPathForName(nodeName) + if err != nil { + return "", err + } + fakememinfo, err := os.Create(fakeMeminfoPath) + defer fakememinfo.Close() + if err != nil { + return "", err + } + + // write content, must be kB + memoryKb := memoryBytes / dockerunits.KB + content := meminfoContent(memoryKb) + _, err = fakememinfo.WriteString(content) + if err != nil { + return "", err + } + + return fakememinfo.Name(), nil +} + +// MakeFakeEdac creates an empty edac folder to force cadvisor +// to use meminfo even for ECC memory +func MakeFakeEdac(nodeName string) (string, error) { + dir, err := GetNodeFakerDirOrCreate(nodeName) + if err != nil { + return "", err + } + edacPath := path.Join(dir, "edac") + // create directories if necessary + if err := createDirIfNotExists(edacPath); err != nil { + log.Errorf("Failed to create fake edac path '%s'", edacPath) + return "", err + } + + return edacPath, nil +} + +// returns a path to (existent or not) fake (mem or cpu)info file for a given node/container name +func fakeInfoPathForName(infoType string, nodeName string) (string, error) { + // this file needs to be kept across reboots, keep it in ~/.k3d + dir, err := GetNodeFakerDirOrCreate(nodeName) + if err != nil { + return "", err + } + return path.Join(dir, infoType), nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 10cde46e..b98a727d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -89,6 +89,7 @@ github.com/docker/go-connections/nat github.com/docker/go-connections/sockets github.com/docker/go-connections/tlsconfig # github.com/docker/go-units v0.4.0 +## explicit github.com/docker/go-units # github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 github.com/docker/libtrust