[FEATURE] Memory Limits (#494, @konradmalik)

pull/552/head
Konrad Malik 3 years ago committed by GitHub
parent 16fa1076ff
commit e495fe83a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 9
      cmd/cluster/clusterCreate.go
  2. 13
      cmd/node/nodeCreate.go
  3. 1
      go.mod
  4. 46
      pkg/client/node.go
  5. 11
      pkg/config/transform.go
  6. 6
      pkg/config/v1alpha2/schema.json
  7. 4
      pkg/config/v1alpha2/types.go
  8. 16
      pkg/config/validate.go
  9. 59
      pkg/runtimes/docker/container.go
  10. 19
      pkg/runtimes/docker/translate.go
  11. 3
      pkg/types/types.go
  12. 124
      pkg/util/infofaker.go
  13. 1
      vendor/modules.txt

@ -24,12 +24,13 @@ package cluster
import (
"fmt"
"github.com/docker/go-connections/nat"
"os"
"runtime"
"strings"
"time"
"github.com/docker/go-connections/nat"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"gopkg.in/yaml.v2"
@ -316,6 +317,12 @@ func NewCmdClusterCreate() *cobra.Command {
cmd.Flags().String("gpus", "", "GPU devices to add to the cluster node containers ('all' to pass all GPUs) [From docker]")
_ = cfgViper.BindPFlag("options.runtime.gpurequest", cmd.Flags().Lookup("gpus"))
cmd.Flags().String("servers-memory", "", "Memory limit imposed on the server nodes [From docker]")
_ = cfgViper.BindPFlag("options.runtime.serversmemory", cmd.Flags().Lookup("servers-memory"))
cmd.Flags().String("agents-memory", "", "Memory limit imposed on the agents nodes [From docker]")
_ = cfgViper.BindPFlag("options.runtime.agentsmemory", cmd.Flags().Lookup("agents-memory"))
/* Image Importing */
cmd.Flags().Bool("no-image-volume", false, "Disable the creation of a volume for importing images")
_ = cfgViper.BindPFlag("options.k3d.disableimagevolume", cmd.Flags().Lookup("no-image-volume"))

@ -27,6 +27,7 @@ import (
"github.com/spf13/cobra"
dockerunits "github.com/docker/go-units"
"github.com/rancher/k3d/v4/cmd/util"
k3dc "github.com/rancher/k3d/v4/pkg/client"
"github.com/rancher/k3d/v4/pkg/runtimes"
@ -67,6 +68,7 @@ func NewCmdNodeCreate() *cobra.Command {
}
cmd.Flags().StringP("image", "i", fmt.Sprintf("%s:%s", k3d.DefaultK3sImageRepo, version.GetK3sVersion(false)), "Specify k3s image used for the node(s)")
cmd.Flags().String("memory", "", "Memory limit imposed on the node [From docker]")
cmd.Flags().BoolVar(&createNodeOpts.Wait, "wait", false, "Wait for the node(s) to be ready before returning.")
cmd.Flags().DurationVar(&createNodeOpts.Timeout, "timeout", 0*time.Second, "Maximum waiting time for '--wait' before canceling/returning.")
@ -112,6 +114,16 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
Name: clusterName,
}
// --memory
memory, err := cmd.Flags().GetString("memory")
if err != nil {
log.Errorln("No memory specified")
log.Fatalln(err)
}
if _, err := dockerunits.RAMInBytes(memory); memory != "" && err != nil {
log.Errorf("Provided memory limit value is invalid")
}
// generate list of nodes
nodes := []*k3d.Node{}
for i := 0; i < replicas; i++ {
@ -123,6 +135,7 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
k3d.LabelRole: roleStr,
},
Restart: true,
Memory: memory,
}
nodes = append(nodes, node)
}

@ -12,6 +12,7 @@ require (
github.com/docker/distribution v2.7.1+incompatible // indirect
github.com/docker/docker v20.10.5+incompatible
github.com/docker/go-connections v0.4.0
github.com/docker/go-units v0.4.0
github.com/go-test/deep v1.0.4
github.com/gogo/protobuf v1.3.2 // indirect
github.com/heroku/docker-registry-client v0.0.0-20190909225348-afc9e1acc3d5

@ -26,13 +26,17 @@ import (
"bytes"
"context"
"fmt"
"os"
"reflect"
"strings"
"time"
dockerunits "github.com/docker/go-units"
"github.com/imdario/mergo"
"github.com/rancher/k3d/v4/pkg/runtimes"
"github.com/rancher/k3d/v4/pkg/runtimes/docker"
k3d "github.com/rancher/k3d/v4/pkg/types"
"github.com/rancher/k3d/v4/pkg/util"
log "github.com/sirupsen/logrus"
"golang.org/x/sync/errgroup"
)
@ -329,6 +333,37 @@ func NodeCreate(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, c
}
}
// memory limits
if node.Memory != "" {
if runtime != runtimes.Docker {
log.Warn("ignoring specified memory limits as runtime is not Docker")
} else {
memory, err := dockerunits.RAMInBytes(node.Memory)
if err != nil {
return fmt.Errorf("Invalid memory limit format: %+v", err)
}
// mount fake meminfo as readonly
fakemempath, err := util.MakeFakeMeminfo(memory, node.Name)
if err != nil {
return fmt.Errorf("Failed to create fake meminfo: %+v", err)
}
node.Volumes = append(node.Volumes, fmt.Sprintf("%s:%s:ro", fakemempath, util.MemInfoPath))
// mount empty edac folder, but only if it exists
exists, err := docker.CheckIfDirectoryExists(ctx, node.Image, util.EdacFolderPath)
if err != nil {
return fmt.Errorf("Failed to check for the existence of edac folder: %+v", err)
}
if exists {
log.Debugln("Found edac folder")
fakeedacpath, err := util.MakeFakeEdac(node.Name)
if err != nil {
return fmt.Errorf("Failed to create fake edac: %+v", err)
}
node.Volumes = append(node.Volumes, fmt.Sprintf("%s:%s:ro", fakeedacpath, util.EdacFolderPath))
}
}
}
/*
* CREATION
*/
@ -346,6 +381,17 @@ func NodeDelete(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, o
log.Error(err)
}
// delete fake folder created for limits
if node.Memory != "" {
log.Debug("Cleaning fake files folder from k3d config dir for this node...")
filepath, err := util.GetNodeFakerDirOrCreate(node.Name)
err = os.RemoveAll(filepath)
if err != nil {
// this err prob should not be fatal, just log it
log.Errorf("Could not remove fake files folder for node %s: %+v", node.Name, err)
}
}
// update the server loadbalancer
if !opts.SkipLBUpdate && (node.Role == k3d.ServerRole || node.Role == k3d.AgentRole) {
cluster, err := ClusterGet(ctx, runtime, &k3d.Cluster{Name: node.Labels[k3d.LabelClusterName]})

@ -107,6 +107,7 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
Image: simpleConfig.Image,
Args: simpleConfig.Options.K3sOptions.ExtraServerArgs,
ServerOpts: k3d.ServerOpts{},
Memory: simpleConfig.Options.Runtime.ServersMemory,
}
// first server node will be init node if we have more than one server specified but no external datastore
@ -120,9 +121,10 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
for i := 0; i < simpleConfig.Agents; i++ {
agentNode := k3d.Node{
Role: k3d.AgentRole,
Image: simpleConfig.Image,
Args: simpleConfig.Options.K3sOptions.ExtraAgentArgs,
Role: k3d.AgentRole,
Image: simpleConfig.Image,
Args: simpleConfig.Options.K3sOptions.ExtraAgentArgs,
Memory: simpleConfig.Options.Runtime.AgentsMemory,
}
newCluster.Nodes = append(newCluster.Nodes, &agentNode)
}
@ -226,6 +228,9 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
DisableLoadBalancer: simpleConfig.Options.K3dOptions.DisableLoadbalancer,
K3sServerArgs: simpleConfig.Options.K3sOptions.ExtraServerArgs,
K3sAgentArgs: simpleConfig.Options.K3sOptions.ExtraAgentArgs,
GPURequest: simpleConfig.Options.Runtime.GPURequest,
ServersMemory: simpleConfig.Options.Runtime.ServersMemory,
AgentsMemory: simpleConfig.Options.Runtime.AgentsMemory,
GlobalLabels: map[string]string{}, // empty init
GlobalEnv: []string{}, // empty init
}

@ -172,6 +172,12 @@
"properties": {
"gpuRequest": {
"type": "string"
},
"serversMemory": {
"type": "string"
},
"agentsMemory": {
"type": "string"
}
}
}

@ -97,7 +97,9 @@ type SimpleConfigOptions struct {
}
type SimpleConfigOptionsRuntime struct {
GPURequest string `mapstructure:"gpuRequest" yaml:"gpuRequest"`
GPURequest string `mapstructure:"gpuRequest" yaml:"gpuRequest"`
ServersMemory string `mapstructure:"serversMemory" yaml:"serversMemory"`
AgentsMemory string `mapstructure:"agentsMemory" yaml:"agentsMemory"`
}
type SimpleConfigOptionsK3d struct {

@ -34,6 +34,7 @@ import (
"fmt"
dockerunits "github.com/docker/go-units"
log "github.com/sirupsen/logrus"
)
@ -63,6 +64,21 @@ func ValidateClusterConfig(ctx context.Context, runtime runtimes.Runtime, config
return fmt.Errorf("The API Port can not be changed when using 'host' network")
}
// memory limits must have proper format
// if empty we don't care about errors in parsing
if config.ClusterCreateOpts.ServersMemory != "" {
if _, err := dockerunits.RAMInBytes(config.ClusterCreateOpts.ServersMemory); err != nil {
return fmt.Errorf("Provided servers memory limit value is invalid")
}
}
if config.ClusterCreateOpts.AgentsMemory != "" {
if _, err := dockerunits.RAMInBytes(config.ClusterCreateOpts.AgentsMemory); err != nil {
return fmt.Errorf("Provided agents memory limit value is invalid")
}
}
// validate nodes one by one
for _, node := range config.Cluster.Nodes {

@ -180,3 +180,62 @@ func getNodeContainer(ctx context.Context, node *k3d.Node) (*types.Container, er
return &containers[0], nil
}
// executes an arbitrary command in a container while returning its exit code.
// useful to check something in docker env
func executeCheckInContainer(ctx context.Context, image string, cmd []string) (int64, error) {
docker, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
log.Errorln("Failed to create docker client")
return -1, err
}
defer docker.Close()
if err = pullImage(ctx, docker, image); err != nil {
return -1, err
}
resp, err := docker.ContainerCreate(ctx, &container.Config{
Image: image,
Cmd: cmd,
Tty: false,
Entrypoint: []string{},
}, nil, nil, nil, "")
if err != nil {
log.Errorf("Failed to create container from image %s with cmd %s", image, cmd)
return -1, err
}
if err = startContainer(ctx, resp.ID); err != nil {
return -1, err
}
exitCode := -1
statusCh, errCh := docker.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning)
select {
case err := <-errCh:
if err != nil {
log.Errorf("Error while waiting for container %s to exit", resp.ID)
return -1, err
}
case status := <-statusCh:
exitCode = int(status.StatusCode)
}
if err = removeContainer(ctx, resp.ID); err != nil {
return -1, err
}
return int64(exitCode), nil
}
// CheckIfDirectoryExists checks for the existence of a given path inside the docker environment
func CheckIfDirectoryExists(ctx context.Context, image string, dir string) (bool, error) {
log.Tracef("checking if dir %s exists in docker environment...", dir)
shellCmd := fmt.Sprintf("[ -d \"%s\" ] && exit 0 || exit 1", dir)
cmd := []string{"sh", "-c", shellCmd}
exitCode, err := executeCheckInContainer(ctx, image, cmd)
log.Tracef("check dir container returned %d exist code", exitCode)
return exitCode == 0, err
}

@ -36,6 +36,7 @@ import (
log "github.com/sirupsen/logrus"
dockercliopts "github.com/docker/cli/opts"
dockerunits "github.com/docker/go-units"
)
// TranslateNodeToContainer translates a k3d node specification to a docker container representation
@ -85,6 +86,16 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) {
hostConfig.DeviceRequests = gpuopts.Value()
}
// memory limits
// fake meminfo is mounted to hostConfig.Binds
if node.Memory != "" {
memory, err := dockerunits.RAMInBytes(node.Memory)
if err != nil {
return nil, fmt.Errorf("Failed to set memory limit: %+v", err)
}
hostConfig.Memory = memory
}
/* They have to run in privileged mode */
// TODO: can we replace this by a reduced set of capabilities?
hostConfig.Privileged = true
@ -233,6 +244,13 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d
Status: containerDetails.ContainerJSONBase.State.Status,
}
// memory limit
memoryStr := dockerunits.HumanSize(float64(containerDetails.HostConfig.Memory))
// no-limit is returned as 0B, filter this out
if memoryStr == "0B" {
memoryStr = ""
}
node := &k3d.Node{
Name: strings.TrimPrefix(containerDetails.Name, "/"), // container name with leading '/' cut off
Role: k3d.NodeRoles[containerDetails.Config.Labels[k3d.LabelRole]],
@ -249,6 +267,7 @@ func TranslateContainerDetailsToNode(containerDetails types.ContainerJSON) (*k3d
ServerOpts: serverOpts,
AgentOpts: k3d.AgentOpts{},
State: nodeState,
Memory: memoryStr,
}
return node, nil
}

@ -184,6 +184,8 @@ type ClusterCreateOpts struct {
K3sServerArgs []string `yaml:"k3sServerArgs" json:"k3sServerArgs,omitempty"`
K3sAgentArgs []string `yaml:"k3sAgentArgs" json:"k3sAgentArgs,omitempty"`
GPURequest string `yaml:"gpuRequest" json:"gpuRequest,omitempty"`
ServersMemory string `yaml:"serversMemory" json:"serversMemory,omitempty"`
AgentsMemory string `yaml:"agentsMemory" json:"agentsMemory,omitempty"`
NodeHooks []NodeHook `yaml:"nodeHooks,omitempty" json:"nodeHooks,omitempty"`
GlobalLabels map[string]string `yaml:"globalLabels,omitempty" json:"globalLabels,omitempty"`
GlobalEnv []string `yaml:"globalEnv,omitempty" json:"globalEnv,omitempty"`
@ -328,6 +330,7 @@ type Node struct {
ServerOpts ServerOpts `yaml:"serverOpts" json:"serverOpts,omitempty"`
AgentOpts AgentOpts `yaml:"agentOpts" json:"agentOpts,omitempty"`
GPURequest string // filled automatically
Memory string // filled automatically
State NodeState // filled automatically
}

@ -0,0 +1,124 @@
/*
Copyright © 2020 The k3d Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package util
import (
"fmt"
"os"
"path"
"strings"
dockerunits "github.com/docker/go-units"
log "github.com/sirupsen/logrus"
)
const (
EdacFolderPath = "/sys/devices/system/edac"
MemInfoPath = "/proc/meminfo"
)
// creates a mininal fake meminfo with fields required by cadvisor (see machine.go in cadvisor)
func meminfoContent(totalKB int64) string {
var lines = []string{
fmt.Sprintf("MemTotal: %d kB", totalKB),
// this may be configurable later
"SwapTotal: 0 kB",
}
return strings.Join(lines, "\n")
}
// GetNodeFakerDirOrCreate creates or gets a hidden folder in k3d home dir
// to keep container(node)-specific fake files in it
func GetNodeFakerDirOrCreate(name string) (string, error) {
// this folder needs to be kept across reboots, keep it in ~/.k3d
configdir, err := GetConfigDirOrCreate()
if err != nil {
return "", err
}
fakeDir := path.Join(configdir, fmt.Sprintf(".%s", name))
// create directories if necessary
if err := createDirIfNotExists(fakeDir); err != nil {
log.Errorf("Failed to create fake files path '%s'", fakeDir)
return "", err
}
return fakeDir, nil
}
// GetFakeMeminfoPathForName returns a path to (existent or not) fake meminfo file for a given node/container name
func GetFakeMeminfoPathForName(nodeName string) (string, error) {
return fakeInfoPathForName("meminfo", nodeName)
}
// MakeFakeMeminfo creates a fake meminfo file to be mounted and provide a specific RAM capacity.
// This file is created on a per specific container/node basis, uniqueName must ensure that.
// Returns a path to the file
func MakeFakeMeminfo(memoryBytes int64, nodeName string) (string, error) {
fakeMeminfoPath, err := GetFakeMeminfoPathForName(nodeName)
if err != nil {
return "", err
}
fakememinfo, err := os.Create(fakeMeminfoPath)
defer fakememinfo.Close()
if err != nil {
return "", err
}
// write content, must be kB
memoryKb := memoryBytes / dockerunits.KB
content := meminfoContent(memoryKb)
_, err = fakememinfo.WriteString(content)
if err != nil {
return "", err
}
return fakememinfo.Name(), nil
}
// MakeFakeEdac creates an empty edac folder to force cadvisor
// to use meminfo even for ECC memory
func MakeFakeEdac(nodeName string) (string, error) {
dir, err := GetNodeFakerDirOrCreate(nodeName)
if err != nil {
return "", err
}
edacPath := path.Join(dir, "edac")
// create directories if necessary
if err := createDirIfNotExists(edacPath); err != nil {
log.Errorf("Failed to create fake edac path '%s'", edacPath)
return "", err
}
return edacPath, nil
}
// returns a path to (existent or not) fake (mem or cpu)info file for a given node/container name
func fakeInfoPathForName(infoType string, nodeName string) (string, error) {
// this file needs to be kept across reboots, keep it in ~/.k3d
dir, err := GetNodeFakerDirOrCreate(nodeName)
if err != nil {
return "", err
}
return path.Join(dir, infoType), nil
}

@ -89,6 +89,7 @@ github.com/docker/go-connections/nat
github.com/docker/go-connections/sockets
github.com/docker/go-connections/tlsconfig
# github.com/docker/go-units v0.4.0
## explicit
github.com/docker/go-units
# github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7
github.com/docker/libtrust

Loading…
Cancel
Save