/ *
Copyright © 2020 - 2021 The k3d Author ( s )
Permission is hereby granted , free of charge , to any person obtaining a copy
of this software and associated documentation files ( the "Software" ) , to deal
in the Software without restriction , including without limitation the rights
to use , copy , modify , merge , publish , distribute , sublicense , and / or sell
copies of the Software , and to permit persons to whom the Software is
furnished to do so , subject to the following conditions :
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software .
THE SOFTWARE IS PROVIDED "AS IS" , WITHOUT WARRANTY OF ANY KIND , EXPRESS OR
IMPLIED , INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY ,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT . IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM , DAMAGES OR OTHER
LIABILITY , WHETHER IN AN ACTION OF CONTRACT , TORT OR OTHERWISE , ARISING FROM ,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE .
* /
package docker
import (
"context"
"fmt"
"os"
"strconv"
"strings"
"github.com/containerd/containerd/log"
"github.com/docker/docker/api/types"
docker "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/network"
"github.com/docker/go-connections/nat"
l "github.com/rancher/k3d/v5/pkg/logger"
runtimeErr "github.com/rancher/k3d/v5/pkg/runtimes/errors"
k3d "github.com/rancher/k3d/v5/pkg/types"
"github.com/rancher/k3d/v5/pkg/types/fixes"
"inet.af/netaddr"
dockercliopts "github.com/docker/cli/opts"
dockerunits "github.com/docker/go-units"
)
// TranslateNodeToContainer translates a k3d node specification to a docker container representation
func TranslateNodeToContainer ( node * k3d . Node ) ( * NodeInDocker , error ) {
init := true
if disableInit , err := strconv . ParseBool ( os . Getenv ( "K3D_DEBUG_DISABLE_DOCKER_INIT" ) ) ; err == nil && disableInit {
l . Log ( ) . Traceln ( "docker-init disabled for all containers" )
init = false
}
/* initialize everything that we need */
containerConfig := docker . Config { }
hostConfig := docker . HostConfig {
Init : & init ,
ExtraHosts : node . ExtraHosts ,
}
networkingConfig := network . NetworkingConfig { }
/* Name & Image */
containerConfig . Hostname = node . Name
containerConfig . Image = node . Image
/* Command & Arguments */
// FIXME: FixCgroupV2 - to be removed when fixed upstream
if fixes . FixEnabledAny ( ) {
if node . Role == k3d . AgentRole || node . Role == k3d . ServerRole {
containerConfig . Entrypoint = [ ] string {
"/bin/k3d-entrypoint.sh" ,
}
}
}
containerConfig . Cmd = [ ] string { }
containerConfig . Cmd = append ( containerConfig . Cmd , node . Cmd ... ) // contains k3s command and role-specific required flags/args
containerConfig . Cmd = append ( containerConfig . Cmd , node . Args ... ) // extra flags/args
/* Environment Variables */
containerConfig . Env = node . Env
/* Labels */
containerConfig . Labels = node . RuntimeLabels // has to include the role
/* Auto-Restart */
if node . Restart {
hostConfig . RestartPolicy = docker . RestartPolicy {
Name : "unless-stopped" ,
}
}
/* Tmpfs Mounts */
hostConfig . Tmpfs = make ( map [ string ] string )
for _ , mnt := range k3d . DefaultTmpfsMounts {
hostConfig . Tmpfs [ mnt ] = ""
}
if node . GPURequest != "" {
gpuopts := dockercliopts . GpuOpts { }
if err := gpuopts . Set ( node . GPURequest ) ; err != nil {
return nil , fmt . Errorf ( "Failed to set GPU Request: %+v" , err )
}
hostConfig . DeviceRequests = gpuopts . Value ( )
}
// memory limits
// fake meminfo is mounted to hostConfig.Binds
if node . Memory != "" {
memory , err := dockerunits . RAMInBytes ( node . Memory )
if err != nil {
return nil , fmt . Errorf ( "Failed to set memory limit: %+v" , err )
}
hostConfig . Memory = memory
}
/* They have to run in privileged mode */
// TODO: can we replace this by a reduced set of capabilities?
hostConfig . Privileged = true
/* Volumes */
hostConfig . Binds = node . Volumes
// containerConfig.Volumes = map[string]struct{}{} // TODO: do we need this? We only used binds before
/* Ports */
exposedPorts := nat . PortSet { }
for ep := range node . Ports {
if _ , exists := exposedPorts [ ep ] ; ! exists {
exposedPorts [ ep ] = struct { } { }
}
}
containerConfig . ExposedPorts = exposedPorts
hostConfig . PortBindings = node . Ports
/* Network */
endpointsConfig := map [ string ] * network . EndpointSettings { }
for _ , net := range node . Networks {
epSettings := & network . EndpointSettings { }
endpointsConfig [ net ] = epSettings
}
networkingConfig . EndpointsConfig = endpointsConfig
/* Static IP */
if ! node . IP . IP . IsZero ( ) && node . IP . Static {
epconf := networkingConfig . EndpointsConfig [ node . Networks [ 0 ] ]
if epconf . IPAMConfig == nil {
epconf . IPAMConfig = & network . EndpointIPAMConfig { }
}
epconf . IPAMConfig . IPv4Address = node . IP . IP . String ( )
}
if len ( node . Networks ) > 0 {
netInfo , err := GetNetwork ( context . Background ( ) , node . Networks [ 0 ] ) // FIXME: only considering first network here, as that's the one k3d creates for a cluster
if err != nil {
l . Log ( ) . Warnf ( "Failed to get network information: %v" , err )
} else if netInfo . Driver == "host" {
hostConfig . NetworkMode = "host"
}
}
return & NodeInDocker {
ContainerConfig : containerConfig ,
HostConfig : hostConfig ,
NetworkingConfig : networkingConfig ,
} , nil
}
// TranslateContainerToNode translates a docker container object into a k3d node representation
func TranslateContainerToNode ( cont * types . Container ) ( * k3d . Node , error ) {
node := & k3d . Node {
Name : strings . TrimPrefix ( cont . Names [ 0 ] , "/" ) , // container name with leading '/' cut off
Image : cont . Image ,
RuntimeLabels : cont . Labels ,
Role : k3d . NodeRoles [ cont . Labels [ k3d . LabelRole ] ] ,
// TODO: all the rest
}
return node , nil
}
// TranslateContainerDetailsToNode translates a docker containerJSON object into a k3d node representation
func TranslateContainerDetailsToNode ( containerDetails types . ContainerJSON ) ( * k3d . Node , error ) {
// first, make sure, that it's actually a k3d managed container by checking if it has all the default labels
for k , v := range k3d . DefaultRuntimeLabels {
l . Log ( ) . Tracef ( "TranslateContainerDetailsToNode: Checking for default object label %s=%s on container %s" , k , v , containerDetails . Name )
found := false
for lk , lv := range containerDetails . Config . Labels {
if lk == k && lv == v {
found = true
break
}
}
if ! found {
l . Log ( ) . Debugf ( "Container %s is missing default label %s=%s in label set %+v" , containerDetails . Name , k , v , containerDetails . Config . Labels )
return nil , runtimeErr . ErrRuntimeContainerUnknown
}
}
// restart -> we only set 'unless-stopped' upon cluster creation
restart := false
if containerDetails . HostConfig . RestartPolicy . IsAlways ( ) || containerDetails . HostConfig . RestartPolicy . IsUnlessStopped ( ) {
restart = true
}
// get networks and ensure that the cluster network is first in list
orderedNetworks := [ ] string { }
otherNetworks := [ ] string { }
for networkName := range containerDetails . NetworkSettings . Networks {
if strings . HasPrefix ( networkName , fmt . Sprintf ( "%s-%s" , k3d . DefaultObjectNamePrefix , containerDetails . Config . Labels [ k3d . LabelClusterName ] ) ) { // FIXME: catch error if label 'k3d.cluster' does not exist, but this should also never be the case
orderedNetworks = append ( orderedNetworks , networkName )
continue
}
otherNetworks = append ( otherNetworks , networkName )
}
orderedNetworks = append ( orderedNetworks , otherNetworks ... )
/ * *
* ServerOpts
* /
// IsInit
serverOpts := k3d . ServerOpts { IsInit : false }
clusterInitFlagSet := false
for _ , arg := range containerDetails . Args {
if strings . Contains ( arg , "--cluster-init" ) {
clusterInitFlagSet = true
break
}
}
if serverIsInitLabel , ok := containerDetails . Config . Labels [ k3d . LabelServerIsInit ] ; ok {
if serverIsInitLabel == "true" {
if ! clusterInitFlagSet {
l . Log ( ) . Errorf ( "Container %s has label %s=true, but the args do not contain the --cluster-init flag" , containerDetails . Name , k3d . LabelServerIsInit )
} else {
serverOpts . IsInit = true
}
}
}
// Kube API
serverOpts . KubeAPI = & k3d . ExposureOpts { }
for k , v := range containerDetails . Config . Labels {
if k == k3d . LabelServerAPIHostIP {
serverOpts . KubeAPI . Binding . HostIP = v
} else if k == k3d . LabelServerAPIHost {
serverOpts . KubeAPI . Host = v
} else if k == k3d . LabelServerAPIPort {
serverOpts . KubeAPI . Binding . HostPort = v
}
}
// labels: only copy k3d.* labels
labels := map [ string ] string { }
for k , v := range containerDetails . Config . Labels {
if strings . HasPrefix ( k , "k3d" ) {
labels [ k ] = v
}
}
// status
nodeState := k3d . NodeState {
Running : containerDetails . ContainerJSONBase . State . Running ,
Status : containerDetails . ContainerJSONBase . State . Status ,
}
// memory limit
memoryStr := dockerunits . HumanSize ( float64 ( containerDetails . HostConfig . Memory ) )
// no-limit is returned as 0B, filter this out
if memoryStr == "0B" {
memoryStr = ""
}
// IP
var nodeIP k3d . NodeIP
var clusterNet * network . EndpointSettings
if netLabel , ok := labels [ k3d . LabelNetwork ] ; ok {
for netName , net := range containerDetails . NetworkSettings . Networks {
if netName == netLabel {
clusterNet = net
}
}
} else {
l . Log ( ) . Debugf ( "no netlabel present on container %s" , containerDetails . Name )
}
if clusterNet != nil {
parsedIP , err := netaddr . ParseIP ( clusterNet . IPAddress )
if err != nil {
if nodeState . Running && nodeState . Status != "restarting" { // if the container is not running or currently restarting, it won't have an IP, so we don't error in that case
return nil , fmt . Errorf ( "failed to parse IP '%s' for container '%s': %s\nStatus: %v\n%+v" , clusterNet . IPAddress , containerDetails . Name , err , nodeState . Status , containerDetails . NetworkSettings )
} else {
log . L . Tracef ( "failed to parse IP '%s' for container '%s', likely because it's not running (or restarting): %v" , clusterNet . IPAddress , containerDetails . Name , err )
}
}
isStaticIP := false
if staticIPLabel , ok := labels [ k3d . LabelNodeStaticIP ] ; ok && staticIPLabel != "" {
isStaticIP = true
}
if ! parsedIP . IsZero ( ) {
nodeIP = k3d . NodeIP {
IP : parsedIP ,
Static : isStaticIP ,
}
}
} else {
l . Log ( ) . Debugf ( "failed to get IP for container %s as we couldn't find the cluster network" , containerDetails . Name )
}
node := & k3d . Node {
Name : strings . TrimPrefix ( containerDetails . Name , "/" ) , // container name with leading '/' cut off
Role : k3d . NodeRoles [ containerDetails . Config . Labels [ k3d . LabelRole ] ] ,
Image : containerDetails . Image ,
Volumes : containerDetails . HostConfig . Binds ,
Env : containerDetails . Config . Env ,
Cmd : containerDetails . Config . Cmd ,
Args : [ ] string { } , // empty, since Cmd already contains flags
Ports : containerDetails . HostConfig . PortBindings ,
Restart : restart ,
Created : containerDetails . Created ,
RuntimeLabels : labels ,
Networks : orderedNetworks ,
ServerOpts : serverOpts ,
AgentOpts : k3d . AgentOpts { } ,
State : nodeState ,
Memory : memoryStr ,
IP : nodeIP , // only valid for the cluster network
}
return node , nil
}