Skip to content

Commit

Permalink
Allow docker container to reach host instead of host knowing where do…
Browse files Browse the repository at this point in the history
…cker VM is. (#2395) (#2400)

(cherry picked from commit c0a7a4a)

Co-authored-by: Ryan Swanson <[email protected]>
  • Loading branch information
loft-bot and zerbitx authored Jan 15, 2025
1 parent c55fcea commit 7327b59
Show file tree
Hide file tree
Showing 3 changed files with 105 additions and 107 deletions.
2 changes: 1 addition & 1 deletion pkg/cli/connect_helm.go
Original file line number Diff line number Diff line change
Expand Up @@ -337,7 +337,7 @@ func (cmd *connectHelm) getVClusterKubeConfig(ctx context.Context, vclusterName
if cmd.Server == "" && cmd.BackgroundProxy {
if localkubernetes.IsDockerInstalledAndUpAndRunning() {
// start background container
cmd.Server, err = localkubernetes.CreateBackgroundProxyContainer(ctx, vclusterName, cmd.Namespace, cmd.kubeClientConfig, kubeConfig, cmd.LocalPort, cmd.Log)
cmd.Server, err = localkubernetes.CreateBackgroundProxyContainer(ctx, vclusterName, cmd.Namespace, cmd.kubeClientConfig, cmd.LocalPort, cmd.Log)
if err != nil {
cmd.Log.Warnf("Error exposing local vcluster, will fallback to port-forwarding: %v", err)
cmd.BackgroundProxy = false
Expand Down
193 changes: 98 additions & 95 deletions pkg/cli/localkubernetes/configure.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,11 @@ package localkubernetes
import (
"context"
"fmt"
"net"
"os"
"os/exec"
"runtime"
"slices"
"strconv"
"strings"
"sync"
"time"

"github.com/loft-sh/log"
Expand Down Expand Up @@ -112,127 +109,133 @@ func directConnection(ctx context.Context, vRawConfig *clientcmdapi.Config, serv
return server, nil
}

func CreateBackgroundProxyContainer(ctx context.Context, vClusterName, vClusterNamespace string, rawConfig clientcmd.ClientConfig, vRawConfig *clientcmdapi.Config, localPort int, log log.Logger) (string, error) {
// CreateBackgroundProxyContainer runs kubectl port-forward in a docker container, forwarding from the vcluster service
// on the host cluster to a port matching the kubernetes context for the virtual cluster.
func CreateBackgroundProxyContainer(_ context.Context, vClusterName, vClusterNamespace string, rawConfig clientcmd.ClientConfig, localPort int, log log.Logger) (string, error) {
rawConfigObj, err := rawConfig.RawConfig()
if err != nil {
return "", err
}

// write kube config to buffer
physicalCluster, err := kubeconfig.ResolveKubeConfig(rawConfig)
physicalRawConfig, err := kubeconfig.ResolveKubeConfig(rawConfig)
if err != nil {
return "", fmt.Errorf("resolve kube config: %w", err)
}

// write a temporary kube file
tempFile, err := os.CreateTemp("", "")
if err != nil {
return "", errors.Wrap(err, "create temp file")
}
_, err = tempFile.Write(physicalCluster)
// construct proxy name
proxyName := find.VClusterConnectBackgroundProxyName(vClusterName, vClusterNamespace, rawConfigObj.CurrentContext)

// check if the background proxy container for this vcluster is running and then remove it.
_ = CleanupBackgroundProxy(proxyName, log)

cmd, err := buildDockerCommand(physicalRawConfig, proxyName, vClusterName, vClusterNamespace, localPort)
if err != nil {
return "", errors.Wrap(err, "write kube config to temp file")
return "", fmt.Errorf("build docker command: %w", err)
}
err = tempFile.Close()
if err != nil {
return "", errors.Wrap(err, "close temp file")

log.Infof("Starting background proxy container...")
if out, err := cmd.CombinedOutput(); err != nil {
return "", errors.Errorf("error starting background proxy: %s %v", string(out), err)
}
kubeConfigPath := tempFile.Name()

// allow permissions for kube config path
err = os.Chmod(kubeConfigPath, 0666)
return fmt.Sprintf("https://127.0.0.1:%v", localPort), nil
}

// build a different docker command for darwin vs. everything else
func buildDockerCommand(physicalRawConfig clientcmdapi.Config, proxyName, vClusterName, vClusterNamespace string, localPort int) (*exec.Cmd, error) {
// write a temporary kube file
tempFile, err := os.CreateTemp("", "")
if err != nil {
return "", fmt.Errorf("chmod temp file: %w", err)
return nil, errors.Wrap(err, "create temp file")
}

// construct proxy name
proxyName := find.VClusterConnectBackgroundProxyName(vClusterName, vClusterNamespace, rawConfigObj.CurrentContext)

// check if the background proxy container for this vcluster is running and then remove it.
_ = CleanupBackgroundProxy(proxyName, log)
kubeConfigPath := tempFile.Name()

// get ips to try to connect to
dockerMachineIPs := []string{
"127.0.0.1",
}
var cmd *exec.Cmd
// For non-linux, update the kube config to point to the special host.docker.internal and don't use
// host networking.
if runtime.GOOS != "linux" {
output, err := exec.Command("docker", "run", "--network=host", "--rm", "alpine", "sh", "-c", "ifconfig | grep 'inet addr:' | sed 's/.*inet addr:\\([^ ]*\\).*/\\1/'").CombinedOutput()
physicalRawConfig, err = updateConfigForDockerToHost(physicalRawConfig)
if err != nil {
log.Warnf("Couldn't find network config for docker machine: %s %v", string(output), err)
}
for _, ip := range strings.Split(string(output), "\n") {
ip = strings.TrimSpace(ip)
if net.ParseIP(ip) != nil && !slices.Contains(dockerMachineIPs, ip) {
dockerMachineIPs = append(dockerMachineIPs, ip)
}
return nil, fmt.Errorf("update config: %w", err)
}

// since the vCluster certificate is only signed for 127.0.0.1 we need to make sure we use insecure here
if len(dockerMachineIPs) > 1 {
for k := range vRawConfig.Clusters {
vRawConfig.Clusters[k].CertificateAuthority = ""
vRawConfig.Clusters[k].CertificateAuthorityData = nil
vRawConfig.Clusters[k].InsecureSkipTLSVerify = true
}
}
cmd = exec.Command(
"docker",
"run",
"--rm",
"-d",
"-v", fmt.Sprintf("%v:%v", kubeConfigPath, "/kube-config"),
fmt.Sprintf("--name=%s", proxyName),
"-p",
fmt.Sprintf("%d:8443", localPort),
"bitnami/kubectl:1.29",
"port-forward",
"svc/"+vClusterName,
"--address=0.0.0.0",
"8443:443",
"--kubeconfig", "/kube-config",
"-n", vClusterNamespace,
)
} else {
cmd = exec.Command(
"docker",
"run",
"--rm",
"-d",
"-v", fmt.Sprintf("%v:%v", kubeConfigPath, "/kube-config"),
fmt.Sprintf("--name=%s", proxyName),
"--network=host",
"bitnami/kubectl:1.29",
"port-forward",
"svc/"+vClusterName,
"--address=0.0.0.0",
strconv.Itoa(localPort)+":443",
"--kubeconfig", "/kube-config",
"-n", vClusterNamespace,
)
}

// build the command
cmd := exec.Command(
"docker",
"run",
"--rm",
"-d",
"-v", fmt.Sprintf("%v:%v", kubeConfigPath, "/kube-config"),
fmt.Sprintf("--name=%s", proxyName),
"--network=host",
"bitnami/kubectl:1.29",
"port-forward",
"svc/"+vClusterName,
"--address=0.0.0.0",
strconv.Itoa(localPort)+":443",
"--kubeconfig", "/kube-config",
"-n", vClusterNamespace,
)
log.Infof("Starting background proxy container...")
out, err := cmd.CombinedOutput()
// write kube config to buffer
physicalCluster, err := clientcmd.Write(physicalRawConfig)
if err != nil {
return "", errors.Errorf("error starting background proxy: %s %v", string(out), err)
return nil, fmt.Errorf("failed to write config: %w", err)
}

server := ""
serverMutex := sync.Mutex{}
waitErr := wait.PollUntilContextTimeout(ctx, time.Second, time.Second*10, true, func(ctx context.Context) (bool, error) {
// try all ips in parallel
waitGroup := sync.WaitGroup{}
for _, ip := range dockerMachineIPs {
waitGroup.Add(1)
go func() {
defer waitGroup.Done()

testServer := fmt.Sprintf("https://%s:%v", ip, localPort)
err := testConnectionWithServer(ctx, vRawConfig, testServer)
if err == nil {
serverMutex.Lock()
if server == "" {
server = testServer
}
serverMutex.Unlock()
} else {
log.Debugf("Attempted to connect to %s: %v", testServer, err)
}
}()
}
if _, err = tempFile.Write(physicalCluster); err != nil {
return nil, errors.Wrap(err, "write kube config to temp file")
}

// wait until we are done
waitGroup.Wait()
return server != "", nil
})
if waitErr != nil {
return "", errors.New("test connection for background proxy failed")
if err = tempFile.Close(); err != nil {
return nil, errors.Wrap(err, "close temp file")
}

return server, nil
// allow permissions for kube config path
if err = os.Chmod(kubeConfigPath, 0666); err != nil {
return nil, fmt.Errorf("chmod temp file: %w", err)
}

return cmd, nil
}

// Update the configuration for the local cluster to be able to reach the host via the special host.docker.internal address
func updateConfigForDockerToHost(rawConfig clientcmdapi.Config) (clientcmdapi.Config, error) {
updated := rawConfig.DeepCopy()

if updated.Clusters == nil {
return clientcmdapi.Config{}, fmt.Errorf("config missing clusters")
}

if _, ok := updated.Clusters["local"]; !ok {
return clientcmdapi.Config{}, fmt.Errorf("config missing local cluster")
}

localCluster := updated.Clusters["local"]
localCluster.InsecureSkipTLSVerify = true
localCluster.CertificateAuthorityData = nil
localCluster.Server = strings.ReplaceAll(localCluster.Server, "127.0.0.1", "host.docker.internal")

return *updated, nil
}

func IsDockerInstalledAndUpAndRunning() bool {
Expand Down
17 changes: 6 additions & 11 deletions pkg/util/kubeconfig/kubeconfig.go
Original file line number Diff line number Diff line change
Expand Up @@ -189,34 +189,29 @@ func ConvertRestConfigToClientConfig(config *rest.Config) (clientcmd.ClientConfi
return clientcmd.NewDefaultClientConfig(*kubeConfig, &clientcmd.ConfigOverrides{}), nil
}

func ResolveKubeConfig(rawConfig clientcmd.ClientConfig) ([]byte, error) {
func ResolveKubeConfig(rawConfig clientcmd.ClientConfig) (clientcmdapi.Config, error) {
restConfig, err := rawConfig.ClientConfig()
if err != nil {
return nil, err
return clientcmdapi.Config{}, err
}

// convert exec auth
if restConfig.ExecProvider != nil {
err = resolveExecCredentials(restConfig)
if err != nil {
return nil, fmt.Errorf("resolve exec credentials: %w", err)
return clientcmdapi.Config{}, fmt.Errorf("resolve exec credentials: %w", err)
}
}
if restConfig.AuthProvider != nil {
return nil, fmt.Errorf("auth provider is not supported")
return clientcmdapi.Config{}, fmt.Errorf("auth provider is not supported")
}

retConfig, err := ConvertRestConfigToClientConfig(restConfig)
if err != nil {
return nil, err
return clientcmdapi.Config{}, err
}

retRawConfig, err := retConfig.RawConfig()
if err != nil {
return nil, err
}

return clientcmd.Write(retRawConfig)
return retConfig.RawConfig()
}

func resolveExecCredentials(restConfig *rest.Config) error {
Expand Down

0 comments on commit 7327b59

Please sign in to comment.