diff --git a/cmd/agent/container_tunnel.go b/cmd/agent/container_tunnel.go index a172673bb..a07d01d24 100644 --- a/cmd/agent/container_tunnel.go +++ b/cmd/agent/container_tunnel.go @@ -58,6 +58,12 @@ func (cmd *ContainerTunnelCmd) Run(ctx context.Context, log log.Logger) error { return nil } + // make sure content folder exists + err = workspace.InitContentFolder(workspaceInfo, log) + if err != nil { + return err + } + // create runner runner, err := workspace.CreateRunner(workspaceInfo, log) if err != nil { diff --git a/cmd/agent/workspace/build.go b/cmd/agent/workspace/build.go index 67e8c7bc9..e64982a10 100644 --- a/cmd/agent/workspace/build.go +++ b/cmd/agent/workspace/build.go @@ -6,7 +6,6 @@ import ( "github.com/loft-sh/devpod/cmd/flags" "github.com/loft-sh/devpod/pkg/agent" - "github.com/loft-sh/devpod/pkg/devcontainer/config" provider2 "github.com/loft-sh/devpod/pkg/provider" "github.com/loft-sh/log" "github.com/pkg/errors" @@ -81,7 +80,7 @@ func (cmd *BuildCmd) Run(ctx context.Context) error { // build and push images for _, platform := range platforms { // build the image - imageName, err := runner.Build(ctx, config.BuildOptions{ + imageName, err := runner.Build(ctx, provider2.BuildOptions{ CLIOptions: workspaceInfo.CLIOptions, Platform: platform, diff --git a/cmd/agent/workspace/up.go b/cmd/agent/workspace/up.go index e28c4b858..13eff553d 100644 --- a/cmd/agent/workspace/up.go +++ b/cmd/agent/workspace/up.go @@ -175,14 +175,50 @@ func (cmd *UpCmd) up(ctx context.Context, workspaceInfo *provider2.AgentWorkspac } func prepareWorkspace(ctx context.Context, workspaceInfo *provider2.AgentWorkspaceInfo, client tunnel.TunnelClient, helper string, log log.Logger) error { + // make sure content folder exists + err := InitContentFolder(workspaceInfo, log) + if err != nil { + return err + } + + // check if we should init + if workspaceInfo.LastDevContainerConfig != nil { + log.Debugf("Workspace was already executed, skip downloading") + return nil + } + + // check what type of workspace this is + if workspaceInfo.Workspace.Source.GitRepository != "" { + log.Debugf("Clone Repository") + err = CloneRepository(ctx, workspaceInfo.Agent.Local == "true", workspaceInfo.ContentFolder, workspaceInfo.Workspace.Source, helper, log) + if err != nil { + // fallback + log.Errorf("Cloning failed: %v. Trying cloning on local machine and uploading folder", err) + return RemoteCloneAndDownload(ctx, workspaceInfo.ContentFolder, client, log) + } + + return nil + } else if workspaceInfo.Workspace.Source.LocalFolder != "" { + log.Debugf("Download Local Folder") + return DownloadLocalFolder(ctx, workspaceInfo.ContentFolder, client, log) + } else if workspaceInfo.Workspace.Source.Image != "" { + log.Debugf("Prepare Image") + return PrepareImage(workspaceInfo.ContentFolder, workspaceInfo.Workspace.Source.Image) + } + + return fmt.Errorf("either workspace repository, image or local-folder is required") +} + +func InitContentFolder(workspaceInfo *provider2.AgentWorkspaceInfo, log log.Logger) error { // check if workspace content folder exists _, err := os.Stat(workspaceInfo.ContentFolder) if err == nil { - log.Debugf("Workspace Folder already exists") + log.Debugf("Workspace Folder already exists %s", workspaceInfo.ContentFolder) return nil } // make content dir + log.Debugf("Create content folder %s", workspaceInfo.ContentFolder) err = os.MkdirAll(workspaceInfo.ContentFolder, 0777) if err != nil { return errors.Wrap(err, "make workspace folder") @@ -202,26 +238,41 @@ func prepareWorkspace(ctx context.Context, workspaceInfo *provider2.AgentWorkspa return fmt.Errorf("error downloading workspace %s binaries: %w", workspaceInfo.Workspace.ID, err) } - // check what type of workspace this is - if workspaceInfo.Workspace.Source.GitRepository != "" { - log.Debugf("Clone Repository") - err = CloneRepository(ctx, workspaceInfo.Agent.Local == "true", workspaceInfo.ContentFolder, workspaceInfo.Workspace.Source, helper, log) + // if workspace was already executed, we skip this part + if workspaceInfo.LastDevContainerConfig != nil { + // make sure the devcontainer.json exists + err = ensureLastDevContainerJson(workspaceInfo) if err != nil { - // fallback - log.Errorf("Cloning failed: %v. Trying cloning on local machine and uploading folder", err) - return RemoteCloneAndDownload(ctx, workspaceInfo.ContentFolder, client, log) + log.Errorf("Ensure devcontainer.json: %v", err) } + } - return nil - } else if workspaceInfo.Workspace.Source.LocalFolder != "" { - log.Debugf("Download Local Folder") - return DownloadLocalFolder(ctx, workspaceInfo.ContentFolder, client, log) - } else if workspaceInfo.Workspace.Source.Image != "" { - log.Debugf("Prepare Image") - return PrepareImage(workspaceInfo.ContentFolder, workspaceInfo.Workspace.Source.Image) + return nil +} + +func ensureLastDevContainerJson(workspaceInfo *provider2.AgentWorkspaceInfo) error { + filePath := filepath.Join(workspaceInfo.ContentFolder, filepath.FromSlash(workspaceInfo.LastDevContainerConfig.Path)) + _, err := os.Stat(filePath) + if os.IsNotExist(err) { + err = os.MkdirAll(filepath.Dir(filePath), 0755) + if err != nil { + return fmt.Errorf("create %s: %w", filepath.Dir(filePath), err) + } + + raw, err := json.Marshal(workspaceInfo.LastDevContainerConfig.Config) + if err != nil { + return fmt.Errorf("marshal devcontainer.json: %w", err) + } + + err = os.WriteFile(filePath, raw, 0666) + if err != nil { + return fmt.Errorf("write %s: %w", filePath, err) + } + } else if err != nil { + return fmt.Errorf("error stating %s: %w", filePath, err) } - return fmt.Errorf("either workspace repository, image or local-folder is required") + return nil } func configureCredentials(ctx context.Context, cancel context.CancelFunc, workspaceInfo *provider2.AgentWorkspaceInfo, client tunnel.TunnelClient, log log.Logger) (string, string, error) { diff --git a/cmd/export.go b/cmd/export.go new file mode 100644 index 000000000..2ec4bf46b --- /dev/null +++ b/cmd/export.go @@ -0,0 +1,95 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/loft-sh/devpod/cmd/flags" + "github.com/loft-sh/devpod/pkg/config" + "github.com/loft-sh/devpod/pkg/provider" + workspace2 "github.com/loft-sh/devpod/pkg/workspace" + "github.com/loft-sh/log" + "github.com/spf13/cobra" +) + +// ExportCmd holds the export cmd flags +type ExportCmd struct { + *flags.GlobalFlags +} + +// NewExportCmd creates a new command +func NewExportCmd(flags *flags.GlobalFlags) *cobra.Command { + cmd := &ExportCmd{ + GlobalFlags: flags, + } + exportCmd := &cobra.Command{ + Use: "export", + Short: "Exports a workspace configuration", + RunE: func(_ *cobra.Command, args []string) error { + ctx := context.Background() + devPodConfig, err := config.LoadConfig(cmd.Context, cmd.Provider) + if err != nil { + return err + } + + return cmd.Run(ctx, devPodConfig, args) + }, + } + + return exportCmd +} + +// Run runs the command logic +func (cmd *ExportCmd) Run(ctx context.Context, devPodConfig *config.Config, args []string) error { + // try to load workspace + logger := log.Default.ErrorStreamOnly() + client, err := workspace2.GetWorkspace(devPodConfig, args, false, logger) + if err != nil { + return err + } + + // export workspace + exportConfig, err := exportWorkspace(devPodConfig, client.WorkspaceConfig()) + if err != nil { + return err + } + + // marshal config + out, err := json.Marshal(exportConfig) + if err != nil { + return err + } + + fmt.Println(string(out)) + return nil +} + +func exportWorkspace(devPodConfig *config.Config, workspaceConfig *provider.Workspace) (*provider.ExportConfig, error) { + var err error + + // create return config + retConfig := &provider.ExportConfig{} + + // export workspace + retConfig.Workspace, err = provider.ExportWorkspace(workspaceConfig.Context, workspaceConfig.ID) + if err != nil { + return nil, fmt.Errorf("export workspace config: %w", err) + } + + // has machine? + if workspaceConfig.Machine.ID != "" { + retConfig.Machine, err = provider.ExportMachine(workspaceConfig.Context, workspaceConfig.Machine.ID) + if err != nil { + return nil, fmt.Errorf("export machine config: %w", err) + } + } + + // export provider + retConfig.Provider, err = provider.ExportProvider(devPodConfig, workspaceConfig.Context, workspaceConfig.Provider.Name) + if err != nil { + return nil, fmt.Errorf("export provider config: %w", err) + } + + return retConfig, nil +} diff --git a/cmd/import.go b/cmd/import.go new file mode 100644 index 000000000..2487c0671 --- /dev/null +++ b/cmd/import.go @@ -0,0 +1,295 @@ +package cmd + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "os" + + "github.com/loft-sh/devpod/cmd/flags" + "github.com/loft-sh/devpod/pkg/config" + "github.com/loft-sh/devpod/pkg/extract" + "github.com/loft-sh/devpod/pkg/provider" + "github.com/loft-sh/devpod/pkg/workspace" + "github.com/loft-sh/log" + "github.com/spf13/cobra" +) + +// ImportCmd holds the export cmd flags +type ImportCmd struct { + *flags.GlobalFlags + + WorkspaceID string + + MachineID string + MachineReuse bool + + ProviderID string + ProviderReuse bool + + Data string +} + +// NewImportCmd creates a new command +func NewImportCmd(flags *flags.GlobalFlags) *cobra.Command { + cmd := &ImportCmd{ + GlobalFlags: flags, + } + importCmd := &cobra.Command{ + Use: "import", + Short: "Imports a workspace configuration", + Args: cobra.NoArgs, + RunE: func(_ *cobra.Command, args []string) error { + ctx := context.Background() + devPodConfig, err := config.LoadConfig(cmd.Context, cmd.Provider) + if err != nil { + return err + } + + return cmd.Run(ctx, devPodConfig, log.Default) + }, + } + + importCmd.Flags().StringVar(&cmd.WorkspaceID, "workspace-id", "", "To workspace id to use") + importCmd.Flags().StringVar(&cmd.MachineID, "machine-id", "", "The machine id to use") + importCmd.Flags().BoolVar(&cmd.MachineReuse, "machine-reuse", false, "If machine already exists, reuse existing machine") + importCmd.Flags().StringVar(&cmd.ProviderID, "provider-id", "", "The provider id to use") + importCmd.Flags().BoolVar(&cmd.ProviderReuse, "provider-reuse", false, "If provider already exists, reuse existing provider") + importCmd.Flags().StringVar(&cmd.Data, "data", "", "The data to import as raw json") + _ = importCmd.MarkFlagRequired("data") + return importCmd +} + +// Run runs the command logic +func (cmd *ImportCmd) Run(ctx context.Context, devPodConfig *config.Config, log log.Logger) error { + exportConfig := &provider.ExportConfig{} + err := json.Unmarshal([]byte(cmd.Data), exportConfig) + if err != nil { + return fmt.Errorf("decode workspace data: %w", err) + } else if exportConfig.Workspace == nil { + return fmt.Errorf("workspace is missing in imported data") + } else if exportConfig.Provider == nil { + return fmt.Errorf("provider is missing in imported data") + } + + // set ids correctly + if cmd.MachineID == "" && exportConfig.Machine != nil { + cmd.MachineID = exportConfig.Machine.ID + } + if cmd.WorkspaceID == "" { + cmd.WorkspaceID = exportConfig.Workspace.ID + } + if cmd.ProviderID == "" { + cmd.ProviderID = exportConfig.Provider.ID + } + + // check if conflicting ids + err = cmd.checkForConflictingIDs(exportConfig, devPodConfig, log) + if err != nil { + return err + } + + // import provider + err = cmd.importProvider(devPodConfig, exportConfig, log) + if err != nil { + return err + } + + // import machine + err = cmd.importMachine(devPodConfig, exportConfig, log) + if err != nil { + return err + } + + // import workspace + err = cmd.importWorkspace(devPodConfig, exportConfig, log) + if err != nil { + return err + } + + return nil +} + +func (cmd *ImportCmd) importWorkspace(devPodConfig *config.Config, exportConfig *provider.ExportConfig, log log.Logger) error { + workspaceDir, err := provider.GetWorkspaceDir(devPodConfig.DefaultContext, cmd.WorkspaceID) + if err != nil { + return fmt.Errorf("get workspace dir: %w", err) + } + + err = os.MkdirAll(workspaceDir, 0755) + if err != nil { + return fmt.Errorf("create workspace dir: %w", err) + } + + decoded, err := base64.RawStdEncoding.DecodeString(exportConfig.Workspace.Data) + if err != nil { + return fmt.Errorf("decode workspace data: %w", err) + } + + err = extract.Extract(bytes.NewReader(decoded), workspaceDir) + if err != nil { + return fmt.Errorf("extract workspace data: %w", err) + } + + // exchange config + workspaceConfig, err := provider.LoadWorkspaceConfig(devPodConfig.DefaultContext, cmd.WorkspaceID) + if err != nil { + return fmt.Errorf("load machine config: %w", err) + } + workspaceConfig.ID = cmd.WorkspaceID + workspaceConfig.Context = devPodConfig.DefaultContext + workspaceConfig.Machine.ID = cmd.MachineID + workspaceConfig.Provider.Name = cmd.ProviderID + + // save machine config + err = provider.SaveWorkspaceConfig(workspaceConfig) + if err != nil { + return fmt.Errorf("save workspace config: %w", err) + } + + log.Donef("Successfully imported workspace %s", cmd.WorkspaceID) + return nil +} + +func (cmd *ImportCmd) importMachine(devPodConfig *config.Config, exportConfig *provider.ExportConfig, log log.Logger) error { + if exportConfig.Machine == nil { + return nil + } + + // if machine already exists we skip + if cmd.MachineReuse && provider.MachineExists(devPodConfig.DefaultContext, cmd.MachineID) { + log.Infof("Reusing existing machine %s", cmd.MachineID) + return nil + } + + machineDir, err := provider.GetMachineDir(devPodConfig.DefaultContext, cmd.MachineID) + if err != nil { + return fmt.Errorf("get machine dir: %w", err) + } + + err = os.MkdirAll(machineDir, 0755) + if err != nil { + return fmt.Errorf("create machine dir: %w", err) + } + + decoded, err := base64.RawStdEncoding.DecodeString(exportConfig.Machine.Data) + if err != nil { + return fmt.Errorf("decode machine data: %w", err) + } + + err = extract.Extract(bytes.NewReader(decoded), machineDir) + if err != nil { + return fmt.Errorf("extract machine data: %w", err) + } + + // exchange config + machineConfig, err := provider.LoadMachineConfig(devPodConfig.DefaultContext, cmd.MachineID) + if err != nil { + return fmt.Errorf("load machine config: %w", err) + } + machineConfig.ID = cmd.MachineID + machineConfig.Context = devPodConfig.DefaultContext + machineConfig.Provider.Name = cmd.ProviderID + + // save machine config + err = provider.SaveMachineConfig(machineConfig) + if err != nil { + return fmt.Errorf("save machine config: %w", err) + } + + log.Donef("Successfully imported machine %s", cmd.MachineID) + return nil +} + +func (cmd *ImportCmd) importProvider(devPodConfig *config.Config, exportConfig *provider.ExportConfig, log log.Logger) error { + // if provider already exists we skip + if cmd.ProviderReuse && provider.ProviderExists(devPodConfig.DefaultContext, cmd.ProviderID) { + log.Infof("Reusing existing provider %s", cmd.ProviderID) + return nil + } + + providerDir, err := provider.GetProviderDir(devPodConfig.DefaultContext, cmd.ProviderID) + if err != nil { + return fmt.Errorf("get provider dir: %w", err) + } + + err = os.MkdirAll(providerDir, 0755) + if err != nil { + return fmt.Errorf("create provider dir: %w", err) + } + + decoded, err := base64.RawStdEncoding.DecodeString(exportConfig.Provider.Data) + if err != nil { + return fmt.Errorf("decode provider data: %w", err) + } + + err = extract.Extract(bytes.NewReader(decoded), providerDir) + if err != nil { + return fmt.Errorf("extract provider data: %w", err) + } + + // exchange config + providerConfig, err := provider.LoadProviderConfig(devPodConfig.DefaultContext, cmd.ProviderID) + if err != nil { + return fmt.Errorf("load provider config: %w", err) + } + providerConfig.Name = cmd.ProviderID + + // save provider config + err = provider.SaveProviderConfig(devPodConfig.DefaultContext, providerConfig) + if err != nil { + return fmt.Errorf("save provider config: %w", err) + } + + // add provider options + if exportConfig.Provider.Config != nil { + if devPodConfig.Current().Providers == nil { + devPodConfig.Current().Providers = map[string]*config.ProviderConfig{} + } + + devPodConfig.Current().Providers[cmd.ProviderID] = exportConfig.Provider.Config + err = config.SaveConfig(devPodConfig) + if err != nil { + return fmt.Errorf("save devpod config: %w", err) + } + } + + log.Donef("Successfully imported provider %s", cmd.ProviderID) + return nil +} + +func (cmd *ImportCmd) checkForConflictingIDs(exportConfig *provider.ExportConfig, devPodConfig *config.Config, log log.Logger) error { + workspaces, err := workspace.ListWorkspaces(devPodConfig, log) + if err != nil { + return fmt.Errorf("error listing workspaces: %w", err) + } + + // check for workspace duplicate + if exportConfig.Workspace != nil { + for _, workspace := range workspaces { + if workspace.ID == cmd.WorkspaceID { + return fmt.Errorf("existing workspace with id %s found, please use --workspace-id to override the workspace id", cmd.WorkspaceID) + } else if workspace.UID == exportConfig.Workspace.UID { + return fmt.Errorf("existing workspace %s with uid %s found, please use --workspace-id to override the workspace id", workspace.ID, workspace.UID) + } + } + } + + // check if machine already exists + if !cmd.MachineReuse && exportConfig.Machine != nil { + if provider.MachineExists(devPodConfig.DefaultContext, cmd.MachineID) { + return fmt.Errorf("existing machine with id %s found, please use --machine-reuse to skip importing the machine or --machine-id to override the machine id", cmd.MachineID) + } + } + + // check if provider already exists + if !cmd.ProviderReuse && exportConfig.Provider != nil { + if provider.ProviderExists(devPodConfig.DefaultContext, cmd.ProviderID) { + return fmt.Errorf("existing provider with id %s found, please use --provider-reuse to skip importing the provider or --provider-id to override the provider id", cmd.ProviderID) + } + } + + return nil +} diff --git a/cmd/list.go b/cmd/list.go index bbbf3254a..eec6ffe87 100644 --- a/cmd/list.go +++ b/cmd/list.go @@ -32,6 +32,7 @@ func NewListCmd(flags *flags.GlobalFlags) *cobra.Command { Use: "list", Aliases: []string{"ls"}, Short: "Lists existing workspaces", + Args: cobra.NoArgs, RunE: func(_ *cobra.Command, args []string) error { if len(args) > 0 { return fmt.Errorf("no arguments are allowed for this command") diff --git a/cmd/pro/import_workspace.go b/cmd/pro/import_workspace.go index f311f6338..539c4dbfa 100644 --- a/cmd/pro/import_workspace.go +++ b/cmd/pro/import_workspace.go @@ -99,15 +99,9 @@ func (cmd *ImportCmd) Run(ctx context.Context, args []string) error { } func (cmd *ImportCmd) writeWorkspaceDefinition(devPodConfig *config.Config, provider *provider2.ProviderConfig) error { - workspaceFolder, err := provider2.GetWorkspaceDir(devPodConfig.DefaultContext, cmd.WorkspaceId) - if err != nil { - return errors.Wrap(err, "get workspace dir") - } - workspaceObj := &provider2.Workspace{ - ID: cmd.WorkspaceId, - UID: cmd.WorkspaceUid, - Folder: workspaceFolder, + ID: cmd.WorkspaceId, + UID: cmd.WorkspaceUid, Provider: provider2.WorkspaceProviderConfig{ Name: provider.Name, Options: map[string]config.OptionValue{}, @@ -122,7 +116,7 @@ func (cmd *ImportCmd) writeWorkspaceDefinition(devPodConfig *config.Config, prov } } - err = provider2.SaveWorkspaceConfig(workspaceObj) + err := provider2.SaveWorkspaceConfig(workspaceObj) if err != nil { return err } diff --git a/cmd/root.go b/cmd/root.go index df44f90da..9f491f67c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -138,5 +138,7 @@ func BuildRoot() *cobra.Command { rootCmd.AddCommand(NewStatusCmd(globalFlags)) rootCmd.AddCommand(NewBuildCmd(globalFlags)) rootCmd.AddCommand(NewLogsDaemonCmd(globalFlags)) + rootCmd.AddCommand(NewExportCmd(globalFlags)) + rootCmd.AddCommand(NewImportCmd(globalFlags)) return rootCmd } diff --git a/cmd/up.go b/cmd/up.go index 5f5b2f2bf..33d5c13e3 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -252,19 +252,29 @@ func (cmd *UpCmd) devPodUp( } defer client.Unlock() - // check if regular workspace client - workspaceClient, ok := client.(client2.WorkspaceClient) - if ok { - return cmd.devPodUpMachine(ctx, workspaceClient, log) + // get result + var result *config2.Result + + // check what client we have + if workspaceClient, ok := client.(client2.WorkspaceClient); ok { + result, err = cmd.devPodUpMachine(ctx, workspaceClient, log) + if err != nil { + return nil, err + } + } else if proxyClient, ok := client.(client2.ProxyClient); ok { + result, err = cmd.devPodUpProxy(ctx, proxyClient, log) + if err != nil { + return nil, err + } } - // check if proxy client - proxyClient, ok := client.(client2.ProxyClient) - if ok { - return cmd.devPodUpProxy(ctx, proxyClient, log) + // save result to file + err = provider2.SaveWorkspaceResult(client.WorkspaceConfig(), result) + if err != nil { + return nil, fmt.Errorf("save workspace result: %w", err) } - return nil, nil + return result, nil } func (cmd *UpCmd) devPodUpProxy( @@ -401,7 +411,18 @@ func (cmd *UpCmd) devPodUpMachine( ) } - return sshtunnel.ExecuteCommand(ctx, client, agentInjectFunc, sshTunnelCmd, agentCommand, cmd.Proxy, false, false, nil, log) + return sshtunnel.ExecuteCommand( + ctx, + client, + agentInjectFunc, + sshTunnelCmd, + agentCommand, + cmd.Proxy, + false, + false, + nil, + log, + ) } func startJupyterNotebookInBrowser( diff --git a/e2e/framework/util.go b/e2e/framework/util.go index 12a579e5d..369620faf 100644 --- a/e2e/framework/util.go +++ b/e2e/framework/util.go @@ -58,9 +58,9 @@ func CopyToTempDirWithoutChdir(relativePath string) (string, error) { return dir, nil } -func CopyToTempDir(relativePath string) (string, error) { +func CopyToTempDirInDir(baseDir, relativePath string) (string, error) { // Create temporary directory - dir, err := os.MkdirTemp("", "temp-*") + dir, err := os.MkdirTemp(baseDir, "temp-*") if err != nil { return "", err } @@ -88,6 +88,10 @@ func CopyToTempDir(relativePath string) (string, error) { return dir, nil } +func CopyToTempDir(relativePath string) (string, error) { + return CopyToTempDirInDir("", relativePath) +} + func CleanupTempDir(initialDir, tempDir string) { err := os.RemoveAll(tempDir) if err != nil { diff --git a/e2e/tests/up/testdata/kubernetes/test_file.txt b/e2e/tests/up/testdata/kubernetes/test_file.txt new file mode 100644 index 000000000..30d74d258 --- /dev/null +++ b/e2e/tests/up/testdata/kubernetes/test_file.txt @@ -0,0 +1 @@ +test \ No newline at end of file diff --git a/e2e/tests/up/up.go b/e2e/tests/up/up.go index 5558d8bb3..9659c5e9c 100644 --- a/e2e/tests/up/up.go +++ b/e2e/tests/up/up.go @@ -10,6 +10,7 @@ import ( "strings" "github.com/loft-sh/devpod/e2e/framework" + config2 "github.com/loft-sh/devpod/pkg/config" "github.com/loft-sh/devpod/pkg/devcontainer/config" docker "github.com/loft-sh/devpod/pkg/docker" "github.com/loft-sh/devpod/pkg/language" @@ -118,7 +119,7 @@ var _ = DevPodDescribe("devpod up test suite", func() { framework.ExpectNoError(err) }) - ginkgo.It("run devpod in Kubernetes", func() { + ginkgo.FIt("run devpod in Kubernetes", func() { ctx := context.Background() f := framework.NewDefaultFramework(initialDir + "/bin") tempDir, err := framework.CopyToTempDir("tests/up/testdata/kubernetes") @@ -188,6 +189,45 @@ var _ = DevPodDescribe("devpod up test suite", func() { err = f.DevPodSSHEchoTestString(ctx, tempDir) framework.ExpectNoError(err) + // export workspace + data, err := f.ExecCommandOutput(ctx, []string{"export", tempDir}) + framework.ExpectNoError(err) + + // check if file is there + out, err := os.ReadFile(filepath.Join(tempDir, "test_file.txt")) + framework.ExpectNoError(err) + framework.ExpectEqual(strings.TrimSpace(string(out)), "test") + + // delete devpod directory & temp dir + configDir, err := config2.GetConfigDir() + framework.ExpectNoError(err) + err = os.RemoveAll(configDir) + framework.ExpectNoError(err) + err = os.RemoveAll(tempDir) + framework.ExpectNoError(err) + + // import workspace + _, err = f.ExecCommandOutput(ctx, []string{"import", "--data", data}) + framework.ExpectNoError(err) + + // check if ssh works + err = f.DevPodSSHEchoTestString(ctx, tempDir) + framework.ExpectNoError(err) + + // make sure file is not there anymore + _, err = os.ReadFile(filepath.Join(tempDir, "test_file.txt")) + framework.ExpectError(err) + _, err = os.ReadFile(filepath.Join(tempDir, ".devcontainer.json")) + framework.ExpectNoError(err) + + // run up + err = f.DevPodUp(ctx, tempDir) + framework.ExpectNoError(err) + + // check if ssh works + err = f.DevPodSSHEchoTestString(ctx, tempDir) + framework.ExpectNoError(err) + // delete workspace err = f.DevPodWorkspaceDelete(ctx, tempDir) framework.ExpectNoError(err) diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 7194ae01d..bb586c629 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -187,7 +187,7 @@ func decodeWorkspaceInfoAndWrite( // check content folder if workspaceInfo.Workspace.Source.LocalFolder != "" { - _, err = os.Stat(workspaceInfo.Workspace.Source.LocalFolder) + _, err = os.Stat(workspaceInfo.WorkspaceOrigin) if err == nil { workspaceInfo.ContentFolder = workspaceInfo.Workspace.Source.LocalFolder } diff --git a/pkg/agent/tunnelserver/tunnelserver.go b/pkg/agent/tunnelserver/tunnelserver.go index d2052b34d..cf19aefaa 100644 --- a/pkg/agent/tunnelserver/tunnelserver.go +++ b/pkg/agent/tunnelserver/tunnelserver.go @@ -6,7 +6,7 @@ import ( "encoding/json" "fmt" "io" - "path/filepath" + "os" "strings" "github.com/loft-sh/devpod/pkg/agent/tunnel" @@ -238,13 +238,20 @@ func (t *tunnelServer) StreamGitClone(message *tunnel.Empty, stream tunnel.Tunne } // clone here - gitCloneDir := filepath.Join(t.workspace.Folder, "source") - cloneArgs := []string{"clone", t.workspace.Source.GitRepository, gitCloneDir} + tempDir, err := os.MkdirTemp("", "devpod-git-clone-*") + if err != nil { + return fmt.Errorf("create temp dir: %w", err) + } + defer os.RemoveAll(tempDir) + + // clone repository + cloneArgs := []string{"clone", t.workspace.Source.GitRepository, tempDir} if t.workspace.Source.GitBranch != "" { cloneArgs = append(cloneArgs, "--branch", t.workspace.Source.GitBranch) } - err := git.CommandContext(context.Background(), cloneArgs...).Run() + // run command + err = git.CommandContext(context.Background(), cloneArgs...).Run() if err != nil { return err } @@ -255,7 +262,7 @@ func (t *tunnelServer) StreamGitClone(message *tunnel.Empty, stream tunnel.Tunne // git fetch origin pull/996/head:PR996 fetchArgs := []string{"fetch", "origin", t.workspace.Source.GitPRReference + ":" + prBranch} fetchCmd := git.CommandContext(context.Background(), fetchArgs...) - fetchCmd.Dir = gitCloneDir + fetchCmd.Dir = tempDir err = fetchCmd.Run() if err != nil { return err @@ -264,7 +271,7 @@ func (t *tunnelServer) StreamGitClone(message *tunnel.Empty, stream tunnel.Tunne // git switch PR996 switchArgs := []string{"switch", prBranch} switchCmd := git.CommandContext(context.Background(), switchArgs...) - switchCmd.Dir = gitCloneDir + switchCmd.Dir = tempDir err = switchCmd.Run() if err != nil { return err @@ -274,7 +281,7 @@ func (t *tunnelServer) StreamGitClone(message *tunnel.Empty, stream tunnel.Tunne // git reset --hard $COMMIT_SHA resetArgs := []string{"reset", "--hard", t.workspace.Source.GitCommit} resetCmd := git.CommandContext(context.Background(), resetArgs...) - resetCmd.Dir = gitCloneDir + resetCmd.Dir = tempDir err = resetCmd.Run() if err != nil { @@ -283,7 +290,7 @@ func (t *tunnelServer) StreamGitClone(message *tunnel.Empty, stream tunnel.Tunne } buf := bufio.NewWriterSize(NewStreamWriter(stream, t.log), 10*1024) - err = extract.WriteTar(buf, gitCloneDir, false) + err = extract.WriteTar(buf, tempDir, false) if err != nil { return err } diff --git a/pkg/client/clientimplementation/workspace_client.go b/pkg/client/clientimplementation/workspace_client.go index badfa7c72..5df0682a2 100644 --- a/pkg/client/clientimplementation/workspace_client.go +++ b/pkg/client/clientimplementation/workspace_client.go @@ -18,6 +18,7 @@ import ( "github.com/loft-sh/devpod/pkg/client" "github.com/loft-sh/devpod/pkg/compress" "github.com/loft-sh/devpod/pkg/config" + config2 "github.com/loft-sh/devpod/pkg/devcontainer/config" "github.com/loft-sh/devpod/pkg/options" "github.com/loft-sh/devpod/pkg/provider" "github.com/loft-sh/devpod/pkg/shell" @@ -155,13 +156,29 @@ func (s *workspaceClient) AgentInfo(cliOptions provider.CLIOptions) (string, *pr } func (s *workspaceClient) agentInfo(cliOptions provider.CLIOptions) (string, *provider.AgentWorkspaceInfo, error) { + // try to load last devcontainer.json + var lastDevContainerConfig *config2.DevContainerConfigWithPath + var workspaceOrigin string + if s.workspace != nil { + result, err := provider.LoadWorkspaceResult(s.workspace.Context, s.workspace.ID) + if err != nil { + s.log.Debugf("Error loading workspace result: %v", err) + } else if result != nil { + lastDevContainerConfig = result.DevContainerConfigWithPath + } + + workspaceOrigin = s.workspace.Origin + } + // build struct agentInfo := &provider.AgentWorkspaceInfo{ - Workspace: s.workspace, - Machine: s.machine, - CLIOptions: cliOptions, - Agent: options.ResolveAgentConfig(s.devPodConfig, s.config, s.workspace, s.machine), - Options: s.devPodConfig.ProviderOptions(s.Provider()), + WorkspaceOrigin: workspaceOrigin, + Workspace: s.workspace, + Machine: s.machine, + LastDevContainerConfig: lastDevContainerConfig, + CLIOptions: cliOptions, + Agent: options.ResolveAgentConfig(s.devPodConfig, s.config, s.workspace, s.machine), + Options: s.devPodConfig.ProviderOptions(s.Provider()), } // we don't send any provider options if proxy because these could contain diff --git a/pkg/devcontainer/build.go b/pkg/devcontainer/build.go index 493956dc2..b64bee363 100644 --- a/pkg/devcontainer/build.go +++ b/pkg/devcontainer/build.go @@ -16,12 +16,18 @@ import ( "github.com/loft-sh/devpod/pkg/driver" "github.com/loft-sh/devpod/pkg/driver/docker" "github.com/loft-sh/devpod/pkg/image" + "github.com/loft-sh/devpod/pkg/provider" "github.com/pkg/errors" ) -func (r *runner) build(ctx context.Context, parsedConfig *config.SubstitutedConfig, options config.BuildOptions) (*config.BuildInfo, error) { +func (r *runner) build( + ctx context.Context, + parsedConfig *config.SubstitutedConfig, + substitutionContext *config.SubstitutionContext, + options provider.BuildOptions, +) (*config.BuildInfo, error) { if isDockerFileConfig(parsedConfig.Config) { - return r.buildAndExtendImage(ctx, parsedConfig, options) + return r.buildAndExtendImage(ctx, parsedConfig, substitutionContext, options) } else if isDockerComposeConfig(parsedConfig.Config) { composeHelper, err := r.composeHelper() if err != nil { @@ -69,7 +75,7 @@ func (r *runner) build(ctx context.Context, parsedConfig *config.SubstitutedConf } } - overrideBuildImageName, _, imageMetadata, _, err := r.buildAndExtendDockerCompose(ctx, parsedConfig, project, composeHelper, &composeService, composeGlobalArgs) + overrideBuildImageName, _, imageMetadata, _, err := r.buildAndExtendDockerCompose(ctx, parsedConfig, substitutionContext, project, composeHelper, &composeService, composeGlobalArgs) if err != nil { return nil, errors.Wrap(err, "build and extend docker-compose") } @@ -92,18 +98,23 @@ func (r *runner) build(ctx context.Context, parsedConfig *config.SubstitutedConf }, nil } - return r.extendImage(ctx, parsedConfig, options) + return r.extendImage(ctx, parsedConfig, substitutionContext, options) } -func (r *runner) extendImage(ctx context.Context, parsedConfig *config.SubstitutedConfig, options config.BuildOptions) (*config.BuildInfo, error) { +func (r *runner) extendImage( + ctx context.Context, + parsedConfig *config.SubstitutedConfig, + substitutionContext *config.SubstitutionContext, + options provider.BuildOptions, +) (*config.BuildInfo, error) { imageBase := parsedConfig.Config.Image - imageBuildInfo, err := r.getImageBuildInfoFromImage(ctx, imageBase) + imageBuildInfo, err := r.getImageBuildInfoFromImage(ctx, substitutionContext, imageBase) if err != nil { return nil, errors.Wrap(err, "get image build info") } // get extend image build info - extendedBuildInfo, err := feature.GetExtendedBuildInfo(r.SubstitutionContext, imageBuildInfo.Metadata, imageBuildInfo.User, imageBase, parsedConfig, r.Log, options.ForceBuild) + extendedBuildInfo, err := feature.GetExtendedBuildInfo(substitutionContext, imageBuildInfo.Metadata, imageBuildInfo.User, imageBase, parsedConfig, r.Log, options.ForceBuild) if err != nil { return nil, errors.Wrap(err, "get extended build info") } @@ -118,10 +129,15 @@ func (r *runner) extendImage(ctx context.Context, parsedConfig *config.Substitut } // build the image - return r.buildImage(ctx, parsedConfig, imageBuildInfo, extendedBuildInfo, "", "", options) + return r.buildImage(ctx, parsedConfig, substitutionContext, imageBuildInfo, extendedBuildInfo, "", "", options) } -func (r *runner) buildAndExtendImage(ctx context.Context, parsedConfig *config.SubstitutedConfig, options config.BuildOptions) (*config.BuildInfo, error) { +func (r *runner) buildAndExtendImage( + ctx context.Context, + parsedConfig *config.SubstitutedConfig, + substitutionContext *config.SubstitutionContext, + options provider.BuildOptions, +) (*config.BuildInfo, error) { dockerFilePath, err := r.getDockerfilePath(parsedConfig.Config) if err != nil { return nil, err @@ -148,19 +164,19 @@ func (r *runner) buildAndExtendImage(ctx context.Context, parsedConfig *config.S } // get image build info - imageBuildInfo, err := r.getImageBuildInfoFromDockerfile(string(dockerFileContent), parsedConfig.Config.GetArgs(), parsedConfig.Config.GetTarget()) + imageBuildInfo, err := r.getImageBuildInfoFromDockerfile(substitutionContext, string(dockerFileContent), parsedConfig.Config.GetArgs(), parsedConfig.Config.GetTarget()) if err != nil { return nil, errors.Wrap(err, "get image build info") } // get extend image build info - extendedBuildInfo, err := feature.GetExtendedBuildInfo(r.SubstitutionContext, imageBuildInfo.Metadata, imageBuildInfo.User, imageBase, parsedConfig, r.Log, options.ForceBuild) + extendedBuildInfo, err := feature.GetExtendedBuildInfo(substitutionContext, imageBuildInfo.Metadata, imageBuildInfo.User, imageBase, parsedConfig, r.Log, options.ForceBuild) if err != nil { return nil, errors.Wrap(err, "get extended build info") } // build the image - return r.buildImage(ctx, parsedConfig, imageBuildInfo, extendedBuildInfo, dockerFilePath, string(dockerFileContent), options) + return r.buildImage(ctx, parsedConfig, substitutionContext, imageBuildInfo, extendedBuildInfo, dockerFilePath, string(dockerFileContent), options) } func (r *runner) getDockerfilePath(parsedConfig *config.DevContainerConfig) (string, error) { @@ -179,7 +195,7 @@ func (r *runner) getDockerfilePath(parsedConfig *config.DevContainerConfig) (str return dockerfilePath, nil } -func (r *runner) getImageBuildInfoFromImage(ctx context.Context, imageName string) (*config.ImageBuildInfo, error) { +func (r *runner) getImageBuildInfoFromImage(ctx context.Context, substitutionContext *config.SubstitutionContext, imageName string) (*config.ImageBuildInfo, error) { imageDetails, err := r.inspectImage(ctx, imageName) if err != nil { return nil, err @@ -190,7 +206,7 @@ func (r *runner) getImageBuildInfoFromImage(ctx context.Context, imageName strin user = imageDetails.Config.User } - imageMetadata, err := metadata.GetImageMetadata(imageDetails, r.SubstitutionContext, r.Log) + imageMetadata, err := metadata.GetImageMetadata(imageDetails, substitutionContext, r.Log) if err != nil { return nil, errors.Wrap(err, "get image metadata") } @@ -202,7 +218,7 @@ func (r *runner) getImageBuildInfoFromImage(ctx context.Context, imageName strin }, nil } -func (r *runner) getImageBuildInfoFromDockerfile(dockerFileContent string, buildArgs map[string]string, target string) (*config.ImageBuildInfo, error) { +func (r *runner) getImageBuildInfoFromDockerfile(substitutionContext *config.SubstitutionContext, dockerFileContent string, buildArgs map[string]string, target string) (*config.ImageBuildInfo, error) { parsedDockerfile, err := dockerfile.Parse(dockerFileContent) if err != nil { return nil, errors.Wrap(err, "parse dockerfile") @@ -228,7 +244,7 @@ func (r *runner) getImageBuildInfoFromDockerfile(dockerFileContent string, build } // parse metadata from image details - imageMetadataConfig, err := metadata.GetImageMetadata(imageDetails, r.SubstitutionContext, r.Log) + imageMetadataConfig, err := metadata.GetImageMetadata(imageDetails, substitutionContext, r.Log) if err != nil { return nil, errors.Wrap(err, "get image metadata") } @@ -243,11 +259,12 @@ func (r *runner) getImageBuildInfoFromDockerfile(dockerFileContent string, build func (r *runner) buildImage( ctx context.Context, parsedConfig *config.SubstitutedConfig, + substitutionContext *config.SubstitutionContext, buildInfo *config.ImageBuildInfo, extendedBuildInfo *feature.ExtendedBuildInfo, dockerfilePath, dockerfileContent string, - options config.BuildOptions, + options provider.BuildOptions, ) (*config.BuildInfo, error) { targetArch, err := r.Driver.TargetArchitecture(ctx, r.ID) if err != nil { @@ -300,7 +317,7 @@ func (r *runner) buildImage( return nil, fmt.Errorf("cannot build devcontainer because driver is non-docker and dockerless fallback is disabled") } - return dockerlessFallback(r.LocalWorkspaceFolder, r.SubstitutionContext.ContainerWorkspaceFolder, parsedConfig, buildInfo, extendedBuildInfo, dockerfileContent) + return dockerlessFallback(r.LocalWorkspaceFolder, substitutionContext.ContainerWorkspaceFolder, parsedConfig, buildInfo, extendedBuildInfo, dockerfileContent) } return dockerDriver.BuildDevContainer(ctx, prebuildHash, parsedConfig, extendedBuildInfo, dockerfilePath, dockerfileContent, r.LocalWorkspaceFolder, options) diff --git a/pkg/devcontainer/compose.go b/pkg/devcontainer/compose.go index f429d5abc..faa981c36 100644 --- a/pkg/devcontainer/compose.go +++ b/pkg/devcontainer/compose.go @@ -45,7 +45,7 @@ func (r *runner) stopDockerCompose(ctx context.Context, projectName string) erro return errors.Wrap(err, "find docker compose") } - parsedConfig, err := r.prepare(r.WorkspaceConfig.CLIOptions) + parsedConfig, _, err := r.prepare(r.WorkspaceConfig.CLIOptions) if err != nil { return errors.Wrap(err, "get parsed config") } @@ -69,7 +69,7 @@ func (r *runner) deleteDockerCompose(ctx context.Context, projectName string) er return errors.Wrap(err, "find docker compose") } - parsedConfig, err := r.prepare(r.WorkspaceConfig.CLIOptions) + parsedConfig, _, err := r.prepare(r.WorkspaceConfig.CLIOptions) if err != nil { return errors.Wrap(err, "get parsed config") } @@ -110,7 +110,12 @@ func (r *runner) dockerComposeProjectFiles(parsedConfig *config.SubstitutedConfi return composeFiles, envFiles, args, nil } -func (r *runner) runDockerCompose(ctx context.Context, parsedConfig *config.SubstitutedConfig, options UpOptions) (*config.Result, error) { +func (r *runner) runDockerCompose( + ctx context.Context, + parsedConfig *config.SubstitutedConfig, + substitutionContext *config.SubstitutionContext, + options UpOptions, +) (*config.Result, error) { composeHelper, err := r.composeHelper() if err != nil { return nil, errors.Wrap(err, "find docker compose") @@ -137,7 +142,7 @@ func (r *runner) runDockerCompose(ctx context.Context, parsedConfig *config.Subs // does the container already exist or is it not running? if containerDetails == nil || containerDetails.State.Status != "running" || options.Recreate { // Start container if not running - containerDetails, err = r.startContainer(ctx, parsedConfig, project, composeHelper, composeGlobalArgs, containerDetails, options) + containerDetails, err = r.startContainer(ctx, parsedConfig, substitutionContext, project, composeHelper, composeGlobalArgs, containerDetails, options) if err != nil { return nil, errors.Wrap(err, "start container") } else if containerDetails == nil { @@ -145,7 +150,7 @@ func (r *runner) runDockerCompose(ctx context.Context, parsedConfig *config.Subs } } - imageMetadataConfig, err := metadata.GetImageMetadataFromContainer(containerDetails, r.SubstitutionContext, r.Log) + imageMetadataConfig, err := metadata.GetImageMetadataFromContainer(containerDetails, substitutionContext, r.Log) if err != nil { return nil, errors.Wrap(err, "get image metadata from container") } @@ -156,7 +161,7 @@ func (r *runner) runDockerCompose(ctx context.Context, parsedConfig *config.Subs } // setup container - return r.setupContainer(ctx, containerDetails, mergedConfig) + return r.setupContainer(ctx, parsedConfig.Raw, containerDetails, mergedConfig, substitutionContext) } func (r *runner) getDockerComposeFilePaths(parsedConfig *config.SubstitutedConfig, envFiles []string) ([]string, error) { @@ -214,6 +219,7 @@ func (r *runner) getEnvFiles() ([]string, error) { func (r *runner) startContainer( ctx context.Context, parsedConfig *config.SubstitutedConfig, + substitutionContext *config.SubstitutionContext, project *composetypes.Project, composeHelper *compose.ComposeHelper, composeGlobalArgs []string, @@ -265,7 +271,7 @@ func (r *runner) startContainer( } if container == nil || !didRestoreFromPersistedShare { - overrideBuildImageName, overrideComposeBuildFilePath, imageMetadata, metadataLabel, err := r.buildAndExtendDockerCompose(ctx, parsedConfig, project, composeHelper, &composeService, composeGlobalArgs) + overrideBuildImageName, overrideComposeBuildFilePath, imageMetadata, metadataLabel, err := r.buildAndExtendDockerCompose(ctx, parsedConfig, substitutionContext, project, composeHelper, &composeService, composeGlobalArgs) if err != nil { return nil, errors.Wrap(err, "build and extend docker-compose") } @@ -350,7 +356,15 @@ func (r *runner) startContainer( } // This extends the build information for docker compose containers -func (r *runner) buildAndExtendDockerCompose(ctx context.Context, parsedConfig *config.SubstitutedConfig, project *composetypes.Project, composeHelper *compose.ComposeHelper, composeService *composetypes.ServiceConfig, globalArgs []string) (string, string, *config.ImageMetadataConfig, string, error) { +func (r *runner) buildAndExtendDockerCompose( + ctx context.Context, + parsedConfig *config.SubstitutedConfig, + substitutionContext *config.SubstitutionContext, + project *composetypes.Project, + composeHelper *compose.ComposeHelper, + composeService *composetypes.ServiceConfig, + globalArgs []string, +) (string, string, *config.ImageMetadataConfig, string, error) { var dockerFilePath, dockerfileContents, dockerComposeFilePath string var imageBuildInfo *config.ImageBuildInfo var err error @@ -386,18 +400,18 @@ func (r *runner) buildAndExtendDockerCompose(ctx context.Context, parsedConfig * dockerfileContents = modifiedDockerfile } } - imageBuildInfo, err = r.getImageBuildInfoFromDockerfile(string(originalDockerfile), mappingToMap(composeService.Build.Args), originalTarget) + imageBuildInfo, err = r.getImageBuildInfoFromDockerfile(substitutionContext, string(originalDockerfile), mappingToMap(composeService.Build.Args), originalTarget) if err != nil { return "", "", nil, "", err } } else { - imageBuildInfo, err = r.getImageBuildInfoFromImage(ctx, composeService.Image) + imageBuildInfo, err = r.getImageBuildInfoFromImage(ctx, substitutionContext, composeService.Image) if err != nil { return "", "", nil, "", err } } - extendImageBuildInfo, err := feature.GetExtendedBuildInfo(r.SubstitutionContext, imageBuildInfo.Metadata, imageBuildInfo.User, buildTarget, parsedConfig, r.Log, false) + extendImageBuildInfo, err := feature.GetExtendedBuildInfo(substitutionContext, imageBuildInfo.Metadata, imageBuildInfo.User, buildTarget, parsedConfig, r.Log, false) if err != nil { return "", "", nil, "", err } @@ -464,7 +478,7 @@ func (r *runner) buildAndExtendDockerCompose(ctx context.Context, parsedConfig * return buildImageName, "", nil, "", err } - imageMetadata, err := metadata.GetDevContainerMetadata(r.SubstitutionContext, imageBuildInfo.Metadata, parsedConfig, extendImageBuildInfo.Features) + imageMetadata, err := metadata.GetDevContainerMetadata(substitutionContext, imageBuildInfo.Metadata, parsedConfig, extendImageBuildInfo.Features) if err != nil { return buildImageName, "", nil, "", err } diff --git a/pkg/devcontainer/config/build.go b/pkg/devcontainer/config/build.go index df9d2923a..69e59fdbf 100644 --- a/pkg/devcontainer/config/build.go +++ b/pkg/devcontainer/config/build.go @@ -2,7 +2,6 @@ package config import ( "github.com/loft-sh/devpod/pkg/dockerfile" - provider2 "github.com/loft-sh/devpod/pkg/provider" ) const ( @@ -17,13 +16,6 @@ func GetDockerLabelForID(id string) []string { return []string{DockerIDLabel + "=" + id} } -type BuildOptions struct { - provider2.CLIOptions - - Platform string - NoBuild bool -} - type BuildInfo struct { ImageDetails *ImageDetails ImageMetadata *ImageMetadataConfig diff --git a/pkg/devcontainer/config/result.go b/pkg/devcontainer/config/result.go index a6ec4ea54..4cb8f267e 100644 --- a/pkg/devcontainer/config/result.go +++ b/pkg/devcontainer/config/result.go @@ -3,9 +3,18 @@ package config const UserLabel = "devpod.user" type Result struct { - MergedConfig *MergedDevContainerConfig `json:"MergedConfig"` - SubstitutionContext *SubstitutionContext `json:"SubstitutionContext"` - ContainerDetails *ContainerDetails `json:"ContainerDetails"` + DevContainerConfigWithPath *DevContainerConfigWithPath `json:"DevContainerConfigWithPath"` + MergedConfig *MergedDevContainerConfig `json:"MergedConfig"` + SubstitutionContext *SubstitutionContext `json:"SubstitutionContext"` + ContainerDetails *ContainerDetails `json:"ContainerDetails"` +} + +type DevContainerConfigWithPath struct { + // Config is the devcontainer.json config + Config *DevContainerConfig `json:"config,omitempty"` + + // Path is the relative path to the devcontainer.json from the workspace folder + Path string `json:"path,omitempty"` } func GetMounts(result *Result) []*Mount { diff --git a/pkg/devcontainer/prebuild.go b/pkg/devcontainer/prebuild.go index 18aa2c0d9..71bc25b80 100644 --- a/pkg/devcontainer/prebuild.go +++ b/pkg/devcontainer/prebuild.go @@ -10,16 +10,17 @@ import ( "github.com/loft-sh/devpod/pkg/driver" "github.com/loft-sh/devpod/pkg/driver/docker" "github.com/loft-sh/devpod/pkg/image" + "github.com/loft-sh/devpod/pkg/provider" "github.com/pkg/errors" ) -func (r *runner) Build(ctx context.Context, options config.BuildOptions) (string, error) { +func (r *runner) Build(ctx context.Context, options provider.BuildOptions) (string, error) { dockerDriver, ok := r.Driver.(driver.DockerDriver) if !ok { return "", fmt.Errorf("building only supported with docker driver") } - substitutedConfig, err := r.prepare(options.CLIOptions) + substitutedConfig, substitutionContext, err := r.prepare(options.CLIOptions) if err != nil { return "", err } @@ -37,7 +38,7 @@ func (r *runner) Build(ctx context.Context, options config.BuildOptions) (string }() // check if we need to build container - buildInfo, err := r.build(ctx, substitutedConfig, options) + buildInfo, err := r.build(ctx, substitutedConfig, substitutionContext, options) if err != nil { return "", errors.Wrap(err, "build image") } diff --git a/pkg/devcontainer/run.go b/pkg/devcontainer/run.go index 978ade42a..31c8c088a 100644 --- a/pkg/devcontainer/run.go +++ b/pkg/devcontainer/run.go @@ -25,7 +25,7 @@ import ( type Runner interface { Up(ctx context.Context, options UpOptions) (*config.Result, error) - Build(ctx context.Context, options config.BuildOptions) (string, error) + Build(ctx context.Context, options provider2.BuildOptions) (string, error) Find(ctx context.Context) (*config.ContainerDetails, error) @@ -75,7 +75,6 @@ type runner struct { AgentDownloadURL string LocalWorkspaceFolder string - SubstitutionContext *config.SubstitutionContext ID string @@ -93,12 +92,12 @@ func (r *runner) Up(ctx context.Context, options UpOptions) (*config.Result, err // download workspace source before recreating container _, isDockerDriver := r.Driver.(driver.DockerDriver) if options.Recreate && !isDockerDriver { - // TODO: for drivers other than docker and recreate is true, we need to download the complete context here first + // TODO: implement this return nil, fmt.Errorf("rebuilding the workspace is currently not supported for non-docker drivers") } // prepare config - substitutedConfig, err := r.prepare(options.CLIOptions) + substitutedConfig, substitutionContext, err := r.prepare(options.CLIOptions) if err != nil { return nil, err } @@ -121,13 +120,14 @@ func (r *runner) Up(ctx context.Context, options UpOptions) (*config.Result, err result, err = r.runSingleContainer( ctx, substitutedConfig, + substitutionContext, options, ) if err != nil { return nil, err } } else if isDockerComposeConfig(substitutedConfig.Config) { - result, err = r.runDockerCompose(ctx, substitutedConfig, options) + result, err = r.runDockerCompose(ctx, substitutedConfig, substitutionContext, options) if err != nil { return nil, err } @@ -144,7 +144,7 @@ func (r *runner) Up(ctx context.Context, options UpOptions) (*config.Result, err } substitutedConfig.Config.ImageContainer = language.MapConfig[lang].ImageContainer - result, err = r.runSingleContainer(ctx, substitutedConfig, options) + result, err = r.runSingleContainer(ctx, substitutedConfig, substitutionContext, options) if err != nil { return nil, err } @@ -156,27 +156,40 @@ func (r *runner) Up(ctx context.Context, options UpOptions) (*config.Result, err func (r *runner) prepare( options provider2.CLIOptions, -) (*config.SubstitutedConfig, error) { - rawParsedConfig, err := config.ParseDevContainerJSON( - r.LocalWorkspaceFolder, - r.WorkspaceConfig.Workspace.DevContainerPath, - ) - - // We want to fail only in case of real errors, non-existing devcontainer.jon - // will be gracefully handled by the auto-detection mechanism - if err != nil && !os.IsNotExist(err) { - return nil, errors.Wrap(err, "parsing devcontainer.json") - } else if rawParsedConfig == nil { - r.Log.Infof("Couldn't find a devcontainer.json") - r.Log.Infof("Try detecting project programming language...") - defaultConfig := language.DefaultConfig(r.LocalWorkspaceFolder, r.Log) - defaultConfig.Origin = path.Join(filepath.ToSlash(r.LocalWorkspaceFolder), ".devcontainer.json") - err = config.SaveDevContainerJSON(defaultConfig) - if err != nil { - return nil, errors.Wrap(err, "write default devcontainer.json") +) (*config.SubstitutedConfig, *config.SubstitutionContext, error) { + var rawParsedConfig *config.DevContainerConfig + if r.WorkspaceConfig.Workspace.DevContainerConfig != nil { + rawParsedConfig = config.CloneDevContainerConfig(r.WorkspaceConfig.Workspace.DevContainerConfig) + if r.WorkspaceConfig.Workspace.DevContainerPath != "" { + rawParsedConfig.Origin = path.Join(filepath.ToSlash(r.LocalWorkspaceFolder), r.WorkspaceConfig.Workspace.DevContainerPath) + } else { + rawParsedConfig.Origin = path.Join(filepath.ToSlash(r.LocalWorkspaceFolder), ".devcontainer.devpod.json") } + } else { + var err error - rawParsedConfig = defaultConfig + // parse the devcontainer json + rawParsedConfig, err = config.ParseDevContainerJSON( + r.LocalWorkspaceFolder, + r.WorkspaceConfig.Workspace.DevContainerPath, + ) + + // We want to fail only in case of real errors, non-existing devcontainer.jon + // will be gracefully handled by the auto-detection mechanism + if err != nil && !os.IsNotExist(err) { + return nil, nil, errors.Wrap(err, "parsing devcontainer.json") + } else if rawParsedConfig == nil { + r.Log.Infof("Couldn't find a devcontainer.json") + r.Log.Infof("Try detecting project programming language...") + defaultConfig := language.DefaultConfig(r.LocalWorkspaceFolder, r.Log) + defaultConfig.Origin = path.Join(filepath.ToSlash(r.LocalWorkspaceFolder), ".devcontainer.json") + err = config.SaveDevContainerJSON(defaultConfig) + if err != nil { + return nil, nil, errors.Wrap(err, "write default devcontainer.json") + } + + rawParsedConfig = defaultConfig + } } configFile := rawParsedConfig.Origin @@ -186,7 +199,7 @@ func (r *runner) prepare( r.WorkspaceConfig.Workspace.ID, rawParsedConfig, ) - r.SubstitutionContext = &config.SubstitutionContext{ + substitutionContext := &config.SubstitutionContext{ DevContainerID: r.ID, LocalWorkspaceFolder: r.LocalWorkspaceFolder, ContainerWorkspaceFolder: containerWorkspaceFolder, @@ -197,15 +210,15 @@ func (r *runner) prepare( // substitute & load parsedConfig := &config.DevContainerConfig{} - err = config.Substitute(r.SubstitutionContext, rawParsedConfig, parsedConfig) + err := config.Substitute(substitutionContext, rawParsedConfig, parsedConfig) if err != nil { - return nil, err + return nil, nil, err } if parsedConfig.WorkspaceFolder != "" { - r.SubstitutionContext.ContainerWorkspaceFolder = parsedConfig.WorkspaceFolder + substitutionContext.ContainerWorkspaceFolder = parsedConfig.WorkspaceFolder } if parsedConfig.WorkspaceMount != "" { - r.SubstitutionContext.WorkspaceMount = parsedConfig.WorkspaceMount + substitutionContext.WorkspaceMount = parsedConfig.WorkspaceMount } if options.DevContainerImage != "" { @@ -219,7 +232,7 @@ func (r *runner) prepare( return &config.SubstitutedConfig{ Config: parsedConfig, Raw: rawParsedConfig, - }, nil + }, substitutionContext, nil } func (r *runner) Command( diff --git a/pkg/devcontainer/setup.go b/pkg/devcontainer/setup.go index a28cd48ca..38abfbc49 100644 --- a/pkg/devcontainer/setup.go +++ b/pkg/devcontainer/setup.go @@ -6,7 +6,9 @@ import ( "fmt" "io" "os" + "path/filepath" "runtime" + "strings" "github.com/loft-sh/devpod/pkg/agent" "github.com/loft-sh/devpod/pkg/compress" @@ -20,8 +22,10 @@ import ( func (r *runner) setupContainer( ctx context.Context, + rawConfig *config.DevContainerConfig, containerDetails *config.ContainerDetails, mergedConfig *config.MergedDevContainerConfig, + substitutionContext *config.SubstitutionContext, ) (*config.Result, error) { // inject agent err := agent.InjectAgent(ctx, func(ctx context.Context, command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error { @@ -35,8 +39,13 @@ func (r *runner) setupContainer( // compress info result := &config.Result{ + DevContainerConfigWithPath: &config.DevContainerConfigWithPath{ + Config: rawConfig, + Path: getRelativeDevContainerJson(rawConfig.Origin, r.LocalWorkspaceFolder), + }, + MergedConfig: mergedConfig, - SubstitutionContext: r.SubstitutionContext, + SubstitutionContext: substitutionContext, ContainerDetails: containerDetails, } marshalled, err := json.Marshal(result) @@ -89,5 +98,21 @@ func (r *runner) setupContainer( return r.Driver.CommandDevContainer(cancelCtx, r.ID, "root", sshCmd, sshTunnelStdinReader, sshTunnelStdoutWriter, writer) } - return sshtunnel.ExecuteCommand(ctx, nil, agentInjectFunc, sshTunnelCmd, setupCommand, false, true, r.WorkspaceConfig.Agent.InjectDockerCredentials != "false", result, r.Log) + return sshtunnel.ExecuteCommand( + ctx, + nil, + agentInjectFunc, + sshTunnelCmd, + setupCommand, + false, + true, + r.WorkspaceConfig.Agent.InjectDockerCredentials != "false", + config.GetMounts(result), + r.Log, + ) +} + +func getRelativeDevContainerJson(origin, localWorkspaceFolder string) string { + relativePath := strings.TrimPrefix(filepath.ToSlash(origin), filepath.ToSlash(localWorkspaceFolder)) + return strings.TrimPrefix(relativePath, "/") } diff --git a/pkg/devcontainer/single.go b/pkg/devcontainer/single.go index 76884a93f..ed3154993 100644 --- a/pkg/devcontainer/single.go +++ b/pkg/devcontainer/single.go @@ -15,7 +15,7 @@ import ( var dockerlessImage = "ghcr.io/loft-sh/dockerless:0.1.4" -func (r *runner) runSingleContainer(ctx context.Context, parsedConfig *config.SubstitutedConfig, options UpOptions) (*config.Result, error) { +func (r *runner) runSingleContainer(ctx context.Context, parsedConfig *config.SubstitutedConfig, substitutionContext *config.SubstitutionContext, options UpOptions) (*config.Result, error) { containerDetails, err := r.Driver.FindDevContainer(ctx, r.ID) if err != nil { return nil, fmt.Errorf("find dev container: %w", err) @@ -34,7 +34,7 @@ func (r *runner) runSingleContainer(ctx context.Context, parsedConfig *config.Su } } - imageMetadataConfig, err := metadata.GetImageMetadataFromContainer(containerDetails, r.SubstitutionContext, r.Log) + imageMetadataConfig, err := metadata.GetImageMetadataFromContainer(containerDetails, substitutionContext, r.Log) if err != nil { return nil, err } @@ -45,7 +45,7 @@ func (r *runner) runSingleContainer(ctx context.Context, parsedConfig *config.Su } } else { // we need to build the container - buildInfo, err := r.build(ctx, parsedConfig, config.BuildOptions{ + buildInfo, err := r.build(ctx, parsedConfig, substitutionContext, provider2.BuildOptions{ CLIOptions: provider2.CLIOptions{ PrebuildRepositories: options.PrebuildRepositories, ForceDockerless: options.ForceDockerless, @@ -71,7 +71,7 @@ func (r *runner) runSingleContainer(ctx context.Context, parsedConfig *config.Su } // run dev container - err = r.runContainer(ctx, parsedConfig, mergedConfig, buildInfo) + err = r.runContainer(ctx, parsedConfig, substitutionContext, mergedConfig, buildInfo) if err != nil { return nil, errors.Wrap(err, "start dev container") } @@ -86,12 +86,13 @@ func (r *runner) runSingleContainer(ctx context.Context, parsedConfig *config.Su } // setup container - return r.setupContainer(ctx, containerDetails, mergedConfig) + return r.setupContainer(ctx, parsedConfig.Raw, containerDetails, mergedConfig, substitutionContext) } func (r *runner) runContainer( ctx context.Context, parsedConfig *config.SubstitutedConfig, + substitutionContext *config.SubstitutionContext, mergedConfig *config.MergedDevContainerConfig, buildInfo *config.BuildInfo, ) error { @@ -100,13 +101,13 @@ func (r *runner) runContainer( // build run options for dockerless mode var runOptions *driver.RunOptions if buildInfo.Dockerless != nil { - runOptions, err = r.getDockerlessRunOptions(mergedConfig, buildInfo) + runOptions, err = r.getDockerlessRunOptions(mergedConfig, substitutionContext, buildInfo) if err != nil { return fmt.Errorf("build dockerless run options: %w", err) } } else { // build run options - runOptions, err = r.getRunOptions(mergedConfig, buildInfo) + runOptions, err = r.getRunOptions(mergedConfig, substitutionContext, buildInfo) if err != nil { return fmt.Errorf("build run options: %w", err) } @@ -132,10 +133,11 @@ func (r *runner) runContainer( func (r *runner) getDockerlessRunOptions( mergedConfig *config.MergedDevContainerConfig, + substitutionContext *config.SubstitutionContext, buildInfo *config.BuildInfo, ) (*driver.RunOptions, error) { // parse workspace mount - workspaceMountParsed := config.ParseMount(r.SubstitutionContext.WorkspaceMount) + workspaceMountParsed := config.ParseMount(substitutionContext.WorkspaceMount) // add metadata as label here marshalled, err := json.Marshal(buildInfo.ImageMetadata.Raw) @@ -202,10 +204,11 @@ func (r *runner) getDockerlessRunOptions( func (r *runner) getRunOptions( mergedConfig *config.MergedDevContainerConfig, + substitutionContext *config.SubstitutionContext, buildInfo *config.BuildInfo, ) (*driver.RunOptions, error) { // parse workspace mount - workspaceMountParsed := config.ParseMount(r.SubstitutionContext.WorkspaceMount) + workspaceMountParsed := config.ParseMount(substitutionContext.WorkspaceMount) // add metadata as label here marshalled, err := json.Marshal(buildInfo.ImageMetadata.Raw) diff --git a/pkg/devcontainer/sshtunnel/sshtunnel.go b/pkg/devcontainer/sshtunnel/sshtunnel.go index a95f1d490..1de6af850 100644 --- a/pkg/devcontainer/sshtunnel/sshtunnel.go +++ b/pkg/devcontainer/sshtunnel/sshtunnel.go @@ -21,7 +21,18 @@ import ( type AgentInjectFunc func(context.Context, string, *os.File, *os.File, io.WriteCloser) error // ExecuteCommand runs the command in an SSH Tunnel and returns the result. -func ExecuteCommand(ctx context.Context, client client2.WorkspaceClient, agentInject AgentInjectFunc, sshCommand, command string, proxy, setupContainer, allowDockerCredentials bool, result *config2.Result, log log.Logger) (*config2.Result, error) { +func ExecuteCommand( + ctx context.Context, + client client2.WorkspaceClient, + agentInject AgentInjectFunc, + sshCommand, + command string, + proxy, + setupContainer, + allowDockerCredentials bool, + mounts []*config2.Mount, + log log.Logger, +) (*config2.Result, error) { // create pipes sshTunnelStdoutReader, sshTunnelStdoutWriter, err := os.Pipe() if err != nil { @@ -115,6 +126,7 @@ func ExecuteCommand(ctx context.Context, client client2.WorkspaceClient, agentIn }() // create container etc. + var result *config2.Result if proxy { // create client on stdin & stdout tunnelClient, err := tunnelserver.NewTunnelClient(os.Stdin, os.Stdout, true) @@ -140,7 +152,7 @@ func ExecuteCommand(ctx context.Context, client client2.WorkspaceClient, agentIn gRPCConnStdoutReader, gRPCConnStdinWriter, allowDockerCredentials, - config2.GetMounts(result), + mounts, log, ) if err != nil { diff --git a/pkg/driver/docker.go b/pkg/driver/docker.go index c1c4d1792..2bd20059d 100644 --- a/pkg/driver/docker.go +++ b/pkg/driver/docker.go @@ -7,6 +7,7 @@ import ( config2 "github.com/loft-sh/devpod/pkg/config" "github.com/loft-sh/devpod/pkg/devcontainer/config" "github.com/loft-sh/devpod/pkg/devcontainer/feature" + "github.com/loft-sh/devpod/pkg/provider" ) type DockerDriver interface { @@ -35,7 +36,7 @@ type DockerDriver interface { dockerfilePath, dockerfileContent string, localWorkspaceFolder string, - options config.BuildOptions, + options provider.BuildOptions, ) (*config.BuildInfo, error) // PushDevContainer pushes the given image to a registry diff --git a/pkg/driver/docker/build.go b/pkg/driver/docker/build.go index 07ff93af8..0291fa806 100644 --- a/pkg/driver/docker/build.go +++ b/pkg/driver/docker/build.go @@ -17,6 +17,7 @@ import ( "github.com/loft-sh/devpod/pkg/docker" "github.com/loft-sh/devpod/pkg/dockerfile" "github.com/loft-sh/devpod/pkg/id" + "github.com/loft-sh/devpod/pkg/provider" "github.com/loft-sh/log/hash" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -30,7 +31,7 @@ func (d *dockerDriver) BuildDevContainer( dockerfilePath, dockerfileContent string, localWorkspaceFolder string, - options config.BuildOptions, + options provider.BuildOptions, ) (*config.BuildInfo, error) { // check if image build is necessary imageName := GetImageName(localWorkspaceFolder, prebuildHash) diff --git a/pkg/extract/compress.go b/pkg/extract/compress.go index 4dc0b54ed..78e98d794 100644 --- a/pkg/extract/compress.go +++ b/pkg/extract/compress.go @@ -8,12 +8,13 @@ import ( "path" "path/filepath" "runtime" + "strings" "time" "github.com/pkg/errors" ) -func WriteTar(writer io.Writer, localPath string, compress bool) error { +func WriteTarExclude(writer io.Writer, localPath string, compress bool, excludedPaths []string) error { absolute, err := filepath.Abs(localPath) if err != nil { return errors.Wrap(err, "absolute") @@ -40,12 +41,16 @@ func WriteTar(writer io.Writer, localPath string, compress bool) error { // When its a file we copy the file to the toplevel of the tar if !stat.IsDir() { - return NewArchiver(filepath.Dir(absolute), tarWriter).AddToArchive(filepath.Base(absolute)) + return NewArchiver(filepath.Dir(absolute), tarWriter, excludedPaths).AddToArchive(filepath.Base(absolute)) } // When its a folder we copy the contents and not the folder itself to the // toplevel of the tar - return NewArchiver(absolute, tarWriter).AddToArchive("") + return NewArchiver(absolute, tarWriter, excludedPaths).AddToArchive("") +} + +func WriteTar(writer io.Writer, localPath string, compress bool) error { + return WriteTarExclude(writer, localPath, compress, nil) } // Archiver is responsible for compressing specific files and folders within a target directory @@ -53,39 +58,61 @@ type Archiver struct { basePath string writer *tar.Writer writtenFiles map[string]bool + + excludedPaths []string } // NewArchiver creates a new archiver -func NewArchiver(basePath string, writer *tar.Writer) *Archiver { +func NewArchiver(basePath string, writer *tar.Writer, excludedPaths []string) *Archiver { return &Archiver{ basePath: basePath, writer: writer, writtenFiles: map[string]bool{}, + + excludedPaths: excludedPaths, } } // AddToArchive adds a new path to the archive func (a *Archiver) AddToArchive(relativePath string) error { - absFilepath := path.Join(a.basePath, relativePath) if a.writtenFiles[relativePath] { return nil } // We skip files that are suddenly not there anymore - stat, err := os.Lstat(absFilepath) + stat, err := os.Lstat(path.Join(a.basePath, relativePath)) if err != nil { // config.Logf("[Upstream] Couldn't stat file %s: %s\n", absFilepath, err.Error()) return nil } if stat.IsDir() { + // check if excluded + if a.isExcluded(path.Clean(relativePath) + "/") { + return nil + } + // Recursively tar folder return a.tarFolder(relativePath, stat) } + // check if excluded + if a.isExcluded(path.Clean(relativePath)) { + return nil + } return a.tarFile(relativePath, stat) } +func (a *Archiver) isExcluded(relativePath string) bool { + for _, excludePath := range a.excludedPaths { + if strings.HasPrefix(relativePath, excludePath) { + return true + } + } + + return false +} + func (a *Archiver) tarFolder(target string, targetStat os.FileInfo) error { filePath := path.Join(a.basePath, target) files, err := os.ReadDir(filePath) diff --git a/pkg/ide/openvscode/openvscode.go b/pkg/ide/openvscode/openvscode.go index 79ba85d0c..ba3367a43 100644 --- a/pkg/ide/openvscode/openvscode.go +++ b/pkg/ide/openvscode/openvscode.go @@ -54,7 +54,7 @@ var Options = ide.Options{ VersionOption: { Name: VersionOption, Description: "The version for the open vscode binary", - Default: "v1.83.0", + Default: "v1.84.2", }, OpenOption: { Name: OpenOption, diff --git a/pkg/provider/dir.go b/pkg/provider/dir.go index ea8d8e1d8..9a40b2841 100644 --- a/pkg/provider/dir.go +++ b/pkg/provider/dir.go @@ -10,11 +10,13 @@ import ( "strings" "github.com/loft-sh/devpod/pkg/config" + config2 "github.com/loft-sh/devpod/pkg/devcontainer/config" "github.com/loft-sh/devpod/pkg/id" ) const ( WorkspaceConfigFile = "workspace.json" + WorkspaceResultFile = "workspace_result.json" MachineConfigFile = "machine.json" ProInstanceConfigFile = "pro.json" ProviderConfigFile = "provider.json" @@ -203,6 +205,31 @@ func SaveProInstanceConfig(context string, proInstance *ProInstance) error { return nil } +func SaveWorkspaceResult(workspace *Workspace, result *config2.Result) error { + workspaceDir, err := GetWorkspaceDir(workspace.Context, workspace.ID) + if err != nil { + return err + } + + err = os.MkdirAll(workspaceDir, 0755) + if err != nil { + return err + } + + resultBytes, err := json.Marshal(result) + if err != nil { + return err + } + + workspaceResultFile := filepath.Join(workspaceDir, WorkspaceResultFile) + err = os.WriteFile(workspaceResultFile, resultBytes, 0666) + if err != nil { + return err + } + + return nil +} + func SaveWorkspaceConfig(workspace *Workspace) error { workspaceDir, err := GetWorkspaceDir(workspace.Context, workspace.ID) if err != nil { @@ -260,7 +287,16 @@ func MachineExists(context, machineID string) bool { } _, err = os.Stat(machineDir) + return err == nil +} +func ProviderExists(context, provider string) bool { + providerDir, err := GetProviderDir(context, provider) + if err != nil { + return false + } + + _, err = os.Stat(providerDir) return err == nil } @@ -350,3 +386,26 @@ func LoadWorkspaceConfig(context, workspaceID string) (*Workspace, error) { workspaceConfig.Origin = workspaceConfigFile return workspaceConfig, nil } + +func LoadWorkspaceResult(context, workspaceID string) (*config2.Result, error) { + workspaceDir, err := GetWorkspaceDir(context, workspaceID) + if err != nil { + return nil, err + } + + workspaceResultFile := filepath.Join(workspaceDir, WorkspaceResultFile) + workspaceResultBytes, err := os.ReadFile(workspaceResultFile) + if os.IsNotExist(err) { + return nil, nil + } else if err != nil { + return nil, err + } + + workspaceResult := &config2.Result{} + err = json.Unmarshal(workspaceResultBytes, workspaceResult) + if err != nil { + return nil, err + } + + return workspaceResult, nil +} diff --git a/pkg/provider/env.go b/pkg/provider/env.go index cc4976bd5..901ba1e51 100644 --- a/pkg/provider/env.go +++ b/pkg/provider/env.go @@ -31,24 +31,10 @@ const ( PROVIDER_FOLDER = "PROVIDER_FOLDER" ) -// driver env const ( DEVCONTAINER_ID = "DEVCONTAINER_ID" ) -// FromEnvironment retrives options from environment and fills a machine with it. This is primarily -// used by provider implementations. -func FromEnvironment() *Machine { - return &Machine{ - ID: os.Getenv(MACHINE_ID), - Folder: os.Getenv(MACHINE_FOLDER), - Provider: MachineProviderConfig{ - Name: os.Getenv(MACHINE_PROVIDER), - }, - Context: os.Getenv(MACHINE_CONTEXT), - } -} - func combineOptions(resolvedOptions map[string]config.OptionValue, otherOptions map[string]config.OptionValue) map[string]config.OptionValue { options := map[string]config.OptionValue{} for k, v := range resolvedOptions { @@ -98,9 +84,8 @@ func ToOptionsWorkspace(workspace *Workspace) map[string]string { if workspace.UID != "" { retVars[WORKSPACE_UID] = workspace.UID } - if workspace.Folder != "" { - retVars[WORKSPACE_FOLDER] = filepath.ToSlash(workspace.Folder) - } + retVars[WORKSPACE_FOLDER], _ = GetWorkspaceDir(workspace.Context, workspace.ID) + retVars[WORKSPACE_FOLDER] = filepath.ToSlash(retVars[WORKSPACE_FOLDER]) if workspace.Context != "" { retVars[WORKSPACE_CONTEXT] = workspace.Context retVars[MACHINE_CONTEXT] = workspace.Context @@ -133,9 +118,8 @@ func ToOptionsMachine(machine *Machine) map[string]string { if machine.ID != "" { retVars[MACHINE_ID] = machine.ID } - if machine.Folder != "" { - retVars[MACHINE_FOLDER] = filepath.ToSlash(machine.Folder) - } + retVars[MACHINE_FOLDER], _ = GetMachineDir(machine.Context, machine.ID) + retVars[MACHINE_FOLDER] = filepath.ToSlash(retVars[MACHINE_FOLDER]) if machine.Context != "" { retVars[MACHINE_CONTEXT] = machine.Context } diff --git a/pkg/provider/export.go b/pkg/provider/export.go new file mode 100644 index 000000000..0a1ad8b5a --- /dev/null +++ b/pkg/provider/export.go @@ -0,0 +1,150 @@ +package provider + +import ( + "bytes" + "encoding/base64" + "fmt" + + "github.com/loft-sh/devpod/pkg/config" + "github.com/loft-sh/devpod/pkg/extract" +) + +var excludedPaths = []string{ + ".cache/", + "cache/", + "binaries/", + "source/", + ".temp/", + "temp/", + ".tmp/", + "tmp/", +} + +type ExportConfig struct { + // Workspace is the workspace that was exported + Workspace *ExportWorkspaceConfig `json:"workspace,omitempty"` + + // Machine is the machine that was exported + Machine *ExportMachineConfig `json:"machine,omitempty"` + + // Provider is the provider that was exported + Provider *ExportProviderConfig `json:"provider,omitempty"` +} + +type ExportWorkspaceConfig struct { + // ID is the workspace id + ID string `json:"id,omitempty"` + + // Context is the workspace context + Context string `json:"context,omitempty"` + + // UID is used to identify this specific workspace + UID string `json:"uid,omitempty"` + + // Data is the workspace folder data + Data string `json:"data,omitempty"` +} + +type ExportMachineConfig struct { + // ID is the machine id + ID string `json:"id,omitempty"` + + // Context is the machine context + Context string `json:"context,omitempty"` + + // Data is the machine folder data + Data string `json:"data,omitempty"` +} + +type ExportProviderConfig struct { + // ID is the provider id + ID string `json:"id,omitempty"` + + // Context is the provider context + Context string `json:"context,omitempty"` + + // Data is the provider folder data + Data string `json:"data,omitempty"` + + // Config is the provider config within the config.yaml + Config *config.ProviderConfig `json:"config,omitempty"` +} + +func ExportWorkspace(context, workspaceID string) (*ExportWorkspaceConfig, error) { + workspaceDir, err := GetWorkspaceDir(context, workspaceID) + if err != nil { + return nil, err + } + + workspaceConfig, err := LoadWorkspaceConfig(context, workspaceID) + if err != nil { + return nil, err + } + + buf := &bytes.Buffer{} + err = extract.WriteTarExclude(buf, workspaceDir, true, excludedPaths) + if err != nil { + return nil, fmt.Errorf("compress workspace dir: %w", err) + } + + return &ExportWorkspaceConfig{ + ID: workspaceID, + UID: workspaceConfig.UID, + Context: context, + Data: base64.RawStdEncoding.EncodeToString(buf.Bytes()), + }, nil +} + +func ExportMachine(context, machineID string) (*ExportMachineConfig, error) { + machineDir, err := GetMachineDir(context, machineID) + if err != nil { + return nil, err + } + + _, err = LoadMachineConfig(context, machineID) + if err != nil { + return nil, err + } + + buf := &bytes.Buffer{} + err = extract.WriteTarExclude(buf, machineDir, true, excludedPaths) + if err != nil { + return nil, fmt.Errorf("compress machine dir: %w", err) + } + + return &ExportMachineConfig{ + ID: machineID, + Context: context, + Data: base64.RawStdEncoding.EncodeToString(buf.Bytes()), + }, nil +} + +func ExportProvider(devPodConfig *config.Config, context, providerID string) (*ExportProviderConfig, error) { + providerDir, err := GetProviderDir(context, providerID) + if err != nil { + return nil, err + } + + _, err = LoadProviderConfig(context, providerID) + if err != nil { + return nil, err + } + + buf := &bytes.Buffer{} + err = extract.WriteTarExclude(buf, providerDir, true, excludedPaths) + if err != nil { + return nil, fmt.Errorf("compress provider dir: %w", err) + } + + var providerConfig *config.ProviderConfig + if devPodConfig != nil && devPodConfig.Contexts[context] != nil && devPodConfig.Contexts[context].Providers != nil && devPodConfig.Contexts[context].Providers[providerID] != nil { + providerConfig = devPodConfig.Contexts[context].Providers[providerID] + } + + return &ExportProviderConfig{ + ID: providerID, + Context: context, + Data: base64.RawStdEncoding.EncodeToString(buf.Bytes()), + Config: providerConfig, + }, nil +} diff --git a/pkg/provider/machine.go b/pkg/provider/machine.go index 11fdecc4b..a00184fed 100644 --- a/pkg/provider/machine.go +++ b/pkg/provider/machine.go @@ -9,9 +9,6 @@ type Machine struct { // ID is the machine id to use ID string `json:"id,omitempty"` - // Folder is the local folder where machine related contents will be stored - Folder string `json:"folder,omitempty"` - // Provider is the provider used to create this workspace Provider MachineProviderConfig `json:"provider,omitempty"` diff --git a/pkg/provider/workspace.go b/pkg/provider/workspace.go index 21d8698d0..e3004d113 100644 --- a/pkg/provider/workspace.go +++ b/pkg/provider/workspace.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/loft-sh/devpod/pkg/config" + devcontainerconfig "github.com/loft-sh/devpod/pkg/devcontainer/config" "github.com/loft-sh/devpod/pkg/git" "github.com/loft-sh/devpod/pkg/types" ) @@ -21,9 +22,6 @@ type Workspace struct { // UID is used to identify this specific workspace UID string `json:"uid,omitempty"` - // Folder is the local folder where workspace related contents will be stored - Folder string `json:"folder,omitempty"` - // Picture is the project social media image Picture string `json:"picture,omitempty"` @@ -45,6 +43,9 @@ type Workspace struct { // DevContainerPath is the relative path where the devcontainer.json is located. DevContainerPath string `json:"devContainerPath,omitempty"` + // DevContainerConfig holds the config for the devcontainer.json. + DevContainerConfig *devcontainerconfig.DevContainerConfig `json:"devContainerConfig,omitempty"` + // CreationTimestamp is the timestamp when this workspace was created CreationTimestamp types.Time `json:"creationTimestamp,omitempty"` @@ -73,9 +74,6 @@ type WorkspaceMachineConfig struct { // ID is the machine ID to use for this workspace ID string `json:"machineId,omitempty"` - // UID is the machine UID to use for this workspace - UID string `json:"machineUid,omitempty"` - // AutoDelete specifies if the machine should get destroyed when // the workspace is destroyed AutoDelete bool `json:"autoDelete,omitempty"` @@ -125,9 +123,16 @@ type ContainerWorkspaceInfo struct { } type AgentWorkspaceInfo struct { + // WorkspaceOrigin is the path where this workspace config originated from + WorkspaceOrigin string `json:"workspaceOrigin,omitempty"` + // Workspace holds the workspace info Workspace *Workspace `json:"workspace,omitempty"` + // LastDevContainerConfig can be used as a fallback if the workspace was already started + // and we lost track of the devcontainer.json + LastDevContainerConfig *devcontainerconfig.DevContainerConfigWithPath `json:"lastDevContainerConfig,omitempty"` + // Machine holds the machine info Machine *Machine `json:"machine,omitempty"` @@ -173,6 +178,13 @@ type CLIOptions struct { ForceInternalBuildKit bool `json:"forceInternalBuildKit,omitempty"` } +type BuildOptions struct { + CLIOptions + + Platform string + NoBuild bool +} + func (w WorkspaceSource) String() string { if w.GitRepository != "" { if w.GitPRReference != "" { diff --git a/pkg/workspace/machine.go b/pkg/workspace/machine.go index 51e4cc544..52ab650b8 100644 --- a/pkg/workspace/machine.go +++ b/pkg/workspace/machine.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "path/filepath" "github.com/loft-sh/devpod/pkg/client" "github.com/loft-sh/devpod/pkg/client/clientimplementation" @@ -85,7 +86,7 @@ func resolveMachine(devPodConfig *config.Config, args []string, log log.Logger) // create a new client machineClient, err := clientimplementation.NewMachineClient(devPodConfig, defaultProvider.Config, machineObj, log) if err != nil { - _ = os.RemoveAll(machineObj.Folder) + _ = os.RemoveAll(filepath.Dir(machineObj.Origin)) return nil, err } diff --git a/pkg/workspace/workspace.go b/pkg/workspace/workspace.go index 68059c020..5f426b9c0 100644 --- a/pkg/workspace/workspace.go +++ b/pkg/workspace/workspace.go @@ -414,7 +414,6 @@ func resolve( workspace := &provider2.Workspace{ ID: workspaceID, UID: uid, - Folder: workspaceFolder, Context: devPodConfig.DefaultContext, Provider: provider2.WorkspaceProviderConfig{ Name: defaultProvider.Config.Name, @@ -634,12 +633,12 @@ func createMachine(context, machineID, providerName string) (*provider2.Machine, // save machine config machine := &provider2.Machine{ ID: machineID, - Folder: machineDir, Context: context, Provider: provider2.MachineProviderConfig{ Name: providerName, }, CreationTimestamp: types.Now(), + Origin: filepath.Join(machineDir, provider2.MachineConfigFile), } // create machine folder