From e19fe1b878881d2b3ce0d679aa27697b4b83ccdb Mon Sep 17 00:00:00 2001 From: razzle Date: Fri, 6 Oct 2023 22:31:01 -0500 Subject: [PATCH 01/70] something to build off of Signed-off-by: razzle --- src/pkg/oci/fetch.go | 12 +- src/pkg/oci/pull.go | 4 +- src/pkg/packager/composer/list.go | 169 ++++++++++++++++++++++++++ src/pkg/packager/composer/override.go | 120 ++++++++++++++++++ src/pkg/packager/create.go | 6 +- 5 files changed, 302 insertions(+), 9 deletions(-) create mode 100644 src/pkg/packager/composer/list.go create mode 100644 src/pkg/packager/composer/override.go diff --git a/src/pkg/oci/fetch.go b/src/pkg/oci/fetch.go index be7406819a..4306e709e1 100644 --- a/src/pkg/oci/fetch.go +++ b/src/pkg/oci/fetch.go @@ -52,12 +52,20 @@ func (o *OrasRemote) FetchLayer(desc ocispec.Descriptor) (bytes []byte, err erro } // FetchZarfYAML fetches the zarf.yaml file from the remote repository. -func (o *OrasRemote) FetchZarfYAML(manifest *ZarfOCIManifest) (pkg types.ZarfPackage, err error) { +func (o *OrasRemote) FetchZarfYAML() (pkg types.ZarfPackage, err error) { + manifest, err := o.FetchRoot() + if err != nil { + return pkg, err + } return FetchYAMLFile[types.ZarfPackage](o.FetchLayer, manifest, layout.ZarfYAML) } // FetchImagesIndex fetches the images/index.json file from the remote repository. -func (o *OrasRemote) FetchImagesIndex(manifest *ZarfOCIManifest) (index *ocispec.Index, err error) { +func (o *OrasRemote) FetchImagesIndex() (index *ocispec.Index, err error) { + manifest, err := o.FetchRoot() + if err != nil { + return index, err + } return FetchJSONFile[*ocispec.Index](o.FetchLayer, manifest, ZarfPackageIndexPath) } diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 15c7417cbf..d981740b4c 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -83,7 +83,7 @@ func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) return nil, err } - pkg, err := o.FetchZarfYAML(root) + pkg, err := o.FetchZarfYAML() if err != nil { return nil, err } @@ -116,7 +116,7 @@ func (o *OrasRemote) LayersFromRequestedComponents(requestedComponents []string) if len(images) > 0 { // Add the image index and the oci-layout layers layers = append(layers, root.Locate(ZarfPackageIndexPath), root.Locate(ZarfPackageLayoutPath)) - index, err := o.FetchImagesIndex(root) + index, err := o.FetchImagesIndex() if err != nil { return nil, err } diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go new file mode 100644 index 0000000000..81e4b10d07 --- /dev/null +++ b/src/pkg/packager/composer/list.go @@ -0,0 +1,169 @@ +package composer + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" +) + +type Node struct { + cwd string + types.ZarfComponent + + prev *Node + next *Node +} + +type ImportChain struct { + head *Node + tail *Node +} + +func (ic *ImportChain) append(c types.ZarfComponent, cwd string) { + node := &Node{ZarfComponent: c, cwd: cwd, prev: nil, next: nil} + if ic.head == nil { + ic.head = node + ic.tail = node + } else { + p := ic.head + for p.next != nil { + p = p.next + } + node.prev = p + + p.next = node + ic.tail = node + } +} + +func (ic *ImportChain) Build(head types.ZarfComponent, arch string) error { + cwd, err := os.Getwd() + if err != nil { + return err + } + ic.append(head, cwd) + + node := ic.head + for node != nil { + isLocal := node.Import.Path != "" && node.Import.URL == "" + isRemote := node.Import.Path == "" && node.Import.URL != "" + + if !isLocal && !isRemote { + // EOL + return nil + } + + if node.prev != nil && node.prev.Import.URL != "" { + return fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") + } + + var pkg types.ZarfPackage + name := node.Name + + if isLocal { + cwd = filepath.Join(cwd, node.Import.Path) + if err := utils.ReadYaml(filepath.Join(cwd, layout.ZarfYAML), &pkg); err != nil { + return err + } + } else if isRemote { + cwd = "" + remote, err := oci.NewOrasRemote(node.Import.URL) + if err != nil { + return err + } + pkg, err = remote.FetchZarfYAML() + if err != nil { + return err + } + } + + if node.Import.ComponentName != "" { + name = node.Import.ComponentName + } + + found := helpers.Find(pkg.Components, func(c types.ZarfComponent) bool { + return c.Name == name + }) + + if found.Name == "" { + if isLocal { + return fmt.Errorf("component %q not found in package %q", name, filepath.Join(cwd, layout.ZarfYAML)) + } else if isRemote { + return fmt.Errorf("component %q not found in package %q", name, node.Import.URL) + } + } + + if node.Only.Cluster.Architecture != "" { + arch = node.Only.Cluster.Architecture + } + + if arch != "" && found.Only.Cluster.Architecture != "" && found.Only.Cluster.Architecture != arch { + if isLocal { + return fmt.Errorf("component %q is not compatible with %q architecture in package %q", name, arch, filepath.Join(cwd, layout.ZarfYAML)) + } else if isRemote { + return fmt.Errorf("component %q is not compatible with %q architecture in package %q", name, arch, node.Import.URL) + } + } + + ic.append(found, cwd) + node = node.next + } + return nil +} + +func (ic *ImportChain) Print() { + // components := []types.ZarfComponent{} + paths := []string{} + node := ic.head + for node != nil { + // components = append(components, node) + paths = append(paths, node.cwd) + if node.cwd == "" && node.Import.URL != "" { + paths = append(paths, node.Import.URL) + } + node = node.next + } + // fmt.Println(message.JSONValue(components)) + fmt.Println(message.JSONValue(paths)) +} + +func (ic *ImportChain) Compose() (composed types.ZarfComponent) { + node := ic.tail + + if ic.tail.Import.URL != "" { + composed = ic.tail.ZarfComponent + // TODO: handle remote components + // this should download the remote component, fix the paths, then compose it + node = node.prev + } + + for node != nil { + // if we are on the last node, set the starting point + if composed.Name == "" { + composed = node.ZarfComponent + node = node.prev + continue + } + + // TODO: fix the paths to be relative to the head node + // use node.cwd for that + + // perform overrides here + overrideMetadata(&composed, node.ZarfComponent) + overrideDeprecated(&composed, node.ZarfComponent) + overrideResources(&composed, node.ZarfComponent) + overrideExtensions(&composed, node.ZarfComponent) + overrideActions(&composed, node.ZarfComponent) + + node = node.prev + } + + return composed +} diff --git a/src/pkg/packager/composer/override.go b/src/pkg/packager/composer/override.go new file mode 100644 index 0000000000..20123f885c --- /dev/null +++ b/src/pkg/packager/composer/override.go @@ -0,0 +1,120 @@ +package composer + +import ( + "github.com/defenseunicorns/zarf/src/types" +) + +func overrideMetadata(c *types.ZarfComponent, override types.ZarfComponent) { + c.Name = override.Name + c.Default = override.Default + c.Required = override.Required + + // Override description if it was provided. + if override.Description != "" { + c.Description = override.Description + } +} + +func overrideDeprecated(c *types.ZarfComponent, override types.ZarfComponent) { + // Override cosign key path if it was provided. + if override.DeprecatedCosignKeyPath != "" { + c.DeprecatedCosignKeyPath = override.DeprecatedCosignKeyPath + } + + c.Group = override.Group + + // Merge deprecated scripts for backwards compatibility with older zarf binaries. + c.DeprecatedScripts.Before = append(c.DeprecatedScripts.Before, override.DeprecatedScripts.Before...) + c.DeprecatedScripts.After = append(c.DeprecatedScripts.After, override.DeprecatedScripts.After...) + + if override.DeprecatedScripts.Retry { + c.DeprecatedScripts.Retry = true + } + if override.DeprecatedScripts.ShowOutput { + c.DeprecatedScripts.ShowOutput = true + } + if override.DeprecatedScripts.TimeoutSeconds > 0 { + c.DeprecatedScripts.TimeoutSeconds = override.DeprecatedScripts.TimeoutSeconds + } +} + +func overrideActions(c *types.ZarfComponent, override types.ZarfComponent) { + // Merge create actions. + c.Actions.OnCreate.Before = append(c.Actions.OnCreate.Before, override.Actions.OnCreate.Before...) + c.Actions.OnCreate.After = append(c.Actions.OnCreate.After, override.Actions.OnCreate.After...) + c.Actions.OnCreate.OnFailure = append(c.Actions.OnCreate.OnFailure, override.Actions.OnCreate.OnFailure...) + c.Actions.OnCreate.OnSuccess = append(c.Actions.OnCreate.OnSuccess, override.Actions.OnCreate.OnSuccess...) + + // Merge deploy actions. + c.Actions.OnDeploy.Before = append(c.Actions.OnDeploy.Before, override.Actions.OnDeploy.Before...) + c.Actions.OnDeploy.After = append(c.Actions.OnDeploy.After, override.Actions.OnDeploy.After...) + c.Actions.OnDeploy.OnFailure = append(c.Actions.OnDeploy.OnFailure, override.Actions.OnDeploy.OnFailure...) + c.Actions.OnDeploy.OnSuccess = append(c.Actions.OnDeploy.OnSuccess, override.Actions.OnDeploy.OnSuccess...) + + // Merge remove actions. + c.Actions.OnRemove.Before = append(c.Actions.OnRemove.Before, override.Actions.OnRemove.Before...) + c.Actions.OnRemove.After = append(c.Actions.OnRemove.After, override.Actions.OnRemove.After...) + c.Actions.OnRemove.OnFailure = append(c.Actions.OnRemove.OnFailure, override.Actions.OnRemove.OnFailure...) + c.Actions.OnRemove.OnSuccess = append(c.Actions.OnRemove.OnSuccess, override.Actions.OnRemove.OnSuccess...) +} + +func overrideResources(c *types.ZarfComponent, override types.ZarfComponent) { + c.DataInjections = append(c.DataInjections, override.DataInjections...) + c.Files = append(c.Files, override.Files...) + c.Images = append(c.Images, override.Images...) + c.Repos = append(c.Repos, override.Repos...) + + // Merge charts with the same name to keep them unique + for _, overrideChart := range override.Charts { + existing := false + for idx := range c.Charts { + if c.Charts[idx].Name == overrideChart.Name { + if overrideChart.Namespace != "" { + c.Charts[idx].Namespace = overrideChart.Namespace + } + if overrideChart.ReleaseName != "" { + c.Charts[idx].ReleaseName = overrideChart.ReleaseName + } + c.Charts[idx].ValuesFiles = append(c.Charts[idx].ValuesFiles, overrideChart.ValuesFiles...) + existing = true + } + } + + if !existing { + c.Charts = append(c.Charts, overrideChart) + } + } + + // Merge manifests with the same name to keep them unique + for _, overrideManifest := range override.Manifests { + existing := false + for idx := range c.Manifests { + if c.Manifests[idx].Name == overrideManifest.Name { + if overrideManifest.Namespace != "" { + c.Manifests[idx].Namespace = overrideManifest.Namespace + } + c.Manifests[idx].Files = append(c.Manifests[idx].Files, overrideManifest.Files...) + c.Manifests[idx].Kustomizations = append(c.Manifests[idx].Kustomizations, overrideManifest.Kustomizations...) + + existing = true + } + } + + if !existing { + c.Manifests = append(c.Manifests, overrideManifest) + } + } +} + +// TODO: improve me w/ an interface system +func overrideExtensions(c *types.ZarfComponent, override types.ZarfComponent) { + // Check for nil array + if override.Extensions.BigBang != nil { + if override.Extensions.BigBang.ValuesFiles != nil { + c.Extensions.BigBang.ValuesFiles = append(c.Extensions.BigBang.ValuesFiles, override.Extensions.BigBang.ValuesFiles...) + } + if override.Extensions.BigBang.FluxPatchFiles != nil { + c.Extensions.BigBang.FluxPatchFiles = append(c.Extensions.BigBang.FluxPatchFiles, override.Extensions.BigBang.FluxPatchFiles...) + } + } +} diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index db9624beae..4eee65fa49 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -630,11 +630,7 @@ func (p *Packager) loadDifferentialData() error { if err != nil { return err } - manifest, err := p.remote.FetchRoot() - if err != nil { - return err - } - pkg, err := p.remote.FetchZarfYAML(manifest) + pkg, err := p.remote.FetchZarfYAML() if err != nil { return err } From d1f90461b14e5e2271559ea0927ef8a1a5e39e6e Mon Sep 17 00:00:00 2001 From: razzle Date: Sat, 7 Oct 2023 21:03:02 -0500 Subject: [PATCH 02/70] naming Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 46 ++++++++++++++++--------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 81e4b10d07..9f86817092 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -13,21 +13,21 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -type Node struct { +type node struct { cwd string types.ZarfComponent - prev *Node - next *Node + prev *node + next *node } -type ImportChain struct { - head *Node - tail *Node +type importChain struct { + head *node + tail *node } -func (ic *ImportChain) append(c types.ZarfComponent, cwd string) { - node := &Node{ZarfComponent: c, cwd: cwd, prev: nil, next: nil} +func (ic *importChain) append(c types.ZarfComponent, cwd string) { + node := &node{ZarfComponent: c, cwd: cwd, prev: nil, next: nil} if ic.head == nil { ic.head = node ic.tail = node @@ -43,10 +43,12 @@ func (ic *ImportChain) append(c types.ZarfComponent, cwd string) { } } -func (ic *ImportChain) Build(head types.ZarfComponent, arch string) error { +func NewImportChain(head types.ZarfComponent, arch string) (*importChain, error) { + ic := &importChain{} + cwd, err := os.Getwd() if err != nil { - return err + return ic, err } ic.append(head, cwd) @@ -57,11 +59,11 @@ func (ic *ImportChain) Build(head types.ZarfComponent, arch string) error { if !isLocal && !isRemote { // EOL - return nil + return ic, nil } if node.prev != nil && node.prev.Import.URL != "" { - return fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") + return ic, fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") } var pkg types.ZarfPackage @@ -70,17 +72,17 @@ func (ic *ImportChain) Build(head types.ZarfComponent, arch string) error { if isLocal { cwd = filepath.Join(cwd, node.Import.Path) if err := utils.ReadYaml(filepath.Join(cwd, layout.ZarfYAML), &pkg); err != nil { - return err + return ic, err } } else if isRemote { cwd = "" remote, err := oci.NewOrasRemote(node.Import.URL) if err != nil { - return err + return ic, err } pkg, err = remote.FetchZarfYAML() if err != nil { - return err + return ic, err } } @@ -94,9 +96,9 @@ func (ic *ImportChain) Build(head types.ZarfComponent, arch string) error { if found.Name == "" { if isLocal { - return fmt.Errorf("component %q not found in package %q", name, filepath.Join(cwd, layout.ZarfYAML)) + return ic, fmt.Errorf("component %q not found in package %q", name, filepath.Join(cwd, layout.ZarfYAML)) } else if isRemote { - return fmt.Errorf("component %q not found in package %q", name, node.Import.URL) + return ic, fmt.Errorf("component %q not found in package %q", name, node.Import.URL) } } @@ -106,19 +108,19 @@ func (ic *ImportChain) Build(head types.ZarfComponent, arch string) error { if arch != "" && found.Only.Cluster.Architecture != "" && found.Only.Cluster.Architecture != arch { if isLocal { - return fmt.Errorf("component %q is not compatible with %q architecture in package %q", name, arch, filepath.Join(cwd, layout.ZarfYAML)) + return ic, fmt.Errorf("component %q is not compatible with %q architecture in package %q", name, arch, filepath.Join(cwd, layout.ZarfYAML)) } else if isRemote { - return fmt.Errorf("component %q is not compatible with %q architecture in package %q", name, arch, node.Import.URL) + return ic, fmt.Errorf("component %q is not compatible with %q architecture in package %q", name, arch, node.Import.URL) } } ic.append(found, cwd) node = node.next } - return nil + return ic, nil } -func (ic *ImportChain) Print() { +func (ic *importChain) Print() { // components := []types.ZarfComponent{} paths := []string{} node := ic.head @@ -134,7 +136,7 @@ func (ic *ImportChain) Print() { fmt.Println(message.JSONValue(paths)) } -func (ic *ImportChain) Compose() (composed types.ZarfComponent) { +func (ic *importChain) Compose() (composed types.ZarfComponent) { node := ic.tail if ic.tail.Import.URL != "" { From 3342ad23abc4f1e95c727358a699b9d30232f43c Mon Sep 17 00:00:00 2001 From: razzle Date: Sat, 7 Oct 2023 21:11:38 -0500 Subject: [PATCH 03/70] revive Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 37 +++++++++++++++++---------- src/pkg/packager/composer/override.go | 4 +++ 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 9f86817092..5f39d60504 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package composer contains functions for composing components within Zarf packages. package composer import ( @@ -13,21 +17,23 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -type node struct { +// Node is a node in the import chain +type Node struct { cwd string types.ZarfComponent - prev *node - next *node + prev *Node + next *Node } -type importChain struct { - head *node - tail *node +// ImportChain is a doubly linked list of components +type ImportChain struct { + head *Node + tail *Node } -func (ic *importChain) append(c types.ZarfComponent, cwd string) { - node := &node{ZarfComponent: c, cwd: cwd, prev: nil, next: nil} +func (ic *ImportChain) append(c types.ZarfComponent, cwd string) { + node := &Node{ZarfComponent: c, cwd: cwd, prev: nil, next: nil} if ic.head == nil { ic.head = node ic.tail = node @@ -43,8 +49,9 @@ func (ic *importChain) append(c types.ZarfComponent, cwd string) { } } -func NewImportChain(head types.ZarfComponent, arch string) (*importChain, error) { - ic := &importChain{} +// NewImportChain creates a new import chain from a component +func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) { + ic := &ImportChain{} cwd, err := os.Getwd() if err != nil { @@ -120,7 +127,9 @@ func NewImportChain(head types.ZarfComponent, arch string) (*importChain, error) return ic, nil } -func (ic *importChain) Print() { +// Print prints the import chain +// TODO: remove when fully implemented || replace w/ a debug flag +func (ic *ImportChain) Print() { // components := []types.ZarfComponent{} paths := []string{} node := ic.head @@ -136,13 +145,15 @@ func (ic *importChain) Print() { fmt.Println(message.JSONValue(paths)) } -func (ic *importChain) Compose() (composed types.ZarfComponent) { +// Compose merges the import chain into a single component +// fixing paths, overriding metadata, etc +func (ic *ImportChain) Compose() (composed types.ZarfComponent) { node := ic.tail if ic.tail.Import.URL != "" { composed = ic.tail.ZarfComponent // TODO: handle remote components - // this should download the remote component, fix the paths, then compose it + // this should download the remote component tarball, fix the paths, then compose it node = node.prev } diff --git a/src/pkg/packager/composer/override.go b/src/pkg/packager/composer/override.go index 20123f885c..690d2c3904 100644 --- a/src/pkg/packager/composer/override.go +++ b/src/pkg/packager/composer/override.go @@ -1,3 +1,7 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package composer contains functions for composing components within Zarf packages. package composer import ( From d17a6e73d77593656a69244b0baf3fd56d6e8ad3 Mon Sep 17 00:00:00 2001 From: razzle Date: Sun, 8 Oct 2023 14:43:44 -0500 Subject: [PATCH 04/70] slightly better Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 51 +++++++++++++------------------ 1 file changed, 21 insertions(+), 30 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 5f39d60504..48149c64b6 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -6,11 +6,9 @@ package composer import ( "fmt" - "os" "path/filepath" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -19,7 +17,6 @@ import ( // Node is a node in the import chain type Node struct { - cwd string types.ZarfComponent prev *Node @@ -32,8 +29,8 @@ type ImportChain struct { tail *Node } -func (ic *ImportChain) append(c types.ZarfComponent, cwd string) { - node := &Node{ZarfComponent: c, cwd: cwd, prev: nil, next: nil} +func (ic *ImportChain) append(c types.ZarfComponent) { + node := &Node{ZarfComponent: c, prev: nil, next: nil} if ic.head == nil { ic.head = node ic.tail = node @@ -53,11 +50,9 @@ func (ic *ImportChain) append(c types.ZarfComponent, cwd string) { func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) { ic := &ImportChain{} - cwd, err := os.Getwd() - if err != nil { - return ic, err - } - ic.append(head, cwd) + ic.append(head) + + history := []string{} node := ic.head for node != nil { @@ -77,12 +72,12 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) name := node.Name if isLocal { - cwd = filepath.Join(cwd, node.Import.Path) - if err := utils.ReadYaml(filepath.Join(cwd, layout.ZarfYAML), &pkg); err != nil { + history = append(history, node.Import.Path) + paths := append(history, layout.ZarfYAML) + if err := utils.ReadYaml(filepath.Join(paths...), &pkg); err != nil { return ic, err } } else if isRemote { - cwd = "" remote, err := oci.NewOrasRemote(node.Import.URL) if err != nil { return ic, err @@ -103,9 +98,9 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) if found.Name == "" { if isLocal { - return ic, fmt.Errorf("component %q not found in package %q", name, filepath.Join(cwd, layout.ZarfYAML)) + return ic, fmt.Errorf("component %q not found in %q", name, filepath.Join(history...)) } else if isRemote { - return ic, fmt.Errorf("component %q not found in package %q", name, node.Import.URL) + return ic, fmt.Errorf("component %q not found in %q", name, node.Import.URL) } } @@ -115,34 +110,30 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) if arch != "" && found.Only.Cluster.Architecture != "" && found.Only.Cluster.Architecture != arch { if isLocal { - return ic, fmt.Errorf("component %q is not compatible with %q architecture in package %q", name, arch, filepath.Join(cwd, layout.ZarfYAML)) + return ic, fmt.Errorf("component %q is not compatible with %q architecture in %q", name, arch, filepath.Join(history...)) } else if isRemote { - return ic, fmt.Errorf("component %q is not compatible with %q architecture in package %q", name, arch, node.Import.URL) + return ic, fmt.Errorf("component %q is not compatible with %q architecture in %q", name, arch, node.Import.URL) } } - ic.append(found, cwd) + ic.append(found) node = node.next } return ic, nil } -// Print prints the import chain -// TODO: remove when fully implemented || replace w/ a debug flag -func (ic *ImportChain) Print() { - // components := []types.ZarfComponent{} - paths := []string{} +// History returns the history of the import chain +func (ic *ImportChain) History() []string { + history := []string{} node := ic.head for node != nil { - // components = append(components, node) - paths = append(paths, node.cwd) - if node.cwd == "" && node.Import.URL != "" { - paths = append(paths, node.Import.URL) + history = append(history, node.Import.Path) + if node.Import.URL != "" { + history = append(history, node.Import.URL) } node = node.next } - // fmt.Println(message.JSONValue(components)) - fmt.Println(message.JSONValue(paths)) + return history } // Compose merges the import chain into a single component @@ -166,7 +157,7 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent) { } // TODO: fix the paths to be relative to the head node - // use node.cwd for that + // use history for that // perform overrides here overrideMetadata(&composed, node.ZarfComponent) From d47874a23f13a6e0103b57fc1ad2a34164816f1f Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 11 Oct 2023 13:39:41 -0500 Subject: [PATCH 05/70] progress Signed-off-by: razzle --- src/pkg/packager/composer/extensions.go | 29 +++++++ src/pkg/packager/composer/list.go | 72 +++++++++-------- src/pkg/packager/composer/override.go | 13 --- src/pkg/packager/composer/pathfixer.go | 101 ++++++++++++++++++++++++ 4 files changed, 167 insertions(+), 48 deletions(-) create mode 100644 src/pkg/packager/composer/extensions.go create mode 100644 src/pkg/packager/composer/pathfixer.go diff --git a/src/pkg/packager/composer/extensions.go b/src/pkg/packager/composer/extensions.go new file mode 100644 index 0000000000..bbc09233c4 --- /dev/null +++ b/src/pkg/packager/composer/extensions.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package composer contains functions for composing components within Zarf packages. +package composer + +import ( + "github.com/defenseunicorns/zarf/src/extensions/bigbang" + "github.com/defenseunicorns/zarf/src/types" +) + +// TODO: improve me +func composeExtensions(c *types.ZarfComponent, override types.ZarfComponent, relativeTo string) { + // fix the file paths + if override.Extensions.BigBang != nil { + component := bigbang.Compose(relativeTo, override) + c = &component + } + + // perform any overrides + if override.Extensions.BigBang != nil { + if override.Extensions.BigBang.ValuesFiles != nil { + c.Extensions.BigBang.ValuesFiles = append(c.Extensions.BigBang.ValuesFiles, override.Extensions.BigBang.ValuesFiles...) + } + if override.Extensions.BigBang.FluxPatchFiles != nil { + c.Extensions.BigBang.FluxPatchFiles = append(c.Extensions.BigBang.FluxPatchFiles, override.Extensions.BigBang.FluxPatchFiles...) + } + } +} diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 48149c64b6..64346e0c16 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -19,6 +19,8 @@ import ( type Node struct { types.ZarfComponent + relativeToHead string + prev *Node next *Node } @@ -29,8 +31,8 @@ type ImportChain struct { tail *Node } -func (ic *ImportChain) append(c types.ZarfComponent) { - node := &Node{ZarfComponent: c, prev: nil, next: nil} +func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string) { + node := &Node{ZarfComponent: c, relativeToHead: relativeToHead, prev: nil, next: nil} if ic.head == nil { ic.head = node ic.tail = node @@ -48,23 +50,32 @@ func (ic *ImportChain) append(c types.ZarfComponent) { // NewImportChain creates a new import chain from a component func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) { + if arch == "" { + return nil, fmt.Errorf("cannot build import chain: architecture must be provided") + } + ic := &ImportChain{} - ic.append(head) + ic.append(head, "") history := []string{} node := ic.head + // todo: verify this behavior + if ic.head.Only.Cluster.Architecture != "" { + arch = node.Only.Cluster.Architecture + } for node != nil { isLocal := node.Import.Path != "" && node.Import.URL == "" isRemote := node.Import.Path == "" && node.Import.URL != "" if !isLocal && !isRemote { - // EOL + // This is the end of the import chain, + // as the current node/component is not importing anything return ic, nil } - if node.prev != nil && node.prev.Import.URL != "" { + if node.prev != nil && node.prev.Import.URL != "" && isRemote { return ic, fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") } @@ -73,8 +84,8 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) if isLocal { history = append(history, node.Import.Path) - paths := append(history, layout.ZarfYAML) - if err := utils.ReadYaml(filepath.Join(paths...), &pkg); err != nil { + relativeToHead := filepath.Join(history...) + if err := utils.ReadYaml(filepath.Join(relativeToHead, layout.ZarfYAML), &pkg); err != nil { return ic, err } } else if isRemote { @@ -92,45 +103,42 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) name = node.Import.ComponentName } - found := helpers.Find(pkg.Components, func(c types.ZarfComponent) bool { - return c.Name == name + found := helpers.Filter(pkg.Components, func(c types.ZarfComponent) bool { + matchesName := c.Name == name + satisfiesArch := c.Only.Cluster.Architecture == "" || c.Only.Cluster.Architecture == arch + return matchesName && satisfiesArch }) - if found.Name == "" { + if len(found) == 0 { if isLocal { return ic, fmt.Errorf("component %q not found in %q", name, filepath.Join(history...)) } else if isRemote { return ic, fmt.Errorf("component %q not found in %q", name, node.Import.URL) } - } - - if node.Only.Cluster.Architecture != "" { - arch = node.Only.Cluster.Architecture - } - - if arch != "" && found.Only.Cluster.Architecture != "" && found.Only.Cluster.Architecture != arch { + } else if len(found) > 1 { + // TODO: improve this error message / figure out the best way to present this error if isLocal { - return ic, fmt.Errorf("component %q is not compatible with %q architecture in %q", name, arch, filepath.Join(history...)) + return ic, fmt.Errorf("multiple components named %q found in %q", name, filepath.Join(history...)) } else if isRemote { - return ic, fmt.Errorf("component %q is not compatible with %q architecture in %q", name, arch, node.Import.URL) + return ic, fmt.Errorf("multiple components named %q found in %q", name, node.Import.URL) } } - ic.append(found) + ic.append(found[0], filepath.Join(history...)) node = node.next } return ic, nil } // History returns the history of the import chain -func (ic *ImportChain) History() []string { - history := []string{} +func (ic *ImportChain) History() (history []string) { node := ic.head for node != nil { - history = append(history, node.Import.Path) if node.Import.URL != "" { history = append(history, node.Import.URL) + continue } + history = append(history, node.relativeToHead) node = node.next } return history @@ -139,33 +147,27 @@ func (ic *ImportChain) History() []string { // Compose merges the import chain into a single component // fixing paths, overriding metadata, etc func (ic *ImportChain) Compose() (composed types.ZarfComponent) { + composed = ic.tail.ZarfComponent + node := ic.tail - if ic.tail.Import.URL != "" { - composed = ic.tail.ZarfComponent + if ic.tail.prev.Import.URL != "" { // TODO: handle remote components // this should download the remote component tarball, fix the paths, then compose it node = node.prev } for node != nil { - // if we are on the last node, set the starting point - if composed.Name == "" { - composed = node.ZarfComponent - node = node.prev - continue - } - - // TODO: fix the paths to be relative to the head node - // use history for that + fixPaths(&node.ZarfComponent, node.relativeToHead) // perform overrides here overrideMetadata(&composed, node.ZarfComponent) overrideDeprecated(&composed, node.ZarfComponent) overrideResources(&composed, node.ZarfComponent) - overrideExtensions(&composed, node.ZarfComponent) overrideActions(&composed, node.ZarfComponent) + composeExtensions(&composed, node.ZarfComponent, node.relativeToHead) + node = node.prev } diff --git a/src/pkg/packager/composer/override.go b/src/pkg/packager/composer/override.go index 690d2c3904..ba8fc747d4 100644 --- a/src/pkg/packager/composer/override.go +++ b/src/pkg/packager/composer/override.go @@ -109,16 +109,3 @@ func overrideResources(c *types.ZarfComponent, override types.ZarfComponent) { } } } - -// TODO: improve me w/ an interface system -func overrideExtensions(c *types.ZarfComponent, override types.ZarfComponent) { - // Check for nil array - if override.Extensions.BigBang != nil { - if override.Extensions.BigBang.ValuesFiles != nil { - c.Extensions.BigBang.ValuesFiles = append(c.Extensions.BigBang.ValuesFiles, override.Extensions.BigBang.ValuesFiles...) - } - if override.Extensions.BigBang.FluxPatchFiles != nil { - c.Extensions.BigBang.FluxPatchFiles = append(c.Extensions.BigBang.FluxPatchFiles, override.Extensions.BigBang.FluxPatchFiles...) - } - } -} diff --git a/src/pkg/packager/composer/pathfixer.go b/src/pkg/packager/composer/pathfixer.go new file mode 100644 index 0000000000..d1c45531c0 --- /dev/null +++ b/src/pkg/packager/composer/pathfixer.go @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package composer contains functions for composing components within Zarf packages. +package composer + +import ( + "path/filepath" + + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/types" +) + +func makePathRelativeTo(path, relativeTo string) string { + if helpers.IsURL(path) { + return path + } + + return filepath.Join(relativeTo, path) +} + +func fixPaths(child *types.ZarfComponent, relativeToHead string) { + for fileIdx, file := range child.Files { + composed := makePathRelativeTo(file.Source, relativeToHead) + child.Files[fileIdx].Source = composed + } + + for chartIdx, chart := range child.Charts { + for valuesIdx, valuesFile := range chart.ValuesFiles { + composed := makePathRelativeTo(valuesFile, relativeToHead) + child.Charts[chartIdx].ValuesFiles[valuesIdx] = composed + } + if child.Charts[chartIdx].LocalPath != "" { + composed := makePathRelativeTo(chart.LocalPath, relativeToHead) + child.Charts[chartIdx].LocalPath = composed + } + } + + for manifestIdx, manifest := range child.Manifests { + for fileIdx, file := range manifest.Files { + composed := makePathRelativeTo(file, relativeToHead) + child.Manifests[manifestIdx].Files[fileIdx] = composed + } + for kustomizeIdx, kustomization := range manifest.Kustomizations { + composed := makePathRelativeTo(kustomization, relativeToHead) + // kustomizations can use non-standard urls, so we need to check if the composed path exists on the local filesystem + abs, _ := filepath.Abs(composed) + invalid := utils.InvalidPath(abs) + if !invalid { + child.Manifests[manifestIdx].Kustomizations[kustomizeIdx] = composed + } + } + } + + for dataInjectionsIdx, dataInjection := range child.DataInjections { + composed := makePathRelativeTo(dataInjection.Source, relativeToHead) + child.DataInjections[dataInjectionsIdx].Source = composed + } + + for actionIdx, action := range child.Actions.OnCreate.Before { + if action.Dir != nil { + composed := makePathRelativeTo(*action.Dir, relativeToHead) + child.Actions.OnCreate.Before[actionIdx].Dir = &composed + } + } + + for actionIdx, action := range child.Actions.OnCreate.After { + if action.Dir != nil { + composed := makePathRelativeTo(*action.Dir, relativeToHead) + child.Actions.OnCreate.After[actionIdx].Dir = &composed + } + } + + for actionIdx, action := range child.Actions.OnCreate.OnFailure { + if action.Dir != nil { + composed := makePathRelativeTo(*action.Dir, relativeToHead) + child.Actions.OnCreate.OnFailure[actionIdx].Dir = &composed + } + } + + for actionIdx, action := range child.Actions.OnCreate.OnSuccess { + if action.Dir != nil { + composed := makePathRelativeTo(*action.Dir, relativeToHead) + child.Actions.OnCreate.OnSuccess[actionIdx].Dir = &composed + } + } + + totalActions := len(child.Actions.OnCreate.Before) + len(child.Actions.OnCreate.After) + len(child.Actions.OnCreate.OnFailure) + len(child.Actions.OnCreate.OnSuccess) + + if totalActions > 0 { + composedDefaultDir := makePathRelativeTo(child.Actions.OnCreate.Defaults.Dir, relativeToHead) + child.Actions.OnCreate.Defaults.Dir = composedDefaultDir + } + + // deprecated + if child.DeprecatedCosignKeyPath != "" { + composed := makePathRelativeTo(child.DeprecatedCosignKeyPath, relativeToHead) + child.DeprecatedCosignKeyPath = composed + } +} From bedc0e2125d4a6a4786a3b2a1db14e2a691de8df Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 12 Oct 2023 12:14:50 -0500 Subject: [PATCH 06/70] progress Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 64346e0c16..b8079ad71f 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -7,6 +7,7 @@ package composer import ( "fmt" "path/filepath" + "strings" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/oci" @@ -89,6 +90,9 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) return ic, err } } else if isRemote { + if !strings.HasSuffix(node.Import.URL, oci.SkeletonSuffix) { + return ic, fmt.Errorf("remote component %q does not have a %q suffix", node.Import.URL, oci.SkeletonSuffix) + } remote, err := oci.NewOrasRemote(node.Import.URL) if err != nil { return ic, err @@ -131,14 +135,14 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) } // History returns the history of the import chain -func (ic *ImportChain) History() (history []string) { +func (ic *ImportChain) History() (history [][]string) { node := ic.head for node != nil { if node.Import.URL != "" { - history = append(history, node.Import.URL) + history = append(history, []string{node.Name, node.Import.URL}) continue } - history = append(history, node.relativeToHead) + history = append(history, []string{node.Name, node.relativeToHead, node.Import.Path}) node = node.next } return history From f035195aaa36dcb12fd2ec49ec6874ce47a7c147 Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 12 Oct 2023 12:36:25 -0500 Subject: [PATCH 07/70] merging vars and consts Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 62 ++++++++++++++++++++++++++++--- 1 file changed, 57 insertions(+), 5 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index b8079ad71f..95e3346e92 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -20,6 +20,9 @@ import ( type Node struct { types.ZarfComponent + vars []types.ZarfPackageVariable + consts []types.ZarfPackageConstant + relativeToHead string prev *Node @@ -32,8 +35,15 @@ type ImportChain struct { tail *Node } -func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string) { - node := &Node{ZarfComponent: c, relativeToHead: relativeToHead, prev: nil, next: nil} +func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { + node := &Node{ + ZarfComponent: c, + relativeToHead: relativeToHead, + vars: vars, + consts: consts, + prev: nil, + next: nil, + } if ic.head == nil { ic.head = node ic.tail = node @@ -50,14 +60,14 @@ func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string) { } // NewImportChain creates a new import chain from a component -func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) { +func NewImportChain(head types.ZarfComponent, arch string, headVars []types.ZarfPackageVariable, headConsts []types.ZarfPackageConstant) (*ImportChain, error) { if arch == "" { return nil, fmt.Errorf("cannot build import chain: architecture must be provided") } ic := &ImportChain{} - ic.append(head, "") + ic.append(head, "", headVars, headConsts) history := []string{} @@ -128,7 +138,7 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) } } - ic.append(found[0], filepath.Join(history...)) + ic.append(found[0], filepath.Join(history...), pkg.Variables, pkg.Constants) node = node.next } return ic, nil @@ -177,3 +187,45 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent) { return composed } + +func (ic *ImportChain) MergeVariables() (vars []types.ZarfPackageVariable) { + node := ic.head + for node != nil { + // // merge the vars + for _, v := range node.vars { + exists := false + for _, vv := range vars { + if v.Name == vv.Name { + exists = true + break + } + } + if !exists { + vars = append(vars, v) + } + } + node = node.next + } + return vars +} + +func (ic *ImportChain) MergeConstants() (consts []types.ZarfPackageConstant) { + node := ic.head + for node != nil { + // merge the consts + for _, c := range node.consts { + exists := false + for _, cc := range consts { + if c.Name == cc.Name { + exists = true + break + } + } + if !exists { + consts = append(consts, c) + } + } + node = node.next + } + return consts +} From c2cb8209145e2a919fbe10706a0ec911eaa7cf5c Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 12 Oct 2023 13:39:00 -0500 Subject: [PATCH 08/70] start using for real Signed-off-by: razzle --- src/pkg/packager/compose.go | 41 ++++++++++++++++++++++--------- src/pkg/packager/composer/list.go | 41 +++++++++++++++++++++++-------- 2 files changed, 61 insertions(+), 21 deletions(-) diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 01aab83748..6bfcb3a2d8 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -9,11 +9,14 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/composer" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" @@ -24,19 +27,35 @@ import ( func (p *Packager) composeComponents() error { components := []types.ZarfComponent{} + pkgVars := p.cfg.Pkg.Variables + pkgConsts := p.cfg.Pkg.Constants + for _, component := range p.cfg.Pkg.Components { - if component.Import.Path == "" && component.Import.URL == "" { - // Migrate any deprecated component configurations now - migratedComponent, warnings := deprecated.MigrateComponent(p.cfg.Pkg.Build, component) - components = append(components, migratedComponent) - p.warnings = append(p.warnings, warnings...) - } else { - composedComponent, err := p.getComposedComponent(component) - if err != nil { - return fmt.Errorf("unable to compose component %s: %w", component.Name, err) - } - components = append(components, composedComponent) + // build the import chain + start := time.Now() + chain, err := composer.NewImportChain(component, p.arch) + if err != nil { + return err } + message.Debugf("Building import chain for %q took %s", component.Name, time.Since(start)) + + // migrate any deprecated component configurations now + start = time.Now() + warnings := chain.Migrate(p.cfg.Pkg.Build) + p.warnings = append(p.warnings, warnings...) + message.Debugf("Migrating %q took %s", component.Name, time.Since(start)) + + // get the composed component + start = time.Now() + composed := chain.Compose() + components = append(components, composed) + message.Debugf("Composing %q took %s", component.Name, time.Since(start)) + + // merge variables and constants + start = time.Now() + pkgVars = chain.MergeVariables(pkgVars) + pkgConsts = chain.MergeConstants(pkgConsts) + message.Debugf("Merging pkg vars and consts from import chain took %s", time.Since(start)) } // Update the parent package config with the expanded sub components. diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 95e3346e92..2861200dc6 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -10,7 +10,9 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" @@ -33,6 +35,8 @@ type Node struct { type ImportChain struct { head *Node tail *Node + + progress *message.ProgressBar } func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { @@ -60,14 +64,14 @@ func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string, vars } // NewImportChain creates a new import chain from a component -func NewImportChain(head types.ZarfComponent, arch string, headVars []types.ZarfPackageVariable, headConsts []types.ZarfPackageConstant) (*ImportChain, error) { +func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) { if arch == "" { return nil, fmt.Errorf("cannot build import chain: architecture must be provided") } ic := &ImportChain{} - ic.append(head, "", headVars, headConsts) + ic.append(head, "", nil, nil) history := []string{} @@ -96,6 +100,7 @@ func NewImportChain(head types.ZarfComponent, arch string, headVars []types.Zarf if isLocal { history = append(history, node.Import.Path) relativeToHead := filepath.Join(history...) + // this assumes the composed package is following the zarf layout if err := utils.ReadYaml(filepath.Join(relativeToHead, layout.ZarfYAML), &pkg); err != nil { return ic, err } @@ -158,6 +163,18 @@ func (ic *ImportChain) History() (history [][]string) { return history } +// TODO: is this the best place to perform migrations? +func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { + node := ic.head + for node != nil { + migrated, w := deprecated.MigrateComponent(build, node.ZarfComponent) + node.ZarfComponent = migrated + warnings = append(warnings, w...) + node = node.next + } + return warnings +} + // Compose merges the import chain into a single component // fixing paths, overriding metadata, etc func (ic *ImportChain) Compose() (composed types.ZarfComponent) { @@ -188,44 +205,48 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent) { return composed } -func (ic *ImportChain) MergeVariables() (vars []types.ZarfPackageVariable) { +func (ic *ImportChain) MergeVariables(vars []types.ZarfPackageVariable) (merged []types.ZarfPackageVariable) { + merged = vars + node := ic.head for node != nil { // // merge the vars for _, v := range node.vars { exists := false - for _, vv := range vars { + for _, vv := range merged { if v.Name == vv.Name { exists = true break } } if !exists { - vars = append(vars, v) + merged = append(merged, v) } } node = node.next } - return vars + return merged } -func (ic *ImportChain) MergeConstants() (consts []types.ZarfPackageConstant) { +func (ic *ImportChain) MergeConstants(consts []types.ZarfPackageConstant) (merged []types.ZarfPackageConstant) { + merged = consts + node := ic.head for node != nil { // merge the consts for _, c := range node.consts { exists := false - for _, cc := range consts { + for _, cc := range merged { if c.Name == cc.Name { exists = true break } } if !exists { - consts = append(consts, c) + merged = append(merged, c) } } node = node.next } - return consts + return merged } From f7cd2b14f98325f86cc524ccbdacf328047b79fd Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 12 Oct 2023 13:51:02 -0500 Subject: [PATCH 09/70] runtime validation of import definitions Signed-off-by: razzle --- src/internal/packager/validate/validate.go | 8 ++++++-- src/pkg/packager/compose.go | 2 +- src/pkg/packager/composer/list.go | 21 ++++++++++++--------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go index 46336e3995..db4b02077a 100644 --- a/src/internal/packager/validate/validate.go +++ b/src/internal/packager/validate/validate.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" @@ -67,8 +68,8 @@ func Run(pkg types.ZarfPackage) error { return nil } -// ImportPackage validates the package trying to be imported. -func ImportPackage(composedComponent *types.ZarfComponent) error { +// ImportDefinition validates the component trying to be imported. +func ImportDefinition(composedComponent *types.ZarfComponent) error { path := composedComponent.Import.Path url := composedComponent.Import.URL @@ -101,6 +102,9 @@ func ImportPackage(composedComponent *types.ZarfComponent) error { if !ok { return fmt.Errorf(lang.PkgValidateErrImportURLInvalid, composedComponent.Import.URL) } + if !strings.HasSuffix(url, oci.SkeletonSuffix) { + return fmt.Errorf("remote component %q does not have a %q suffix", url, oci.SkeletonSuffix) + } } return nil diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 6bfcb3a2d8..868c95b0fa 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -72,7 +72,7 @@ func (p *Packager) composeComponents() error { // where 1 component parent is made up of 0...n composite or leaf children. func (p *Packager) getComposedComponent(parentComponent types.ZarfComponent) (child types.ZarfComponent, err error) { // Make sure the component we're trying to import can't be accessed. - if err := validate.ImportPackage(&parentComponent); err != nil { + if err := validate.ImportDefinition(&parentComponent); err != nil { return child, fmt.Errorf("invalid import definition in the %s component: %w", parentComponent.Name, err) } diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 2861200dc6..4bb9469bfb 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -7,10 +7,9 @@ package composer import ( "fmt" "path/filepath" - "strings" + "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -35,8 +34,6 @@ type Node struct { type ImportChain struct { head *Node tail *Node - - progress *message.ProgressBar } func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { @@ -81,8 +78,8 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) arch = node.Only.Cluster.Architecture } for node != nil { - isLocal := node.Import.Path != "" && node.Import.URL == "" - isRemote := node.Import.Path == "" && node.Import.URL != "" + isLocal := node.Import.Path != "" + isRemote := node.Import.URL != "" if !isLocal && !isRemote { // This is the end of the import chain, @@ -90,6 +87,11 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) return ic, nil } + // TODO: stuff like this should also happen in linting + if err := validate.ImportDefinition(&node.ZarfComponent); err != nil { + return ic, err + } + if node.prev != nil && node.prev.Import.URL != "" && isRemote { return ic, fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") } @@ -105,9 +107,6 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) return ic, err } } else if isRemote { - if !strings.HasSuffix(node.Import.URL, oci.SkeletonSuffix) { - return ic, fmt.Errorf("remote component %q does not have a %q suffix", node.Import.URL, oci.SkeletonSuffix) - } remote, err := oci.NewOrasRemote(node.Import.URL) if err != nil { return ic, err @@ -163,6 +162,8 @@ func (ic *ImportChain) History() (history [][]string) { return history } +// Migrate performs migrations on the import chain +// // TODO: is this the best place to perform migrations? func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { node := ic.head @@ -205,6 +206,7 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent) { return composed } +// MergeVariables merges variables from the import chain func (ic *ImportChain) MergeVariables(vars []types.ZarfPackageVariable) (merged []types.ZarfPackageVariable) { merged = vars @@ -228,6 +230,7 @@ func (ic *ImportChain) MergeVariables(vars []types.ZarfPackageVariable) (merged return merged } +// MergeConstants merges constants from the import chain func (ic *ImportChain) MergeConstants(consts []types.ZarfPackageConstant) (merged []types.ZarfPackageConstant) { merged = consts From b32bfa23bbf5899675be1a74c826f3047725444c Mon Sep 17 00:00:00 2001 From: razzle Date: Fri, 13 Oct 2023 11:06:41 -0500 Subject: [PATCH 10/70] more progress Signed-off-by: razzle --- src/internal/packager/validate/validate.go | 33 ++++--------------- src/pkg/packager/compose.go | 10 +++++- src/pkg/packager/composer/list.go | 38 +++++++++++++++++----- 3 files changed, 46 insertions(+), 35 deletions(-) diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go index db4b02077a..dbb8973043 100644 --- a/src/internal/packager/validate/validate.go +++ b/src/internal/packager/validate/validate.go @@ -6,16 +6,12 @@ package validate import ( "fmt" - "os" - "path/filepath" "regexp" "strings" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/config/lang" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/oci" - "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -69,38 +65,23 @@ func Run(pkg types.ZarfPackage) error { } // ImportDefinition validates the component trying to be imported. -func ImportDefinition(composedComponent *types.ZarfComponent) error { - path := composedComponent.Import.Path - url := composedComponent.Import.URL +func ImportDefinition(component *types.ZarfComponent) error { + path := component.Import.Path + url := component.Import.URL if url == "" { - // ensure path exists + // ensure path is not empty if path == "" { - return fmt.Errorf(lang.PkgValidateErrImportPathMissing, composedComponent.Name) - } - - // remove zarf.yaml from path if path has zarf.yaml suffix - if strings.HasSuffix(path, layout.ZarfYAML) { - path = strings.Split(path, layout.ZarfYAML)[0] - } - - // add a forward slash to end of path if it does not have one - if !strings.HasSuffix(path, string(os.PathSeparator)) { - path = filepath.Clean(path) + string(os.PathSeparator) - } - - // ensure there is a zarf.yaml in provided path - if utils.InvalidPath(filepath.Join(path, layout.ZarfYAML)) { - return fmt.Errorf(lang.PkgValidateErrImportPathInvalid, composedComponent.Import.Path) + return fmt.Errorf(lang.PkgValidateErrImportPathMissing, component.Name) } } else { // ensure path is empty if path != "" { - return fmt.Errorf(lang.PkgValidateErrImportOptions, composedComponent.Name) + return fmt.Errorf(lang.PkgValidateErrImportOptions, component.Name) } ok := helpers.IsOCIURL(url) if !ok { - return fmt.Errorf(lang.PkgValidateErrImportURLInvalid, composedComponent.Import.URL) + return fmt.Errorf(lang.PkgValidateErrImportURLInvalid, component.Import.URL) } if !strings.HasSuffix(url, oci.SkeletonSuffix) { return fmt.Errorf("remote component %q does not have a %q suffix", url, oci.SkeletonSuffix) diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 868c95b0fa..15e0c6f3f6 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -30,6 +30,11 @@ func (p *Packager) composeComponents() error { pkgVars := p.cfg.Pkg.Variables pkgConsts := p.cfg.Pkg.Constants + // TODO: filter components by arch outside of composeComponents + + // if ic.head.Only.Cluster.Architecture != "" { + // arch = node.Only.Cluster.Architecture + // } for _, component := range p.cfg.Pkg.Components { // build the import chain start := time.Now() @@ -47,7 +52,10 @@ func (p *Packager) composeComponents() error { // get the composed component start = time.Now() - composed := chain.Compose() + composed, err := chain.Compose() + if err != nil { + return err + } components = append(components, composed) message.Debugf("Composing %q took %s", component.Name, time.Since(start)) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 4bb9469bfb..f881457cbd 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -7,7 +7,9 @@ package composer import ( "fmt" "path/filepath" + "strings" + "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/oci" @@ -73,10 +75,6 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) history := []string{} node := ic.head - // todo: verify this behavior - if ic.head.Only.Cluster.Architecture != "" { - arch = node.Only.Cluster.Architecture - } for node != nil { isLocal := node.Import.Path != "" isRemote := node.Import.URL != "" @@ -92,6 +90,7 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) return ic, err } + // todo: explain me if node.prev != nil && node.prev.Import.URL != "" && isRemote { return ic, fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") } @@ -173,20 +172,43 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { warnings = append(warnings, w...) node = node.next } + // TODO: make a final warning if warnings are found return warnings } // Compose merges the import chain into a single component // fixing paths, overriding metadata, etc -func (ic *ImportChain) Compose() (composed types.ZarfComponent) { +func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { composed = ic.tail.ZarfComponent + if ic.tail.prev == nil { + return composed, nil + } + node := ic.tail if ic.tail.prev.Import.URL != "" { // TODO: handle remote components - // this should download the remote component tarball, fix the paths, then compose it - node = node.prev + ociImport := ic.tail.prev.Import + skelURL := strings.TrimPrefix(ociImport.URL, helpers.OCIURLPrefix) + cachePath := filepath.Join(config.GetAbsCachePath(), "oci", skelURL) + if err := utils.CreateDirectory(cachePath, 0755); err != nil { + return composed, err + } + // cwd, err := os.Getwd() + // if err != nil { + // return composed, err + // } + // rel, err = filepath.Rel(cwd, cachePath) + // if err != nil { + // return composed, err + // } + // ic.tail.relativeToHead = filepath.Join(rel, layout.ComponentsDir, ic.tail.Name) + + // remote, err := oci.NewOrasRemote(ociImport.URL) + // if err != nil { + // return composed, err + // } } for node != nil { @@ -203,7 +225,7 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent) { node = node.prev } - return composed + return composed, nil } // MergeVariables merges variables from the import chain From cb4f324a1730feebe798eaf95283f0fe8303ddaa Mon Sep 17 00:00:00 2001 From: razzle Date: Sun, 15 Oct 2023 22:05:22 -0500 Subject: [PATCH 11/70] progress on oci compose Signed-off-by: razzle --- src/pkg/oci/pull.go | 3 + src/pkg/packager/composer/list.go | 92 ++++++++++++++++++++++++++++--- 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index d981740b4c..14b9b3646b 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -199,6 +199,9 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta } } + if copyOpts == nil { + copyOpts = &o.CopyOpts + } copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { nodes, err := content.Successors(ctx, fetcher, desc) if err != nil { diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index f881457cbd..29c3fd4d31 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -5,7 +5,9 @@ package composer import ( + "context" "fmt" + "os" "path/filepath" "strings" @@ -17,6 +19,9 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" + "github.com/mholt/archiver/v3" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + ocistore "oras.land/oras-go/v2/content/oci" ) // Node is a node in the import chain @@ -36,6 +41,8 @@ type Node struct { type ImportChain struct { head *Node tail *Node + + remotes map[string]*oci.OrasRemote } func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { @@ -68,7 +75,9 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) return nil, fmt.Errorf("cannot build import chain: architecture must be provided") } - ic := &ImportChain{} + ic := &ImportChain{ + remotes: make(map[string]*oci.OrasRemote), + } ic.append(head, "", nil, nil) @@ -106,7 +115,7 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) return ic, err } } else if isRemote { - remote, err := oci.NewOrasRemote(node.Import.URL) + remote, err := ic.getRemote(node.Import.URL) if err != nil { return ic, err } @@ -176,25 +185,94 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { return warnings } +func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { + if remote, ok := ic.remotes[url]; ok { + return remote, nil + } else { + remote, err := oci.NewOrasRemote(url) + if err != nil { + return nil, err + } + ic.remotes[url] = remote + return remote, nil + } +} + +func (ic *ImportChain) fetchOCISkeleton(node *Node) error { + if node.Import.URL == "" { + return fmt.Errorf("cannot fetch remote component skeleton: no URL provided") + } + remote, err := ic.getRemote(node.Import.URL) + if err != nil { + return err + } + + manifest, err := remote.FetchRoot() + if err != nil { + return err + } + + componentDesc := manifest.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf("%s.tar", node.Name))) + + if oci.IsEmptyDescriptor(componentDesc) { + // nothing to fetch + return nil + } + + cache := filepath.Join(config.GetAbsCachePath(), "oci") + store, err := ocistore.New(cache) + if err != nil { + return err + } + + tb := filepath.Join(cache, "blobs", "sha256", componentDesc.Digest.String()) + + if ok, err := store.Exists(context.TODO(), componentDesc); err != nil { + return err + } else if ok { + // already exists in the cache + return nil + } + + cwd, err := os.Getwd() + if err != nil { + return err + } + rel, err := filepath.Rel(cwd, tb) + if err != nil { + return err + } + node.relativeToHead = rel + + if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, nil, cache); err != nil { + return err + } + + if !utils.InvalidPath(strings.TrimSuffix(tb, ".tar")) { + // already extracted + return nil + } + tu := archiver.Tar{} + return tu.Unarchive(tb, strings.TrimSuffix(tb, ".tar")) +} + // Compose merges the import chain into a single component // fixing paths, overriding metadata, etc func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { composed = ic.tail.ZarfComponent if ic.tail.prev == nil { + // only had one component in the import chain return composed, nil } node := ic.tail if ic.tail.prev.Import.URL != "" { - // TODO: handle remote components - ociImport := ic.tail.prev.Import - skelURL := strings.TrimPrefix(ociImport.URL, helpers.OCIURLPrefix) - cachePath := filepath.Join(config.GetAbsCachePath(), "oci", skelURL) - if err := utils.CreateDirectory(cachePath, 0755); err != nil { + if err := ic.fetchOCISkeleton(ic.tail.prev); err != nil { return composed, err } + // TODO: handle remote components // cwd, err := os.Getwd() // if err != nil { // return composed, err From 70fc996aed214bbe04159d998fcfd0802a80201d Mon Sep 17 00:00:00 2001 From: razzle Date: Sun, 15 Oct 2023 22:08:51 -0500 Subject: [PATCH 12/70] progress on oci compose Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 29c3fd4d31..063d79e907 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -227,13 +227,6 @@ func (ic *ImportChain) fetchOCISkeleton(node *Node) error { tb := filepath.Join(cache, "blobs", "sha256", componentDesc.Digest.String()) - if ok, err := store.Exists(context.TODO(), componentDesc); err != nil { - return err - } else if ok { - // already exists in the cache - return nil - } - cwd, err := os.Getwd() if err != nil { return err @@ -244,8 +237,12 @@ func (ic *ImportChain) fetchOCISkeleton(node *Node) error { } node.relativeToHead = rel - if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, nil, cache); err != nil { + if exists, err := store.Exists(context.TODO(), componentDesc); err != nil { return err + } else if !exists { + if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, nil, cache); err != nil { + return err + } } if !utils.InvalidPath(strings.TrimSuffix(tb, ".tar")) { From 762356d90d22138f2c0b52665c8055bbbe836bf5 Mon Sep 17 00:00:00 2001 From: razzle Date: Sun, 15 Oct 2023 22:17:13 -0500 Subject: [PATCH 13/70] fix tb:dir relationship Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 063d79e907..77c136bc1d 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -231,11 +231,15 @@ func (ic *ImportChain) fetchOCISkeleton(node *Node) error { if err != nil { return err } - rel, err := filepath.Rel(cwd, tb) + dir := strings.TrimSuffix(tb, ".tar") + rel, err := filepath.Rel(cwd, dir) if err != nil { return err } - node.relativeToHead = rel + // this node is the node importing the remote component + // and has a filepath relative to the head of the import chain + // the next (tail) node will have a filepath relative from cwd to the tarball in cache + node.next.relativeToHead = rel if exists, err := store.Exists(context.TODO(), componentDesc); err != nil { return err @@ -245,12 +249,12 @@ func (ic *ImportChain) fetchOCISkeleton(node *Node) error { } } - if !utils.InvalidPath(strings.TrimSuffix(tb, ".tar")) { + if !utils.InvalidPath(dir) { // already extracted return nil } tu := archiver.Tar{} - return tu.Unarchive(tb, strings.TrimSuffix(tb, ".tar")) + return tu.Unarchive(tb, dir) } // Compose merges the import chain into a single component @@ -269,21 +273,6 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { if err := ic.fetchOCISkeleton(ic.tail.prev); err != nil { return composed, err } - // TODO: handle remote components - // cwd, err := os.Getwd() - // if err != nil { - // return composed, err - // } - // rel, err = filepath.Rel(cwd, cachePath) - // if err != nil { - // return composed, err - // } - // ic.tail.relativeToHead = filepath.Join(rel, layout.ComponentsDir, ic.tail.Name) - - // remote, err := oci.NewOrasRemote(ociImport.URL) - // if err != nil { - // return composed, err - // } } for node != nil { From 96d59c098da332e65ee871ca0ac079786ff3e504 Mon Sep 17 00:00:00 2001 From: razzle Date: Sun, 15 Oct 2023 22:28:40 -0500 Subject: [PATCH 14/70] derp Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 77c136bc1d..82fa3483e2 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -42,7 +42,7 @@ type ImportChain struct { head *Node tail *Node - remotes map[string]*oci.OrasRemote + remote *oci.OrasRemote } func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string, vars []types.ZarfPackageVariable, consts []types.ZarfPackageConstant) { @@ -75,9 +75,7 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) return nil, fmt.Errorf("cannot build import chain: architecture must be provided") } - ic := &ImportChain{ - remotes: make(map[string]*oci.OrasRemote), - } + ic := &ImportChain{} ic.append(head, "", nil, nil) @@ -186,16 +184,15 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { } func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { - if remote, ok := ic.remotes[url]; ok { - return remote, nil - } else { - remote, err := oci.NewOrasRemote(url) - if err != nil { - return nil, err - } - ic.remotes[url] = remote - return remote, nil + if ic.remote != nil { + return ic.remote, nil + } + var err error + ic.remote, err = oci.NewOrasRemote(url) + if err != nil { + return nil, err } + return ic.remote, nil } func (ic *ImportChain) fetchOCISkeleton(node *Node) error { From 543928589b4f57de8279f00ee5e339b044c5a866 Mon Sep 17 00:00:00 2001 From: razzle Date: Sun, 15 Oct 2023 22:36:58 -0500 Subject: [PATCH 15/70] cleanup Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 82fa3483e2..06d06dc7ba 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -195,9 +195,12 @@ func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { return ic.remote, nil } -func (ic *ImportChain) fetchOCISkeleton(node *Node) error { +func (ic *ImportChain) fetchOCISkeleton() error { + // only the 2nd to last node will have a remote import + node := ic.tail.prev if node.Import.URL == "" { - return fmt.Errorf("cannot fetch remote component skeleton: no URL provided") + // nothing to fetch + return nil } remote, err := ic.getRemote(node.Import.URL) if err != nil { @@ -236,7 +239,7 @@ func (ic *ImportChain) fetchOCISkeleton(node *Node) error { // this node is the node importing the remote component // and has a filepath relative to the head of the import chain // the next (tail) node will have a filepath relative from cwd to the tarball in cache - node.next.relativeToHead = rel + ic.tail.relativeToHead = rel if exists, err := store.Exists(context.TODO(), componentDesc); err != nil { return err @@ -264,14 +267,11 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { return composed, nil } - node := ic.tail - - if ic.tail.prev.Import.URL != "" { - if err := ic.fetchOCISkeleton(ic.tail.prev); err != nil { - return composed, err - } + if err := ic.fetchOCISkeleton(); err != nil { + return composed, err } + node := ic.tail for node != nil { fixPaths(&node.ZarfComponent, node.relativeToHead) From a49aec6e005fbbb1ba6727cd85e05a530c64a409 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 16 Oct 2023 11:42:41 -0500 Subject: [PATCH 16/70] fix doubling Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 06d06dc7ba..0bcc3dd9a2 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -260,8 +260,6 @@ func (ic *ImportChain) fetchOCISkeleton() error { // Compose merges the import chain into a single component // fixing paths, overriding metadata, etc func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { - composed = ic.tail.ZarfComponent - if ic.tail.prev == nil { // only had one component in the import chain return composed, nil From 707fb30d168f8bbe19cc2e76af4f1c46ea853986 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 16 Oct 2023 13:29:07 -0500 Subject: [PATCH 17/70] string instead of history Signed-off-by: razzle --- src/pkg/packager/compose.go | 23 +++++---------- src/pkg/packager/composer/list.go | 49 +++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 15e0c6f3f6..c2c8df8ac4 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -9,12 +9,10 @@ import ( "os" "path/filepath" "strings" - "time" "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/composer" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" @@ -30,40 +28,33 @@ func (p *Packager) composeComponents() error { pkgVars := p.cfg.Pkg.Variables pkgConsts := p.cfg.Pkg.Constants - // TODO: filter components by arch outside of composeComponents - - // if ic.head.Only.Cluster.Architecture != "" { - // arch = node.Only.Cluster.Architecture - // } for _, component := range p.cfg.Pkg.Components { + arch := p.arch + // filter by architecture if set, otherwise use system architecture + if component.Only.Cluster.Architecture != "" { + arch = component.Only.Cluster.Architecture + } + // build the import chain - start := time.Now() - chain, err := composer.NewImportChain(component, p.arch) + chain, err := composer.NewImportChain(component, arch) if err != nil { return err } - message.Debugf("Building import chain for %q took %s", component.Name, time.Since(start)) // migrate any deprecated component configurations now - start = time.Now() warnings := chain.Migrate(p.cfg.Pkg.Build) p.warnings = append(p.warnings, warnings...) - message.Debugf("Migrating %q took %s", component.Name, time.Since(start)) // get the composed component - start = time.Now() composed, err := chain.Compose() if err != nil { return err } components = append(components, composed) - message.Debugf("Composing %q took %s", component.Name, time.Since(start)) // merge variables and constants - start = time.Now() pkgVars = chain.MergeVariables(pkgVars) pkgConsts = chain.MergeConstants(pkgConsts) - message.Debugf("Merging pkg vars and consts from import chain took %s", time.Since(start)) } // Update the parent package config with the expanded sub components. diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 0bcc3dd9a2..fe8592f426 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -97,10 +97,14 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) return ic, err } - // todo: explain me + // ensure that remote components are not importing other remote components if node.prev != nil && node.prev.Import.URL != "" && isRemote { return ic, fmt.Errorf("detected malformed import chain, cannot import remote components from remote components") } + // ensure that remote components are not importing local components + if node.prev != nil && node.prev.Import.URL != "" && isLocal { + return ic, fmt.Errorf("detected malformed import chain, cannot import local components from remote components") + } var pkg types.ZarfPackage name := node.Name @@ -154,23 +158,39 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) return ic, nil } -// History returns the history of the import chain -func (ic *ImportChain) History() (history [][]string) { - node := ic.head - for node != nil { - if node.Import.URL != "" { - history = append(history, []string{node.Name, node.Import.URL}) - continue +func (ic *ImportChain) String() string { + if ic.head.next == nil { + return fmt.Sprintf("[%s]", ic.head.Name) + } + + s := strings.Builder{} + + if ic.head.Import.Path != "" { + s.WriteString(fmt.Sprintf("[%q imports %q] --> ", ic.head.Name, ic.head.Import.Path)) + } else { + s.WriteString(fmt.Sprintf("[%q imports %q] --> ", ic.head.Name, ic.head.Import.URL)) + } + + node := ic.head.next + for node != ic.tail { + name := node.Name + if node.Import.ComponentName != "" { + name = node.Import.ComponentName + } + if node.Import.Path != "" { + s.WriteString(fmt.Sprintf("[%q imports %q] --> ", name, node.Import.Path)) + } else { + s.WriteString(fmt.Sprintf("[%q imports %q] --> ", name, node.Import.URL)) } - history = append(history, []string{node.Name, node.relativeToHead, node.Import.Path}) node = node.next } - return history + + s.WriteString(fmt.Sprintf("[%q]", ic.tail.Name)) + + return s.String() } // Migrate performs migrations on the import chain -// -// TODO: is this the best place to perform migrations? func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { node := ic.head for node != nil { @@ -179,7 +199,10 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { warnings = append(warnings, w...) node = node.next } - // TODO: make a final warning if warnings are found + if len(warnings) > 0 { + final := fmt.Sprintf("migrations were performed on the import chain of: %q", ic.head.Name) + warnings = append(warnings, final) + } return warnings } From 9b5942a9af47cf186b5bb2c91205a2354297ffd2 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 16 Oct 2023 14:50:22 -0500 Subject: [PATCH 18/70] fix merge vars and consts Signed-off-by: razzle --- src/pkg/packager/compose.go | 3 +++ src/pkg/packager/composer/list.go | 43 ++++++++++--------------------- src/pkg/utils/helpers/slice.go | 30 +++++++++++++++++++++ 3 files changed, 47 insertions(+), 29 deletions(-) create mode 100644 src/pkg/utils/helpers/slice.go diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index c2c8df8ac4..a48ab306f5 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -61,6 +61,9 @@ func (p *Packager) composeComponents() error { // This is important when the deploy package is created. p.cfg.Pkg.Components = components + p.cfg.Pkg.Variables = pkgVars + p.cfg.Pkg.Constants = pkgConsts + return nil } diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index fe8592f426..ee04200bb4 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -158,6 +158,7 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) return ic, nil } +// String returns a string representation of the import chain func (ic *ImportChain) String() string { if ic.head.next == nil { return fmt.Sprintf("[%s]", ic.head.Name) @@ -311,48 +312,32 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { } // MergeVariables merges variables from the import chain -func (ic *ImportChain) MergeVariables(vars []types.ZarfPackageVariable) (merged []types.ZarfPackageVariable) { - merged = vars +func (ic *ImportChain) MergeVariables(existing []types.ZarfPackageVariable) (merged []types.ZarfPackageVariable) { + exists := func(v1 types.ZarfPackageVariable, v2 types.ZarfPackageVariable) bool { + return v1.Name == v2.Name + } + merged = helpers.MergeSlices(existing, merged, exists) node := ic.head for node != nil { - // // merge the vars - for _, v := range node.vars { - exists := false - for _, vv := range merged { - if v.Name == vv.Name { - exists = true - break - } - } - if !exists { - merged = append(merged, v) - } - } + // merge the vars + merged = helpers.MergeSlices(node.vars, merged, exists) node = node.next } return merged } // MergeConstants merges constants from the import chain -func (ic *ImportChain) MergeConstants(consts []types.ZarfPackageConstant) (merged []types.ZarfPackageConstant) { - merged = consts +func (ic *ImportChain) MergeConstants(existing []types.ZarfPackageConstant) (merged []types.ZarfPackageConstant) { + exists := func(c1 types.ZarfPackageConstant, c2 types.ZarfPackageConstant) bool { + return c1.Name == c2.Name + } + merged = helpers.MergeSlices(existing, merged, exists) node := ic.head for node != nil { // merge the consts - for _, c := range node.consts { - exists := false - for _, cc := range merged { - if c.Name == cc.Name { - exists = true - break - } - } - if !exists { - merged = append(merged, c) - } - } + merged = helpers.MergeSlices(node.consts, merged, exists) node = node.next } return merged diff --git a/src/pkg/utils/helpers/slice.go b/src/pkg/utils/helpers/slice.go new file mode 100644 index 0000000000..989fde3d30 --- /dev/null +++ b/src/pkg/utils/helpers/slice.go @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package helpers provides generic helper functions with no external imports +package helpers + +// EqualFunc defines a type for a function that determines equality between two elements of type T. +type EqualFunc[T any] func(a, b T) bool + +// MergeSlices merges two slices, s1 and s2, and returns a new slice containing all elements from s1 +// and only those elements from s2 that do not exist in s1 based on the provided equal function. +func MergeSlices[T any](s1, s2 []T, equals EqualFunc[T]) []T { + merged := make([]T, 0, len(s1)+len(s2)) + merged = append(merged, s1...) + + for _, v2 := range s2 { + exists := false + for _, v1 := range s1 { + if equals(v1, v2) { + exists = true + break + } + } + if !exists { + merged = append(merged, v2) + } + } + + return merged +} From 8da6550374b3f01abe650479fa0631e731ac2f83 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 16 Oct 2023 16:13:53 -0500 Subject: [PATCH 19/70] fix single component compose Signed-off-by: razzle --- src/pkg/packager/compose.go | 2 ++ src/pkg/packager/composer/list.go | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index a48ab306f5..3eba64909d 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -13,6 +13,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/composer" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" @@ -50,6 +51,7 @@ func (p *Packager) composeComponents() error { if err != nil { return err } + message.Debug("composed: ", message.JSONValue(composed)) components = append(components, composed) // merge variables and constants diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index ee04200bb4..3305fe74c9 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -284,6 +284,8 @@ func (ic *ImportChain) fetchOCISkeleton() error { // Compose merges the import chain into a single component // fixing paths, overriding metadata, etc func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { + composed = ic.tail.ZarfComponent + if ic.tail.prev == nil { // only had one component in the import chain return composed, nil @@ -293,7 +295,11 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { return composed, err } - node := ic.tail + // fix paths on the tail + fixPaths(&ic.tail.ZarfComponent, ic.tail.prev.relativeToHead) + + // dont compose 2x + node := ic.tail.prev for node != nil { fixPaths(&node.ZarfComponent, node.relativeToHead) From 95c997ca4679a196a63fafbf37977f1953ce355b Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 16 Oct 2023 16:28:56 -0500 Subject: [PATCH 20/70] derp Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 3305fe74c9..6e54e26e01 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -296,7 +296,7 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { } // fix paths on the tail - fixPaths(&ic.tail.ZarfComponent, ic.tail.prev.relativeToHead) + fixPaths(&ic.tail.ZarfComponent, ic.tail.relativeToHead) // dont compose 2x node := ic.tail.prev From b072230a4041e2b7ce2bdade8fd03754395621c8 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 16 Oct 2023 17:59:54 -0500 Subject: [PATCH 21/70] lets try this Signed-off-by: razzle --- src/pkg/packager/compose.go | 2 +- src/pkg/packager/composer/list.go | 44 ++++++++++++++++++++++--------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 3eba64909d..0fd969cb0f 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -41,6 +41,7 @@ func (p *Packager) composeComponents() error { if err != nil { return err } + message.Debugf("%s", chain) // migrate any deprecated component configurations now warnings := chain.Migrate(p.cfg.Pkg.Build) @@ -51,7 +52,6 @@ func (p *Packager) composeComponents() error { if err != nil { return err } - message.Debug("composed: ", message.JSONValue(composed)) components = append(components, composed) // merge variables and constants diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 6e54e26e01..9a5638763b 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -6,6 +6,7 @@ package composer import ( "context" + "crypto/sha256" "fmt" "os" "path/filepath" @@ -236,26 +237,33 @@ func (ic *ImportChain) fetchOCISkeleton() error { return err } - componentDesc := manifest.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf("%s.tar", node.Name))) - - if oci.IsEmptyDescriptor(componentDesc) { - // nothing to fetch - return nil + name := node.Name + if node.Import.ComponentName != "" { + name = node.Import.ComponentName } + componentDesc := manifest.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf("%s.tar", name))) + cache := filepath.Join(config.GetAbsCachePath(), "oci") - store, err := ocistore.New(cache) - if err != nil { + if err := utils.CreateDirectory(cache, 0700); err != nil { return err } - tb := filepath.Join(cache, "blobs", "sha256", componentDesc.Digest.String()) + tb := filepath.Join(cache, "blobs", "sha256", componentDesc.Digest.Encoded()) + + h := sha256.New() + h.Write([]byte(node.Import.URL + name)) + id := fmt.Sprintf("%x", h.Sum(nil)) + + dir := filepath.Join(cache, "dirs", id) + if err := utils.CreateDirectory(dir, 0700); err != nil { + return err + } cwd, err := os.Getwd() if err != nil { return err } - dir := strings.TrimSuffix(tb, ".tar") rel, err := filepath.Rel(cwd, dir) if err != nil { return err @@ -265,6 +273,16 @@ func (ic *ImportChain) fetchOCISkeleton() error { // the next (tail) node will have a filepath relative from cwd to the tarball in cache ic.tail.relativeToHead = rel + if oci.IsEmptyDescriptor(componentDesc) { + // nothing to fetch + return nil + } + + store, err := ocistore.New(cache) + if err != nil { + return err + } + if exists, err := store.Exists(context.TODO(), componentDesc); err != nil { return err } else if !exists { @@ -273,11 +291,11 @@ func (ic *ImportChain) fetchOCISkeleton() error { } } - if !utils.InvalidPath(dir) { - // already extracted - return nil + tu := archiver.Tar{ + OverwriteExisting: true, + // removes // from the paths + StripComponents: 1, } - tu := archiver.Tar{} return tu.Unarchive(tb, dir) } From 6c16f7e78a4fd3e5deb0cf021f35d52ce49aa9a5 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 16 Oct 2023 19:40:55 -0500 Subject: [PATCH 22/70] mutate tail, kinda ugly ngl Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 9a5638763b..536ed0e2f5 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -315,6 +315,8 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { // fix paths on the tail fixPaths(&ic.tail.ZarfComponent, ic.tail.relativeToHead) + // save mutated tail + composed = ic.tail.ZarfComponent // dont compose 2x node := ic.tail.prev From 2c3969f860d4f874eca2cad45234d8d37a2faa09 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 16 Oct 2023 20:11:36 -0500 Subject: [PATCH 23/70] better string Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 32 ++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 536ed0e2f5..edaae25f27 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -162,15 +162,20 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) // String returns a string representation of the import chain func (ic *ImportChain) String() string { if ic.head.next == nil { - return fmt.Sprintf("[%s]", ic.head.Name) + return fmt.Sprintf("component %q imports nothing", ic.head.Name) } s := strings.Builder{} + name := ic.head.Name + if ic.head.Import.ComponentName != "" { + name = ic.head.Import.ComponentName + } + if ic.head.Import.Path != "" { - s.WriteString(fmt.Sprintf("[%q imports %q] --> ", ic.head.Name, ic.head.Import.Path)) + s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.head.Name, name, ic.head.Import.Path)) } else { - s.WriteString(fmt.Sprintf("[%q imports %q] --> ", ic.head.Name, ic.head.Import.URL)) + s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.head.Name, name, ic.head.Import.URL)) } node := ic.head.next @@ -180,15 +185,14 @@ func (ic *ImportChain) String() string { name = node.Import.ComponentName } if node.Import.Path != "" { - s.WriteString(fmt.Sprintf("[%q imports %q] --> ", name, node.Import.Path)) + s.WriteString(fmt.Sprintf(", which imports %q in %s", name, node.Import.Path)) } else { - s.WriteString(fmt.Sprintf("[%q imports %q] --> ", name, node.Import.URL)) + s.WriteString(fmt.Sprintf(", which imports %q in %s", name, node.Import.URL)) } + node = node.next } - s.WriteString(fmt.Sprintf("[%q]", ic.tail.Name)) - return s.String() } @@ -250,12 +254,18 @@ func (ic *ImportChain) fetchOCISkeleton() error { } tb := filepath.Join(cache, "blobs", "sha256", componentDesc.Digest.Encoded()) + dir := filepath.Join(cache, "dirs", componentDesc.Digest.Encoded()) - h := sha256.New() - h.Write([]byte(node.Import.URL + name)) - id := fmt.Sprintf("%x", h.Sum(nil)) + // if there is not tarball to fetch, create a directory named based upon + // the import url and the component name + if oci.IsEmptyDescriptor(componentDesc) { + h := sha256.New() + h.Write([]byte(node.Import.URL + name)) + id := fmt.Sprintf("%x", h.Sum(nil)) + + dir = filepath.Join(cache, "dirs", id) + } - dir := filepath.Join(cache, "dirs", id) if err := utils.CreateDirectory(dir, 0700); err != nil { return err } From a1466b2049bcabbfd7a675b65815050558e00ec4 Mon Sep 17 00:00:00 2001 From: razzle Date: Mon, 16 Oct 2023 20:30:18 -0500 Subject: [PATCH 24/70] better string Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index edaae25f27..82506ee059 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -184,10 +184,11 @@ func (ic *ImportChain) String() string { if node.Import.ComponentName != "" { name = node.Import.ComponentName } + s.WriteString(", which imports ") if node.Import.Path != "" { - s.WriteString(fmt.Sprintf(", which imports %q in %s", name, node.Import.Path)) + s.WriteString(fmt.Sprintf("%q in %s", name, node.Import.Path)) } else { - s.WriteString(fmt.Sprintf(", which imports %q in %s", name, node.Import.URL)) + s.WriteString(fmt.Sprintf("%q in %s", name, node.Import.URL)) } node = node.next From 4b0ef4a7f14dfd6beae7e7b100138d214ba3db0f Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 17 Oct 2023 13:30:19 -0500 Subject: [PATCH 25/70] fix oci compose Signed-off-by: razzle --- src/pkg/oci/pull.go | 31 +++++++++-------- src/pkg/packager/composer/list.go | 52 +++++++++++++++++++---------- src/test/e2e/50_oci_package_test.go | 2 +- 3 files changed, 51 insertions(+), 34 deletions(-) diff --git a/src/pkg/oci/pull.go b/src/pkg/oci/pull.go index 14b9b3646b..a8203345e7 100644 --- a/src/pkg/oci/pull.go +++ b/src/pkg/oci/pull.go @@ -185,11 +185,11 @@ func (o *OrasRemote) PullPackage(destinationDir string, concurrency int, layersT copyOpts := o.CopyOpts copyOpts.Concurrency = concurrency - return layersToPull, o.CopyWithProgress(layersToPull, dst, ©Opts, destinationDir) + return layersToPull, o.CopyWithProgress(layersToPull, dst, copyOpts, destinationDir) } // CopyWithProgress copies the given layers from the remote repository to the given store. -func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Target, copyOpts *oras.CopyOptions, destinationDir string) error { +func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Target, copyOpts oras.CopyOptions, destinationDir string) error { estimatedBytes := int64(0) shas := []string{} for _, layer := range layers { @@ -199,21 +199,20 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta } } - if copyOpts == nil { - copyOpts = &o.CopyOpts - } - copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - nodes, err := content.Successors(ctx, fetcher, desc) - if err != nil { - return nil, err - } - var ret []ocispec.Descriptor - for _, node := range nodes { - if slices.Contains(shas, node.Digest.Encoded()) { - ret = append(ret, node) + if copyOpts.FindSuccessors == nil { + copyOpts.FindSuccessors = func(ctx context.Context, fetcher content.Fetcher, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + nodes, err := content.Successors(ctx, fetcher, desc) + if err != nil { + return nil, err + } + var ret []ocispec.Descriptor + for _, node := range nodes { + if slices.Contains(shas, node.Digest.Encoded()) { + ret = append(ret, node) + } } + return ret, nil } - return ret, nil } // Create a thread to update a progress bar as we save the package to disk @@ -222,7 +221,7 @@ func (o *OrasRemote) CopyWithProgress(layers []ocispec.Descriptor, store oras.Ta wg.Add(1) successText := fmt.Sprintf("Pulling %q", helpers.OCIURLPrefix+o.repo.Reference.String()) go utils.RenderProgressBarForLocalDirWrite(destinationDir, estimatedBytes, &wg, doneSaving, "Pulling", successText) - _, err := oras.Copy(o.ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), *copyOpts) + _, err := oras.Copy(o.ctx, o.repo, o.repo.Reference.String(), store, o.repo.Reference.String(), copyOpts) if err != nil { return err } diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 82506ee059..ab68c0f322 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -15,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" @@ -22,6 +23,7 @@ import ( "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" ocistore "oras.land/oras-go/v2/content/oci" ) @@ -226,7 +228,7 @@ func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { } func (ic *ImportChain) fetchOCISkeleton() error { - // only the 2nd to last node will have a remote import + // only the 2nd to last node may have a remote import node := ic.tail.prev if node.Import.URL == "" { // nothing to fetch @@ -254,8 +256,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { return err } - tb := filepath.Join(cache, "blobs", "sha256", componentDesc.Digest.Encoded()) - dir := filepath.Join(cache, "dirs", componentDesc.Digest.Encoded()) + var tb, dir string // if there is not tarball to fetch, create a directory named based upon // the import url and the component name @@ -265,6 +266,36 @@ func (ic *ImportChain) fetchOCISkeleton() error { id := fmt.Sprintf("%x", h.Sum(nil)) dir = filepath.Join(cache, "dirs", id) + + message.Debug("creating empty directory for remote component:", filepath.Join("", "oci", "dirs", id)) + } else { + tb = filepath.Join(cache, "blobs", "sha256", componentDesc.Digest.Encoded()) + dir = filepath.Join(cache, "dirs", componentDesc.Digest.Encoded()) + + store, err := ocistore.New(cache) + if err != nil { + return err + } + + ctx := context.TODO() + // ensure the tarball is in the cache + exists, err := store.Exists(ctx, componentDesc) + if err != nil { + return err + } else if !exists { + copyOpts := remote.CopyOpts + // TODO: investigate why the default FindSuccessors in CopyWithProgress is not working + copyOpts.FindSuccessors = content.Successors + if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, copyOpts, cache); err != nil { + return err + } + exists, err := store.Exists(ctx, componentDesc) + if err != nil { + return err + } else if !exists { + return fmt.Errorf("failed to fetch remote component: %+v", componentDesc) + } + } } if err := utils.CreateDirectory(dir, 0700); err != nil { @@ -285,23 +316,10 @@ func (ic *ImportChain) fetchOCISkeleton() error { ic.tail.relativeToHead = rel if oci.IsEmptyDescriptor(componentDesc) { - // nothing to fetch + // nothing was fetched, nothing to extract return nil } - store, err := ocistore.New(cache) - if err != nil { - return err - } - - if exists, err := store.Exists(context.TODO(), componentDesc); err != nil { - return err - } else if !exists { - if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, nil, cache); err != nil { - return err - } - } - tu := archiver.Tar{ OverwriteExisting: true, // removes // from the paths diff --git a/src/test/e2e/50_oci_package_test.go b/src/test/e2e/50_oci_package_test.go index 888b9e09e4..b698ba6207 100644 --- a/src/test/e2e/50_oci_package_test.go +++ b/src/test/e2e/50_oci_package_test.go @@ -196,7 +196,7 @@ func (suite *RegistryClientTestSuite) Test_5_Copy() { } } -func TestRegistryClientTestSuite(t *testing.T) { +func TestRegistryClientSuite(t *testing.T) { e2e.SetupWithCluster(t) suite.Run(t, new(RegistryClientTestSuite)) From 77a37c142d73a13be8d4f3df0262b5cdea32f2e6 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 17 Oct 2023 21:42:12 -0500 Subject: [PATCH 26/70] save oci import for differential stuffs Signed-off-by: razzle --- src/pkg/packager/compose.go | 6 ++++++ src/pkg/packager/composer/list.go | 19 ++++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 0fd969cb0f..3492fa3f52 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -43,6 +43,12 @@ func (p *Packager) composeComponents() error { } message.Debugf("%s", chain) + if chain.ContainsOCIImport() { + url, name := chain.OCIImportDefinition() + // Save all the OCI imported components into our build data + p.cfg.Pkg.Build.OCIImportedComponents[url] = name + } + // migrate any deprecated component configurations now warnings := chain.Migrate(p.cfg.Pkg.Build) p.warnings = append(p.warnings, warnings...) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index ab68c0f322..3a5dbcaf20 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -227,11 +227,24 @@ func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { return ic.remote, nil } -func (ic *ImportChain) fetchOCISkeleton() error { +// ContainsOCIImport returns true if the import chain contains a remote import +func (ic *ImportChain) ContainsOCIImport() bool { // only the 2nd to last node may have a remote import + return ic.tail.prev != nil && ic.tail.prev.Import.URL != "" +} + +// OCIImportDefinition returns the url and name of the remote import +func (ic *ImportChain) OCIImportDefinition() (string, string) { + name := ic.tail.prev.Name + if ic.tail.prev.Import.ComponentName != "" { + name = ic.tail.prev.Import.ComponentName + } + return ic.tail.prev.Import.URL, name +} + +func (ic *ImportChain) fetchOCISkeleton() error { node := ic.tail.prev - if node.Import.URL == "" { - // nothing to fetch + if !ic.ContainsOCIImport() { return nil } remote, err := ic.getRemote(node.Import.URL) From 213896a47d2efb672abc02ef92e5d54ecacd07cb Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 17 Oct 2023 22:06:22 -0500 Subject: [PATCH 27/70] remove double nil check Signed-off-by: razzle --- src/pkg/packager/yaml.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/pkg/packager/yaml.go b/src/pkg/packager/yaml.go index 32c04ebb60..e7c991d3d8 100644 --- a/src/pkg/packager/yaml.go +++ b/src/pkg/packager/yaml.go @@ -29,10 +29,6 @@ func (p *Packager) readZarfYAML(path string) error { p.warnings = append(p.warnings, warning) } - if p.cfg.Pkg.Build.OCIImportedComponents == nil { - p.cfg.Pkg.Build.OCIImportedComponents = make(map[string]string) - } - if len(p.cfg.Pkg.Build.Migrations) > 0 { for idx, component := range p.cfg.Pkg.Components { // Handle component configuration deprecations From 943c2ad1b7652946f9ca9aeb9bada3cd430cba7b Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 17 Oct 2023 22:19:24 -0500 Subject: [PATCH 28/70] cleanup Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 47 ++++++++++++++----------------- 1 file changed, 21 insertions(+), 26 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 3a5dbcaf20..8690c1afec 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -40,7 +40,19 @@ type Node struct { next *Node } -// ImportChain is a doubly linked list of components +// ImportName returns the name of the component to import +// +// If the component import has a ComponentName defined, that will be used +// otherwise the name of the component will be used +func (n *Node) ImportName() string { + name := n.ZarfComponent.Name + if n.Import.ComponentName != "" { + name = n.Import.ComponentName + } + return name +} + +// ImportChain is a doubly linked list of component import definitions type ImportChain struct { head *Node tail *Node @@ -110,7 +122,6 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) } var pkg types.ZarfPackage - name := node.Name if isLocal { history = append(history, node.Import.Path) @@ -130,9 +141,7 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) } } - if node.Import.ComponentName != "" { - name = node.Import.ComponentName - } + name := node.ImportName() found := helpers.Filter(pkg.Components, func(c types.ZarfComponent) bool { matchesName := c.Name == name @@ -147,11 +156,10 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) return ic, fmt.Errorf("component %q not found in %q", name, node.Import.URL) } } else if len(found) > 1 { - // TODO: improve this error message / figure out the best way to present this error if isLocal { - return ic, fmt.Errorf("multiple components named %q found in %q", name, filepath.Join(history...)) + return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, filepath.Join(history...), arch) } else if isRemote { - return ic, fmt.Errorf("multiple components named %q found in %q", name, node.Import.URL) + return ic, fmt.Errorf("multiple components named %q found in %q satisfying %q", name, node.Import.URL, arch) } } @@ -169,10 +177,7 @@ func (ic *ImportChain) String() string { s := strings.Builder{} - name := ic.head.Name - if ic.head.Import.ComponentName != "" { - name = ic.head.Import.ComponentName - } + name := ic.head.ImportName() if ic.head.Import.Path != "" { s.WriteString(fmt.Sprintf("component %q imports %q in %s", ic.head.Name, name, ic.head.Import.Path)) @@ -182,10 +187,7 @@ func (ic *ImportChain) String() string { node := ic.head.next for node != ic.tail { - name := node.Name - if node.Import.ComponentName != "" { - name = node.Import.ComponentName - } + name := node.ImportName() s.WriteString(", which imports ") if node.Import.Path != "" { s.WriteString(fmt.Sprintf("%q in %s", name, node.Import.Path)) @@ -235,11 +237,7 @@ func (ic *ImportChain) ContainsOCIImport() bool { // OCIImportDefinition returns the url and name of the remote import func (ic *ImportChain) OCIImportDefinition() (string, string) { - name := ic.tail.prev.Name - if ic.tail.prev.Import.ComponentName != "" { - name = ic.tail.prev.Import.ComponentName - } - return ic.tail.prev.Import.URL, name + return ic.tail.prev.Import.URL, ic.tail.prev.ImportName() } func (ic *ImportChain) fetchOCISkeleton() error { @@ -257,10 +255,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { return err } - name := node.Name - if node.Import.ComponentName != "" { - name = node.Import.ComponentName - } + name := node.ImportName() componentDesc := manifest.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf("%s.tar", name))) @@ -271,7 +266,7 @@ func (ic *ImportChain) fetchOCISkeleton() error { var tb, dir string - // if there is not tarball to fetch, create a directory named based upon + // if there is not a tarball to fetch, create a directory named based upon // the import url and the component name if oci.IsEmptyDescriptor(componentDesc) { h := sha256.New() From dde69b977617a7a3aab146f7139d15185fe2dbab Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 17 Oct 2023 22:38:02 -0500 Subject: [PATCH 29/70] cleanup Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 8690c1afec..8ea41bd844 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -241,10 +241,10 @@ func (ic *ImportChain) OCIImportDefinition() (string, string) { } func (ic *ImportChain) fetchOCISkeleton() error { - node := ic.tail.prev if !ic.ContainsOCIImport() { return nil } + node := ic.tail.prev remote, err := ic.getRemote(node.Import.URL) if err != nil { return err @@ -318,9 +318,8 @@ func (ic *ImportChain) fetchOCISkeleton() error { if err != nil { return err } - // this node is the node importing the remote component - // and has a filepath relative to the head of the import chain - // the next (tail) node will have a filepath relative from cwd to the tarball in cache + // the tail node is the only node whose relativeToHead is based solely upon cwd<->cache + // contrary to the other nodes, which are based upon the previous node ic.tail.relativeToHead = rel if oci.IsEmptyDescriptor(componentDesc) { From aacc98a900ca5d8db2ed0cdcf44184e4e7c15f04 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 17 Oct 2023 23:11:04 -0500 Subject: [PATCH 30/70] prevent circular imports Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 8ea41bd844..5ffef11461 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -92,7 +92,7 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) ic := &ImportChain{} - ic.append(head, "", nil, nil) + ic.append(head, ".", nil, nil) history := []string{} @@ -126,6 +126,17 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) if isLocal { history = append(history, node.Import.Path) relativeToHead := filepath.Join(history...) + + // prevent circular imports (including self-imports) + // this is O(n^2) but the import chain should be small + prev := node.prev + for prev != nil { + if prev.relativeToHead == relativeToHead { + return ic, fmt.Errorf("detected circular import chain: %s", strings.Join(history, " -> ")) + } + prev = prev.prev + } + // this assumes the composed package is following the zarf layout if err := utils.ReadYaml(filepath.Join(relativeToHead, layout.ZarfYAML), &pkg); err != nil { return ic, err From 84a6022f57fa910dd2ebaa985aff9c894d0d1731 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 17 Oct 2023 23:25:24 -0500 Subject: [PATCH 31/70] better validation error messages Signed-off-by: razzle --- src/config/lang/english.go | 5 +--- src/internal/packager/validate/validate.go | 33 ++++++++++++++-------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 1642e605d1..246ce74caf 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -587,10 +587,7 @@ const ( PkgValidateErrComponentReqGrouped = "component %s cannot be both required and grouped" PkgValidateErrComponentYOLO = "component %s incompatible with the online-only package flag (metadata.yolo): %w" PkgValidateErrConstant = "invalid package constant: %w" - PkgValidateErrImportPathInvalid = "invalid file path '%s' provided directory must contain a valid zarf.yaml file" - PkgValidateErrImportURLInvalid = "invalid url '%s' provided" - PkgValidateErrImportOptions = "imported package %s must have either a url or a path" - PkgValidateErrImportPathMissing = "imported package %s must include a path" + PkgValidateErrImportDefinition = "invalid imported defintion for %s: %s" PkgValidateErrInitNoYOLO = "sorry, you can't YOLO an init package" PkgValidateErrManifest = "invalid manifest definition: %w" PkgValidateErrManifestFileOrKustomize = "manifest %s must have at least one file or kustomization" diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go index dbb8973043..854811ff0c 100644 --- a/src/internal/packager/validate/validate.go +++ b/src/internal/packager/validate/validate.go @@ -6,6 +6,7 @@ package validate import ( "fmt" + "path/filepath" "regexp" "strings" @@ -69,22 +70,32 @@ func ImportDefinition(component *types.ZarfComponent) error { path := component.Import.Path url := component.Import.URL - if url == "" { - // ensure path is not empty - if path == "" { - return fmt.Errorf(lang.PkgValidateErrImportPathMissing, component.Name) - } - } else { - // ensure path is empty - if path != "" { - return fmt.Errorf(lang.PkgValidateErrImportOptions, component.Name) + // ensure path or url is provided + if path == "" && url == "" { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "neither a path nor a URL was provided") + } + + // ensure path and url are not both provided + if path != "" && url != "" { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "both a path and a URL were provided") + } + + // validation for path + if url == "" && path != "" { + // ensure path is not an absolute path + if filepath.IsAbs(path) { + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name) } + } + + // validation for url + if url != "" && path == "" { ok := helpers.IsOCIURL(url) if !ok { - return fmt.Errorf(lang.PkgValidateErrImportURLInvalid, component.Import.URL) + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "URL is not a valid OCI URL") } if !strings.HasSuffix(url, oci.SkeletonSuffix) { - return fmt.Errorf("remote component %q does not have a %q suffix", url, oci.SkeletonSuffix) + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "OCI import URL must end with -skeleton") } } From 65005355699175f3e263ccf88080b8464021ec55 Mon Sep 17 00:00:00 2001 From: razzle Date: Tue, 17 Oct 2023 23:28:14 -0500 Subject: [PATCH 32/70] fix Signed-off-by: razzle --- src/internal/packager/validate/validate.go | 2 +- src/pkg/utils/helpers/url.go | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go index 854811ff0c..aaeb5aebca 100644 --- a/src/internal/packager/validate/validate.go +++ b/src/internal/packager/validate/validate.go @@ -84,7 +84,7 @@ func ImportDefinition(component *types.ZarfComponent) error { if url == "" && path != "" { // ensure path is not an absolute path if filepath.IsAbs(path) { - return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name) + return fmt.Errorf(lang.PkgValidateErrImportDefinition, component.Name, "path cannot be an absolute path") } } diff --git a/src/pkg/utils/helpers/url.go b/src/pkg/utils/helpers/url.go index d2df69f01f..d98ceb9f5b 100644 --- a/src/pkg/utils/helpers/url.go +++ b/src/pkg/utils/helpers/url.go @@ -12,8 +12,6 @@ import ( "path" "regexp" "strconv" - - "github.com/defenseunicorns/zarf/src/config/lang" ) // Nonstandard URL schemes or prefixes @@ -55,7 +53,7 @@ func DoHostnamesMatch(url1 string, url2 string) (bool, error) { // ExtractBasePathFromURL returns filename from URL string func ExtractBasePathFromURL(urlStr string) (string, error) { if !IsURL(urlStr) { - return "", fmt.Errorf(lang.PkgValidateErrImportURLInvalid, urlStr) + return "", fmt.Errorf("%s is not a valid URL", urlStr) } parsedURL, err := url.Parse(urlStr) if err != nil { From b274ff37e207ff99ed1a5b18b7bc28800f5e2f7f Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 18 Oct 2023 11:59:29 -0500 Subject: [PATCH 33/70] out with the old Signed-off-by: razzle --- src/pkg/packager/compose.go | 385 --------------------------------- src/pkg/packager/extensions.go | 10 - src/pkg/packager/variables.go | 28 --- 3 files changed, 423 deletions(-) diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 3492fa3f52..67e008f757 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -5,20 +5,8 @@ package packager import ( - "fmt" - "os" - "path/filepath" - "strings" - - "github.com/defenseunicorns/zarf/src/config" - "github.com/defenseunicorns/zarf/src/internal/packager/validate" - "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" - "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/composer" - "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" ) @@ -74,376 +62,3 @@ func (p *Packager) composeComponents() error { return nil } - -// getComposedComponent recursively retrieves a composed Zarf component -// -------------------------------------------------------------------- -// For composed components, we build the tree of components starting at the root and adding children as we go; -// this follows the composite design pattern outlined here: https://en.wikipedia.org/wiki/Composite_pattern -// where 1 component parent is made up of 0...n composite or leaf children. -func (p *Packager) getComposedComponent(parentComponent types.ZarfComponent) (child types.ZarfComponent, err error) { - // Make sure the component we're trying to import can't be accessed. - if err := validate.ImportDefinition(&parentComponent); err != nil { - return child, fmt.Errorf("invalid import definition in the %s component: %w", parentComponent.Name, err) - } - - // Keep track of the composed components import path to build nested composed components. - pathAncestry := "" - - // Get the component that we are trying to import. - // NOTE: This function is recursive and will continue getting the children until there are no more 'imported' components left. - child, err = p.getChildComponent(parentComponent, pathAncestry) - if err != nil { - return child, fmt.Errorf("unable to get child component: %w", err) - } - - // Merge the overrides from the child that we just received with the parent we were provided. - p.mergeComponentOverrides(&child, parentComponent) - - return -} - -func (p *Packager) getChildComponent(parent types.ZarfComponent, pathAncestry string) (child types.ZarfComponent, err error) { - // Figure out which component we are actually importing. - // NOTE: Default to the component name if a custom one was not provided. - childComponentName := parent.Import.ComponentName - if childComponentName == "" { - childComponentName = parent.Name - } - - var cachePath string - - subPkgPaths := layout.New(parent.Import.Path) - - if parent.Import.URL != "" { - if !strings.HasSuffix(parent.Import.URL, oci.SkeletonSuffix) { - return child, fmt.Errorf("import URL must be a 'skeleton' package: %s", parent.Import.URL) - } - - // Save all the OCI imported components into our build data - p.cfg.Pkg.Build.OCIImportedComponents[parent.Import.URL] = childComponentName - - skelURL := strings.TrimPrefix(parent.Import.URL, helpers.OCIURLPrefix) - cachePath = filepath.Join(config.GetAbsCachePath(), "oci", skelURL) - err = os.MkdirAll(cachePath, 0755) - if err != nil { - return child, fmt.Errorf("unable to create cache path %s: %w", cachePath, err) - } - - err = p.setOCIRemote(parent.Import.URL) - if err != nil { - return child, err - } - manifest, err := p.remote.FetchRoot() - if err != nil { - return child, err - } - tb := filepath.Join(layout.ComponentsDir, fmt.Sprintf("%s.tar", childComponentName)) - fetchedLayers, err := p.remote.PullPackage(cachePath, config.CommonOptions.OCIConcurrency, manifest.Locate(tb)) - if err != nil { - return child, fmt.Errorf("unable to pull skeleton from %s: %w", skelURL, err) - } - cwd, err := os.Getwd() - if err != nil { - return child, fmt.Errorf("unable to get current working directory: %w", err) - } - - rel, err := filepath.Rel(cwd, cachePath) - if err != nil { - return child, fmt.Errorf("unable to get relative path: %w", err) - } - parent.Import.Path = rel - subPkgPaths = layout.New(parent.Import.Path) - subPkgPaths.SetFromLayers(fetchedLayers) - } - - var subPkg types.ZarfPackage - if err := utils.ReadYaml(filepath.Join(pathAncestry, subPkgPaths.ZarfYAML), &subPkg); err != nil { - return child, err - } - - // Merge in child package variables (only if the variable does not exist in parent). - for _, importedVariable := range subPkg.Variables { - p.injectImportedVariable(importedVariable) - } - - // Merge in child package constants (only if the constant does not exist in parent). - for _, importedConstant := range subPkg.Constants { - p.injectImportedConstant(importedConstant) - } - - // Find the child component from the imported package that matches our arch. - for _, component := range subPkg.Components { - if component.Name == childComponentName { - filterArch := component.Only.Cluster.Architecture - - // Override the filter if it is set by the parent component. - if parent.Only.Cluster.Architecture != "" { - filterArch = parent.Only.Cluster.Architecture - } - - // Only add this component if it is valid for the target architecture. - if filterArch == "" || filterArch == p.arch { - child = component - break - } - } - } - - // If we didn't find a child component, bail. - if child.Name == "" { - return child, fmt.Errorf("unable to find the component %s in the imported package", childComponentName) - } - - // If it's OCI, we need to unpack the component tarball - if parent.Import.URL != "" { - parent.Import.Path = filepath.Join(parent.Import.Path, layout.ComponentsDir, child.Name) - if err := subPkgPaths.Components.Unarchive(child); err != nil { - if layout.IsNotLoaded(err) { - // If the tarball doesn't exist (skeleton component had no local resources), we need to create the directory anyways in case there are actions - _, err := subPkgPaths.Components.Create(child) - if err != nil { - return child, fmt.Errorf("unable to create composed component cache path %s: %w", cachePath, err) - } - } else { - return child, fmt.Errorf("unable to unarchive component: %w", err) - } - } - } - - pathAncestry = filepath.Join(pathAncestry, parent.Import.Path) - // Check if we need to get more of children. - if child.Import.Path != "" { - // Recursively call this function to get the next layer of children. - grandchildComponent, err := p.getChildComponent(child, pathAncestry) - if err != nil { - return child, err - } - - // Merge the grandchild values into the child. - p.mergeComponentOverrides(&grandchildComponent, child) - - // Set the grandchild as the child component now that we're done with recursively importing. - child = grandchildComponent - } else { - // Fix the filePaths of imported components to be accessible from our current location. - child, err = p.fixComposedFilepaths(pathAncestry, child) - if err != nil { - return child, fmt.Errorf("unable to fix composed filepaths: %s", err.Error()) - } - } - - // Migrate any deprecated component configurations now - var warnings []string - child, warnings = deprecated.MigrateComponent(p.cfg.Pkg.Build, child) - p.warnings = append(p.warnings, warnings...) - - return -} - -func (p *Packager) fixComposedFilepaths(pathAncestry string, child types.ZarfComponent) (types.ZarfComponent, error) { - for fileIdx, file := range child.Files { - composed := p.getComposedFilePath(pathAncestry, file.Source) - child.Files[fileIdx].Source = composed - } - - for chartIdx, chart := range child.Charts { - for valuesIdx, valuesFile := range chart.ValuesFiles { - composed := p.getComposedFilePath(pathAncestry, valuesFile) - child.Charts[chartIdx].ValuesFiles[valuesIdx] = composed - } - if child.Charts[chartIdx].LocalPath != "" { - composed := p.getComposedFilePath(pathAncestry, child.Charts[chartIdx].LocalPath) - child.Charts[chartIdx].LocalPath = composed - } - } - - for manifestIdx, manifest := range child.Manifests { - for fileIdx, file := range manifest.Files { - composed := p.getComposedFilePath(pathAncestry, file) - child.Manifests[manifestIdx].Files[fileIdx] = composed - } - for kustomizeIdx, kustomization := range manifest.Kustomizations { - composed := p.getComposedFilePath(pathAncestry, kustomization) - // kustomizations can use non-standard urls, so we need to check if the composed path exists on the local filesystem - abs, _ := filepath.Abs(composed) - invalid := utils.InvalidPath(abs) - if !invalid { - child.Manifests[manifestIdx].Kustomizations[kustomizeIdx] = composed - } - } - } - - for dataInjectionsIdx, dataInjection := range child.DataInjections { - composed := p.getComposedFilePath(pathAncestry, dataInjection.Source) - child.DataInjections[dataInjectionsIdx].Source = composed - } - - var err error - - if child.Actions.OnCreate.OnSuccess, err = p.fixComposedActionFilepaths(pathAncestry, child.Actions.OnCreate.OnSuccess); err != nil { - return child, err - } - if child.Actions.OnCreate.OnFailure, err = p.fixComposedActionFilepaths(pathAncestry, child.Actions.OnCreate.OnFailure); err != nil { - return child, err - } - if child.Actions.OnCreate.Before, err = p.fixComposedActionFilepaths(pathAncestry, child.Actions.OnCreate.Before); err != nil { - return child, err - } - if child.Actions.OnCreate.After, err = p.fixComposedActionFilepaths(pathAncestry, child.Actions.OnCreate.After); err != nil { - return child, err - } - - totalActions := len(child.Actions.OnCreate.OnSuccess) + len(child.Actions.OnCreate.OnFailure) + len(child.Actions.OnCreate.Before) + len(child.Actions.OnCreate.After) - - if totalActions > 0 { - composedDefaultDir := p.getComposedFilePath(pathAncestry, child.Actions.OnCreate.Defaults.Dir) - child.Actions.OnCreate.Defaults.Dir = composedDefaultDir - } - - if child.DeprecatedCosignKeyPath != "" { - composed := p.getComposedFilePath(pathAncestry, child.DeprecatedCosignKeyPath) - child.DeprecatedCosignKeyPath = composed - } - - child = p.composeExtensions(pathAncestry, child) - - return child, nil -} - -func (p *Packager) fixComposedActionFilepaths(pathAncestry string, actions []types.ZarfComponentAction) ([]types.ZarfComponentAction, error) { - for actionIdx, action := range actions { - if action.Dir != nil { - composedActionDir := p.getComposedFilePath(pathAncestry, *action.Dir) - actions[actionIdx].Dir = &composedActionDir - } - } - - return actions, nil -} - -// Sets Name, Default, Required and Description to the original components values. -func (p *Packager) mergeComponentOverrides(target *types.ZarfComponent, override types.ZarfComponent) { - target.Name = override.Name - target.Group = override.Group - target.Default = override.Default - target.Required = override.Required - - // Override description if it was provided. - if override.Description != "" { - target.Description = override.Description - } - - // Override cosign key path if it was provided. - if override.DeprecatedCosignKeyPath != "" { - target.DeprecatedCosignKeyPath = override.DeprecatedCosignKeyPath - } - - // Append slices where they exist. - target.DataInjections = append(target.DataInjections, override.DataInjections...) - target.Files = append(target.Files, override.Files...) - target.Images = append(target.Images, override.Images...) - target.Repos = append(target.Repos, override.Repos...) - - // Merge charts with the same name to keep them unique - for _, overrideChart := range override.Charts { - existing := false - for idx := range target.Charts { - if target.Charts[idx].Name == overrideChart.Name { - if overrideChart.Namespace != "" { - target.Charts[idx].Namespace = overrideChart.Namespace - } - if overrideChart.ReleaseName != "" { - target.Charts[idx].ReleaseName = overrideChart.ReleaseName - } - target.Charts[idx].ValuesFiles = append(target.Charts[idx].ValuesFiles, overrideChart.ValuesFiles...) - existing = true - } - } - - if !existing { - target.Charts = append(target.Charts, overrideChart) - } - } - - // Merge manifests with the same name to keep them unique - for _, overrideManifest := range override.Manifests { - existing := false - for idx := range target.Manifests { - if target.Manifests[idx].Name == overrideManifest.Name { - if overrideManifest.Namespace != "" { - target.Manifests[idx].Namespace = overrideManifest.Namespace - } - target.Manifests[idx].Files = append(target.Manifests[idx].Files, overrideManifest.Files...) - target.Manifests[idx].Kustomizations = append(target.Manifests[idx].Kustomizations, overrideManifest.Kustomizations...) - - existing = true - } - } - - if !existing { - target.Manifests = append(target.Manifests, overrideManifest) - } - } - - // Check for nil array - if override.Extensions.BigBang != nil { - if override.Extensions.BigBang.ValuesFiles != nil { - target.Extensions.BigBang.ValuesFiles = append(target.Extensions.BigBang.ValuesFiles, override.Extensions.BigBang.ValuesFiles...) - } - if override.Extensions.BigBang.FluxPatchFiles != nil { - target.Extensions.BigBang.FluxPatchFiles = append(target.Extensions.BigBang.FluxPatchFiles, override.Extensions.BigBang.FluxPatchFiles...) - } - } - - // Merge deprecated scripts for backwards compatibility with older zarf binaries. - target.DeprecatedScripts.Before = append(target.DeprecatedScripts.Before, override.DeprecatedScripts.Before...) - target.DeprecatedScripts.After = append(target.DeprecatedScripts.After, override.DeprecatedScripts.After...) - - if override.DeprecatedScripts.Retry { - target.DeprecatedScripts.Retry = true - } - if override.DeprecatedScripts.ShowOutput { - target.DeprecatedScripts.ShowOutput = true - } - if override.DeprecatedScripts.TimeoutSeconds > 0 { - target.DeprecatedScripts.TimeoutSeconds = override.DeprecatedScripts.TimeoutSeconds - } - - // Merge create actions. - target.Actions.OnCreate.Before = append(target.Actions.OnCreate.Before, override.Actions.OnCreate.Before...) - target.Actions.OnCreate.After = append(target.Actions.OnCreate.After, override.Actions.OnCreate.After...) - target.Actions.OnCreate.OnFailure = append(target.Actions.OnCreate.OnFailure, override.Actions.OnCreate.OnFailure...) - target.Actions.OnCreate.OnSuccess = append(target.Actions.OnCreate.OnSuccess, override.Actions.OnCreate.OnSuccess...) - - // Merge deploy actions. - target.Actions.OnDeploy.Before = append(target.Actions.OnDeploy.Before, override.Actions.OnDeploy.Before...) - target.Actions.OnDeploy.After = append(target.Actions.OnDeploy.After, override.Actions.OnDeploy.After...) - target.Actions.OnDeploy.OnFailure = append(target.Actions.OnDeploy.OnFailure, override.Actions.OnDeploy.OnFailure...) - target.Actions.OnDeploy.OnSuccess = append(target.Actions.OnDeploy.OnSuccess, override.Actions.OnDeploy.OnSuccess...) - - // Merge remove actions. - target.Actions.OnRemove.Before = append(target.Actions.OnRemove.Before, override.Actions.OnRemove.Before...) - target.Actions.OnRemove.After = append(target.Actions.OnRemove.After, override.Actions.OnRemove.After...) - target.Actions.OnRemove.OnFailure = append(target.Actions.OnRemove.OnFailure, override.Actions.OnRemove.OnFailure...) - target.Actions.OnRemove.OnSuccess = append(target.Actions.OnRemove.OnSuccess, override.Actions.OnRemove.OnSuccess...) - - // Merge Only filters. - target.Only.Cluster.Distros = append(target.Only.Cluster.Distros, override.Only.Cluster.Distros...) - if override.Only.Cluster.Architecture != "" { - target.Only.Cluster.Architecture = override.Only.Cluster.Architecture - } - if override.Only.LocalOS != "" { - target.Only.LocalOS = override.Only.LocalOS - } -} - -// Prefix file path with importPath if original file path is not a url. -func (p *Packager) getComposedFilePath(prefix string, path string) string { - // Return original if it is a remote file. - if helpers.IsURL(path) { - return path - } - - // Add prefix for local files. - return filepath.Join(prefix, path) -} diff --git a/src/pkg/packager/extensions.go b/src/pkg/packager/extensions.go index 3591e54a80..b5cc0ab17d 100644 --- a/src/pkg/packager/extensions.go +++ b/src/pkg/packager/extensions.go @@ -39,16 +39,6 @@ func (p *Packager) processExtensions() (err error) { return nil } -// Mutate any local files to be relative to the parent -func (p *Packager) composeExtensions(pathAncestry string, component types.ZarfComponent) types.ZarfComponent { - // Big Bang - if component.Extensions.BigBang != nil { - component = bigbang.Compose(pathAncestry, component) - } - - return component -} - // Check for any extensions in use and skeletonize their local files. func (p *Packager) skeletonizeExtensions() (err error) { components := []types.ZarfComponent{} diff --git a/src/pkg/packager/variables.go b/src/pkg/packager/variables.go index 620dd71f4c..f8f9dd20b4 100644 --- a/src/pkg/packager/variables.go +++ b/src/pkg/packager/variables.go @@ -126,34 +126,6 @@ func (p *Packager) setVariableInConfig(name, value string, sensitive bool, autoI } } -// injectImportedVariable determines if an imported package variable exists in the active config and adds it if not. -func (p *Packager) injectImportedVariable(importedVariable types.ZarfPackageVariable) { - presentInActive := false - for _, configVariable := range p.cfg.Pkg.Variables { - if configVariable.Name == importedVariable.Name { - presentInActive = true - } - } - - if !presentInActive { - p.cfg.Pkg.Variables = append(p.cfg.Pkg.Variables, importedVariable) - } -} - -// injectImportedConstant determines if an imported package constant exists in the active config and adds it if not. -func (p *Packager) injectImportedConstant(importedConstant types.ZarfPackageConstant) { - presentInActive := false - for _, configVariable := range p.cfg.Pkg.Constants { - if configVariable.Name == importedConstant.Name { - presentInActive = true - } - } - - if !presentInActive { - p.cfg.Pkg.Constants = append(p.cfg.Pkg.Constants, importedConstant) - } -} - // findComponentTemplatesAndReload appends ###ZARF_COMPONENT_NAME### for each component, assigns value, and reloads func (p *Packager) findComponentTemplatesAndReload() error { // iterate through components to and find all ###ZARF_COMPONENT_NAME, assign to component Name and value From 2db4a43c6a086466128d5e10c9a3a65f388fec24 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 18 Oct 2023 22:33:16 -0500 Subject: [PATCH 34/70] prevent nil pointer Signed-off-by: razzle --- src/pkg/packager/composer/extensions.go | 1 - src/pkg/packager/composer/list.go | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pkg/packager/composer/extensions.go b/src/pkg/packager/composer/extensions.go index bbc09233c4..cd0d029b4f 100644 --- a/src/pkg/packager/composer/extensions.go +++ b/src/pkg/packager/composer/extensions.go @@ -9,7 +9,6 @@ import ( "github.com/defenseunicorns/zarf/src/types" ) -// TODO: improve me func composeExtensions(c *types.ZarfComponent, override types.ZarfComponent, relativeTo string) { // fix the file paths if override.Extensions.BigBang != nil { diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 5ffef11461..2102478742 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -248,6 +248,9 @@ func (ic *ImportChain) ContainsOCIImport() bool { // OCIImportDefinition returns the url and name of the remote import func (ic *ImportChain) OCIImportDefinition() (string, string) { + if !ic.ContainsOCIImport() { + return "", "" + } return ic.tail.prev.Import.URL, ic.tail.prev.ImportName() } From c3ab1ac3ccbe54105206ab439305251b5fdb8d67 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 25 Oct 2023 15:00:28 -0500 Subject: [PATCH 35/70] Update src/pkg/packager/composer/extensions.go Co-authored-by: Wayne Starr --- src/pkg/packager/composer/extensions.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/pkg/packager/composer/extensions.go b/src/pkg/packager/composer/extensions.go index cd0d029b4f..b5d3577ad1 100644 --- a/src/pkg/packager/composer/extensions.go +++ b/src/pkg/packager/composer/extensions.go @@ -18,11 +18,16 @@ func composeExtensions(c *types.ZarfComponent, override types.ZarfComponent, rel // perform any overrides if override.Extensions.BigBang != nil { - if override.Extensions.BigBang.ValuesFiles != nil { - c.Extensions.BigBang.ValuesFiles = append(c.Extensions.BigBang.ValuesFiles, override.Extensions.BigBang.ValuesFiles...) - } - if override.Extensions.BigBang.FluxPatchFiles != nil { - c.Extensions.BigBang.FluxPatchFiles = append(c.Extensions.BigBang.FluxPatchFiles, override.Extensions.BigBang.FluxPatchFiles...) + if c.Extensions.BigBang == nil { + c.Extensions.BigBang = override.Extensions.BigBang + } else { + if override.Extensions.BigBang.ValuesFiles != nil { + c.Extensions.BigBang.ValuesFiles = append(c.Extensions.BigBang.ValuesFiles, override.Extensions.BigBang.ValuesFiles...) + message.Warnf("%s", c.Extensions.BigBang.ValuesFiles) + } + if override.Extensions.BigBang.FluxPatchFiles != nil { + c.Extensions.BigBang.FluxPatchFiles = append(c.Extensions.BigBang.FluxPatchFiles, override.Extensions.BigBang.FluxPatchFiles...) + } } } } From a9ba54be1f2f670c4f3c5ed20748763d543b356d Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 25 Oct 2023 15:02:51 -0500 Subject: [PATCH 36/70] Update src/pkg/packager/composer/list.go Co-authored-by: Wayne Starr --- src/pkg/packager/composer/list.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 2102478742..b710d3c782 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -363,13 +363,11 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { return composed, err } - // fix paths on the tail - fixPaths(&ic.tail.ZarfComponent, ic.tail.relativeToHead) - // save mutated tail - composed = ic.tail.ZarfComponent + // start with an empty component to compose into + composed = types.ZarfComponent{} - // dont compose 2x - node := ic.tail.prev + // start overriding with the tail node + node := ic.tail for node != nil { fixPaths(&node.ZarfComponent, node.relativeToHead) @@ -384,6 +382,9 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { node = node.prev } + node = node.prev + } + return composed, nil } From 384b825022e4f1c3ed939d1dda55f4a4fd401078 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 25 Oct 2023 15:03:18 -0500 Subject: [PATCH 37/70] fix merge Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index b710d3c782..ff10cd633e 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -382,9 +382,6 @@ func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { node = node.prev } - node = node.prev - } - return composed, nil } From 8a4b843278d6bb31a75ad7df55b417b02ebf91de Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 25 Oct 2023 15:03:53 -0500 Subject: [PATCH 38/70] fix merge Signed-off-by: razzle --- src/pkg/packager/composer/extensions.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/packager/composer/extensions.go b/src/pkg/packager/composer/extensions.go index b5d3577ad1..1fc6c35a33 100644 --- a/src/pkg/packager/composer/extensions.go +++ b/src/pkg/packager/composer/extensions.go @@ -6,6 +6,7 @@ package composer import ( "github.com/defenseunicorns/zarf/src/extensions/bigbang" + "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" ) From 5701fc74702c29118eac6d7462ebe5c6c544c222 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 25 Oct 2023 15:09:20 -0500 Subject: [PATCH 39/70] changes from review Signed-off-by: razzle --- src/pkg/packager/composer/list.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index ff10cd633e..dba19c1ebb 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -73,12 +73,8 @@ func (ic *ImportChain) append(c types.ZarfComponent, relativeToHead string, vars ic.head = node ic.tail = node } else { - p := ic.head - for p.next != nil { - p = p.next - } + p := ic.tail node.prev = p - p.next = node ic.tail = node } From 25f66976a062685071a86e596b839843d908a94d Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 25 Oct 2023 15:27:28 -0500 Subject: [PATCH 40/70] changes from review Signed-off-by: razzle --- src/pkg/packager/compose.go | 6 +- src/pkg/packager/composer/extensions.go | 2 - src/pkg/packager/composer/list.go | 130 --------------------- src/pkg/packager/composer/oci.go | 144 ++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 135 deletions(-) create mode 100644 src/pkg/packager/composer/oci.go diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 67e008f757..3011a3167d 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -19,9 +19,9 @@ func (p *Packager) composeComponents() error { for _, component := range p.cfg.Pkg.Components { arch := p.arch - // filter by architecture if set, otherwise use system architecture - if component.Only.Cluster.Architecture != "" { - arch = component.Only.Cluster.Architecture + // filter by architecture + if component.Only.Cluster.Architecture != "" && component.Only.Cluster.Architecture != arch { + continue } // build the import chain diff --git a/src/pkg/packager/composer/extensions.go b/src/pkg/packager/composer/extensions.go index 1fc6c35a33..9ef977ac21 100644 --- a/src/pkg/packager/composer/extensions.go +++ b/src/pkg/packager/composer/extensions.go @@ -6,7 +6,6 @@ package composer import ( "github.com/defenseunicorns/zarf/src/extensions/bigbang" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/types" ) @@ -24,7 +23,6 @@ func composeExtensions(c *types.ZarfComponent, override types.ZarfComponent, rel } else { if override.Extensions.BigBang.ValuesFiles != nil { c.Extensions.BigBang.ValuesFiles = append(c.Extensions.BigBang.ValuesFiles, override.Extensions.BigBang.ValuesFiles...) - message.Warnf("%s", c.Extensions.BigBang.ValuesFiles) } if override.Extensions.BigBang.FluxPatchFiles != nil { c.Extensions.BigBang.FluxPatchFiles = append(c.Extensions.BigBang.FluxPatchFiles, override.Extensions.BigBang.FluxPatchFiles...) diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index dba19c1ebb..436825feca 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -5,26 +5,17 @@ package composer import ( - "context" - "crypto/sha256" "fmt" - "os" "path/filepath" "strings" - "github.com/defenseunicorns/zarf/src/config" "github.com/defenseunicorns/zarf/src/internal/packager/validate" "github.com/defenseunicorns/zarf/src/pkg/layout" - "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/deprecated" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" - "github.com/mholt/archiver/v3" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2/content" - ocistore "oras.land/oras-go/v2/content/oci" ) // Node is a node in the import chain @@ -224,127 +215,6 @@ func (ic *ImportChain) Migrate(build types.ZarfBuildData) (warnings []string) { return warnings } -func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { - if ic.remote != nil { - return ic.remote, nil - } - var err error - ic.remote, err = oci.NewOrasRemote(url) - if err != nil { - return nil, err - } - return ic.remote, nil -} - -// ContainsOCIImport returns true if the import chain contains a remote import -func (ic *ImportChain) ContainsOCIImport() bool { - // only the 2nd to last node may have a remote import - return ic.tail.prev != nil && ic.tail.prev.Import.URL != "" -} - -// OCIImportDefinition returns the url and name of the remote import -func (ic *ImportChain) OCIImportDefinition() (string, string) { - if !ic.ContainsOCIImport() { - return "", "" - } - return ic.tail.prev.Import.URL, ic.tail.prev.ImportName() -} - -func (ic *ImportChain) fetchOCISkeleton() error { - if !ic.ContainsOCIImport() { - return nil - } - node := ic.tail.prev - remote, err := ic.getRemote(node.Import.URL) - if err != nil { - return err - } - - manifest, err := remote.FetchRoot() - if err != nil { - return err - } - - name := node.ImportName() - - componentDesc := manifest.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf("%s.tar", name))) - - cache := filepath.Join(config.GetAbsCachePath(), "oci") - if err := utils.CreateDirectory(cache, 0700); err != nil { - return err - } - - var tb, dir string - - // if there is not a tarball to fetch, create a directory named based upon - // the import url and the component name - if oci.IsEmptyDescriptor(componentDesc) { - h := sha256.New() - h.Write([]byte(node.Import.URL + name)) - id := fmt.Sprintf("%x", h.Sum(nil)) - - dir = filepath.Join(cache, "dirs", id) - - message.Debug("creating empty directory for remote component:", filepath.Join("", "oci", "dirs", id)) - } else { - tb = filepath.Join(cache, "blobs", "sha256", componentDesc.Digest.Encoded()) - dir = filepath.Join(cache, "dirs", componentDesc.Digest.Encoded()) - - store, err := ocistore.New(cache) - if err != nil { - return err - } - - ctx := context.TODO() - // ensure the tarball is in the cache - exists, err := store.Exists(ctx, componentDesc) - if err != nil { - return err - } else if !exists { - copyOpts := remote.CopyOpts - // TODO: investigate why the default FindSuccessors in CopyWithProgress is not working - copyOpts.FindSuccessors = content.Successors - if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, copyOpts, cache); err != nil { - return err - } - exists, err := store.Exists(ctx, componentDesc) - if err != nil { - return err - } else if !exists { - return fmt.Errorf("failed to fetch remote component: %+v", componentDesc) - } - } - } - - if err := utils.CreateDirectory(dir, 0700); err != nil { - return err - } - - cwd, err := os.Getwd() - if err != nil { - return err - } - rel, err := filepath.Rel(cwd, dir) - if err != nil { - return err - } - // the tail node is the only node whose relativeToHead is based solely upon cwd<->cache - // contrary to the other nodes, which are based upon the previous node - ic.tail.relativeToHead = rel - - if oci.IsEmptyDescriptor(componentDesc) { - // nothing was fetched, nothing to extract - return nil - } - - tu := archiver.Tar{ - OverwriteExisting: true, - // removes // from the paths - StripComponents: 1, - } - return tu.Unarchive(tb, dir) -} - // Compose merges the import chain into a single component // fixing paths, overriding metadata, etc func (ic *ImportChain) Compose() (composed types.ZarfComponent, err error) { diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go new file mode 100644 index 0000000000..afa8d2af9b --- /dev/null +++ b/src/pkg/packager/composer/oci.go @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package composer contains functions for composing components within Zarf packages. +package composer + +import ( + "context" + "crypto/sha256" + "fmt" + "os" + "path/filepath" + + "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/pkg/layout" + "github.com/defenseunicorns/zarf/src/pkg/message" + "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/mholt/archiver/v3" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" + ocistore "oras.land/oras-go/v2/content/oci" +) + +func (ic *ImportChain) getRemote(url string) (*oci.OrasRemote, error) { + if ic.remote != nil { + return ic.remote, nil + } + var err error + ic.remote, err = oci.NewOrasRemote(url) + if err != nil { + return nil, err + } + return ic.remote, nil +} + +// ContainsOCIImport returns true if the import chain contains a remote import +func (ic *ImportChain) ContainsOCIImport() bool { + // only the 2nd to last node may have a remote import + return ic.tail.prev != nil && ic.tail.prev.Import.URL != "" +} + +// OCIImportDefinition returns the url and name of the remote import +func (ic *ImportChain) OCIImportDefinition() (string, string) { + if !ic.ContainsOCIImport() { + return "", "" + } + return ic.tail.prev.Import.URL, ic.tail.prev.ImportName() +} + +func (ic *ImportChain) fetchOCISkeleton() error { + if !ic.ContainsOCIImport() { + return nil + } + node := ic.tail.prev + remote, err := ic.getRemote(node.Import.URL) + if err != nil { + return err + } + + manifest, err := remote.FetchRoot() + if err != nil { + return err + } + + name := node.ImportName() + + componentDesc := manifest.Locate(filepath.Join(layout.ComponentsDir, fmt.Sprintf("%s.tar", name))) + + cache := filepath.Join(config.GetAbsCachePath(), "oci") + if err := utils.CreateDirectory(cache, 0700); err != nil { + return err + } + + var tb, dir string + + // if there is not a tarball to fetch, create a directory named based upon + // the import url and the component name + if oci.IsEmptyDescriptor(componentDesc) { + h := sha256.New() + h.Write([]byte(node.Import.URL + name)) + id := fmt.Sprintf("%x", h.Sum(nil)) + + dir = filepath.Join(cache, "dirs", id) + + message.Debug("creating empty directory for remote component:", filepath.Join("", "oci", "dirs", id)) + } else { + tb = filepath.Join(cache, "blobs", "sha256", componentDesc.Digest.Encoded()) + dir = filepath.Join(cache, "dirs", componentDesc.Digest.Encoded()) + + store, err := ocistore.New(cache) + if err != nil { + return err + } + + ctx := context.TODO() + // ensure the tarball is in the cache + exists, err := store.Exists(ctx, componentDesc) + if err != nil { + return err + } else if !exists { + copyOpts := remote.CopyOpts + // TODO: investigate why the default FindSuccessors in CopyWithProgress is not working + copyOpts.FindSuccessors = content.Successors + if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, copyOpts, cache); err != nil { + return err + } + exists, err := store.Exists(ctx, componentDesc) + if err != nil { + return err + } else if !exists { + return fmt.Errorf("failed to fetch remote component: %+v", componentDesc) + } + } + } + + if err := utils.CreateDirectory(dir, 0700); err != nil { + return err + } + + cwd, err := os.Getwd() + if err != nil { + return err + } + rel, err := filepath.Rel(cwd, dir) + if err != nil { + return err + } + // the tail node is the only node whose relativeToHead is based solely upon cwd<->cache + // contrary to the other nodes, which are based upon the previous node + ic.tail.relativeToHead = rel + + if oci.IsEmptyDescriptor(componentDesc) { + // nothing was fetched, nothing to extract + return nil + } + + tu := archiver.Tar{ + OverwriteExisting: true, + // removes // from the paths + StripComponents: 1, + } + return tu.Unarchive(tb, dir) +} From 620feb896334484177c0a28f63d2ba63bbb81724 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 25 Oct 2023 16:24:19 -0500 Subject: [PATCH 41/70] remove oci differential components Signed-off-by: razzle --- src/pkg/packager/common.go | 4 -- src/pkg/packager/compose.go | 6 --- src/pkg/packager/composer/oci.go | 8 ---- src/pkg/packager/create.go | 43 ------------------- .../e2e/52_oci_compose_differential_test.go | 3 -- src/types/package.go | 1 - src/types/runtime.go | 1 - 7 files changed, 66 deletions(-) diff --git a/src/pkg/packager/common.go b/src/pkg/packager/common.go index 2adb24cc73..1d3c2f877a 100644 --- a/src/pkg/packager/common.go +++ b/src/pkg/packager/common.go @@ -94,10 +94,6 @@ func New(cfg *types.PackagerConfig, mods ...Modifier) (*Packager, error) { cfg.SetVariableMap = make(map[string]*types.ZarfSetVariable) } - if cfg.Pkg.Build.OCIImportedComponents == nil { - cfg.Pkg.Build.OCIImportedComponents = make(map[string]string) - } - var ( err error pkgr = &Packager{ diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index 3011a3167d..f4466f5ec5 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -31,12 +31,6 @@ func (p *Packager) composeComponents() error { } message.Debugf("%s", chain) - if chain.ContainsOCIImport() { - url, name := chain.OCIImportDefinition() - // Save all the OCI imported components into our build data - p.cfg.Pkg.Build.OCIImportedComponents[url] = name - } - // migrate any deprecated component configurations now warnings := chain.Migrate(p.cfg.Pkg.Build) p.warnings = append(p.warnings, warnings...) diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index afa8d2af9b..3cdafb51a8 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -40,14 +40,6 @@ func (ic *ImportChain) ContainsOCIImport() bool { return ic.tail.prev != nil && ic.tail.prev.Import.URL != "" } -// OCIImportDefinition returns the url and name of the remote import -func (ic *ImportChain) OCIImportDefinition() (string, string) { - if !ic.ContainsOCIImport() { - return "", "" - } - return ic.tail.prev.Import.URL, ic.tail.prev.ImportName() -} - func (ic *ImportChain) fetchOCISkeleton() error { if !ic.ContainsOCIImport() { return nil diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 064dd76a74..2b7a70d8b8 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -58,11 +58,6 @@ func (p *Packager) Create() (err error) { p.cfg.Pkg.Metadata.Version = config.CLIVersion } - // Before we compose the components (and render the imported OCI components), we need to remove any components that are not needed for a differential build - if err := p.removeDifferentialComponentsFromPackage(); err != nil { - return err - } - // Compose components into a single zarf.yaml file if err := p.composeComponents(); err != nil { return err @@ -667,44 +662,6 @@ func (p *Packager) loadDifferentialData() error { p.cfg.CreateOpts.DifferentialData.DifferentialImages = allIncludedImagesMap p.cfg.CreateOpts.DifferentialData.DifferentialRepos = allIncludedReposMap p.cfg.CreateOpts.DifferentialData.DifferentialPackageVersion = differentialZarfConfig.Metadata.Version - p.cfg.CreateOpts.DifferentialData.DifferentialOCIComponents = differentialZarfConfig.Build.OCIImportedComponents - - return nil -} - -// removeDifferentialComponentsFromPackage will remove unchanged OCI imported components from a differential package creation -func (p *Packager) removeDifferentialComponentsFromPackage() error { - // Remove components that were imported and already built into the reference package - if len(p.cfg.CreateOpts.DifferentialData.DifferentialOCIComponents) > 0 { - componentsToRemove := []int{} - - for idx, component := range p.cfg.Pkg.Components { - // if the component is imported from an OCI package and everything is the same, don't include this package - if helpers.IsOCIURL(component.Import.URL) { - if _, alsoExists := p.cfg.CreateOpts.DifferentialData.DifferentialOCIComponents[component.Import.URL]; alsoExists { - - // If the component spec is not empty, we will still include it in the differential package - // NOTE: We are ignoring fields that are not relevant to the differential build - if component.IsEmpty([]string{"Name", "Required", "Description", "Default", "Import"}) { - componentsToRemove = append(componentsToRemove, idx) - } - } - } - } - - // Remove the components that are already included (via OCI Import) in the reference package - if len(componentsToRemove) > 0 { - for i, componentIndex := range componentsToRemove { - indexToRemove := componentIndex - i - componentToRemove := p.cfg.Pkg.Components[indexToRemove] - - // If we are removing a component, add it to the build metadata and remove it from the list of OCI components for this package - p.cfg.Pkg.Build.DifferentialMissing = append(p.cfg.Pkg.Build.DifferentialMissing, componentToRemove.Name) - - p.cfg.Pkg.Components = append(p.cfg.Pkg.Components[:indexToRemove], p.cfg.Pkg.Components[indexToRemove+1:]...) - } - } - } return nil } diff --git a/src/test/e2e/52_oci_compose_differential_test.go b/src/test/e2e/52_oci_compose_differential_test.go index 78f6c815a8..5404fed7e5 100644 --- a/src/test/e2e/52_oci_compose_differential_test.go +++ b/src/test/e2e/52_oci_compose_differential_test.go @@ -92,8 +92,6 @@ func (suite *OCIDifferentialSuite) Test_0_Create_Differential_OCI() { // Check the metadata and build data for the normal package suite.Equal(normalZarfConfig.Metadata.Version, "v0.24.0") suite.False(normalZarfConfig.Build.Differential) - suite.Len(normalZarfConfig.Build.OCIImportedComponents, 1) - suite.Equal(normalZarfConfig.Build.OCIImportedComponents["oci://127.0.0.1:555/helm-charts:0.0.1-skeleton"], "demo-helm-oci-chart") // Check the component data for the normal package suite.Len(normalZarfConfig.Components, 3) @@ -111,7 +109,6 @@ func (suite *OCIDifferentialSuite) Test_0_Create_Differential_OCI() { suite.True(differentialZarfConfig.Build.Differential) suite.Len(differentialZarfConfig.Build.DifferentialMissing, 1) suite.Equal(differentialZarfConfig.Build.DifferentialMissing[0], "demo-helm-oci-chart") - suite.Len(differentialZarfConfig.Build.OCIImportedComponents, 0) // Check the component data for the differential package suite.Len(differentialZarfConfig.Components, 2) diff --git a/src/types/package.go b/src/types/package.go index b089081a82..6b44fe8a16 100644 --- a/src/types/package.go +++ b/src/types/package.go @@ -52,7 +52,6 @@ type ZarfBuildData struct { Differential bool `json:"differential,omitempty" jsonschema:"description=Whether this package was created with differential components"` RegistryOverrides map[string]string `json:"registryOverrides,omitempty" jsonschema:"description=Any registry domains that were overridden on package create when pulling images"` DifferentialMissing []string `json:"differentialMissing,omitempty" jsonschema:"description=List of components that were not included in this package due to differential packaging"` - OCIImportedComponents map[string]string `json:"OCIImportedComponents,omitempty" jsonschema:"description=Map of components that were imported via OCI. The keys are OCI Package URLs and values are the component names"` LastNonBreakingVersion string `json:"lastNonBreakingVersion,omitempty" jsonschema:"description=The minimum version of Zarf that does not have breaking package structure changes"` } diff --git a/src/types/runtime.go b/src/types/runtime.go index 65c347b882..2a29484f19 100644 --- a/src/types/runtime.go +++ b/src/types/runtime.go @@ -127,5 +127,4 @@ type DifferentialData struct { DifferentialPackageVersion string DifferentialImages map[string]bool DifferentialRepos map[string]bool - DifferentialOCIComponents map[string]string } From da8affcec06165d1c6da3946585c7df9bc70cb7e Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 25 Oct 2023 16:30:42 -0500 Subject: [PATCH 42/70] cleanup test Signed-off-by: razzle --- src/test/e2e/52_oci_compose_differential_test.go | 6 ++---- .../{52-oci-differential => 52-differential}/zarf.yaml | 5 ----- src/test/packages/52-oci-differential/README.md | 7 ------- 3 files changed, 2 insertions(+), 16 deletions(-) rename src/test/packages/{52-oci-differential => 52-differential}/zarf.yaml (77%) delete mode 100644 src/test/packages/52-oci-differential/README.md diff --git a/src/test/e2e/52_oci_compose_differential_test.go b/src/test/e2e/52_oci_compose_differential_test.go index 5404fed7e5..fa3d116c23 100644 --- a/src/test/e2e/52_oci_compose_differential_test.go +++ b/src/test/e2e/52_oci_compose_differential_test.go @@ -11,7 +11,6 @@ import ( "testing" "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/exec" "github.com/defenseunicorns/zarf/src/types" "github.com/mholt/archiver/v3" "github.com/stretchr/testify/require" @@ -31,7 +30,7 @@ var ( differentialPackageName = "" normalPackageName = "" examplePackagePath = filepath.Join("examples", "helm-charts") - anotherPackagePath = filepath.Join("src", "test", "packages", "52-oci-differential") + anotherPackagePath = filepath.Join("src", "test", "packages", "52-differential") ) func (suite *OCIDifferentialSuite) SetupSuite() { @@ -46,8 +45,7 @@ func (suite *OCIDifferentialSuite) SetupSuite() { } func (suite *OCIDifferentialSuite) TearDownSuite() { - _, _, err := exec.Cmd("docker", "rm", "-f", "registry") - suite.NoError(err) + e2e.TeardownRegistry(suite.T(), 555) } func (suite *OCIDifferentialSuite) Test_0_Create_Differential_OCI() { diff --git a/src/test/packages/52-oci-differential/zarf.yaml b/src/test/packages/52-differential/zarf.yaml similarity index 77% rename from src/test/packages/52-oci-differential/zarf.yaml rename to src/test/packages/52-differential/zarf.yaml index 75e194f5e6..ad44bf3f58 100644 --- a/src/test/packages/52-oci-differential/zarf.yaml +++ b/src/test/packages/52-differential/zarf.yaml @@ -5,11 +5,6 @@ metadata: version: "###ZARF_PKG_TMPL_PACKAGE_VERSION###" components: - - name: demo-helm-oci-chart - required: true - import: - url: oci://127.0.0.1:555/helm-charts:0.0.1-skeleton - - name: versioned-assets import: path: ../../packages/08-differential-package diff --git a/src/test/packages/52-oci-differential/README.md b/src/test/packages/52-oci-differential/README.md deleted file mode 100644 index 0da88acc22..0000000000 --- a/src/test/packages/52-oci-differential/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# OCI Differential - -This test package is used to test the functionality of creating a Zarf Package while using the `--differential` flag where one of the components in the package we're creating is a component that has been imported from an OCI registry. - -This test package demonstrates that OCI imported components will not be included during differential package creation and that the proper build metadata will be added to the finalized package to ensure users of the package know which OCI imported components were not included. - -This test also includes components for more standard differential package creation to make sure all of that expected functionality remains the same when there are also OCI imported components in the package. From 0fb1995749828b7abeeaad97a1afc680b0c74159 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 25 Oct 2023 16:40:13 -0500 Subject: [PATCH 43/70] remove todo Signed-off-by: razzle --- src/pkg/packager/compose.go | 3 +-- src/pkg/packager/composer/oci.go | 9 --------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/src/pkg/packager/compose.go b/src/pkg/packager/compose.go index f4466f5ec5..b390dfb5f4 100644 --- a/src/pkg/packager/compose.go +++ b/src/pkg/packager/compose.go @@ -47,8 +47,7 @@ func (p *Packager) composeComponents() error { pkgConsts = chain.MergeConstants(pkgConsts) } - // Update the parent package config with the expanded sub components. - // This is important when the deploy package is created. + // set the filtered + composed components p.cfg.Pkg.Components = components p.cfg.Pkg.Variables = pkgVars diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index 3cdafb51a8..a2782166a5 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -18,7 +18,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/mholt/archiver/v3" ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/v2/content" ocistore "oras.land/oras-go/v2/content/oci" ) @@ -92,17 +91,9 @@ func (ic *ImportChain) fetchOCISkeleton() error { return err } else if !exists { copyOpts := remote.CopyOpts - // TODO: investigate why the default FindSuccessors in CopyWithProgress is not working - copyOpts.FindSuccessors = content.Successors if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, copyOpts, cache); err != nil { return err } - exists, err := store.Exists(ctx, componentDesc) - if err != nil { - return err - } else if !exists { - return fmt.Errorf("failed to fetch remote component: %+v", componentDesc) - } } } From e160dd93cf4ccec73140764bdac756c5b1ba7582 Mon Sep 17 00:00:00 2001 From: razzle Date: Wed, 25 Oct 2023 16:44:22 -0500 Subject: [PATCH 44/70] cleanup helpers Signed-off-by: razzle --- src/pkg/utils/helpers/misc.go | 64 --------------------------------- src/pkg/utils/helpers/slice.go | 65 ++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 64 deletions(-) diff --git a/src/pkg/utils/helpers/misc.go b/src/pkg/utils/helpers/misc.go index 06f168c1d2..6031e21262 100644 --- a/src/pkg/utils/helpers/misc.go +++ b/src/pkg/utils/helpers/misc.go @@ -8,60 +8,9 @@ import ( "fmt" "reflect" "regexp" - "strings" "time" ) -// Unique returns a new slice with only unique elements. -func Unique[T comparable](s []T) (r []T) { - exists := make(map[T]bool) - for _, str := range s { - if _, ok := exists[str]; !ok { - exists[str] = true - r = append(r, str) - } - } - return r -} - -// Reverse returns a new slice with the elements in reverse order. -func Reverse[T any](s []T) (r []T) { - for i := len(s) - 1; i >= 0; i-- { - r = append(r, s[i]) - } - return r -} - -// Filter returns a new slice with only the elements that pass the test. -func Filter[T any](ss []T, test func(T) bool) (r []T) { - for _, s := range ss { - if test(s) { - r = append(r, s) - } - } - return r -} - -// Find returns the first element that passes the test. -func Find[T any](ss []T, test func(T) bool) (r T) { - for _, s := range ss { - if test(s) { - return s - } - } - return r -} - -// RemoveMatches removes the given element from the slice that matches the test. -func RemoveMatches[T any](ss []T, test func(T) bool) (r []T) { - for _, s := range ss { - if !test(s) { - r = append(r, s) - } - } - return r -} - // Retry will retry a function until it succeeds or the timeout is reached, timeout == retries * delay. func Retry(fn func() error, retries int, delay time.Duration) (err error) { for r := 0; r < retries; r++ { @@ -180,16 +129,3 @@ func MergeNonZero[T any](original T, overrides T) T { } return originalValue.Interface().(T) } - -// StringToSlice converts a comma-separated string to a slice of lowercase strings. -func StringToSlice(s string) []string { - if s != "" { - split := strings.Split(s, ",") - for idx, element := range split { - split[idx] = strings.ToLower(strings.TrimSpace(element)) - } - return split - } - - return []string{} -} diff --git a/src/pkg/utils/helpers/slice.go b/src/pkg/utils/helpers/slice.go index 989fde3d30..12b662b157 100644 --- a/src/pkg/utils/helpers/slice.go +++ b/src/pkg/utils/helpers/slice.go @@ -4,6 +4,71 @@ // Package helpers provides generic helper functions with no external imports package helpers +import "strings" + +// Unique returns a new slice with only unique elements. +func Unique[T comparable](s []T) (r []T) { + exists := make(map[T]bool) + for _, str := range s { + if _, ok := exists[str]; !ok { + exists[str] = true + r = append(r, str) + } + } + return r +} + +// Reverse returns a new slice with the elements in reverse order. +func Reverse[T any](s []T) (r []T) { + for i := len(s) - 1; i >= 0; i-- { + r = append(r, s[i]) + } + return r +} + +// Filter returns a new slice with only the elements that pass the test. +func Filter[T any](ss []T, test func(T) bool) (r []T) { + for _, s := range ss { + if test(s) { + r = append(r, s) + } + } + return r +} + +// Find returns the first element that passes the test. +func Find[T any](ss []T, test func(T) bool) (r T) { + for _, s := range ss { + if test(s) { + return s + } + } + return r +} + +// RemoveMatches removes the given element from the slice that matches the test. +func RemoveMatches[T any](ss []T, test func(T) bool) (r []T) { + for _, s := range ss { + if !test(s) { + r = append(r, s) + } + } + return r +} + +// StringToSlice converts a comma-separated string to a slice of lowercase strings. +func StringToSlice(s string) []string { + if s != "" { + split := strings.Split(s, ",") + for idx, element := range split { + split[idx] = strings.ToLower(strings.TrimSpace(element)) + } + return split + } + + return []string{} +} + // EqualFunc defines a type for a function that determines equality between two elements of type T. type EqualFunc[T any] func(a, b T) bool From 46f54b8d6c7a1c522e447a27240bc179a92de5e8 Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 26 Oct 2023 12:35:43 -0500 Subject: [PATCH 45/70] cleaner bb compose Signed-off-by: razzle --- src/extensions/bigbang/bigbang.go | 36 ++++++++++++++++++++----- src/pkg/packager/composer/extensions.go | 16 +---------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index c15fe80d5a..1ebe615916 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -291,16 +291,40 @@ func Skeletonize(tmpPaths *layout.ComponentPaths, c types.ZarfComponent) (types. return c, nil } -// Compose mutates a component so that the valuesFiles are relative to the parent importing component -func Compose(pathAncestry string, c types.ZarfComponent) types.ZarfComponent { +// Compose mutates a component so that its local paths are relative to the provided path +// +// additionally, it will merge any overrides +func Compose(c types.ZarfComponent, override types.ZarfComponent, relativeTo string) types.ZarfComponent { for valuesIdx, valuesFile := range c.Extensions.BigBang.ValuesFiles { - parentRel := filepath.Join(pathAncestry, valuesFile) - c.Extensions.BigBang.ValuesFiles[valuesIdx] = parentRel + if helpers.IsURL(valuesFile) { + continue + } + + fixed := filepath.Join(relativeTo, valuesFile) + c.Extensions.BigBang.ValuesFiles[valuesIdx] = fixed } for fluxPatchFileIdx, fluxPatchFile := range c.Extensions.BigBang.FluxPatchFiles { - parentRel := filepath.Join(pathAncestry, fluxPatchFile) - c.Extensions.BigBang.FluxPatchFiles[fluxPatchFileIdx] = parentRel + if helpers.IsURL(fluxPatchFile) { + continue + } + + fixed := filepath.Join(relativeTo, fluxPatchFile) + c.Extensions.BigBang.FluxPatchFiles[fluxPatchFileIdx] = fixed + } + + // perform any overrides + if override.Extensions.BigBang != nil { + if c.Extensions.BigBang == nil { + c.Extensions.BigBang = override.Extensions.BigBang + } else { + if override.Extensions.BigBang.ValuesFiles != nil { + c.Extensions.BigBang.ValuesFiles = append(c.Extensions.BigBang.ValuesFiles, override.Extensions.BigBang.ValuesFiles...) + } + if override.Extensions.BigBang.FluxPatchFiles != nil { + c.Extensions.BigBang.FluxPatchFiles = append(c.Extensions.BigBang.FluxPatchFiles, override.Extensions.BigBang.FluxPatchFiles...) + } + } } return c diff --git a/src/pkg/packager/composer/extensions.go b/src/pkg/packager/composer/extensions.go index 9ef977ac21..976e1bbb6d 100644 --- a/src/pkg/packager/composer/extensions.go +++ b/src/pkg/packager/composer/extensions.go @@ -12,21 +12,7 @@ import ( func composeExtensions(c *types.ZarfComponent, override types.ZarfComponent, relativeTo string) { // fix the file paths if override.Extensions.BigBang != nil { - component := bigbang.Compose(relativeTo, override) + component := bigbang.Compose(*c, override, relativeTo) c = &component } - - // perform any overrides - if override.Extensions.BigBang != nil { - if c.Extensions.BigBang == nil { - c.Extensions.BigBang = override.Extensions.BigBang - } else { - if override.Extensions.BigBang.ValuesFiles != nil { - c.Extensions.BigBang.ValuesFiles = append(c.Extensions.BigBang.ValuesFiles, override.Extensions.BigBang.ValuesFiles...) - } - if override.Extensions.BigBang.FluxPatchFiles != nil { - c.Extensions.BigBang.FluxPatchFiles = append(c.Extensions.BigBang.FluxPatchFiles, override.Extensions.BigBang.FluxPatchFiles...) - } - } - } } From 601e61f2ffa1b65efd230bf9ab73ad9bae06dd97 Mon Sep 17 00:00:00 2001 From: razzle Date: Thu, 26 Oct 2023 12:37:39 -0500 Subject: [PATCH 46/70] docs and schema Signed-off-by: razzle --- docs/3-create-a-zarf-package/4-zarf-schema.md | 39 ------------------- zarf.schema.json | 9 ----- 2 files changed, 48 deletions(-) diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index b0659ede67..ac63e9dff7 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -491,45 +491,6 @@ must respect the following conditions -
- - OCIImportedComponents - -  -
- - ## build > OCIImportedComponents - -**Description:** Map of components that were imported via OCI. The keys are OCI Package URLs and values are the component names - -| | | -| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| **Type** | `object` | -| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") | - -
- - Pattern Property .* - -  -
- -:::note -All properties whose name matches the regular expression -```.*``` ([Test](https://regex101.com/?regex=.%2A)) -must respect the following conditions -::: - -| | | -| -------- | -------- | -| **Type** | `string` | - -
-
- -
-
-
lastNonBreakingVersion diff --git a/zarf.schema.json b/zarf.schema.json index 483c37ea02..46343d8e8d 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -132,15 +132,6 @@ "type": "array", "description": "List of components that were not included in this package due to differential packaging" }, - "OCIImportedComponents": { - "patternProperties": { - ".*": { - "type": "string" - } - }, - "type": "object", - "description": "Map of components that were imported via OCI. The keys are OCI Package URLs and values are the component names" - }, "lastNonBreakingVersion": { "type": "string", "description": "The minimum version of Zarf that does not have breaking package structure changes" From 8dffec06c7356b74f3661d256861082a0b3dc652 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Thu, 26 Oct 2023 14:57:58 -0600 Subject: [PATCH 47/70] Fix fix paths for actions to properly mutate Dirs --- src/pkg/packager/composer/pathfixer.go | 53 +++++++++----------------- 1 file changed, 19 insertions(+), 34 deletions(-) diff --git a/src/pkg/packager/composer/pathfixer.go b/src/pkg/packager/composer/pathfixer.go index d1c45531c0..b52c2c3257 100644 --- a/src/pkg/packager/composer/pathfixer.go +++ b/src/pkg/packager/composer/pathfixer.go @@ -58,40 +58,11 @@ func fixPaths(child *types.ZarfComponent, relativeToHead string) { child.DataInjections[dataInjectionsIdx].Source = composed } - for actionIdx, action := range child.Actions.OnCreate.Before { - if action.Dir != nil { - composed := makePathRelativeTo(*action.Dir, relativeToHead) - child.Actions.OnCreate.Before[actionIdx].Dir = &composed - } - } - - for actionIdx, action := range child.Actions.OnCreate.After { - if action.Dir != nil { - composed := makePathRelativeTo(*action.Dir, relativeToHead) - child.Actions.OnCreate.After[actionIdx].Dir = &composed - } - } - - for actionIdx, action := range child.Actions.OnCreate.OnFailure { - if action.Dir != nil { - composed := makePathRelativeTo(*action.Dir, relativeToHead) - child.Actions.OnCreate.OnFailure[actionIdx].Dir = &composed - } - } - - for actionIdx, action := range child.Actions.OnCreate.OnSuccess { - if action.Dir != nil { - composed := makePathRelativeTo(*action.Dir, relativeToHead) - child.Actions.OnCreate.OnSuccess[actionIdx].Dir = &composed - } - } - - totalActions := len(child.Actions.OnCreate.Before) + len(child.Actions.OnCreate.After) + len(child.Actions.OnCreate.OnFailure) + len(child.Actions.OnCreate.OnSuccess) - - if totalActions > 0 { - composedDefaultDir := makePathRelativeTo(child.Actions.OnCreate.Defaults.Dir, relativeToHead) - child.Actions.OnCreate.Defaults.Dir = composedDefaultDir - } + defaultDir := child.Actions.OnCreate.Defaults.Dir + child.Actions.OnCreate.Before = fixActionPaths(child.Actions.OnCreate.Before, defaultDir, relativeToHead) + child.Actions.OnCreate.After = fixActionPaths(child.Actions.OnCreate.After, defaultDir, relativeToHead) + child.Actions.OnCreate.OnFailure = fixActionPaths(child.Actions.OnCreate.OnFailure, defaultDir, relativeToHead) + child.Actions.OnCreate.OnSuccess = fixActionPaths(child.Actions.OnCreate.OnSuccess, defaultDir, relativeToHead) // deprecated if child.DeprecatedCosignKeyPath != "" { @@ -99,3 +70,17 @@ func fixPaths(child *types.ZarfComponent, relativeToHead string) { child.DeprecatedCosignKeyPath = composed } } + +// fixActionPaths takes a slice of actions and mutates the Dir to be relative to the head node +func fixActionPaths(actions []types.ZarfComponentAction, defaultDir, relativeToHead string) []types.ZarfComponentAction { + for actionIdx, action := range actions { + var composed string + if action.Dir != nil { + composed = makePathRelativeTo(*action.Dir, relativeToHead) + } else { + composed = makePathRelativeTo(defaultDir, relativeToHead) + } + actions[actionIdx].Dir = &composed + } + return actions +} From 73a5678bc08bd6d5db97d50f7cdc2496d1deaf6a Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Thu, 26 Oct 2023 16:50:43 -0600 Subject: [PATCH 48/70] Add back copyOpts.FindSuccessors override --- src/pkg/packager/composer/oci.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/pkg/packager/composer/oci.go b/src/pkg/packager/composer/oci.go index a2782166a5..5a7b4637ed 100644 --- a/src/pkg/packager/composer/oci.go +++ b/src/pkg/packager/composer/oci.go @@ -18,6 +18,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/mholt/archiver/v3" ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2/content" ocistore "oras.land/oras-go/v2/content/oci" ) @@ -91,6 +92,8 @@ func (ic *ImportChain) fetchOCISkeleton() error { return err } else if !exists { copyOpts := remote.CopyOpts + // TODO (@WSTARR): This overrides the FindSuccessors function to no longer filter nodes when pulling which is necessary when caching - once we implement caching more thoroughly we will need to reevaluate this. + copyOpts.FindSuccessors = content.Successors if err := remote.CopyWithProgress([]ocispec.Descriptor{componentDesc}, store, copyOpts, cache); err != nil { return err } From 5e0d7aac8424019161a982d9ef8d297410404dc6 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Thu, 26 Oct 2023 17:44:33 -0600 Subject: [PATCH 49/70] Add an ADR draft --- adr/0021-composable-components.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 adr/0021-composable-components.md diff --git a/adr/0021-composable-components.md b/adr/0021-composable-components.md new file mode 100644 index 0000000000..eb26be6788 --- /dev/null +++ b/adr/0021-composable-components.md @@ -0,0 +1,23 @@ +# 21. Composable Components + +Date: 2023-10-26 + +## Status + +Accepted + +## Context + +Zarf has supports composing components together between packages on `zarf package create` since v0.16.0. This has allowed package creators to make more complex packages from smaller reusable bits. As this functionality grew however there were a few problems that developed: + +1. Import chains did not handle scaling to larger numbers of layers with test coverage usually only covering the first import. +2. When OCI skeletons were added they were largely bolted on after the fact without rethinking how they would impact composability. +3. Component filtering via the `only` filter was not implemented in a central location leading to bugs with create-time filters. + +## Decision + +We decided to separate composability into its own package that represents a composability import chain as a doubly linked list. This allows us to represent the whole chain as it exists relative to the "head" Zarf package (the definition that Zarf was asked to build) to more easily handle packages that are in different locations (such as OCI skeletons in one's cache). We also run the compose functions on all components so that the additional filter logic that is needed for these components can be handled more concisely and built upon (as it might for `flavor` https://github.com/defenseunicorns/zarf/issues/2101). + +## Consequences + +Maintaining the full context within a linked list does use more memory and some operations on it are less efficient than they could be if we one-shotted the compose. This is a decent tradeoff however as most import chains won't be longer than 4 or 5 elements in practice and these structs and operations are relatively small. From 6930d03de2803ed21827fe5ef28c43acd8ec766b Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Thu, 26 Oct 2023 17:50:18 -0600 Subject: [PATCH 50/70] Remove OCI differential tests --- .../e2e/52_oci_compose_differential_test.go | 128 ------------------ src/test/packages/52-differential/zarf.yaml | 16 --- 2 files changed, 144 deletions(-) delete mode 100644 src/test/e2e/52_oci_compose_differential_test.go delete mode 100644 src/test/packages/52-differential/zarf.yaml diff --git a/src/test/e2e/52_oci_compose_differential_test.go b/src/test/e2e/52_oci_compose_differential_test.go deleted file mode 100644 index fa3d116c23..0000000000 --- a/src/test/e2e/52_oci_compose_differential_test.go +++ /dev/null @@ -1,128 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: 2021-Present The Zarf Authors - -// Package test provides e2e tests for Zarf. -package test - -import ( - "fmt" - "os" - "path/filepath" - "testing" - - "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/types" - "github.com/mholt/archiver/v3" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/suite" - "oras.land/oras-go/v2/registry" -) - -// OCIDifferentialSuite validates that OCI imported components get handled correctly when performing a `zarf package create --differential` -type OCIDifferentialSuite struct { - suite.Suite - *require.Assertions - Reference registry.Reference - tmpdir string -} - -var ( - differentialPackageName = "" - normalPackageName = "" - examplePackagePath = filepath.Join("examples", "helm-charts") - anotherPackagePath = filepath.Join("src", "test", "packages", "52-differential") -) - -func (suite *OCIDifferentialSuite) SetupSuite() { - suite.tmpdir = suite.T().TempDir() - suite.Assertions = require.New(suite.T()) - - differentialPackageName = fmt.Sprintf("zarf-package-podinfo-with-oci-flux-%s-v0.24.0-differential-v0.25.0.tar.zst", e2e.Arch) - normalPackageName = fmt.Sprintf("zarf-package-podinfo-with-oci-flux-%s-v0.24.0.tar.zst", e2e.Arch) - - e2e.SetupDockerRegistry(suite.T(), 555) - suite.Reference.Registry = "localhost:555" -} - -func (suite *OCIDifferentialSuite) TearDownSuite() { - e2e.TeardownRegistry(suite.T(), 555) -} - -func (suite *OCIDifferentialSuite) Test_0_Create_Differential_OCI() { - suite.T().Log("E2E: Test Differential Packages w/ OCI Imports") - - // publish one of the example packages to the registry - stdOut, stdErr, err := e2e.Zarf("package", "publish", examplePackagePath, "oci://"+suite.Reference.String(), "--insecure") - suite.NoError(err, stdOut, stdErr) - - // build the package that we are going to publish - stdOut, stdErr, err = e2e.Zarf("package", "create", anotherPackagePath, "--insecure", "--set=PACKAGE_VERSION=v0.24.0", "-o", suite.tmpdir, "--confirm") - suite.NoError(err, stdOut, stdErr) - - // publish the package that we just built - normalPackagePath := filepath.Join(suite.tmpdir, normalPackageName) - stdOut, stdErr, err = e2e.Zarf("package", "publish", normalPackagePath, "oci://"+suite.Reference.String(), "--insecure") - suite.NoError(err, stdOut, stdErr) - - // Build without differential - stdOut, stdErr, err = e2e.Zarf("package", "create", anotherPackagePath, "--insecure", "--set=PACKAGE_VERSION=v0.25.0", "-o", suite.tmpdir, "--confirm") - suite.NoError(err, stdOut, stdErr) - - // Extract and load the zarf.yaml config for the normally built package - err = archiver.Extract(filepath.Join(suite.tmpdir, normalPackageName), "zarf.yaml", suite.tmpdir) - suite.NoError(err, "unable to extract zarf.yaml from the differential git package") - var normalZarfConfig types.ZarfPackage - err = utils.ReadYaml(filepath.Join(suite.tmpdir, "zarf.yaml"), &normalZarfConfig) - suite.NoError(err, "unable to read zarf.yaml from the differential git package") - os.Remove(filepath.Join(suite.tmpdir, "zarf.yaml")) - - stdOut, stdErr, err = e2e.Zarf("package", "create", anotherPackagePath, "--differential", "oci://"+suite.Reference.String()+"/podinfo-with-oci-flux:v0.24.0-amd64", "--insecure", "--set=PACKAGE_VERSION=v0.25.0", "-o", suite.tmpdir, "--confirm") - suite.NoError(err, stdOut, stdErr) - - // Extract and load the zarf.yaml config for the differentially built package - err = archiver.Extract(filepath.Join(suite.tmpdir, differentialPackageName), "zarf.yaml", suite.tmpdir) - suite.NoError(err, "unable to extract zarf.yaml from the differential git package") - var differentialZarfConfig types.ZarfPackage - err = utils.ReadYaml(filepath.Join(suite.tmpdir, "zarf.yaml"), &differentialZarfConfig) - suite.NoError(err, "unable to read zarf.yaml from the differential git package") - - /* Perform a bunch of asserts around the non-differential package */ - // Check the metadata and build data for the normal package - suite.Equal(normalZarfConfig.Metadata.Version, "v0.24.0") - suite.False(normalZarfConfig.Build.Differential) - - // Check the component data for the normal package - suite.Len(normalZarfConfig.Components, 3) - suite.Equal(normalZarfConfig.Components[0].Name, "demo-helm-oci-chart") - suite.Equal(normalZarfConfig.Components[0].Charts[0].URL, "oci://ghcr.io/stefanprodan/charts/podinfo") - suite.Equal(normalZarfConfig.Components[0].Images[0], "ghcr.io/stefanprodan/podinfo:6.4.0") - suite.Len(normalZarfConfig.Components[1].Images, 2) - suite.Len(normalZarfConfig.Components[1].Repos, 2) - suite.Len(normalZarfConfig.Components[2].Images, 1) - suite.Len(normalZarfConfig.Components[2].Repos, 3) - - /* Perform a bunch of asserts around the differential package */ - // Check the metadata and build data for the differential package - suite.Equal(differentialZarfConfig.Metadata.Version, "v0.25.0") - suite.True(differentialZarfConfig.Build.Differential) - suite.Len(differentialZarfConfig.Build.DifferentialMissing, 1) - suite.Equal(differentialZarfConfig.Build.DifferentialMissing[0], "demo-helm-oci-chart") - - // Check the component data for the differential package - suite.Len(differentialZarfConfig.Components, 2) - suite.Equal(differentialZarfConfig.Components[0].Name, "versioned-assets") - suite.Len(differentialZarfConfig.Components[0].Images, 1) - suite.Equal(differentialZarfConfig.Components[0].Images[0], "ghcr.io/defenseunicorns/zarf/agent:v0.25.0") - suite.Len(differentialZarfConfig.Components[0].Repos, 1) - suite.Equal(differentialZarfConfig.Components[0].Repos[0], "https://github.com/defenseunicorns/zarf.git@refs/tags/v0.25.0") - suite.Len(differentialZarfConfig.Components[1].Images, 1) - suite.Len(differentialZarfConfig.Components[1].Repos, 3) - suite.Equal(differentialZarfConfig.Components[1].Images[0], "ghcr.io/stefanprodan/podinfo:latest") - suite.Equal(differentialZarfConfig.Components[1].Repos[0], "https://github.com/stefanprodan/podinfo.git") -} - -func TestOCIDifferentialSuite(t *testing.T) { - e2e.SetupWithCluster(t) - - suite.Run(t, new(OCIDifferentialSuite)) -} diff --git a/src/test/packages/52-differential/zarf.yaml b/src/test/packages/52-differential/zarf.yaml deleted file mode 100644 index ad44bf3f58..0000000000 --- a/src/test/packages/52-differential/zarf.yaml +++ /dev/null @@ -1,16 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: podinfo-with-oci-flux - description: Deploy flux and then podinfo via flux - version: "###ZARF_PKG_TMPL_PACKAGE_VERSION###" - -components: - - name: versioned-assets - import: - path: ../../packages/08-differential-package - name: versioned-assets - - - name: generalized-assets - import: - path: ../../packages/08-differential-package - name: generalized-assets From ad4f8e003644f365729167f66df2e67833b34d96 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 27 Oct 2023 17:12:42 -0600 Subject: [PATCH 51/70] actually make extensions work --- src/extensions/bigbang/bigbang.go | 42 ++- src/pkg/layout/package_test.go | 2 +- src/pkg/packager/composer/extensions.go | 6 +- src/pkg/packager/composer/list.go | 2 +- src/pkg/packager/composer/list_test.go | 361 ++++++++++++++++++++++++ 5 files changed, 382 insertions(+), 31 deletions(-) create mode 100644 src/pkg/packager/composer/list_test.go diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index 1ebe615916..1050cc1d1f 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -294,40 +294,34 @@ func Skeletonize(tmpPaths *layout.ComponentPaths, c types.ZarfComponent) (types. // Compose mutates a component so that its local paths are relative to the provided path // // additionally, it will merge any overrides -func Compose(c types.ZarfComponent, override types.ZarfComponent, relativeTo string) types.ZarfComponent { - for valuesIdx, valuesFile := range c.Extensions.BigBang.ValuesFiles { - if helpers.IsURL(valuesFile) { - continue +func Compose(c *types.ZarfComponent, override types.ZarfComponent, relativeTo string) { + // perform any overrides + if override.Extensions.BigBang != nil { + for valuesIdx, valuesFile := range override.Extensions.BigBang.ValuesFiles { + if helpers.IsURL(valuesFile) { + continue + } + + fixed := filepath.Join(relativeTo, valuesFile) + override.Extensions.BigBang.ValuesFiles[valuesIdx] = fixed } - fixed := filepath.Join(relativeTo, valuesFile) - c.Extensions.BigBang.ValuesFiles[valuesIdx] = fixed - } + for fluxPatchFileIdx, fluxPatchFile := range override.Extensions.BigBang.FluxPatchFiles { + if helpers.IsURL(fluxPatchFile) { + continue + } - for fluxPatchFileIdx, fluxPatchFile := range c.Extensions.BigBang.FluxPatchFiles { - if helpers.IsURL(fluxPatchFile) { - continue + fixed := filepath.Join(relativeTo, fluxPatchFile) + override.Extensions.BigBang.FluxPatchFiles[fluxPatchFileIdx] = fixed } - fixed := filepath.Join(relativeTo, fluxPatchFile) - c.Extensions.BigBang.FluxPatchFiles[fluxPatchFileIdx] = fixed - } - - // perform any overrides - if override.Extensions.BigBang != nil { if c.Extensions.BigBang == nil { c.Extensions.BigBang = override.Extensions.BigBang } else { - if override.Extensions.BigBang.ValuesFiles != nil { - c.Extensions.BigBang.ValuesFiles = append(c.Extensions.BigBang.ValuesFiles, override.Extensions.BigBang.ValuesFiles...) - } - if override.Extensions.BigBang.FluxPatchFiles != nil { - c.Extensions.BigBang.FluxPatchFiles = append(c.Extensions.BigBang.FluxPatchFiles, override.Extensions.BigBang.FluxPatchFiles...) - } + c.Extensions.BigBang.ValuesFiles = append(c.Extensions.BigBang.ValuesFiles, override.Extensions.BigBang.ValuesFiles...) + c.Extensions.BigBang.FluxPatchFiles = append(c.Extensions.BigBang.FluxPatchFiles, override.Extensions.BigBang.FluxPatchFiles...) } } - - return c } // isValidVersion check if the version is 1.54.0 or greater. diff --git a/src/pkg/layout/package_test.go b/src/pkg/layout/package_test.go index c009496581..dc98286775 100644 --- a/src/pkg/layout/package_test.go +++ b/src/pkg/layout/package_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestPackage_Files(t *testing.T) { +func TestPackageFiles(t *testing.T) { pp := New("test") raw := &PackagePaths{ diff --git a/src/pkg/packager/composer/extensions.go b/src/pkg/packager/composer/extensions.go index 976e1bbb6d..77ac73c038 100644 --- a/src/pkg/packager/composer/extensions.go +++ b/src/pkg/packager/composer/extensions.go @@ -10,9 +10,5 @@ import ( ) func composeExtensions(c *types.ZarfComponent, override types.ZarfComponent, relativeTo string) { - // fix the file paths - if override.Extensions.BigBang != nil { - component := bigbang.Compose(*c, override, relativeTo) - c = &component - } + bigbang.Compose(c, override, relativeTo) } diff --git a/src/pkg/packager/composer/list.go b/src/pkg/packager/composer/list.go index 436825feca..0db1aaffb0 100644 --- a/src/pkg/packager/composer/list.go +++ b/src/pkg/packager/composer/list.go @@ -116,7 +116,7 @@ func NewImportChain(head types.ZarfComponent, arch string) (*ImportChain, error) // prevent circular imports (including self-imports) // this is O(n^2) but the import chain should be small - prev := node.prev + prev := node for prev != nil { if prev.relativeToHead == relativeToHead { return ic, fmt.Errorf("detected circular import chain: %s", strings.Join(history, " -> ")) diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go new file mode 100644 index 0000000000..8a0c9981c1 --- /dev/null +++ b/src/pkg/packager/composer/list_test.go @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package composer contains functions for composing components within Zarf packages. +package composer + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/defenseunicorns/zarf/src/types" + "github.com/defenseunicorns/zarf/src/types/extensions" + "github.com/stretchr/testify/require" +) + +func TestNewImportChain(t *testing.T) { + t.Parallel() + + type testCase struct { + name string + head types.ZarfComponent + arch string + expectedErrorMessage string + } + + testCases := []testCase{ + { + name: "No Architecture", + head: types.ZarfComponent{}, + expectedErrorMessage: "architecture must be provided", + }, + { + name: "Circular Import", + head: types.ZarfComponent{ + Import: types.ZarfComponentImport{ + Path: ".", + }, + }, + arch: "amd64", + expectedErrorMessage: "detected circular import chain", + }, + } + + for _, testCase := range testCases { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + _, err := NewImportChain(testCase.head, testCase.arch) + require.Contains(t, err.Error(), testCase.expectedErrorMessage) + }) + } +} + +func TestCompose(t *testing.T) { + t.Parallel() + + type testCase struct { + name string + ic *ImportChain + returnError bool + expectedComposed types.ZarfComponent + expectedErrorMessage string + } + + importDirectory := "hello" + currentDirectory := "." + + testCases := []testCase{ + { + name: "Single Component", + ic: createChainFromSlice([]types.ZarfComponent{ + { + Name: "no-import", + }, + }), + returnError: false, + expectedComposed: types.ZarfComponent{ + Name: "no-import", + }, + }, + { + name: "Multiple Components", + ic: createChainFromSlice([]types.ZarfComponent{ + { + Name: "import-hello", + Import: types.ZarfComponentImport{ + Path: importDirectory, + ComponentName: "end-world", + }, + Files: []types.ZarfFile{ + { + Source: "hello.txt", + }, + }, + Charts: []types.ZarfChart{ + { + Name: "hello", + LocalPath: "chart", + ValuesFiles: []string{ + "values.yaml", + }, + }, + }, + Manifests: []types.ZarfManifest{ + { + Name: "hello", + Files: []string{ + "manifest.yaml", + }, + }, + }, + DataInjections: []types.ZarfDataInjection{ + { + Source: "hello", + }, + }, + Actions: types.ZarfComponentActions{ + OnCreate: types.ZarfComponentActionSet{ + Before: []types.ZarfComponentAction{ + { + Cmd: "hello", + }, + }, + After: []types.ZarfComponentAction{ + { + Cmd: "hello", + }, + }, + OnSuccess: []types.ZarfComponentAction{ + { + Cmd: "hello", + }, + }, + OnFailure: []types.ZarfComponentAction{ + { + Cmd: "hello", + }, + }, + }, + }, + Extensions: extensions.ZarfComponentExtensions{ + BigBang: &extensions.BigBang{ + ValuesFiles: []string{ + "values.yaml", + }, + FluxPatchFiles: []string{ + "patch.yaml", + }, + }, + }, + }, + { + Name: "end-world", + Files: []types.ZarfFile{ + { + Source: "world.txt", + }, + }, + Charts: []types.ZarfChart{ + { + Name: "world", + LocalPath: "chart", + ValuesFiles: []string{ + "values.yaml", + }, + }, + }, + Manifests: []types.ZarfManifest{ + { + Name: "world", + Files: []string{ + "manifest.yaml", + }, + }, + }, + DataInjections: []types.ZarfDataInjection{ + { + Source: "world", + }, + }, + Actions: types.ZarfComponentActions{ + OnCreate: types.ZarfComponentActionSet{ + Before: []types.ZarfComponentAction{ + { + Cmd: "world", + }, + }, + After: []types.ZarfComponentAction{ + { + Cmd: "world", + }, + }, + OnSuccess: []types.ZarfComponentAction{ + { + Cmd: "world", + }, + }, + OnFailure: []types.ZarfComponentAction{ + { + Cmd: "world", + }, + }, + }, + }, + Extensions: extensions.ZarfComponentExtensions{ + BigBang: &extensions.BigBang{ + ValuesFiles: []string{ + "values.yaml", + }, + FluxPatchFiles: []string{ + "patch.yaml", + }, + }, + }, + }, + }), + returnError: false, + expectedComposed: types.ZarfComponent{ + Name: "import-hello", + Files: []types.ZarfFile{ + { + Source: fmt.Sprintf("%s/world.txt", importDirectory), + }, + { + Source: "hello.txt", + }, + }, + Charts: []types.ZarfChart{ + { + Name: "world", + LocalPath: fmt.Sprintf("%s/chart", importDirectory), + ValuesFiles: []string{ + fmt.Sprintf("%s/values.yaml", importDirectory), + }, + }, + { + Name: "hello", + LocalPath: "chart", + ValuesFiles: []string{ + "values.yaml", + }, + }, + }, + Manifests: []types.ZarfManifest{ + { + Name: "world", + Files: []string{ + fmt.Sprintf("%s/manifest.yaml", importDirectory), + }, + }, + { + Name: "hello", + Files: []string{ + "manifest.yaml", + }, + }, + }, + DataInjections: []types.ZarfDataInjection{ + { + Source: fmt.Sprintf("%s/world", importDirectory), + }, + { + Source: "hello", + }, + }, + Actions: types.ZarfComponentActions{ + OnCreate: types.ZarfComponentActionSet{ + Before: []types.ZarfComponentAction{ + { + Cmd: "world", + Dir: &importDirectory, + }, + { + Cmd: "hello", + Dir: ¤tDirectory, + }, + }, + After: []types.ZarfComponentAction{ + { + Cmd: "world", + Dir: &importDirectory, + }, + { + Cmd: "hello", + Dir: ¤tDirectory, + }, + }, + OnSuccess: []types.ZarfComponentAction{ + { + Cmd: "world", + Dir: &importDirectory, + }, + { + Cmd: "hello", + Dir: ¤tDirectory, + }, + }, + OnFailure: []types.ZarfComponentAction{ + { + Cmd: "world", + Dir: &importDirectory, + }, + { + Cmd: "hello", + Dir: ¤tDirectory, + }, + }, + }, + }, + Extensions: extensions.ZarfComponentExtensions{ + BigBang: &extensions.BigBang{ + ValuesFiles: []string{ + fmt.Sprintf("%s/values.yaml", importDirectory), + "values.yaml", + }, + FluxPatchFiles: []string{ + fmt.Sprintf("%s/patch.yaml", importDirectory), + "patch.yaml", + }, + }, + }, + }, + }, + } + + for _, testCase := range testCases { + testCase := testCase + + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + composed, err := testCase.ic.Compose() + if testCase.returnError { + require.Contains(t, err.Error(), testCase.expectedErrorMessage) + } else { + require.EqualValues(t, testCase.expectedComposed, composed) + } + }) + } +} + +func createChainFromSlice(components []types.ZarfComponent) (ic *ImportChain) { + ic = &ImportChain{} + + if len(components) == 0 { + return ic + } + + ic.append(components[0], ".", nil, nil) + history := []string{} + + for idx := 1; idx < len(components); idx++ { + history = append(history, components[idx-1].Import.Path) + ic.append(components[idx], filepath.Join(history...), nil, nil) + } + + return ic +} From bb558ba084de55da2df672df7a9f261de7cb80b4 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Fri, 27 Oct 2023 17:40:40 -0600 Subject: [PATCH 52/70] Fixup and expand tests --- src/pkg/packager/composer/list_test.go | 272 +++++++++++-------------- 1 file changed, 121 insertions(+), 151 deletions(-) diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go index 8a0c9981c1..ffae3d773a 100644 --- a/src/pkg/packager/composer/list_test.go +++ b/src/pkg/packager/composer/list_test.go @@ -6,6 +6,7 @@ package composer import ( "fmt" + "os" "path/filepath" "testing" @@ -65,8 +66,10 @@ func TestCompose(t *testing.T) { expectedErrorMessage string } - importDirectory := "hello" currentDirectory := "." + firstDirectory := "hello" + secondDirectory := "world" + finalDirectory := filepath.Join(firstDirectory, secondDirectory) testCases := []testCase{ { @@ -84,145 +87,19 @@ func TestCompose(t *testing.T) { { name: "Multiple Components", ic: createChainFromSlice([]types.ZarfComponent{ - { - Name: "import-hello", - Import: types.ZarfComponentImport{ - Path: importDirectory, - ComponentName: "end-world", - }, - Files: []types.ZarfFile{ - { - Source: "hello.txt", - }, - }, - Charts: []types.ZarfChart{ - { - Name: "hello", - LocalPath: "chart", - ValuesFiles: []string{ - "values.yaml", - }, - }, - }, - Manifests: []types.ZarfManifest{ - { - Name: "hello", - Files: []string{ - "manifest.yaml", - }, - }, - }, - DataInjections: []types.ZarfDataInjection{ - { - Source: "hello", - }, - }, - Actions: types.ZarfComponentActions{ - OnCreate: types.ZarfComponentActionSet{ - Before: []types.ZarfComponentAction{ - { - Cmd: "hello", - }, - }, - After: []types.ZarfComponentAction{ - { - Cmd: "hello", - }, - }, - OnSuccess: []types.ZarfComponentAction{ - { - Cmd: "hello", - }, - }, - OnFailure: []types.ZarfComponentAction{ - { - Cmd: "hello", - }, - }, - }, - }, - Extensions: extensions.ZarfComponentExtensions{ - BigBang: &extensions.BigBang{ - ValuesFiles: []string{ - "values.yaml", - }, - FluxPatchFiles: []string{ - "patch.yaml", - }, - }, - }, - }, - { - Name: "end-world", - Files: []types.ZarfFile{ - { - Source: "world.txt", - }, - }, - Charts: []types.ZarfChart{ - { - Name: "world", - LocalPath: "chart", - ValuesFiles: []string{ - "values.yaml", - }, - }, - }, - Manifests: []types.ZarfManifest{ - { - Name: "world", - Files: []string{ - "manifest.yaml", - }, - }, - }, - DataInjections: []types.ZarfDataInjection{ - { - Source: "world", - }, - }, - Actions: types.ZarfComponentActions{ - OnCreate: types.ZarfComponentActionSet{ - Before: []types.ZarfComponentAction{ - { - Cmd: "world", - }, - }, - After: []types.ZarfComponentAction{ - { - Cmd: "world", - }, - }, - OnSuccess: []types.ZarfComponentAction{ - { - Cmd: "world", - }, - }, - OnFailure: []types.ZarfComponentAction{ - { - Cmd: "world", - }, - }, - }, - }, - Extensions: extensions.ZarfComponentExtensions{ - BigBang: &extensions.BigBang{ - ValuesFiles: []string{ - "values.yaml", - }, - FluxPatchFiles: []string{ - "patch.yaml", - }, - }, - }, - }, + createDummyComponent("hello", firstDirectory, "hello"), + createDummyComponent("world", secondDirectory, "world"), + createDummyComponent("today", "", "hello"), }), returnError: false, expectedComposed: types.ZarfComponent{ Name: "import-hello", Files: []types.ZarfFile{ { - Source: fmt.Sprintf("%s/world.txt", importDirectory), + Source: fmt.Sprintf("%s%stoday.txt", finalDirectory, string(os.PathSeparator)), + }, + { + Source: fmt.Sprintf("%s%sworld.txt", firstDirectory, string(os.PathSeparator)), }, { Source: "hello.txt", @@ -230,37 +107,42 @@ func TestCompose(t *testing.T) { }, Charts: []types.ZarfChart{ { - Name: "world", - LocalPath: fmt.Sprintf("%s/chart", importDirectory), + Name: "hello", + LocalPath: fmt.Sprintf("%s%schart", finalDirectory, string(os.PathSeparator)), ValuesFiles: []string{ - fmt.Sprintf("%s/values.yaml", importDirectory), + fmt.Sprintf("%s%svalues.yaml", finalDirectory, string(os.PathSeparator)), + "values.yaml", }, }, { - Name: "hello", - LocalPath: "chart", + Name: "world", + LocalPath: fmt.Sprintf("%s%schart", firstDirectory, string(os.PathSeparator)), ValuesFiles: []string{ - "values.yaml", + fmt.Sprintf("%s%svalues.yaml", firstDirectory, string(os.PathSeparator)), }, }, }, Manifests: []types.ZarfManifest{ { - Name: "world", + Name: "hello", Files: []string{ - fmt.Sprintf("%s/manifest.yaml", importDirectory), + fmt.Sprintf("%s%smanifest.yaml", finalDirectory, string(os.PathSeparator)), + "manifest.yaml", }, }, { - Name: "hello", + Name: "world", Files: []string{ - "manifest.yaml", + fmt.Sprintf("%s%smanifest.yaml", firstDirectory, string(os.PathSeparator)), }, }, }, DataInjections: []types.ZarfDataInjection{ { - Source: fmt.Sprintf("%s/world", importDirectory), + Source: fmt.Sprintf("%s%stoday", finalDirectory, string(os.PathSeparator)), + }, + { + Source: fmt.Sprintf("%s%sworld", firstDirectory, string(os.PathSeparator)), }, { Source: "hello", @@ -269,9 +151,13 @@ func TestCompose(t *testing.T) { Actions: types.ZarfComponentActions{ OnCreate: types.ZarfComponentActionSet{ Before: []types.ZarfComponentAction{ + { + Cmd: "today", + Dir: &finalDirectory, + }, { Cmd: "world", - Dir: &importDirectory, + Dir: &firstDirectory, }, { Cmd: "hello", @@ -279,9 +165,13 @@ func TestCompose(t *testing.T) { }, }, After: []types.ZarfComponentAction{ + { + Cmd: "today", + Dir: &finalDirectory, + }, { Cmd: "world", - Dir: &importDirectory, + Dir: &firstDirectory, }, { Cmd: "hello", @@ -289,9 +179,13 @@ func TestCompose(t *testing.T) { }, }, OnSuccess: []types.ZarfComponentAction{ + { + Cmd: "today", + Dir: &finalDirectory, + }, { Cmd: "world", - Dir: &importDirectory, + Dir: &firstDirectory, }, { Cmd: "hello", @@ -299,9 +193,13 @@ func TestCompose(t *testing.T) { }, }, OnFailure: []types.ZarfComponentAction{ + { + Cmd: "today", + Dir: &finalDirectory, + }, { Cmd: "world", - Dir: &importDirectory, + Dir: &firstDirectory, }, { Cmd: "hello", @@ -313,11 +211,13 @@ func TestCompose(t *testing.T) { Extensions: extensions.ZarfComponentExtensions{ BigBang: &extensions.BigBang{ ValuesFiles: []string{ - fmt.Sprintf("%s/values.yaml", importDirectory), + fmt.Sprintf("%s%svalues.yaml", finalDirectory, string(os.PathSeparator)), + fmt.Sprintf("%s%svalues.yaml", firstDirectory, string(os.PathSeparator)), "values.yaml", }, FluxPatchFiles: []string{ - fmt.Sprintf("%s/patch.yaml", importDirectory), + fmt.Sprintf("%s%spatch.yaml", finalDirectory, string(os.PathSeparator)), + fmt.Sprintf("%s%spatch.yaml", firstDirectory, string(os.PathSeparator)), "patch.yaml", }, }, @@ -359,3 +259,73 @@ func createChainFromSlice(components []types.ZarfComponent) (ic *ImportChain) { return ic } + +func createDummyComponent(name, importDir, subName string) types.ZarfComponent { + return types.ZarfComponent{ + Name: fmt.Sprintf("import-%s", name), + Import: types.ZarfComponentImport{ + Path: importDir, + }, + Files: []types.ZarfFile{ + { + Source: fmt.Sprintf("%s.txt", name), + }, + }, + Charts: []types.ZarfChart{ + { + Name: subName, + LocalPath: "chart", + ValuesFiles: []string{ + "values.yaml", + }, + }, + }, + Manifests: []types.ZarfManifest{ + { + Name: subName, + Files: []string{ + "manifest.yaml", + }, + }, + }, + DataInjections: []types.ZarfDataInjection{ + { + Source: name, + }, + }, + Actions: types.ZarfComponentActions{ + OnCreate: types.ZarfComponentActionSet{ + Before: []types.ZarfComponentAction{ + { + Cmd: name, + }, + }, + After: []types.ZarfComponentAction{ + { + Cmd: name, + }, + }, + OnSuccess: []types.ZarfComponentAction{ + { + Cmd: name, + }, + }, + OnFailure: []types.ZarfComponentAction{ + { + Cmd: name, + }, + }, + }, + }, + Extensions: extensions.ZarfComponentExtensions{ + BigBang: &extensions.BigBang{ + ValuesFiles: []string{ + "values.yaml", + }, + FluxPatchFiles: []string{ + "patch.yaml", + }, + }, + }, + } +} From 34537607ebd7f9ccaddc31dd250427dabd3e8155 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 30 Oct 2023 13:44:39 -0600 Subject: [PATCH 53/70] Initial laydown of improiving the compose example --- .../publish-application-packages.yml | 1 + examples/composable-packages/README.md | 23 ++---------- examples/composable-packages/zarf.yaml | 8 ++--- src/test/e2e/09_component_compose_test.go | 35 +++++++++++++++++++ 4 files changed, 42 insertions(+), 25 deletions(-) create mode 100644 src/test/e2e/09_component_compose_test.go diff --git a/.github/workflows/publish-application-packages.yml b/.github/workflows/publish-application-packages.yml index b79a455686..6245fd9f75 100644 --- a/.github/workflows/publish-application-packages.yml +++ b/.github/workflows/publish-application-packages.yml @@ -40,6 +40,7 @@ jobs: zarf package publish ./build/zarf-package-dos-games-amd64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages zarf package publish ./build/zarf-package-dos-games-arm64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages + zarf package publish examples/dos-games oci://ghcr.io/defenseunicorns/packages zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-amd64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-x86_64 zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-arm64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-aarch64 diff --git a/examples/composable-packages/README.md b/examples/composable-packages/README.md index 764242f40d..657fd03b60 100644 --- a/examples/composable-packages/README.md +++ b/examples/composable-packages/README.md @@ -2,13 +2,13 @@ import ExampleYAML from "@site/src/components/ExampleYAML"; # Composable Packages -This example demonstrates using Zarf to import components from existing Zarf package definitions while merging overrides to add or change functionality. It uses the existing [DOS games](../dos-games/README.md) and [WordPress](../wordpress/README.md) examples by simply adding `import` keys in the new [zarf.yaml](zarf.yaml) file. +This example demonstrates using Zarf to import components from existing Zarf package definitions while merging overrides to add or change functionality. It uses the existing [DOS games](../dos-games/README.md) example by simply adding `import` keys in the new [zarf.yaml](zarf.yaml) file. The `import` key in Zarf supports two modes to pull in a component: -1. The `path` key allows you to specify a path to a directory that contains the `zarf.yaml` that you wish to import on your local filesystem. This allows you to have a common component that you can reuse across multiple packages *within* a project. +1. The `path` key allows you to specify a path to a directory that contains the `zarf.yaml` that you wish to import on your local filesystem. This allows you to have a common component that you can reuse across multiple packages *within* a project (i.e. within one team/codebase). -2. The `url` key allows you to specify an `oci://` URL to a skeleton package that was published to an OCI registry. Skeleton packages are special package bundles that contain the `zarf.yaml` package definition and any local files referenced by that definition at publish time. This allows you to version a set of components and import them into multiple packages *across* projects. +2. The `url` key allows you to specify an `oci://` URL to a skeleton package that was published to an OCI registry. Skeleton packages are special package bundles that contain the `zarf.yaml` package definition and any local files referenced by that definition at publish time. This allows you to version a set of reusable components and import them into multiple packages *across* projects (i.e. across teams/codebases). :::tip @@ -40,23 +40,6 @@ To view the example in its entirety, select the `Edit this page` link below the ::: -:::note - -Creating this example requires a locally hosted container registry that has the `wordpress` skeleton package published and available. You can do this by running the following commands: - -```bash -docker run -d -p 555:5000 --restart=always --name registry registry:2 -zarf package publish examples/wordpress oci://127.0.0.1:555 --insecure -``` - -You will also need to pass the `--insecure` flag to `zarf package create` to pull from the `http` registry: - -```bash -zarf package create examples/composable-packages/ --insecure -``` - -::: - :::info diff --git a/examples/composable-packages/zarf.yaml b/examples/composable-packages/zarf.yaml index ab5890fddd..34f499d670 100644 --- a/examples/composable-packages/zarf.yaml +++ b/examples/composable-packages/zarf.yaml @@ -24,20 +24,18 @@ components: files: - quake-service.yaml - - name: oci-wordpress-url + - name: oci-games-url # The component logic keys ('required', 'group', and 'default') always override those of the imported package # required: false # the initial value overrides the child component # group: "" # the initial value overrides the child component # default: false # the initial value overrides the child component import: # The URL to the skeleton package containing this component's package definition - url: oci://localhost:555/wordpress:16.0.4-skeleton + url: oci://🦄/dos-games:1.0.0-skeleton # Example optional custom name to point to in the imported package (default is to use this component's name) - name: wordpress + name: baseline # Un'name'd Zarf primitives will be appended to the end of the primitive's list for that component. actions: onDeploy: before: - cmd: ./zarf tools kubectl get -n dos-games deployment -o jsonpath={.items[0].metadata.creationTimestamp} - setVariables: - - name: WORDPRESS_BLOG_NAME diff --git a/src/test/e2e/09_component_compose_test.go b/src/test/e2e/09_component_compose_test.go new file mode 100644 index 0000000000..dbcf3571ad --- /dev/null +++ b/src/test/e2e/09_component_compose_test.go @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package test provides e2e tests for Zarf. +package test + +import ( + "testing" + + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +type CompositionSuite struct { + suite.Suite + *require.Assertions +} + +func (suite *CompositionSuite) SetupSuite() { + suite.Assertions = require.New(suite.T()) +} + +// func (suite *CompositionSuite) TearDownSuite() { + +// } + +func (suite *SkeletonSuite) Test_0_ComposabilityExample() { + +} + +func TestCompositionSuite(t *testing.T) { + e2e.SetupWithCluster(t) + + suite.Run(t, new(CompositionSuite)) +} From ff11f7326a723c3a4a6de046efde88d50f223e0d Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 30 Oct 2023 13:48:11 -0600 Subject: [PATCH 54/70] Add public key to publish of tarball package --- .github/workflows/publish-application-packages.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-application-packages.yml b/.github/workflows/publish-application-packages.yml index 6245fd9f75..cf069c2a67 100644 --- a/.github/workflows/publish-application-packages.yml +++ b/.github/workflows/publish-application-packages.yml @@ -38,8 +38,8 @@ jobs: zarf package create -o build -a amd64 examples/dos-games --key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm zarf package create -o build -a arm64 examples/dos-games --key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm - zarf package publish ./build/zarf-package-dos-games-amd64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages - zarf package publish ./build/zarf-package-dos-games-arm64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages + zarf package publish ./build/zarf-package-dos-games-amd64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub + zarf package publish ./build/zarf-package-dos-games-arm64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub zarf package publish examples/dos-games oci://ghcr.io/defenseunicorns/packages zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-amd64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-x86_64 From 2b11dbf9b3eac68a261967cac6407f877707bc5e Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 30 Oct 2023 14:01:59 -0600 Subject: [PATCH 55/70] Update signing key flag and temporarily change publish workflow --- .../publish-application-packages.yml | 30 +++++++++++++------ .../100-cli-commands/zarf_package_create.md | 4 +-- src/cmd/package.go | 8 ++++- .../e2e/32_checksum_and_signature_test.go | 2 +- src/test/nightly/ecr_publish_test.go | 2 +- 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/.github/workflows/publish-application-packages.yml b/.github/workflows/publish-application-packages.yml index cf069c2a67..48559fcfd3 100644 --- a/.github/workflows/publish-application-packages.yml +++ b/.github/workflows/publish-application-packages.yml @@ -22,8 +22,17 @@ jobs: with: ref: ${{ github.event.inputs.branchName }} - - name: Install The Latest Release Version of Zarf - uses: defenseunicorns/setup-zarf@main + # - name: Install The Latest Release Version of Zarf + # uses: defenseunicorns/setup-zarf@main + + - name: Setup golang + uses: ./.github/actions/golang + + - name: Build binary and zarf packages + uses: ./.github/actions/packages + with: + build-examples: "false" + init-package: "false" - name: "Login to GHCR" uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 @@ -35,15 +44,18 @@ jobs: - name: Build And Publish Application Packages # Create the dos-games package with the cosign signature, publish to ghcr and copy the tags to allow 'uname -m' to work run: | - zarf package create -o build -a amd64 examples/dos-games --key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm - zarf package create -o build -a arm64 examples/dos-games --key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm + ./build/zarf package create -o build -a amd64 examples/dos-games --signing-key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm + ./build/zarf package create -o build -a arm64 examples/dos-games --signing-key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm + + # Publish a the signed dos-games package + ./build/zarf package publish ./build/zarf-package-dos-games-amd64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub + ./build/zarf package publish ./build/zarf-package-dos-games-arm64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub - zarf package publish ./build/zarf-package-dos-games-amd64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub - zarf package publish ./build/zarf-package-dos-games-arm64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub - zarf package publish examples/dos-games oci://ghcr.io/defenseunicorns/packages + # Publish a skeleton of the dos-games package + ./build/zarf package publish examples/dos-games oci://ghcr.io/defenseunicorns/packages - zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-amd64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-x86_64 - zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-arm64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-aarch64 + ./build/zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-amd64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-x86_64 + ./build/zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-arm64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-aarch64 env: AWS_REGION: ${{ secrets.COSIGN_AWS_REGION }} AWS_ACCESS_KEY_ID: ${{ secrets.COSIGN_AWS_KEY_ID }} diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md index 4d7f3f9933..8bd70e500d 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md @@ -19,14 +19,14 @@ zarf package create [ DIRECTORY ] [flags] --confirm Confirm package creation without prompting --differential string [beta] Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package -h, --help help for create - -k, --key string Path to private key file for signing packages - --key-pass string Password to the private key file used for signing packages -m, --max-package-size int Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts to be loaded onto smaller media (i.e. DVDs). Use 0 to disable splitting. -o, --output string Specify the output (either a directory or an oci:// URL) for the created Zarf package --registry-override stringToString Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet) (default []) -s, --sbom View SBOM contents after creating the package --sbom-out string Specify an output directory for the SBOMs from the created Zarf package --set stringToString Specify package variables to set on the command line (KEY=value) (default []) + --signing-key string Path to private key file for signing packages + --signing-key-pass string Password to the private key file used for signing packages --skip-sbom Skip generating SBOM for this package ``` diff --git a/src/cmd/package.go b/src/cmd/package.go index e448667b39..ae3c34697f 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -349,11 +349,17 @@ func bindCreateFlags(v *viper.Viper) { createFlags.StringVar(&pkgConfig.CreateOpts.SBOMOutputDir, "sbom-out", v.GetString(common.VPkgCreateSbomOutput), lang.CmdPackageCreateFlagSbomOut) createFlags.BoolVar(&pkgConfig.CreateOpts.SkipSBOM, "skip-sbom", v.GetBool(common.VPkgCreateSkipSbom), lang.CmdPackageCreateFlagSkipSbom) createFlags.IntVarP(&pkgConfig.CreateOpts.MaxPackageSizeMB, "max-package-size", "m", v.GetInt(common.VPkgCreateMaxPackageSize), lang.CmdPackageCreateFlagMaxPackageSize) + createFlags.StringToStringVar(&pkgConfig.CreateOpts.RegistryOverrides, "registry-override", v.GetStringMapString(common.VPkgCreateRegistryOverride), lang.CmdPackageCreateFlagRegistryOverride) + + createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPath, "signing-key", v.GetString(common.VPkgCreateSigningKey), lang.CmdPackageCreateFlagSigningKey) + createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "signing-key-pass", v.GetString(common.VPkgCreateSigningKeyPassword), lang.CmdPackageCreateFlagSigningKeyPassword) + createFlags.StringVarP(&pkgConfig.CreateOpts.SigningKeyPath, "key", "k", v.GetString(common.VPkgCreateSigningKey), lang.CmdPackageCreateFlagSigningKey) createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "key-pass", v.GetString(common.VPkgCreateSigningKeyPassword), lang.CmdPackageCreateFlagSigningKeyPassword) - createFlags.StringToStringVar(&pkgConfig.CreateOpts.RegistryOverrides, "registry-override", v.GetStringMapString(common.VPkgCreateRegistryOverride), lang.CmdPackageCreateFlagRegistryOverride) createFlags.MarkHidden("output-directory") + createFlags.MarkHidden("key") + createFlags.MarkHidden("key-pass") } func bindDeployFlags(v *viper.Viper) { diff --git a/src/test/e2e/32_checksum_and_signature_test.go b/src/test/e2e/32_checksum_and_signature_test.go index a4f443ff83..08f7f2cdde 100644 --- a/src/test/e2e/32_checksum_and_signature_test.go +++ b/src/test/e2e/32_checksum_and_signature_test.go @@ -17,7 +17,7 @@ func TestChecksumAndSignature(t *testing.T) { testPackageDirPath := "examples/dos-games" pkgName := fmt.Sprintf("zarf-package-dos-games-%s-1.0.0.tar.zst", e2e.Arch) - privateKeyFlag := "--key=src/test/packages/zarf-test.prv-key" + privateKeyFlag := "--signing-key=src/test/packages/zarf-test.prv-key" publicKeyFlag := "--key=src/test/packages/zarf-test.pub" stdOut, stdErr, err := e2e.Zarf("package", "create", testPackageDirPath, privateKeyFlag, "--confirm") diff --git a/src/test/nightly/ecr_publish_test.go b/src/test/nightly/ecr_publish_test.go index 8995c66095..aff3bf2aef 100644 --- a/src/test/nightly/ecr_publish_test.go +++ b/src/test/nightly/ecr_publish_test.go @@ -51,7 +51,7 @@ func TestECRPublishing(t *testing.T) { keyFlag := fmt.Sprintf("--key=%s", "./src/test/packages/zarf-test.pub") // Build the package with our test signature - stdOut, stdErr, err := e2e.Zarf("package", "create", "examples/helm-charts", "--key=./src/test/packages/zarf-test.prv-key", "--confirm", fmt.Sprintf("-o=%s", tmpDir)) + stdOut, stdErr, err := e2e.Zarf("package", "create", "examples/helm-charts", "--signing-key=./src/test/packages/zarf-test.prv-key", "--confirm", fmt.Sprintf("-o=%s", tmpDir)) require.NoError(t, err, stdOut, stdErr) require.FileExists(t, testPackageLocation) From 5e691e268db40f853e367c08c2750f02e5a9033f Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 30 Oct 2023 14:07:37 -0600 Subject: [PATCH 56/70] Revert application packages workflow --- .../publish-application-packages.yml | 27 +++++++------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/.github/workflows/publish-application-packages.yml b/.github/workflows/publish-application-packages.yml index 48559fcfd3..175543b466 100644 --- a/.github/workflows/publish-application-packages.yml +++ b/.github/workflows/publish-application-packages.yml @@ -22,17 +22,8 @@ jobs: with: ref: ${{ github.event.inputs.branchName }} - # - name: Install The Latest Release Version of Zarf - # uses: defenseunicorns/setup-zarf@main - - - name: Setup golang - uses: ./.github/actions/golang - - - name: Build binary and zarf packages - uses: ./.github/actions/packages - with: - build-examples: "false" - init-package: "false" + - name: Install The Latest Release Version of Zarf + uses: defenseunicorns/setup-zarf@main - name: "Login to GHCR" uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 @@ -44,18 +35,18 @@ jobs: - name: Build And Publish Application Packages # Create the dos-games package with the cosign signature, publish to ghcr and copy the tags to allow 'uname -m' to work run: | - ./build/zarf package create -o build -a amd64 examples/dos-games --signing-key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm - ./build/zarf package create -o build -a arm64 examples/dos-games --signing-key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm + zarf package create -o build -a amd64 examples/dos-games --signing-key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm + zarf package create -o build -a arm64 examples/dos-games --signing-key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm # Publish a the signed dos-games package - ./build/zarf package publish ./build/zarf-package-dos-games-amd64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub - ./build/zarf package publish ./build/zarf-package-dos-games-arm64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub + zarf package publish ./build/zarf-package-dos-games-amd64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub + zarf package publish ./build/zarf-package-dos-games-arm64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub # Publish a skeleton of the dos-games package - ./build/zarf package publish examples/dos-games oci://ghcr.io/defenseunicorns/packages + zarf package publish examples/dos-games oci://ghcr.io/defenseunicorns/packages - ./build/zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-amd64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-x86_64 - ./build/zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-arm64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-aarch64 + zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-amd64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-x86_64 + zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-arm64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-aarch64 env: AWS_REGION: ${{ secrets.COSIGN_AWS_REGION }} AWS_ACCESS_KEY_ID: ${{ secrets.COSIGN_AWS_KEY_ID }} From 418541dfc28bdcbf5cc0da5c2432dbf69d308a47 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 30 Oct 2023 18:09:10 -0600 Subject: [PATCH 57/70] Refactor the import everything tests and add a dedicated compose test --- docs/3-create-a-zarf-package/4-zarf-schema.md | 42 +----- src/config/lang/english.go | 42 +++--- src/internal/packager/validate/validate.go | 4 +- src/test/e2e/09_component_compose_test.go | 124 +++++++++++++++++- src/test/e2e/51_oci_compose_test.go | 86 ++---------- .../files/coffee-ipsum.txt | 0 .../files/kustomization.yaml | 0 .../09-composable-packages/files/service.yaml | 17 +++ .../files/test-values.yaml | 10 ++ .../sub-package/zarf.yaml | 44 +++++++ .../packages/09-composable-packages/zarf.yaml | 65 +++++++++ .../51-import-everything/bar/zarf.yaml | 23 ---- .../51-import-everything/foo/zarf.yaml | 9 -- .../51-import-everything/inception/zarf.yaml | 20 +-- .../packages/51-import-everything/zarf.yaml | 112 +++------------- src/types/component.go | 4 +- zarf.schema.json | 16 +-- 17 files changed, 313 insertions(+), 305 deletions(-) rename src/test/packages/{51-import-everything => 09-composable-packages}/files/coffee-ipsum.txt (100%) rename src/test/packages/{51-import-everything => 09-composable-packages}/files/kustomization.yaml (100%) create mode 100644 src/test/packages/09-composable-packages/files/service.yaml create mode 100644 src/test/packages/09-composable-packages/files/test-values.yaml create mode 100644 src/test/packages/09-composable-packages/sub-package/zarf.yaml create mode 100644 src/test/packages/09-composable-packages/zarf.yaml delete mode 100644 src/test/packages/51-import-everything/bar/zarf.yaml delete mode 100644 src/test/packages/51-import-everything/foo/zarf.yaml diff --git a/docs/3-create-a-zarf-package/4-zarf-schema.md b/docs/3-create-a-zarf-package/4-zarf-schema.md index ac63e9dff7..478a863c26 100644 --- a/docs/3-create-a-zarf-package/4-zarf-schema.md +++ b/docs/3-create-a-zarf-package/4-zarf-schema.md @@ -963,50 +963,10 @@ Must be one of: | | | | ------------------------- | -------------------------------------------------------------------------------------------------------- | -| **Type** | `combining` | +| **Type** | `object` | | **Additional properties** | [![Not allowed](https://img.shields.io/badge/Not%20allowed-red)](# "Additional Properties not allowed.") | | **Defined in** | #/definitions/ZarfChart | -
- -| One of(Option) | -| ---------------------------------------------------- | -| [url](#components_items_charts_items_oneOf_i0) | -| [localPath](#components_items_charts_items_oneOf_i1) | - -
- -### Property `url` - -**Title:** url - -| | | -| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| **Type** | `object` | -| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") | - -#### The following properties are required -* url - -
-
- -### Property `localPath` - -**Title:** localPath - -| | | -| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- | -| **Type** | `object` | -| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") | - -#### The following properties are required -* localPath - -
- -
-
name * diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 14d6758223..dc3b2880db 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -569,36 +569,36 @@ const ( // src/internal/packager/validate. const ( - PkgValidateTemplateDeprecation = "Package template '%s' is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." - PkgValidateMustBeUppercase = "variable name '%s' must be all uppercase and contain no special characters except _" + PkgValidateTemplateDeprecation = "Package template %q is using the deprecated syntax ###ZARF_PKG_VAR_%s###. This will be removed in Zarf v1.0.0. Please update to ###ZARF_PKG_TMPL_%s###." + PkgValidateMustBeUppercase = "variable name %q must be all uppercase and contain no special characters except _" PkgValidateErrAction = "invalid action: %w" - PkgValidateErrActionVariables = "component %s cannot contain setVariables outside of onDeploy in actions" - PkgValidateErrActionCmdWait = "action %s cannot be both a command and wait action" + PkgValidateErrActionVariables = "component %q cannot contain setVariables outside of onDeploy in actions" + PkgValidateErrActionCmdWait = "action %q cannot be both a command and wait action" PkgValidateErrActionClusterNetwork = "a single wait action must contain only one of cluster or network" PkgValidateErrChart = "invalid chart definition: %w" - PkgValidateErrChartName = "chart %s exceed the maximum length of %d characters" - PkgValidateErrChartNameMissing = "chart %s must include a name" + PkgValidateErrChartName = "chart %q exceed the maximum length of %d characters" + PkgValidateErrChartNameMissing = "chart %q must include a name" PkgValidateErrChartNameNotUnique = "chart name %q is not unique" - PkgValidateErrChartNamespaceMissing = "chart %s must include a namespace" - PkgValidateErrChartURLOrPath = "chart %s must only have a url or localPath" - PkgValidateErrChartVersion = "chart %s must include a chart version" - PkgValidateErrComponentNameNotUnique = "component name '%s' is not unique" - PkgValidateErrComponent = "invalid component: %w" - PkgValidateErrComponentReqDefault = "component %s cannot be both required and default" - PkgValidateErrComponentReqGrouped = "component %s cannot be both required and grouped" - PkgValidateErrComponentYOLO = "component %s incompatible with the online-only package flag (metadata.yolo): %w" + PkgValidateErrChartNamespaceMissing = "chart %q must include a namespace" + PkgValidateErrChartURLOrPath = "chart %q must have either a url or localPath" + PkgValidateErrChartVersion = "chart %q must include a chart version" + PkgValidateErrComponentNameNotUnique = "component name %q is not unique" + PkgValidateErrComponent = "invalid component %q: %w" + PkgValidateErrComponentReqDefault = "component %q cannot be both required and default" + PkgValidateErrComponentReqGrouped = "component %q cannot be both required and grouped" + PkgValidateErrComponentYOLO = "component %q incompatible with the online-only package flag (metadata.yolo): %w" PkgValidateErrConstant = "invalid package constant: %w" - PkgValidateErrImportDefinition = "invalid imported defintion for %s: %s" + PkgValidateErrImportDefinition = "invalid imported definition for %s: %s" PkgValidateErrInitNoYOLO = "sorry, you can't YOLO an init package" PkgValidateErrManifest = "invalid manifest definition: %w" - PkgValidateErrManifestFileOrKustomize = "manifest %s must have at least one file or kustomization" - PkgValidateErrManifestNameLength = "manifest %s exceed the maximum length of %d characters" - PkgValidateErrManifestNameMissing = "manifest %s must include a name" + PkgValidateErrManifestFileOrKustomize = "manifest %q must have at least one file or kustomization" + PkgValidateErrManifestNameLength = "manifest %q exceed the maximum length of %d characters" + PkgValidateErrManifestNameMissing = "manifest %q must include a name" PkgValidateErrManifestNameNotUnique = "manifest name %q is not unique" PkgValidateErrName = "invalid package name: %w" - PkgValidateErrPkgConstantName = "constant name '%s' must be all uppercase and contain no special characters except _" - PkgValidateErrPkgConstantPattern = "provided value for constant %q does not match pattern \"%s\"" - PkgValidateErrPkgName = "package name '%s' must be all lowercase and contain no special characters except -" + PkgValidateErrPkgConstantName = "constant name %q must be all uppercase and contain no special characters except _" + PkgValidateErrPkgConstantPattern = "provided value for constant %q does not match pattern %q" + PkgValidateErrPkgName = "package name %q must be all lowercase and contain no special characters except -" PkgValidateErrVariable = "invalid package variable: %w" PkgValidateErrYOLONoArch = "cluster architecture not allowed" PkgValidateErrYOLONoDistro = "cluster distros not allowed" diff --git a/src/internal/packager/validate/validate.go b/src/internal/packager/validate/validate.go index aaeb5aebca..9a1d57db1a 100644 --- a/src/internal/packager/validate/validate.go +++ b/src/internal/packager/validate/validate.go @@ -58,7 +58,7 @@ func Run(pkg types.ZarfPackage) error { uniqueComponentNames[component.Name] = true if err := validateComponent(pkg, component); err != nil { - return fmt.Errorf(lang.PkgValidateErrComponent, err) + return fmt.Errorf(lang.PkgValidateErrComponent, component.Name, err) } } @@ -299,7 +299,7 @@ func validateChart(chart types.ZarfChart) error { return fmt.Errorf(lang.PkgValidateErrChartNamespaceMissing, chart.Name) } - // Must only have a url or localPath + // Must have a url or localPath (and not both) count := oneIfNotEmpty(chart.URL) + oneIfNotEmpty(chart.LocalPath) if count != 1 { return fmt.Errorf(lang.PkgValidateErrChartURLOrPath, chart.Name) diff --git a/src/test/e2e/09_component_compose_test.go b/src/test/e2e/09_component_compose_test.go index dbcf3571ad..ef668dc2f3 100644 --- a/src/test/e2e/09_component_compose_test.go +++ b/src/test/e2e/09_component_compose_test.go @@ -5,6 +5,9 @@ package test import ( + "fmt" + "os" + "path/filepath" "testing" "github.com/stretchr/testify/require" @@ -16,16 +19,131 @@ type CompositionSuite struct { *require.Assertions } +var ( + composeExample = filepath.Join("examples", "composable-packages") + composeExamplePath string +) + func (suite *CompositionSuite) SetupSuite() { suite.Assertions = require.New(suite.T()) + + // Setup the package paths after e2e has been initialized + composeExamplePath = filepath.Join("build", fmt.Sprintf("zarf-package-composable-packages-%s.tar.zst", e2e.Arch)) + +} + +func (suite *CompositionSuite) TearDownSuite() { + err := os.RemoveAll(composeExamplePath) + suite.NoError(err) } -// func (suite *CompositionSuite) TearDownSuite() { +func (suite *CompositionSuite) Test_0_ComposabilityExample() { + suite.T().Log("E2E: Package Compose Example") + + _, stdErr, err := e2e.Zarf("package", "create", composeExample, "-o", "build", "--insecure", "--no-color", "--confirm") + suite.NoError(err) + + // Ensure that common names merge + suite.Contains(stdErr, ` + manifests: + - name: multi-games + namespace: dos-games + files: + - ../dos-games/manifests/deployment.yaml + - ../dos-games/manifests/service.yaml + - quake-service.yaml`) + + // Ensure that the action was appended + suite.Contains(stdErr, ` + - defenseunicorns/zarf-game:multi-tile-dark + actions: + onDeploy: + before: + - cmd: ./zarf tools kubectl get -n dos-games deployment -o jsonpath={.items[0].metadata.creationTimestamp}`) +} -// } +func (suite *CompositionSuite) Test_1_FullComposability() { + suite.T().Log("E2E: Full Package Compose") -func (suite *SkeletonSuite) Test_0_ComposabilityExample() { + _, stdErr, err := e2e.Zarf("package", "create", composeExample, "-o", "build", "--insecure", "--no-color", "--confirm") + suite.NoError(err) + // Ensure that names merge and that composition is added appropriately + suite.Contains(stdErr, ` +- name: test-compose-package + description: A contrived example for podinfo using many Zarf primitives for compose testing + required: true + files: + - source: files/coffee-ipsum.txt + target: coffee-ipsum.txt + - source: files/coffee-ipsum.txt + target: coffee-ipsum.txt + charts: + - name: podinfo-compose + releaseName: podinfo-override + url: oci://ghcr.io/stefanprodan/charts/podinfo + version: 6.4.0 + namespace: podinfo-override + valuesFiles: + - files/test-values.yaml + - files/test-values.yaml + - name: podinfo-compose-two + releaseName: podinfo-compose-two + url: oci://ghcr.io/stefanprodan/charts/podinfo + version: 6.4.0 + namespace: podinfo-compose-two + valuesFiles: + - files/test-values.yaml + manifests: + - name: connect-service + namespace: podinfo-override + files: + - files/service.yaml + - files/service.yaml + kustomizations: + - files + - files + - name: connect-service-two + namespace: podinfo-compose-two + files: + - files/service.yaml + kustomizations: + - files + images: + - ghcr.io/stefanprodan/podinfo:6.4.0 + - ghcr.io/stefanprodan/podinfo:6.4.1 + repos: + - https://github.com/defenseunicorns/zarf-public-test.git + - https://github.com/defenseunicorns/zarf-public-test.git@refs/heads/dragons + dataInjections: + - source: files + target: + namespace: podinfo-upgrade + selector: app.kubernetes.io/name=podinfo-upgrade + container: podinfo + path: /home/app/service.yaml + - source: files + target: + namespace: podinfo-upgrade + selector: app.kubernetes.io/name=podinfo-upgrade + container: podinfo + path: /home/app/service.yaml + actions: + onCreate: + before: + - dir: sub-package + cmd: cat ../files/coffee-ipsum.txt + - dir: . + cmd: cat files/coffee-ipsum.txt + onDeploy: + after: + - cmd: cat coffee-ipsum.txt + - wait: + cluster: + kind: deployment + name: podinfo-compose-two + namespace: podinfo-compose-two + condition: available`) } func TestCompositionSuite(t *testing.T) { diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index ba87329bf4..70a13ab873 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -15,7 +15,6 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/transform" "github.com/defenseunicorns/zarf/src/pkg/utils" - "github.com/defenseunicorns/zarf/src/pkg/utils/exec" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" "github.com/defenseunicorns/zarf/src/types" "github.com/stretchr/testify/require" @@ -30,52 +29,27 @@ type SkeletonSuite struct { } var ( - composeExample = filepath.Join("examples", "composable-packages") - composeExamplePath string importEverything = filepath.Join("src", "test", "packages", "51-import-everything") importEverythingPath string importception = filepath.Join("src", "test", "packages", "51-import-everything", "inception") importceptionPath string - everythingExternal = filepath.Join("src", "test", "packages", "everything-external") - absNoCode = filepath.Join("/", "tmp", "nocode") ) func (suite *SkeletonSuite) SetupSuite() { suite.Assertions = require.New(suite.T()) - err := os.MkdirAll(filepath.Join("src", "test", "packages", "51-import-everything", "charts"), 0755) - suite.NoError(err) - err = utils.CreatePathAndCopy(filepath.Join("examples", "helm-charts", "chart"), filepath.Join("src", "test", "packages", "51-import-everything", "charts", "local")) - suite.NoError(err) - suite.DirExists(filepath.Join("src", "test", "packages", "51-import-everything", "charts", "local")) - - err = utils.CreatePathAndCopy(importEverything, everythingExternal) - suite.NoError(err) - suite.DirExists(everythingExternal) - - err = exec.CmdWithPrint("git", "clone", "https://github.com/kelseyhightower/nocode", absNoCode) - suite.NoError(err) - suite.DirExists(absNoCode) e2e.SetupDockerRegistry(suite.T(), 555) suite.Reference.Registry = "localhost:555" // Setup the package paths after e2e has been initialized - composeExamplePath = filepath.Join("build", fmt.Sprintf("zarf-package-composable-packages-%s.tar.zst", e2e.Arch)) importEverythingPath = filepath.Join("build", fmt.Sprintf("zarf-package-import-everything-%s-0.0.1.tar.zst", e2e.Arch)) importceptionPath = filepath.Join("build", fmt.Sprintf("zarf-package-importception-%s-0.0.1.tar.zst", e2e.Arch)) } func (suite *SkeletonSuite) TearDownSuite() { e2e.TeardownRegistry(suite.T(), 555) - err := os.RemoveAll(everythingExternal) - suite.NoError(err) - err = os.RemoveAll(absNoCode) - suite.NoError(err) - err = os.RemoveAll(filepath.Join("src", "test", "packages", "51-import-everything", "charts", "local")) - suite.NoError(err) - err = os.RemoveAll("files") - suite.NoError(err) - err = os.RemoveAll(composeExamplePath) + + err := os.RemoveAll(filepath.Join("src", "test", "packages", "51-import-everything", "charts", "local")) suite.NoError(err) err = os.RemoveAll(importEverythingPath) suite.NoError(err) @@ -87,13 +61,8 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { suite.T().Log("E2E: Skeleton Package Publish oci://") ref := suite.Reference.String() - wordpress := filepath.Join("examples", "wordpress") - _, stdErr, err := e2e.Zarf("package", "publish", wordpress, "oci://"+ref, "--insecure") - suite.NoError(err) - suite.Contains(stdErr, "Published "+ref) - helmCharts := filepath.Join("examples", "helm-charts") - _, stdErr, err = e2e.Zarf("package", "publish", helmCharts, "oci://"+ref, "--insecure") + _, stdErr, err := e2e.Zarf("package", "publish", helmCharts, "oci://"+ref, "--insecure") suite.NoError(err) suite.Contains(stdErr, "Published "+ref) @@ -119,41 +88,7 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { suite.NoError(err) } -func (suite *SkeletonSuite) Test_1_Compose_Example() { - suite.T().Log("E2E: Skeleton Package Compose oci://") - - _, stdErr, err := e2e.Zarf("package", "create", composeExample, "-o", "build", "--insecure", "--no-color", "--confirm") - suite.NoError(err) - - // Ensure that common names merge - suite.Contains(stdErr, ` - manifests: - - name: multi-games - namespace: dos-games - files: - - ../dos-games/manifests/deployment.yaml - - ../dos-games/manifests/service.yaml - - quake-service.yaml`) - - // Ensure that the action was appended - suite.Contains(stdErr, ` - - docker.io/bitnami/wordpress:6.2.0-debian-11-r18 - actions: - onDeploy: - before: - - cmd: ./zarf tools kubectl get -n dos-games deployment -o jsonpath={.items[0].metadata.creationTimestamp} - setVariables: - - name: WORDPRESS_BLOG_NAME`) - - // Ensure that the variables were merged - suite.Contains(stdErr, ` -- name: WORDPRESS_BLOG_NAME - description: The blog name that is used for the WordPress admin account - default: The Zarf Blog - prompt: true`) -} - -func (suite *SkeletonSuite) Test_2_Compose_Everything_Inception() { +func (suite *SkeletonSuite) Test_1_Compose_Everything_Inception() { suite.T().Log("E2E: Skeleton Package Compose oci://") _, _, err := e2e.Zarf("package", "create", importEverything, "-o", "build", "--insecure", "--confirm") @@ -167,15 +102,10 @@ func (suite *SkeletonSuite) Test_2_Compose_Everything_Inception() { targets := []string{ "import-component-local == import-component-local", - "import-component-local-relative == import-component-local-relative", - "import-component-wordpress == import-component-wordpress", "import-component-oci == import-component-oci", + "import-big-bang == import-big-bang", "file-imports == file-imports", - "import-helm-local == import-helm-local", - "import-helm-local-relative == import-helm-local-relative", - "import-helm-oci == import-helm-oci", - "import-repos == import-repos", - "import-images == import-images", + "local-chart-import == local-chart-import", } for _, target := range targets { @@ -183,8 +113,8 @@ func (suite *SkeletonSuite) Test_2_Compose_Everything_Inception() { } } -func (suite *SkeletonSuite) Test_3_FilePaths() { - suite.T().Log("E2E: Skeleton Package File Paths") +func (suite *SkeletonSuite) Test_2_FilePaths() { + suite.T().Log("E2E: Skeleton + Package File Paths") pkgTars := []string{ filepath.Join("build", fmt.Sprintf("zarf-package-import-everything-%s-0.0.1.tar.zst", e2e.Arch)), diff --git a/src/test/packages/51-import-everything/files/coffee-ipsum.txt b/src/test/packages/09-composable-packages/files/coffee-ipsum.txt similarity index 100% rename from src/test/packages/51-import-everything/files/coffee-ipsum.txt rename to src/test/packages/09-composable-packages/files/coffee-ipsum.txt diff --git a/src/test/packages/51-import-everything/files/kustomization.yaml b/src/test/packages/09-composable-packages/files/kustomization.yaml similarity index 100% rename from src/test/packages/51-import-everything/files/kustomization.yaml rename to src/test/packages/09-composable-packages/files/kustomization.yaml diff --git a/src/test/packages/09-composable-packages/files/service.yaml b/src/test/packages/09-composable-packages/files/service.yaml new file mode 100644 index 0000000000..4547b3dd48 --- /dev/null +++ b/src/test/packages/09-composable-packages/files/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: podinfo + annotations: + zarf.dev/connect-description: Access Podinfo + labels: + # Enables "zarf connect podinfo" + zarf.dev/connect-name: podinfo +spec: + selector: + app.kubernetes.io/name: podinfo-upgrade + ports: + - name: http + port: 9898 + protocol: TCP + targetPort: 9898 diff --git a/src/test/packages/09-composable-packages/files/test-values.yaml b/src/test/packages/09-composable-packages/files/test-values.yaml new file mode 100644 index 0000000000..b574e0dec9 --- /dev/null +++ b/src/test/packages/09-composable-packages/files/test-values.yaml @@ -0,0 +1,10 @@ +podAnnotations: + zarf.dev/dataInjections: ###ZARF_DATA_INJECTION_MARKER### + +resources: + limits: + requests: + cpu: 1m + memory: 16Mi + # Add some ephemeral storage for data injections + ephemeral-storage: 16Mi diff --git a/src/test/packages/09-composable-packages/sub-package/zarf.yaml b/src/test/packages/09-composable-packages/sub-package/zarf.yaml new file mode 100644 index 0000000000..4474ee9955 --- /dev/null +++ b/src/test/packages/09-composable-packages/sub-package/zarf.yaml @@ -0,0 +1,44 @@ +kind: ZarfPackageConfig +metadata: + name: test-compose-sub-package + description: Deploy podinfo using a Helm OCI chart + version: 6.4.0 + +components: + - name: test-compose-sub-package + charts: + - name: podinfo-compose + releaseName: podinfo-compose + version: 6.4.0 + namespace: podinfo-compose + url: oci://ghcr.io/stefanprodan/charts/podinfo + valuesFiles: + - ../files/test-values.yaml + manifests: + - name: connect-service + namespace: podinfo-compose + files: + - ../files/service.yaml + kustomizations: + - ../files/ + images: + - ghcr.io/stefanprodan/podinfo:6.4.0 + repos: + - https://github.com/defenseunicorns/zarf-public-test.git + files: + - source: ../files/coffee-ipsum.txt + target: coffee-ipsum.txt + dataInjections: + - source: ../files + target: + selector: app.kubernetes.io/name=podinfo-upgrade + namespace: podinfo-upgrade + container: podinfo + path: /home/app/service.yaml + actions: + onCreate: + before: + - cmd: cat ../files/coffee-ipsum.txt + onDeploy: + after: + - cmd: cat coffee-ipsum.txt diff --git a/src/test/packages/09-composable-packages/zarf.yaml b/src/test/packages/09-composable-packages/zarf.yaml new file mode 100644 index 0000000000..96b406991a --- /dev/null +++ b/src/test/packages/09-composable-packages/zarf.yaml @@ -0,0 +1,65 @@ +kind: ZarfPackageConfig +metadata: + name: test-compose-package + description: Deploy podinfo using a Helm OCI chart + version: 6.4.0 + +components: + - name: test-compose-package + description: A contrived example for podinfo using many Zarf primitives for compose testing + required: true + import: + path: sub-package + name: test-compose-sub-package + charts: + - name: podinfo-compose + releaseName: podinfo-override + namespace: podinfo-override + valuesFiles: + - files/test-values.yaml + - name: podinfo-compose-two + releaseName: podinfo-compose-two + version: 6.4.0 + namespace: podinfo-compose-two + url: oci://ghcr.io/stefanprodan/charts/podinfo + valuesFiles: + - files/test-values.yaml + manifests: + - name: connect-service + namespace: podinfo-override + files: + - files/service.yaml + kustomizations: + - files/ + - name: connect-service-two + namespace: podinfo-compose-two + files: + - files/service.yaml + kustomizations: + - files/ + images: + - ghcr.io/stefanprodan/podinfo:6.4.1 + repos: + - https://github.com/defenseunicorns/zarf-public-test.git@refs/heads/dragons + files: + - source: files/coffee-ipsum.txt + target: coffee-ipsum.txt + dataInjections: + - source: files + target: + selector: app.kubernetes.io/name=podinfo-upgrade + namespace: podinfo-upgrade + container: podinfo + path: /home/app/service.yaml + actions: + onCreate: + before: + - cmd: cat files/coffee-ipsum.txt + onDeploy: + after: + - wait: + cluster: + kind: deployment + name: podinfo-compose-two + namespace: podinfo-compose-two + condition: available diff --git a/src/test/packages/51-import-everything/bar/zarf.yaml b/src/test/packages/51-import-everything/bar/zarf.yaml deleted file mode 100644 index 8df24ef40a..0000000000 --- a/src/test/packages/51-import-everything/bar/zarf.yaml +++ /dev/null @@ -1,23 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: bar - version: 0.0.1 - -components: - - name: baz - required: true - files: - - source: ../files/coffee-ipsum.txt - target: files/coffee-ipsum.txt - actions: - onRemove: - after: - - cmd: rm files/coffee-ipsum.txt - onCreate: - before: - - cmd: cat files/coffee-ipsum.txt - dir: ../ - after: - - cmd: cat ../files/coffee-ipsum.txt - onSuccess: - - cmd: cat ../files/coffee-ipsum.txt diff --git a/src/test/packages/51-import-everything/foo/zarf.yaml b/src/test/packages/51-import-everything/foo/zarf.yaml deleted file mode 100644 index ed203cf430..0000000000 --- a/src/test/packages/51-import-everything/foo/zarf.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: foo - version: 0.0.1 - -components: - - name: baz - import: - path: ../bar diff --git a/src/test/packages/51-import-everything/inception/zarf.yaml b/src/test/packages/51-import-everything/inception/zarf.yaml index 032125ac10..297f4b8b41 100644 --- a/src/test/packages/51-import-everything/inception/zarf.yaml +++ b/src/test/packages/51-import-everything/inception/zarf.yaml @@ -5,37 +5,27 @@ metadata: version: 0.0.1 components: - - name: import-component-local-relative + - name: import-component-local required: true import: url: oci://localhost:555/import-everything:0.0.1-skeleton - - name: import-component-wordpress + - name: import-component-oci required: true import: url: oci://localhost:555/import-everything:0.0.1-skeleton - - name: file-imports - required: true - import: - url: oci://localhost:555/import-everything:0.0.1-skeleton - - - name: import-helm-local + - name: import-big-bang required: true import: url: oci://localhost:555/import-everything:0.0.1-skeleton - - name: import-helm-local-relative - required: true - import: - url: oci://localhost:555/import-everything:0.0.1-skeleton - - - name: import-helm-oci + - name: file-imports required: true import: url: oci://localhost:555/import-everything:0.0.1-skeleton - - name: import-images + - name: local-chart-import required: true import: url: oci://localhost:555/import-everything:0.0.1-skeleton diff --git a/src/test/packages/51-import-everything/zarf.yaml b/src/test/packages/51-import-everything/zarf.yaml index e24239ceca..b6c1ebee66 100644 --- a/src/test/packages/51-import-everything/zarf.yaml +++ b/src/test/packages/51-import-everything/zarf.yaml @@ -9,46 +9,31 @@ components: description: "import-component-local == ###ZARF_COMPONENT_NAME###" required: false import: - path: foo - name: baz - - - name: import-component-local-relative - description: "import-component-local-relative == ###ZARF_COMPONENT_NAME###" - required: false - import: - path: ../../../../examples/dos-games - name: baseline - manifests: - - name: override - kustomizations: - - files - - - name: import-component-wordpress - description: "import-component-wordpress == ###ZARF_COMPONENT_NAME###" - required: false - import: - path: ../../../../examples/wordpress - name: wordpress + path: ../09-composable-packages + name: test-compose-package - name: import-component-oci description: "import-component-oci == ###ZARF_COMPONENT_NAME###" required: false import: - # name is optional, if not provided the name of the component will be used to import name: demo-helm-local-chart url: oci://localhost:555/helm-charts:0.0.1-skeleton + - name: import-big-bang + description: "import-big-bang-oci == ###ZARF_COMPONENT_NAME###" + required: false + import: + name: bigbang + url: oci://localhost:555/big-bang-example:2.12.0-skeleton + - name: file-imports - description: "file-imports == ###ZARF_COMPONENT_NAME###" + description: "file-imports == ###ZARF_COMPONENT_NAME###" required: false cosignKeyPath: ../../../../cosign.pub files: - # Import of a file within the current directory - - source: files/coffee-ipsum.txt - target: files/coffee-ipsum.txt - # Import of a file from a relative path external to the current directory - - source: ../everything-external/files/coffee-ipsum.txt - target: files/latte.txt + # Import of a local file + - source: ../09-composable-packages/files/coffee-ipsum.txt + target: ../09-composable-packages/coffee-ipsum.txt # Import of a file from a URL - source: https://raw.githubusercontent.com/defenseunicorns/zarf/main/README.md target: files/zarf-readme.md @@ -56,24 +41,21 @@ components: onDeploy: after: - cmd: test -f files/coffee-ipsum.txt - - cmd: test -f files/latte.txt - cmd: test -f files/zarf-readme.md onRemove: before: - cmd: rm files/coffee-ipsum.txt - - cmd: rm files/latte.txt - cmd: rm files/zarf-readme.md after: - cmd: test ! -f files/coffee-ipsum.txt - - cmd: test ! -f files/latte.txt - cmd: test ! -f files/zarf-readme.md - - name: import-helm-local - description: "import-helm-local == ###ZARF_COMPONENT_NAME###" + - name: local-chart-import + description: "local-chart-import == ###ZARF_COMPONENT_NAME###" required: false charts: - name: podinfo - localPath: charts/local + localPath: ../../../../examples/helm-charts/chart namespace: local-chart version: 6.4.0 images: @@ -87,65 +69,3 @@ components: name: podinfo namespace: local-chart condition: available - - - name: import-helm-local-relative - description: "import-helm-local-relative == ###ZARF_COMPONENT_NAME###" - required: false - charts: - - name: podinfo - localPath: ../everything-external/charts/local - namespace: local-chart-relative - version: 6.4.0 - images: - - ghcr.io/stefanprodan/podinfo:6.4.0 - actions: - onDeploy: - after: - - wait: - cluster: - kind: deployment - name: podinfo - namespace: local-chart-relative - condition: available - - - name: import-helm-oci - description: "import-helm-oci == ###ZARF_COMPONENT_NAME###" - required: false - charts: - - name: oci-demo - version: 6.4.0 - namespace: podinfo - url: oci://ghcr.io/stefanprodan/charts/podinfo - images: - - ghcr.io/stefanprodan/podinfo:6.4.0 - dataInjections: - - source: files/coffee-ipsum.txt - target: - namespace: podinfo - selector: app.kubernetes.io/name=oci-demo-podinfo - container: podinfo - path: /files/coffee-ipsum.txt - actions: - onDeploy: - after: - - wait: - cluster: - kind: pod - name: app.kubernetes.io/name=oci-demo-podinfo - namespace: podinfo - condition: ready - - - name: import-repos - description: "import-repos == ###ZARF_COMPONENT_NAME###" - required: false - repos: - # Import a full repo via HTTPS - - https://github.com/kelseyhightower/nocode.git - # Import a full repo via file:// + absolute path - - file:///tmp/nocode - - - name: import-images - description: "import-images == ###ZARF_COMPONENT_NAME###" - required: false - images: - - ghcr.io/stefanprodan/podinfo:6.4.0 diff --git a/src/types/component.go b/src/types/component.go index 89b8e00111..df5f37c836 100644 --- a/src/types/component.go +++ b/src/types/component.go @@ -91,12 +91,12 @@ type ZarfFile struct { type ZarfChart struct { Name string `json:"name" jsonschema:"description=The name of the chart to deploy; this should be the name of the chart as it is installed in the helm repo"` ReleaseName string `json:"releaseName,omitempty" jsonschema:"description=The name of the release to create; defaults to the name of the chart"` - URL string `json:"url,omitempty" jsonschema:"oneof_required=url,example=OCI registry: oci://ghcr.io/stefanprodan/charts/podinfo,example=helm chart repo: https://stefanprodan.github.io/podinfo,example=git repo: https://github.com/stefanprodan/podinfo" jsonschema_description:"The URL of the OCI registry, chart repository, or git repo where the helm chart is stored"` + URL string `json:"url,omitempty" jsonschema:"example=OCI registry: oci://ghcr.io/stefanprodan/charts/podinfo,example=helm chart repo: https://stefanprodan.github.io/podinfo,example=git repo: https://github.com/stefanprodan/podinfo" jsonschema_description:"The URL of the OCI registry, chart repository, or git repo where the helm chart is stored"` Version string `json:"version,omitempty" jsonschema:"description=The version of the chart to deploy; for git-based charts this is also the tag of the git repo"` Namespace string `json:"namespace" jsonschema:"description=The namespace to deploy the chart to"` ValuesFiles []string `json:"valuesFiles,omitempty" jsonschema:"description=List of local values file paths or remote URLs to include in the package; these will be merged together"` GitPath string `json:"gitPath,omitempty" jsonschema:"description=The path to the chart in the repo if using a git repo instead of a helm repo,example=charts/your-chart"` - LocalPath string `json:"localPath,omitempty" jsonschema:"oneof_required=localPath,description=The path to the chart folder"` + LocalPath string `json:"localPath,omitempty" jsonschema:"description=The path to the chart folder"` NoWait bool `json:"noWait,omitempty" jsonschema:"description=Whether to not wait for chart resources to be ready before continuing"` } diff --git a/zarf.schema.json b/zarf.schema.json index 46343d8e8d..f2ff08fd64 100644 --- a/zarf.schema.json +++ b/zarf.schema.json @@ -195,21 +195,7 @@ } }, "additionalProperties": false, - "type": "object", - "oneOf": [ - { - "required": [ - "url" - ], - "title": "url" - }, - { - "required": [ - "localPath" - ], - "title": "localPath" - } - ] + "type": "object" }, "ZarfComponent": { "required": [ From e92c7376c19afe2440339e8dba5e7677a0cd31c4 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 30 Oct 2023 18:32:23 -0600 Subject: [PATCH 58/70] Actually run the test --- src/test/e2e/09_component_compose_test.go | 33 ++++++++++++++++++- .../packages/09-composable-packages/zarf.yaml | 3 +- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/test/e2e/09_component_compose_test.go b/src/test/e2e/09_component_compose_test.go index ef668dc2f3..ba5790f0c5 100644 --- a/src/test/e2e/09_component_compose_test.go +++ b/src/test/e2e/09_component_compose_test.go @@ -22,6 +22,8 @@ type CompositionSuite struct { var ( composeExample = filepath.Join("examples", "composable-packages") composeExamplePath string + composeTest = filepath.Join("src", "test", "packages", "09-composable-packages") + composeTestPath string ) func (suite *CompositionSuite) SetupSuite() { @@ -29,12 +31,15 @@ func (suite *CompositionSuite) SetupSuite() { // Setup the package paths after e2e has been initialized composeExamplePath = filepath.Join("build", fmt.Sprintf("zarf-package-composable-packages-%s.tar.zst", e2e.Arch)) + composeTestPath = filepath.Join("build", fmt.Sprintf("zarf-package-test-compose-package-%s.tar.zst", e2e.Arch)) } func (suite *CompositionSuite) TearDownSuite() { err := os.RemoveAll(composeExamplePath) suite.NoError(err) + err = os.RemoveAll(composeTestPath) + suite.NoError(err) } func (suite *CompositionSuite) Test_0_ComposabilityExample() { @@ -65,19 +70,29 @@ func (suite *CompositionSuite) Test_0_ComposabilityExample() { func (suite *CompositionSuite) Test_1_FullComposability() { suite.T().Log("E2E: Full Package Compose") - _, stdErr, err := e2e.Zarf("package", "create", composeExample, "-o", "build", "--insecure", "--no-color", "--confirm") + _, stdErr, err := e2e.Zarf("package", "create", composeTest, "-o", "build", "--insecure", "--no-color", "--confirm") suite.NoError(err) // Ensure that names merge and that composition is added appropriately + + // Check metadata suite.Contains(stdErr, ` - name: test-compose-package description: A contrived example for podinfo using many Zarf primitives for compose testing required: true +`) + + // Check files + suite.Contains(stdErr, ` files: - source: files/coffee-ipsum.txt target: coffee-ipsum.txt - source: files/coffee-ipsum.txt target: coffee-ipsum.txt +`) + + // Check charts + suite.Contains(stdErr, ` charts: - name: podinfo-compose releaseName: podinfo-override @@ -94,6 +109,10 @@ func (suite *CompositionSuite) Test_1_FullComposability() { namespace: podinfo-compose-two valuesFiles: - files/test-values.yaml +`) + + // Check manifests + suite.Contains(stdErr, ` manifests: - name: connect-service namespace: podinfo-override @@ -109,12 +128,20 @@ func (suite *CompositionSuite) Test_1_FullComposability() { - files/service.yaml kustomizations: - files +`) + + // Check images + repos + suite.Contains(stdErr, ` images: - ghcr.io/stefanprodan/podinfo:6.4.0 - ghcr.io/stefanprodan/podinfo:6.4.1 repos: - https://github.com/defenseunicorns/zarf-public-test.git - https://github.com/defenseunicorns/zarf-public-test.git@refs/heads/dragons +`) + + // Check dataInjections + suite.Contains(stdErr, ` dataInjections: - source: files target: @@ -128,6 +155,10 @@ func (suite *CompositionSuite) Test_1_FullComposability() { selector: app.kubernetes.io/name=podinfo-upgrade container: podinfo path: /home/app/service.yaml +`) + + // Check actions + suite.Contains(stdErr, ` actions: onCreate: before: diff --git a/src/test/packages/09-composable-packages/zarf.yaml b/src/test/packages/09-composable-packages/zarf.yaml index 96b406991a..39ccaf88df 100644 --- a/src/test/packages/09-composable-packages/zarf.yaml +++ b/src/test/packages/09-composable-packages/zarf.yaml @@ -1,8 +1,7 @@ kind: ZarfPackageConfig metadata: name: test-compose-package - description: Deploy podinfo using a Helm OCI chart - version: 6.4.0 + description: A contrived example for podinfo using many Zarf primitives for compose testing components: - name: test-compose-package From 426e3395b00fa7966815184ac516139bc8d5d7b6 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 30 Oct 2023 20:19:48 -0600 Subject: [PATCH 59/70] fix tests and more edge cases --- src/extensions/bigbang/bigbang.go | 24 +-- src/extensions/bigbang/manifests.go | 15 +- src/pkg/packager/composer/list_test.go | 195 +++++++++++------- src/pkg/packager/composer/override.go | 3 + src/pkg/packager/create.go | 15 ++ src/test/e2e/51_oci_compose_test.go | 10 +- .../sub-package/zarf.yaml | 2 +- .../packages/09-composable-packages/zarf.yaml | 2 +- .../flux-overrides-helm-controller.yaml | 17 ++ .../big-bang-min/zarf.yaml | 24 +++ .../packages/51-import-everything/zarf.yaml | 4 +- 11 files changed, 202 insertions(+), 109 deletions(-) create mode 100644 src/test/packages/51-import-everything/big-bang-min/flux-overrides-helm-controller.yaml create mode 100644 src/test/packages/51-import-everything/big-bang-min/zarf.yaml diff --git a/src/extensions/bigbang/bigbang.go b/src/extensions/bigbang/bigbang.go index 1050cc1d1f..3462017858 100644 --- a/src/extensions/bigbang/bigbang.go +++ b/src/extensions/bigbang/bigbang.go @@ -249,14 +249,14 @@ func Run(YOLO bool, tmpPaths *layout.ComponentPaths, c types.ZarfComponent) (typ // Skeletonize mutates a component so that the valuesFiles can be contained inside a skeleton package func Skeletonize(tmpPaths *layout.ComponentPaths, c types.ZarfComponent) (types.ZarfComponent, error) { for valuesIdx, valuesFile := range c.Extensions.BigBang.ValuesFiles { - // Define the name as the file name without the extension. - baseName := strings.TrimSuffix(valuesFile, filepath.Ext(valuesFile)) + // Get the base file name for this file. + baseName := filepath.Base(valuesFile) - // Replace non-alphanumeric characters with a dash. - baseName = nonAlphnumeric.ReplaceAllString(baseName, "-") + // Define the name as the file name without the extension. + baseName = strings.TrimSuffix(baseName, filepath.Ext(baseName)) // Add the skeleton name prefix. - skelName := fmt.Sprintf("bb-ext-skeleton-values-%s.yaml", baseName) + skelName := fmt.Sprintf("bb-skel-vals-%d-%s.yaml", valuesIdx, baseName) rel := filepath.Join(layout.TempDir, skelName) dst := filepath.Join(tmpPaths.Base, rel) @@ -269,14 +269,14 @@ func Skeletonize(tmpPaths *layout.ComponentPaths, c types.ZarfComponent) (types. } for fluxPatchFileIdx, fluxPatchFile := range c.Extensions.BigBang.FluxPatchFiles { - // Define the name as the file name without the extension. - baseName := strings.TrimSuffix(fluxPatchFile, filepath.Ext(fluxPatchFile)) + // Get the base file name for this file. + baseName := filepath.Base(fluxPatchFile) - // Replace non-alphanumeric characters with a dash. - baseName = nonAlphnumeric.ReplaceAllString(baseName, "-") + // Define the name as the file name without the extension. + baseName = strings.TrimSuffix(baseName, filepath.Ext(baseName)) // Add the skeleton name prefix. - skelName := fmt.Sprintf("bb-ext-skeleton-flux-patches-%s.yaml", baseName) + skelName := fmt.Sprintf("bb-skel-flux-patch-%d-%s.yaml", fluxPatchFileIdx, baseName) rel := filepath.Join(layout.TempDir, skelName) dst := filepath.Join(tmpPaths.Base, rel) @@ -495,8 +495,8 @@ func addBigBangManifests(YOLO bool, manifestDir string, cfg *extensions.BigBang) } // Loop through the valuesFrom list and create a manifest for each. - for _, path := range cfg.ValuesFiles { - data, err := manifestValuesFile(path) + for valuesIdx, valuesFile := range cfg.ValuesFiles { + data, err := manifestValuesFile(valuesIdx, valuesFile) if err != nil { return manifest, err } diff --git a/src/extensions/bigbang/manifests.go b/src/extensions/bigbang/manifests.go index 6aea42e421..cec481d6cb 100644 --- a/src/extensions/bigbang/manifests.go +++ b/src/extensions/bigbang/manifests.go @@ -8,7 +8,6 @@ import ( "fmt" "os" "path/filepath" - "regexp" "strings" "github.com/Masterminds/semver/v3" @@ -19,8 +18,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -var nonAlphnumeric = regexp.MustCompile("[^a-zA-Z0-9]+") - const bbV1ZarfCredentialsValues = ` registryCredentials: registry: "###ZARF_REGISTRY###" @@ -122,21 +119,21 @@ func manifestGitRepo(cfg *extensions.BigBang) fluxSrcCtrl.GitRepository { } // manifestValuesFile generates a Secret object for the Big Bang umbrella repo. -func manifestValuesFile(path string) (secret corev1.Secret, err error) { +func manifestValuesFile(idx int, path string) (secret corev1.Secret, err error) { // Read the file from the path. file, err := os.ReadFile(path) if err != nil { return secret, err } - // Define the name as the file name without the extension. - baseName := strings.TrimSuffix(path, filepath.Ext(path)) + // Get the base file name for this file. + baseName := filepath.Base(path) - // Replace non-alphanumeric characters with a dash. - baseName = nonAlphnumeric.ReplaceAllString(baseName, "-") + // Define the name as the file name without the extension. + baseName = strings.TrimSuffix(baseName, filepath.Ext(baseName)) // Add the name prefix. - name := fmt.Sprintf("bb-ext-user-values-%s", baseName) + name := fmt.Sprintf("bb-usr-vals-%d-%s", idx, baseName) // Create a secret with the file contents. secret = corev1.Secret{ diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go index ffae3d773a..3a703c5ed1 100644 --- a/src/pkg/packager/composer/list_test.go +++ b/src/pkg/packager/composer/list_test.go @@ -66,11 +66,14 @@ func TestCompose(t *testing.T) { expectedErrorMessage string } - currentDirectory := "." firstDirectory := "hello" secondDirectory := "world" finalDirectory := filepath.Join(firstDirectory, secondDirectory) + finalDirectoryActionDefault := filepath.Join(firstDirectory, secondDirectory, "today-dc") + secondDirectoryActionDefault := filepath.Join(firstDirectory, "world-dc") + firstDirectoryActionDefault := "hello-dc" + testCases := []testCase{ { name: "Single Component", @@ -95,15 +98,9 @@ func TestCompose(t *testing.T) { expectedComposed: types.ZarfComponent{ Name: "import-hello", Files: []types.ZarfFile{ - { - Source: fmt.Sprintf("%s%stoday.txt", finalDirectory, string(os.PathSeparator)), - }, - { - Source: fmt.Sprintf("%s%sworld.txt", firstDirectory, string(os.PathSeparator)), - }, - { - Source: "hello.txt", - }, + {Source: fmt.Sprintf("%s%stoday.txt", finalDirectory, string(os.PathSeparator))}, + {Source: fmt.Sprintf("%s%sworld.txt", firstDirectory, string(os.PathSeparator))}, + {Source: "hello.txt"}, }, Charts: []types.ZarfChart{ { @@ -138,73 +135,84 @@ func TestCompose(t *testing.T) { }, }, DataInjections: []types.ZarfDataInjection{ - { - Source: fmt.Sprintf("%s%stoday", finalDirectory, string(os.PathSeparator)), - }, - { - Source: fmt.Sprintf("%s%sworld", firstDirectory, string(os.PathSeparator)), - }, - { - Source: "hello", - }, + {Source: fmt.Sprintf("%s%stoday", finalDirectory, string(os.PathSeparator))}, + {Source: fmt.Sprintf("%s%sworld", firstDirectory, string(os.PathSeparator))}, + {Source: "hello"}, }, Actions: types.ZarfComponentActions{ OnCreate: types.ZarfComponentActionSet{ + Defaults: types.ZarfComponentActionDefaults{ + Dir: "hello-dc", + }, + Before: []types.ZarfComponentAction{ + {Cmd: "today-bc", Dir: &finalDirectoryActionDefault}, + {Cmd: "world-bc", Dir: &secondDirectoryActionDefault}, + {Cmd: "hello-bc", Dir: &firstDirectoryActionDefault}, + }, + After: []types.ZarfComponentAction{ + {Cmd: "today-ac", Dir: &finalDirectoryActionDefault}, + {Cmd: "world-ac", Dir: &secondDirectoryActionDefault}, + {Cmd: "hello-ac", Dir: &firstDirectoryActionDefault}, + }, + OnSuccess: []types.ZarfComponentAction{ + {Cmd: "today-sc", Dir: &finalDirectoryActionDefault}, + {Cmd: "world-sc", Dir: &secondDirectoryActionDefault}, + {Cmd: "hello-sc", Dir: &firstDirectoryActionDefault}, + }, + OnFailure: []types.ZarfComponentAction{ + {Cmd: "today-fc", Dir: &finalDirectoryActionDefault}, + {Cmd: "world-fc", Dir: &secondDirectoryActionDefault}, + {Cmd: "hello-fc", Dir: &firstDirectoryActionDefault}, + }, + }, + OnDeploy: types.ZarfComponentActionSet{ + Defaults: types.ZarfComponentActionDefaults{ + Dir: "hello-dd", + }, Before: []types.ZarfComponentAction{ - { - Cmd: "today", - Dir: &finalDirectory, - }, - { - Cmd: "world", - Dir: &firstDirectory, - }, - { - Cmd: "hello", - Dir: ¤tDirectory, - }, + {Cmd: "today-bd"}, + {Cmd: "world-bd"}, + {Cmd: "hello-bd"}, }, After: []types.ZarfComponentAction{ - { - Cmd: "today", - Dir: &finalDirectory, - }, - { - Cmd: "world", - Dir: &firstDirectory, - }, - { - Cmd: "hello", - Dir: ¤tDirectory, - }, + {Cmd: "today-ad"}, + {Cmd: "world-ad"}, + {Cmd: "hello-ad"}, }, OnSuccess: []types.ZarfComponentAction{ - { - Cmd: "today", - Dir: &finalDirectory, - }, - { - Cmd: "world", - Dir: &firstDirectory, - }, - { - Cmd: "hello", - Dir: ¤tDirectory, - }, + {Cmd: "today-sd"}, + {Cmd: "world-sd"}, + {Cmd: "hello-sd"}, }, OnFailure: []types.ZarfComponentAction{ - { - Cmd: "today", - Dir: &finalDirectory, - }, - { - Cmd: "world", - Dir: &firstDirectory, - }, - { - Cmd: "hello", - Dir: ¤tDirectory, - }, + {Cmd: "today-fd"}, + {Cmd: "world-fd"}, + {Cmd: "hello-fd"}, + }, + }, + OnRemove: types.ZarfComponentActionSet{ + Defaults: types.ZarfComponentActionDefaults{ + Dir: "hello-dr", + }, + Before: []types.ZarfComponentAction{ + {Cmd: "today-br"}, + {Cmd: "world-br"}, + {Cmd: "hello-br"}, + }, + After: []types.ZarfComponentAction{ + {Cmd: "today-ar"}, + {Cmd: "world-ar"}, + {Cmd: "hello-ar"}, + }, + OnSuccess: []types.ZarfComponentAction{ + {Cmd: "today-sr"}, + {Cmd: "world-sr"}, + {Cmd: "hello-sr"}, + }, + OnFailure: []types.ZarfComponentAction{ + {Cmd: "today-fr"}, + {Cmd: "world-fr"}, + {Cmd: "hello-fr"}, }, }, }, @@ -295,25 +303,54 @@ func createDummyComponent(name, importDir, subName string) types.ZarfComponent { }, Actions: types.ZarfComponentActions{ OnCreate: types.ZarfComponentActionSet{ + Defaults: types.ZarfComponentActionDefaults{ + Dir: name + "-dc", + }, Before: []types.ZarfComponentAction{ - { - Cmd: name, - }, + {Cmd: name + "-bc"}, }, After: []types.ZarfComponentAction{ - { - Cmd: name, - }, + {Cmd: name + "-ac"}, }, OnSuccess: []types.ZarfComponentAction{ - { - Cmd: name, - }, + {Cmd: name + "-sc"}, }, OnFailure: []types.ZarfComponentAction{ - { - Cmd: name, - }, + {Cmd: name + "-fc"}, + }, + }, + OnDeploy: types.ZarfComponentActionSet{ + Defaults: types.ZarfComponentActionDefaults{ + Dir: name + "-dd", + }, + Before: []types.ZarfComponentAction{ + {Cmd: name + "-bd"}, + }, + After: []types.ZarfComponentAction{ + {Cmd: name + "-ad"}, + }, + OnSuccess: []types.ZarfComponentAction{ + {Cmd: name + "-sd"}, + }, + OnFailure: []types.ZarfComponentAction{ + {Cmd: name + "-fd"}, + }, + }, + OnRemove: types.ZarfComponentActionSet{ + Defaults: types.ZarfComponentActionDefaults{ + Dir: name + "-dr", + }, + Before: []types.ZarfComponentAction{ + {Cmd: name + "-br"}, + }, + After: []types.ZarfComponentAction{ + {Cmd: name + "-ar"}, + }, + OnSuccess: []types.ZarfComponentAction{ + {Cmd: name + "-sr"}, + }, + OnFailure: []types.ZarfComponentAction{ + {Cmd: name + "-fr"}, }, }, }, diff --git a/src/pkg/packager/composer/override.go b/src/pkg/packager/composer/override.go index ba8fc747d4..b45426dc96 100644 --- a/src/pkg/packager/composer/override.go +++ b/src/pkg/packager/composer/override.go @@ -44,18 +44,21 @@ func overrideDeprecated(c *types.ZarfComponent, override types.ZarfComponent) { func overrideActions(c *types.ZarfComponent, override types.ZarfComponent) { // Merge create actions. + c.Actions.OnCreate.Defaults = override.Actions.OnCreate.Defaults c.Actions.OnCreate.Before = append(c.Actions.OnCreate.Before, override.Actions.OnCreate.Before...) c.Actions.OnCreate.After = append(c.Actions.OnCreate.After, override.Actions.OnCreate.After...) c.Actions.OnCreate.OnFailure = append(c.Actions.OnCreate.OnFailure, override.Actions.OnCreate.OnFailure...) c.Actions.OnCreate.OnSuccess = append(c.Actions.OnCreate.OnSuccess, override.Actions.OnCreate.OnSuccess...) // Merge deploy actions. + c.Actions.OnDeploy.Defaults = override.Actions.OnDeploy.Defaults c.Actions.OnDeploy.Before = append(c.Actions.OnDeploy.Before, override.Actions.OnDeploy.Before...) c.Actions.OnDeploy.After = append(c.Actions.OnDeploy.After, override.Actions.OnDeploy.After...) c.Actions.OnDeploy.OnFailure = append(c.Actions.OnDeploy.OnFailure, override.Actions.OnDeploy.OnFailure...) c.Actions.OnDeploy.OnSuccess = append(c.Actions.OnDeploy.OnSuccess, override.Actions.OnDeploy.OnSuccess...) // Merge remove actions. + c.Actions.OnRemove.Defaults = override.Actions.OnRemove.Defaults c.Actions.OnRemove.Before = append(c.Actions.OnRemove.Before, override.Actions.OnRemove.Before...) c.Actions.OnRemove.After = append(c.Actions.OnRemove.After, override.Actions.OnRemove.After...) c.Actions.OnRemove.OnFailure = append(c.Actions.OnRemove.OnFailure, override.Actions.OnRemove.OnFailure...) diff --git a/src/pkg/packager/create.go b/src/pkg/packager/create.go index 4c8f4e6ee2..2e09168c02 100755 --- a/src/pkg/packager/create.go +++ b/src/pkg/packager/create.go @@ -333,6 +333,21 @@ func (p *Packager) addComponent(index int, component types.ZarfComponent, isSkel p.cfg.Pkg.Components[index].DeprecatedCosignKeyPath = "cosign.pub" } + // TODO: (@WSTARR) Shim the skeleton component's create action dirs to be empty. This prevents actions from failing by cd'ing into directories that will be flattened. + if isSkeleton { + component.Actions.OnCreate.Defaults.Dir = "" + resetActions := func(actions []types.ZarfComponentAction) []types.ZarfComponentAction { + for idx := range actions { + actions[idx].Dir = nil + } + return actions + } + component.Actions.OnCreate.Before = resetActions(component.Actions.OnCreate.Before) + component.Actions.OnCreate.After = resetActions(component.Actions.OnCreate.After) + component.Actions.OnCreate.OnSuccess = resetActions(component.Actions.OnCreate.OnSuccess) + component.Actions.OnCreate.OnFailure = resetActions(component.Actions.OnCreate.OnFailure) + } + onCreate := component.Actions.OnCreate if !isSkeleton { if err := p.runActions(onCreate.Defaults, onCreate.Before, nil); err != nil { diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index 70a13ab873..588a976b87 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -66,7 +66,7 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { suite.NoError(err) suite.Contains(stdErr, "Published "+ref) - bigBang := filepath.Join("examples", "big-bang") + bigBang := filepath.Join("src", "test", "packages", "51-import-everything", "big-bang-min") _, stdErr, err = e2e.Zarf("package", "publish", bigBang, "oci://"+ref, "--insecure") suite.NoError(err) suite.Contains(stdErr, "Published "+ref) @@ -84,17 +84,17 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/helm-charts:0.0.1-skeleton", "-o", "build", "--insecure") suite.NoError(err) - _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/big-bang-example:2.12.0-skeleton", "-o", "build", "--insecure") + _, _, err = e2e.Zarf("package", "pull", "oci://"+ref+"/big-bang-min:2.10.0-skeleton", "-o", "build", "--insecure") suite.NoError(err) } func (suite *SkeletonSuite) Test_1_Compose_Everything_Inception() { suite.T().Log("E2E: Skeleton Package Compose oci://") - _, _, err := e2e.Zarf("package", "create", importEverything, "-o", "build", "--insecure", "--confirm") + _, _, err := e2e.Zarf("package", "create", importEverything, "-o", "build", "--insecure", "--set=BB_VERSION=2.10.0", "--set=BB_MAJOR=2", "--confirm") suite.NoError(err) - _, _, err = e2e.Zarf("package", "create", importception, "-o", "build", "--insecure", "--confirm") + _, _, err = e2e.Zarf("package", "create", importception, "-o", "build", "--insecure", "--set=BB_VERSION=2.10.0", "--set=BB_MAJOR=2", "--confirm") suite.NoError(err) _, stdErr, err := e2e.Zarf("package", "inspect", importEverythingPath) @@ -121,7 +121,7 @@ func (suite *SkeletonSuite) Test_2_FilePaths() { filepath.Join("build", "zarf-package-import-everything-skeleton-0.0.1.tar.zst"), filepath.Join("build", fmt.Sprintf("zarf-package-importception-%s-0.0.1.tar.zst", e2e.Arch)), filepath.Join("build", "zarf-package-helm-charts-skeleton-0.0.1.tar.zst"), - filepath.Join("build", "zarf-package-big-bang-example-skeleton-2.12.0.tar.zst"), + filepath.Join("build", "zarf-package-big-bang-min-skeleton-2.10.0.tar.zst"), } for _, pkgTar := range pkgTars { diff --git a/src/test/packages/09-composable-packages/sub-package/zarf.yaml b/src/test/packages/09-composable-packages/sub-package/zarf.yaml index 4474ee9955..2a7a2d082f 100644 --- a/src/test/packages/09-composable-packages/sub-package/zarf.yaml +++ b/src/test/packages/09-composable-packages/sub-package/zarf.yaml @@ -38,7 +38,7 @@ components: actions: onCreate: before: - - cmd: cat ../files/coffee-ipsum.txt + - cmd: ls onDeploy: after: - cmd: cat coffee-ipsum.txt diff --git a/src/test/packages/09-composable-packages/zarf.yaml b/src/test/packages/09-composable-packages/zarf.yaml index 39ccaf88df..9c8e7f7894 100644 --- a/src/test/packages/09-composable-packages/zarf.yaml +++ b/src/test/packages/09-composable-packages/zarf.yaml @@ -53,7 +53,7 @@ components: actions: onCreate: before: - - cmd: cat files/coffee-ipsum.txt + - cmd: ls onDeploy: after: - wait: diff --git a/src/test/packages/51-import-everything/big-bang-min/flux-overrides-helm-controller.yaml b/src/test/packages/51-import-everything/big-bang-min/flux-overrides-helm-controller.yaml new file mode 100644 index 0000000000..d5e68feaee --- /dev/null +++ b/src/test/packages/51-import-everything/big-bang-min/flux-overrides-helm-controller.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: helm-controller + namespace: flux-system +spec: + template: + spec: + containers: + - name: manager + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 100m + memory: 64Mi diff --git a/src/test/packages/51-import-everything/big-bang-min/zarf.yaml b/src/test/packages/51-import-everything/big-bang-min/zarf.yaml new file mode 100644 index 0000000000..2cf15f29b1 --- /dev/null +++ b/src/test/packages/51-import-everything/big-bang-min/zarf.yaml @@ -0,0 +1,24 @@ +kind: ZarfPackageConfig +metadata: + name: big-bang-min + description: A minimal Big Bang package for use in testing + version: 2.10.0 + url: https://p1.dso.mil/products/big-bang + # Big Bang / Iron Bank are only amd64 + architecture: amd64 + +variables: + - name: DOMAIN + default: bigbang.dev + prompt: false + +components: + - name: bigbang + required: true + extensions: + bigbang: + version: 2.10.0 + fluxPatchFiles: + - flux-overrides-helm-controller.yaml + valuesFiles: + - ../../../../extensions/bigbang/test/package/disable-all-bb2.yaml diff --git a/src/test/packages/51-import-everything/zarf.yaml b/src/test/packages/51-import-everything/zarf.yaml index b6c1ebee66..7c1bf3e094 100644 --- a/src/test/packages/51-import-everything/zarf.yaml +++ b/src/test/packages/51-import-everything/zarf.yaml @@ -20,11 +20,11 @@ components: url: oci://localhost:555/helm-charts:0.0.1-skeleton - name: import-big-bang - description: "import-big-bang-oci == ###ZARF_COMPONENT_NAME###" + description: "import-big-bang == ###ZARF_COMPONENT_NAME###" required: false import: name: bigbang - url: oci://localhost:555/big-bang-example:2.12.0-skeleton + url: oci://localhost:555/big-bang-min:2.10.0-skeleton - name: file-imports description: "file-imports == ###ZARF_COMPONENT_NAME###" From 011f2b8081c10a2f8c52047d5a22bbe8b9b213af Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 30 Oct 2023 20:33:22 -0600 Subject: [PATCH 60/70] Mark key and key pass deprecated --- src/cmd/package.go | 4 ++-- src/config/lang/english.go | 28 +++++++++++++++------------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/cmd/package.go b/src/cmd/package.go index ae3c34697f..14eb104ab2 100644 --- a/src/cmd/package.go +++ b/src/cmd/package.go @@ -354,8 +354,8 @@ func bindCreateFlags(v *viper.Viper) { createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPath, "signing-key", v.GetString(common.VPkgCreateSigningKey), lang.CmdPackageCreateFlagSigningKey) createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "signing-key-pass", v.GetString(common.VPkgCreateSigningKeyPassword), lang.CmdPackageCreateFlagSigningKeyPassword) - createFlags.StringVarP(&pkgConfig.CreateOpts.SigningKeyPath, "key", "k", v.GetString(common.VPkgCreateSigningKey), lang.CmdPackageCreateFlagSigningKey) - createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "key-pass", v.GetString(common.VPkgCreateSigningKeyPassword), lang.CmdPackageCreateFlagSigningKeyPassword) + createFlags.StringVarP(&pkgConfig.CreateOpts.SigningKeyPath, "key", "k", v.GetString(common.VPkgCreateSigningKey), lang.CmdPackageCreateFlagDeprecatedKey) + createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "key-pass", v.GetString(common.VPkgCreateSigningKeyPassword), lang.CmdPackageCreateFlagDeprecatedKeyPassword) createFlags.MarkHidden("output-directory") createFlags.MarkHidden("key") diff --git a/src/config/lang/english.go b/src/config/lang/english.go index dc3b2880db..94380e542f 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -238,19 +238,21 @@ const ( CmdPackageListNoPackageWarn = "Unable to get the packages deployed to the cluster" CmdPackageListUnmarshalErr = "Unable to read all of the packages deployed to the cluster" - CmdPackageCreateFlagConfirm = "Confirm package creation without prompting" - CmdPackageCreateFlagSet = "Specify package variables to set on the command line (KEY=value)" - CmdPackageCreateFlagOutput = "Specify the output (either a directory or an oci:// URL) for the created Zarf package" - CmdPackageCreateFlagSbom = "View SBOM contents after creating the package" - CmdPackageCreateFlagSbomOut = "Specify an output directory for the SBOMs from the created Zarf package" - CmdPackageCreateFlagSkipSbom = "Skip generating SBOM for this package" - CmdPackageCreateFlagMaxPackageSize = "Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts to be loaded onto smaller media (i.e. DVDs). Use 0 to disable splitting." - CmdPackageCreateFlagSigningKey = "Path to private key file for signing packages" - CmdPackageCreateFlagSigningKeyPassword = "Password to the private key file used for signing packages" - CmdPackageCreateFlagDifferential = "[beta] Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package" - CmdPackageCreateFlagRegistryOverride = "Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet)" - CmdPackageCreateCleanPathErr = "Invalid characters in Zarf cache path, defaulting to %s" - CmdPackageCreateErr = "Failed to create package: %s" + CmdPackageCreateFlagConfirm = "Confirm package creation without prompting" + CmdPackageCreateFlagSet = "Specify package variables to set on the command line (KEY=value)" + CmdPackageCreateFlagOutput = "Specify the output (either a directory or an oci:// URL) for the created Zarf package" + CmdPackageCreateFlagSbom = "View SBOM contents after creating the package" + CmdPackageCreateFlagSbomOut = "Specify an output directory for the SBOMs from the created Zarf package" + CmdPackageCreateFlagSkipSbom = "Skip generating SBOM for this package" + CmdPackageCreateFlagMaxPackageSize = "Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts to be loaded onto smaller media (i.e. DVDs). Use 0 to disable splitting." + CmdPackageCreateFlagSigningKey = "Path to private key file for signing packages" + CmdPackageCreateFlagSigningKeyPassword = "Password to the private key file used for signing packages" + CmdPackageCreateFlagDeprecatedKey = "[Deprecated] Path to private key file for signing packages (use --signing-key instead)" + CmdPackageCreateFlagDeprecatedKeyPassword = "[Deprecated] Password to the private key file used for signing packages (use --signing-key-pass instead)" + CmdPackageCreateFlagDifferential = "[beta] Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package" + CmdPackageCreateFlagRegistryOverride = "Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet)" + CmdPackageCreateCleanPathErr = "Invalid characters in Zarf cache path, defaulting to %s" + CmdPackageCreateErr = "Failed to create package: %s" CmdPackageDeployFlagConfirm = "Confirms package deployment without prompting. ONLY use with packages you trust. Skips prompts to review SBOM, configure variables, select optional components and review potential breaking changes." CmdPackageDeployFlagAdoptExistingResources = "Adopts any pre-existing K8s resources into the Helm charts managed by Zarf. ONLY use when you have existing deployments you want Zarf to takeover." From b5d8a9d09868ae670837d5a49f6458e91e11f6e7 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 30 Oct 2023 20:40:56 -0600 Subject: [PATCH 61/70] Remove unused variables --- src/test/e2e/51_oci_compose_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index 588a976b87..e0cae59427 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -91,10 +91,10 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { func (suite *SkeletonSuite) Test_1_Compose_Everything_Inception() { suite.T().Log("E2E: Skeleton Package Compose oci://") - _, _, err := e2e.Zarf("package", "create", importEverything, "-o", "build", "--insecure", "--set=BB_VERSION=2.10.0", "--set=BB_MAJOR=2", "--confirm") + _, _, err := e2e.Zarf("package", "create", importEverything, "-o", "build", "--insecure", "--confirm") suite.NoError(err) - _, _, err = e2e.Zarf("package", "create", importception, "-o", "build", "--insecure", "--set=BB_VERSION=2.10.0", "--set=BB_MAJOR=2", "--confirm") + _, _, err = e2e.Zarf("package", "create", importception, "-o", "build", "--insecure", "--confirm") suite.NoError(err) _, stdErr, err := e2e.Zarf("package", "inspect", importEverythingPath) From 7f90dfa737e800dfbdc44884d2159ddf33611ad6 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 30 Oct 2023 20:44:00 -0600 Subject: [PATCH 62/70] Cleanup compose test --- src/test/e2e/09_component_compose_test.go | 8 ++++---- .../packages/09-composable-packages/files/service.yaml | 2 +- .../packages/09-composable-packages/sub-package/zarf.yaml | 4 ++-- src/test/packages/09-composable-packages/zarf.yaml | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/test/e2e/09_component_compose_test.go b/src/test/e2e/09_component_compose_test.go index ba5790f0c5..2558375981 100644 --- a/src/test/e2e/09_component_compose_test.go +++ b/src/test/e2e/09_component_compose_test.go @@ -145,14 +145,14 @@ func (suite *CompositionSuite) Test_1_FullComposability() { dataInjections: - source: files target: - namespace: podinfo-upgrade - selector: app.kubernetes.io/name=podinfo-upgrade + namespace: podinfo-compose + selector: app.kubernetes.io/name=podinfo-compose container: podinfo path: /home/app/service.yaml - source: files target: - namespace: podinfo-upgrade - selector: app.kubernetes.io/name=podinfo-upgrade + namespace: podinfo-compose + selector: app.kubernetes.io/name=podinfo-compose container: podinfo path: /home/app/service.yaml `) diff --git a/src/test/packages/09-composable-packages/files/service.yaml b/src/test/packages/09-composable-packages/files/service.yaml index 4547b3dd48..9f25e96acd 100644 --- a/src/test/packages/09-composable-packages/files/service.yaml +++ b/src/test/packages/09-composable-packages/files/service.yaml @@ -9,7 +9,7 @@ metadata: zarf.dev/connect-name: podinfo spec: selector: - app.kubernetes.io/name: podinfo-upgrade + app.kubernetes.io/name: podinfo-compose ports: - name: http port: 9898 diff --git a/src/test/packages/09-composable-packages/sub-package/zarf.yaml b/src/test/packages/09-composable-packages/sub-package/zarf.yaml index 2a7a2d082f..b4ebd4511c 100644 --- a/src/test/packages/09-composable-packages/sub-package/zarf.yaml +++ b/src/test/packages/09-composable-packages/sub-package/zarf.yaml @@ -31,8 +31,8 @@ components: dataInjections: - source: ../files target: - selector: app.kubernetes.io/name=podinfo-upgrade - namespace: podinfo-upgrade + selector: app.kubernetes.io/name=podinfo-compose + namespace: podinfo-compose container: podinfo path: /home/app/service.yaml actions: diff --git a/src/test/packages/09-composable-packages/zarf.yaml b/src/test/packages/09-composable-packages/zarf.yaml index 9c8e7f7894..21e40e5df2 100644 --- a/src/test/packages/09-composable-packages/zarf.yaml +++ b/src/test/packages/09-composable-packages/zarf.yaml @@ -46,8 +46,8 @@ components: dataInjections: - source: files target: - selector: app.kubernetes.io/name=podinfo-upgrade - namespace: podinfo-upgrade + selector: app.kubernetes.io/name=podinfo-compose + namespace: podinfo-compose container: podinfo path: /home/app/service.yaml actions: From e2af04cfb75fbac1d483224d645cd940cf432189 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 30 Oct 2023 20:45:05 -0600 Subject: [PATCH 63/70] Fix compose test --- src/test/e2e/09_component_compose_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/e2e/09_component_compose_test.go b/src/test/e2e/09_component_compose_test.go index 2558375981..7afb5ac0c3 100644 --- a/src/test/e2e/09_component_compose_test.go +++ b/src/test/e2e/09_component_compose_test.go @@ -163,9 +163,9 @@ func (suite *CompositionSuite) Test_1_FullComposability() { onCreate: before: - dir: sub-package - cmd: cat ../files/coffee-ipsum.txt + cmd: ls - dir: . - cmd: cat files/coffee-ipsum.txt + cmd: ls onDeploy: after: - cmd: cat coffee-ipsum.txt From 116284548bfd412e855d3d731e0e6d82d32e7e05 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 30 Oct 2023 21:27:31 -0600 Subject: [PATCH 64/70] Remove dependence on registry 1 --- src/test/e2e/51_oci_compose_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index e0cae59427..04b25ad217 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -91,10 +91,10 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { func (suite *SkeletonSuite) Test_1_Compose_Everything_Inception() { suite.T().Log("E2E: Skeleton Package Compose oci://") - _, _, err := e2e.Zarf("package", "create", importEverything, "-o", "build", "--insecure", "--confirm") + _, _, err := e2e.Zarf("package", "create", importEverything, "-o", "build", "--insecure", "--registry-override", "registry1.dso.mil/ironbank=ghcr.io", "--confirm") suite.NoError(err) - _, _, err = e2e.Zarf("package", "create", importception, "-o", "build", "--insecure", "--confirm") + _, _, err = e2e.Zarf("package", "create", importception, "-o", "build", "--insecure", "--registry-override", "registry1.dso.mil/ironbank=ghcr.io", "--confirm") suite.NoError(err) _, stdErr, err := e2e.Zarf("package", "inspect", importEverythingPath) From 6ed005e92c87b9526658f97c4f7dae79fddec5ca Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Mon, 30 Oct 2023 21:29:00 -0600 Subject: [PATCH 65/70] Remove dependence on repo 1 --- src/test/packages/51-import-everything/big-bang-min/zarf.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/packages/51-import-everything/big-bang-min/zarf.yaml b/src/test/packages/51-import-everything/big-bang-min/zarf.yaml index 2cf15f29b1..901e83aba2 100644 --- a/src/test/packages/51-import-everything/big-bang-min/zarf.yaml +++ b/src/test/packages/51-import-everything/big-bang-min/zarf.yaml @@ -17,6 +17,7 @@ components: required: true extensions: bigbang: + repo: https://github.com/DoD-Platform-One/big-bang.git version: 2.10.0 fluxPatchFiles: - flux-overrides-helm-controller.yaml From 6c80833cb77f3431ac505714ceeb759a75dc5868 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 31 Oct 2023 08:40:22 -0600 Subject: [PATCH 66/70] Remove flux images --- src/test/e2e/51_oci_compose_test.go | 4 ++-- src/test/packages/51-import-everything/big-bang-min/zarf.yaml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/test/e2e/51_oci_compose_test.go b/src/test/e2e/51_oci_compose_test.go index 04b25ad217..e0cae59427 100644 --- a/src/test/e2e/51_oci_compose_test.go +++ b/src/test/e2e/51_oci_compose_test.go @@ -91,10 +91,10 @@ func (suite *SkeletonSuite) Test_0_Publish_Skeletons() { func (suite *SkeletonSuite) Test_1_Compose_Everything_Inception() { suite.T().Log("E2E: Skeleton Package Compose oci://") - _, _, err := e2e.Zarf("package", "create", importEverything, "-o", "build", "--insecure", "--registry-override", "registry1.dso.mil/ironbank=ghcr.io", "--confirm") + _, _, err := e2e.Zarf("package", "create", importEverything, "-o", "build", "--insecure", "--confirm") suite.NoError(err) - _, _, err = e2e.Zarf("package", "create", importception, "-o", "build", "--insecure", "--registry-override", "registry1.dso.mil/ironbank=ghcr.io", "--confirm") + _, _, err = e2e.Zarf("package", "create", importception, "-o", "build", "--insecure", "--confirm") suite.NoError(err) _, stdErr, err := e2e.Zarf("package", "inspect", importEverythingPath) diff --git a/src/test/packages/51-import-everything/big-bang-min/zarf.yaml b/src/test/packages/51-import-everything/big-bang-min/zarf.yaml index 901e83aba2..6fd483c848 100644 --- a/src/test/packages/51-import-everything/big-bang-min/zarf.yaml +++ b/src/test/packages/51-import-everything/big-bang-min/zarf.yaml @@ -19,6 +19,7 @@ components: bigbang: repo: https://github.com/DoD-Platform-One/big-bang.git version: 2.10.0 + skipFlux: true fluxPatchFiles: - flux-overrides-helm-controller.yaml valuesFiles: From 2a3104054039a08519f260ae26b0546559e05d8e Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 31 Oct 2023 11:49:18 -0600 Subject: [PATCH 67/70] Improve nested test case for #1827 --- .../51-import-everything/oci-import/zarf.yaml | 13 +++++++++++++ src/test/packages/51-import-everything/zarf.yaml | 9 +++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 src/test/packages/51-import-everything/oci-import/zarf.yaml diff --git a/src/test/packages/51-import-everything/oci-import/zarf.yaml b/src/test/packages/51-import-everything/oci-import/zarf.yaml new file mode 100644 index 0000000000..1173eb6e5d --- /dev/null +++ b/src/test/packages/51-import-everything/oci-import/zarf.yaml @@ -0,0 +1,13 @@ +kind: ZarfPackageConfig +metadata: + name: import-oci + description: Test OCI import of helm charts + version: 0.0.1 + +components: + - name: import-component-oci + description: "import-component-oci == ###ZARF_COMPONENT_NAME###" + required: false + import: + name: demo-helm-local-chart + url: oci://localhost:555/helm-charts:0.0.1-skeleton diff --git a/src/test/packages/51-import-everything/zarf.yaml b/src/test/packages/51-import-everything/zarf.yaml index 7c1bf3e094..5807c1431e 100644 --- a/src/test/packages/51-import-everything/zarf.yaml +++ b/src/test/packages/51-import-everything/zarf.yaml @@ -5,6 +5,7 @@ metadata: version: 0.0.1 components: + # Test every simple primitive that Zarf has through a local import - name: import-component-local description: "import-component-local == ###ZARF_COMPONENT_NAME###" required: false @@ -12,13 +13,15 @@ components: path: ../09-composable-packages name: test-compose-package + # Test nested local to oci imports - name: import-component-oci description: "import-component-oci == ###ZARF_COMPONENT_NAME###" required: false import: - name: demo-helm-local-chart - url: oci://localhost:555/helm-charts:0.0.1-skeleton + name: import-component-oci + path: oci-import + # Test big bang extension files - name: import-big-bang description: "import-big-bang == ###ZARF_COMPONENT_NAME###" required: false @@ -26,6 +29,7 @@ components: name: bigbang url: oci://localhost:555/big-bang-min:2.10.0-skeleton + # Test file imports including cosignKeyPath - name: file-imports description: "file-imports == ###ZARF_COMPONENT_NAME###" required: false @@ -50,6 +54,7 @@ components: - cmd: test ! -f files/coffee-ipsum.txt - cmd: test ! -f files/zarf-readme.md + # Test local charts (for skeletons) - name: local-chart-import description: "local-chart-import == ###ZARF_COMPONENT_NAME###" required: false From 22518479fb2ba926105e53df5008cccd5db94aeb Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 31 Oct 2023 11:58:59 -0600 Subject: [PATCH 68/70] Add comments for unit test test cases --- src/pkg/packager/composer/list_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/pkg/packager/composer/list_test.go b/src/pkg/packager/composer/list_test.go index 3a703c5ed1..6d250f6e44 100644 --- a/src/pkg/packager/composer/list_test.go +++ b/src/pkg/packager/composer/list_test.go @@ -97,11 +97,13 @@ func TestCompose(t *testing.T) { returnError: false, expectedComposed: types.ZarfComponent{ Name: "import-hello", + // Files should always be appended with corrected directories Files: []types.ZarfFile{ {Source: fmt.Sprintf("%s%stoday.txt", finalDirectory, string(os.PathSeparator))}, {Source: fmt.Sprintf("%s%sworld.txt", firstDirectory, string(os.PathSeparator))}, {Source: "hello.txt"}, }, + // Charts should be merged if names match and appended if not with corrected directories Charts: []types.ZarfChart{ { Name: "hello", @@ -119,6 +121,7 @@ func TestCompose(t *testing.T) { }, }, }, + // Manifests should be merged if names match and appended if not with corrected directories Manifests: []types.ZarfManifest{ { Name: "hello", @@ -134,12 +137,14 @@ func TestCompose(t *testing.T) { }, }, }, + // DataInjections should always be appended with corrected directories DataInjections: []types.ZarfDataInjection{ {Source: fmt.Sprintf("%s%stoday", finalDirectory, string(os.PathSeparator))}, {Source: fmt.Sprintf("%s%sworld", firstDirectory, string(os.PathSeparator))}, {Source: "hello"}, }, Actions: types.ZarfComponentActions{ + // OnCreate actions should be appended with corrected directories that properly handle default directories OnCreate: types.ZarfComponentActionSet{ Defaults: types.ZarfComponentActionDefaults{ Dir: "hello-dc", @@ -165,6 +170,7 @@ func TestCompose(t *testing.T) { {Cmd: "hello-fc", Dir: &firstDirectoryActionDefault}, }, }, + // OnDeploy actions should be appended without corrected directories OnDeploy: types.ZarfComponentActionSet{ Defaults: types.ZarfComponentActionDefaults{ Dir: "hello-dd", @@ -190,6 +196,7 @@ func TestCompose(t *testing.T) { {Cmd: "hello-fd"}, }, }, + // OnRemove actions should be appended without corrected directories OnRemove: types.ZarfComponentActionSet{ Defaults: types.ZarfComponentActionDefaults{ Dir: "hello-dr", @@ -216,6 +223,7 @@ func TestCompose(t *testing.T) { }, }, }, + // Extensions should be appended with corrected directories Extensions: extensions.ZarfComponentExtensions{ BigBang: &extensions.BigBang{ ValuesFiles: []string{ From e6cc81c8378f0b477cd293aa9cfe6d036d526d84 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 31 Oct 2023 12:12:54 -0600 Subject: [PATCH 69/70] Remove references to deprecated libraries --- go.mod | 3 +-- go.sum | 2 -- src/pkg/transform/image.go | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index cf09b1dcf8..615fdd9a2b 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/anchore/stereoscope v0.0.0-20231027135531-5909e353ee88 github.com/anchore/syft v0.84.1 github.com/derailed/k9s v0.27.4 - github.com/distribution/distribution v2.8.3+incompatible + github.com/distribution/reference v0.5.0 github.com/docker/cli v24.0.6+incompatible github.com/fairwindsops/pluto/v5 v5.18.4 github.com/fatih/color v1.15.0 @@ -186,7 +186,6 @@ require ( github.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352 // indirect github.com/digitorus/timestamp v0.0.0-20230821155606-d1ad5ca9624c // indirect github.com/dimchansky/utfbom v1.1.1 // indirect - github.com/distribution/reference v0.5.0 // indirect github.com/docker/distribution v2.8.3+incompatible // indirect github.com/docker/docker v24.0.6+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect diff --git a/go.sum b/go.sum index 31af02f262..a4189b2d5d 100644 --- a/go.sum +++ b/go.sum @@ -510,8 +510,6 @@ github.com/digitorus/timestamp v0.0.0-20230821155606-d1ad5ca9624c/go.mod h1:GvWn github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/distribution/distribution v2.8.3+incompatible h1:RlpEXBLq/WPXYvBYMDAmBX/SnhD67qwtvW/DzKc8pAo= -github.com/distribution/distribution v2.8.3+incompatible/go.mod h1:EgLm2NgWtdKgzF9NpMzUKgzmR7AMmb0VQi2B+ZzDRjc= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= diff --git a/src/pkg/transform/image.go b/src/pkg/transform/image.go index 2d511c1233..7cff153c34 100644 --- a/src/pkg/transform/image.go +++ b/src/pkg/transform/image.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" - "github.com/distribution/distribution/reference" + "github.com/distribution/reference" ) // Image represents a config for an OCI image. From 1f406e8410a497845de929c6c8fad8f454bd1821 Mon Sep 17 00:00:00 2001 From: Wayne Starr Date: Tue, 31 Oct 2023 12:45:15 -0600 Subject: [PATCH 70/70] Address vulnerability --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 615fdd9a2b..3f3df240eb 100644 --- a/go.mod +++ b/go.mod @@ -187,7 +187,7 @@ require ( github.com/digitorus/timestamp v0.0.0-20230821155606-d1ad5ca9624c // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v24.0.6+incompatible // indirect + github.com/docker/docker v24.0.7+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect diff --git a/go.sum b/go.sum index a4189b2d5d..848584069a 100644 --- a/go.sum +++ b/go.sum @@ -520,8 +520,8 @@ github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWT github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= -github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= +github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=