chore: refactor Zarf Component composition

.github/workflows/publish-application-packages.yml

@@ -35,11 +35,15 @@ jobs:
       - name: Build And Publish Application Packages
         # Create the dos-games package with the cosign signature, publish to ghcr and copy the tags to allow 'uname -m' to work
         run: |
-          zarf package create -o build -a amd64 examples/dos-games --key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm
-          zarf package create -o build -a arm64 examples/dos-games --key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm
+          zarf package create -o build -a amd64 examples/dos-games --signing-key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm
+          zarf package create -o build -a arm64 examples/dos-games --signing-key=awskms:///${{ secrets.COSIGN_AWS_KMS_KEY }} --confirm
 
-          zarf package publish ./build/zarf-package-dos-games-amd64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages
-          zarf package publish ./build/zarf-package-dos-games-arm64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages
+          # Publish a the signed dos-games package
+          zarf package publish ./build/zarf-package-dos-games-amd64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub
+          zarf package publish ./build/zarf-package-dos-games-arm64-1.0.0.tar.zst oci://ghcr.io/defenseunicorns/packages --key=https://zarf.dev/cosign.pub
+
+          # Publish a skeleton of the dos-games package
+          zarf package publish examples/dos-games oci://ghcr.io/defenseunicorns/packages
 
           zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-amd64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-x86_64
           zarf tools registry copy ghcr.io/defenseunicorns/packages/dos-games:1.0.0-arm64 ghcr.io/defenseunicorns/packages/dos-games:1.0.0-aarch64

adr/0021-composable-components.md

@@ -0,0 +1,23 @@
+# 21. Composable Components
+
+Date: 2023-10-26
+
+## Status
+
+Accepted
+
+## Context
+
+Zarf has supports composing components together between packages on `zarf package create` since v0.16.0.  This has allowed package creators to make more complex packages from smaller reusable bits.  As this functionality grew however there were a few problems that developed:
+
+1. Import chains did not handle scaling to larger numbers of layers with test coverage usually only covering the first import.
+2. When OCI skeletons were added they were largely bolted on after the fact without rethinking how they would impact composability.
+3. Component filtering via the `only` filter was not implemented in a central location leading to bugs with create-time filters.
+
+## Decision
+
+We decided to separate composability into its own package that represents a composability import chain as a doubly linked list.  This allows us to represent the whole chain as it exists relative to the "head" Zarf package (the definition that Zarf was asked to build) to more easily handle packages that are in different locations (such as OCI skeletons in one's cache).  We also run the compose functions on all components so that the additional filter logic that is needed for these components can be handled more concisely and built upon (as it might for `flavor` https://github.com/defenseunicorns/zarf/issues/2101).
+
+## Consequences
+
+Maintaining the full context within a linked list does use more memory and some operations on it are less efficient than they could be if we one-shotted the compose.  This is a decent tradeoff however as most import chains won't be longer than 4 or 5 elements in practice and these structs and operations are relatively small.

docs/2-the-zarf-cli/100-cli-commands/zarf_package_create.md

@@ -19,14 +19,14 @@ zarf package create [ DIRECTORY ] [flags]
       --confirm                            Confirm package creation without prompting
       --differential string                [beta] Build a package that only contains the differential changes from local resources and differing remote resources from the specified previously built package
   -h, --help                               help for create
-  -k, --key string                         Path to private key file for signing packages
-      --key-pass string                    Password to the private key file used for signing packages
   -m, --max-package-size int               Specify the maximum size of the package in megabytes, packages larger than this will be split into multiple parts to be loaded onto smaller media (i.e. DVDs). Use 0 to disable splitting.
   -o, --output string                      Specify the output (either a directory or an oci:// URL) for the created Zarf package
       --registry-override stringToString   Specify a map of domains to override on package create when pulling images (e.g. --registry-override docker.io=dockerio-reg.enterprise.intranet) (default [])
   -s, --sbom                               View SBOM contents after creating the package
       --sbom-out string                    Specify an output directory for the SBOMs from the created Zarf package
       --set stringToString                 Specify package variables to set on the command line (KEY=value) (default [])
+      --signing-key string                 Path to private key file for signing packages
+      --signing-key-pass string            Password to the private key file used for signing packages
       --skip-sbom                          Skip generating SBOM for this package
 ```
 

docs/3-create-a-zarf-package/4-zarf-schema.md

@@ -491,45 +491,6 @@ must respect the following conditions
 </blockquote>
 </details>
 
-<details open>
-<summary>
-<strong> <a name="build_OCIImportedComponents"></a>OCIImportedComponents</strong>
-</summary>
-&nbsp;
-<blockquote>
-
-  ## build > OCIImportedComponents
-
-**Description:** Map of components that were imported via OCI. The keys are OCI Package URLs and values are the component names
-
-|                           |                                                                                                                                   |
-| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- |
-| **Type**                  | `object`                                                                                                                          |
-| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") |
-
-<details>
-<summary>
-<strong> <a name="build_OCIImportedComponents_pattern1"></a>Pattern Property .*</strong>
-</summary>
-&nbsp;
-<blockquote>
-
-:::note
-All properties whose name matches the regular expression
-```.*``` ([Test](https://regex101.com/?regex=.%2A))
-must respect the following conditions
-:::
-
-|          |          |
-| -------- | -------- |
-| **Type** | `string` |
-
-</blockquote>
-</details>
-
-</blockquote>
-</details>
-
 <details>
 <summary>
 <strong> <a name="build_lastNonBreakingVersion"></a>lastNonBreakingVersion</strong>
@@ -1002,50 +963,10 @@ Must be one of:
 
 |                           |                                                                                                          |
 | ------------------------- | -------------------------------------------------------------------------------------------------------- |
-| **Type**                  | `combining`                                                                                              |
+| **Type**                  | `object`                                                                                                 |
 | **Additional properties** | [![Not allowed](https://img.shields.io/badge/Not%20allowed-red)](# "Additional Properties not allowed.") |
 | **Defined in**            | #/definitions/ZarfChart                                                                                  |
 
-<blockquote>
-
-| One of(Option)                                       |
-| ---------------------------------------------------- |
-| [url](#components_items_charts_items_oneOf_i0)       |
-| [localPath](#components_items_charts_items_oneOf_i1) |
-
-<blockquote>
-
-### <a name="components_items_charts_items_oneOf_i0"></a>Property `url`
-
-**Title:** url
-
-|                           |                                                                                                                                   |
-| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- |
-| **Type**                  | `object`                                                                                                                          |
-| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") |
-
-#### <a name="autogenerated_heading_2"></a>The following properties are required
-* url
-
-</blockquote>
-<blockquote>
-
-### <a name="components_items_charts_items_oneOf_i1"></a>Property `localPath`
-
-**Title:** localPath
-
-|                           |                                                                                                                                   |
-| ------------------------- | --------------------------------------------------------------------------------------------------------------------------------- |
-| **Type**                  | `object`                                                                                                                          |
-| **Additional properties** | [![Any type: allowed](https://img.shields.io/badge/Any%20type-allowed-green)](# "Additional Properties of any type are allowed.") |
-
-#### <a name="autogenerated_heading_2"></a>The following properties are required
-* localPath
-
-</blockquote>
-
-</blockquote>
-
 <details>
 <summary>
 <strong> <a name="components_items_charts_items_name"></a>name *</strong>

examples/composable-packages/README.md

@@ -2,13 +2,13 @@ import ExampleYAML from "@site/src/components/ExampleYAML";
 
 # Composable Packages
 
-This example demonstrates using Zarf to import components from existing Zarf package definitions while merging overrides to add or change functionality.  It uses the existing [DOS games](../dos-games/README.md) and [WordPress](../wordpress/README.md) examples by simply adding `import` keys in the new [zarf.yaml](zarf.yaml) file.
+This example demonstrates using Zarf to import components from existing Zarf package definitions while merging overrides to add or change functionality.  It uses the existing [DOS games](../dos-games/README.md) example by simply adding `import` keys in the new [zarf.yaml](zarf.yaml) file.
 
 The `import` key in Zarf supports two modes to pull in a component:
 
-1. The `path` key allows you to specify a path to a directory that contains the `zarf.yaml` that you wish to import on your local filesystem.  This allows you to have a common component that you can reuse across multiple packages *within* a project.
+1. The `path` key allows you to specify a path to a directory that contains the `zarf.yaml` that you wish to import on your local filesystem.  This allows you to have a common component that you can reuse across multiple packages *within* a project (i.e. within one team/codebase).
 
-2. The `url` key allows you to specify an `oci://` URL to a skeleton package that was published to an OCI registry.  Skeleton packages are special package bundles that contain the `zarf.yaml` package definition and any local files referenced by that definition at publish time.  This allows you to version a set of components and import them into multiple packages *across* projects.
+2. The `url` key allows you to specify an `oci://` URL to a skeleton package that was published to an OCI registry.  Skeleton packages are special package bundles that contain the `zarf.yaml` package definition and any local files referenced by that definition at publish time.  This allows you to version a set of reusable components and import them into multiple packages *across* projects (i.e. across teams/codebases).
 
 :::tip
 
@@ -40,23 +40,6 @@ To view the example in its entirety, select the `Edit this page` link below the
 
 :::
 
-:::note
-
-Creating this example requires a locally hosted container registry that has the `wordpress` skeleton package published and available. You can do this by running the following commands:
-
-```bash
-docker run -d -p 555:5000 --restart=always --name registry registry:2
-zarf package publish examples/wordpress oci://127.0.0.1:555 --insecure
-```
-
-You will also need to pass the `--insecure` flag to `zarf package create` to pull from the `http` registry:
-
-```bash
-zarf package create examples/composable-packages/ --insecure
-```
-
-:::
-
 <ExampleYAML src={require('./zarf.yaml')} showLink={false} />
 
 :::info

examples/composable-packages/zarf.yaml

@@ -24,20 +24,18 @@ components:
         files:
           - quake-service.yaml
 
-  - name: oci-wordpress-url
+  - name: oci-games-url
     # The component logic keys ('required', 'group', and 'default') always override those of the imported package
     # required: false   # the initial value overrides the child component
     # group: ""         # the initial value overrides the child component
     # default: false    # the initial value overrides the child component
     import:
       # The URL to the skeleton package containing this component's package definition
-      url: oci://localhost:555/wordpress:16.0.4-skeleton
+      url: oci://🦄/dos-games:1.0.0-skeleton
       # Example optional custom name to point to in the imported package (default is to use this component's name)
-      name: wordpress
+      name: baseline
     # Un'name'd Zarf primitives will be appended to the end of the primitive's list for that component.
     actions:
       onDeploy:
         before:
           - cmd: ./zarf tools kubectl get -n dos-games deployment -o jsonpath={.items[0].metadata.creationTimestamp}
-            setVariables:
-              - name: WORDPRESS_BLOG_NAME

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/anchore/stereoscope v0.0.0-20231027135531-5909e353ee88
 	github.com/anchore/syft v0.84.1
 	github.com/derailed/k9s v0.27.4
-	github.com/distribution/distribution v2.8.3+incompatible
+	github.com/distribution/reference v0.5.0
 	github.com/docker/cli v24.0.6+incompatible
 	github.com/fairwindsops/pluto/v5 v5.18.4
 	github.com/fatih/color v1.15.0
@@ -186,9 +186,8 @@ require (
 	github.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352 // indirect
 	github.com/digitorus/timestamp v0.0.0-20230821155606-d1ad5ca9624c // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
-	github.com/distribution/reference v0.5.0 // indirect
 	github.com/docker/distribution v2.8.3+incompatible // indirect
-	github.com/docker/docker v24.0.6+incompatible // indirect
+	github.com/docker/docker v24.0.7+incompatible // indirect
 	github.com/docker/docker-credential-helpers v0.7.0 // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect

go.sum

@@ -510,8 +510,6 @@ github.com/digitorus/timestamp v0.0.0-20230821155606-d1ad5ca9624c/go.mod h1:GvWn
 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
-github.com/distribution/distribution v2.8.3+incompatible h1:RlpEXBLq/WPXYvBYMDAmBX/SnhD67qwtvW/DzKc8pAo=
-github.com/distribution/distribution v2.8.3+incompatible/go.mod h1:EgLm2NgWtdKgzF9NpMzUKgzmR7AMmb0VQi2B+ZzDRjc=
 github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc=
 github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2/go.mod h1:WHNsWjnIn2V1LYOrME7e8KxSeKunYHsxEm4am0BUtcI=
 github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
@@ -522,8 +520,8 @@ github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWT
 github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
 github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE=
-github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
 github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A=
 github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=

src/cmd/package.go

@@ -349,11 +349,17 @@ func bindCreateFlags(v *viper.Viper) {
 	createFlags.StringVar(&pkgConfig.CreateOpts.SBOMOutputDir, "sbom-out", v.GetString(common.VPkgCreateSbomOutput), lang.CmdPackageCreateFlagSbomOut)
 	createFlags.BoolVar(&pkgConfig.CreateOpts.SkipSBOM, "skip-sbom", v.GetBool(common.VPkgCreateSkipSbom), lang.CmdPackageCreateFlagSkipSbom)
 	createFlags.IntVarP(&pkgConfig.CreateOpts.MaxPackageSizeMB, "max-package-size", "m", v.GetInt(common.VPkgCreateMaxPackageSize), lang.CmdPackageCreateFlagMaxPackageSize)
-	createFlags.StringVarP(&pkgConfig.CreateOpts.SigningKeyPath, "key", "k", v.GetString(common.VPkgCreateSigningKey), lang.CmdPackageCreateFlagSigningKey)
-	createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "key-pass", v.GetString(common.VPkgCreateSigningKeyPassword), lang.CmdPackageCreateFlagSigningKeyPassword)
 	createFlags.StringToStringVar(&pkgConfig.CreateOpts.RegistryOverrides, "registry-override", v.GetStringMapString(common.VPkgCreateRegistryOverride), lang.CmdPackageCreateFlagRegistryOverride)
 
+	createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPath, "signing-key", v.GetString(common.VPkgCreateSigningKey), lang.CmdPackageCreateFlagSigningKey)
+	createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "signing-key-pass", v.GetString(common.VPkgCreateSigningKeyPassword), lang.CmdPackageCreateFlagSigningKeyPassword)
+
+	createFlags.StringVarP(&pkgConfig.CreateOpts.SigningKeyPath, "key", "k", v.GetString(common.VPkgCreateSigningKey), lang.CmdPackageCreateFlagDeprecatedKey)
+	createFlags.StringVar(&pkgConfig.CreateOpts.SigningKeyPassword, "key-pass", v.GetString(common.VPkgCreateSigningKeyPassword), lang.CmdPackageCreateFlagDeprecatedKeyPassword)
+
 	createFlags.MarkHidden("output-directory")
+	createFlags.MarkHidden("key")
+	createFlags.MarkHidden("key-pass")
 }
 
 func bindDeployFlags(v *viper.Viper) {