Merge branch 'openbao:main' into main

.github/workflows/ci.yml

@@ -109,79 +109,78 @@ jobs:
       checkout-ref: ${{ needs.setup.outputs.checkout-ref }}
     secrets: inherit
 
-#  test-ui:
-#    name: Test UI
-#    # The test-ui job is only run on:
-#    # - pushes to main and branches starting with "release/"
-#    # - PRs where the branch starts with "ui/", "backport/ui/", "merge", or when base branch starts with "release/"
-#    # - PRs with the "ui" label on GitHub
-#    if: |
-#      github.ref_name == 'main' ||
-#      startsWith(github.ref_name, 'release/') ||
-#      startsWith(github.head_ref, 'ui/') ||
-#      startsWith(github.head_ref, 'backport/ui/') ||
-#      startsWith(github.head_ref, 'merge') ||
-#      contains(github.event.pull_request.labels.*.name, 'ui')
-#    needs:
-#      - setup
-#    runs-on: ubuntu-latest
-#    steps:
-#      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
-#      - uses: ./.github/actions/set-up-go
-#      # Setup node.js without caching to allow running npm install -g yarn (next step)
-#      - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
-#        with:
-#          node-version-file: "./ui/package.json"
-#      - id: install-yarn
-#        run: |
-#          npm install -g yarn
-#      # Setup node.js with caching using the yarn.lock file
-#      - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
-#        with:
-#          node-version-file: "./ui/package.json"
-#          cache: yarn
-#          cache-dependency-path: ui/yarn.lock
-#      - id: install-browser
-#        uses: browser-actions/setup-chrome@db1b524c26f20a8d1a10f7fc385c92387e2d0477 # v1.7.1
-#      - id: ui-dependencies
-#        name: ui-dependencies
-#        working-directory: ./ui
-#        run: |
-#          yarn install --frozen-lockfile
-#          npm rebuild node-sass
-#      - id: build-go-dev
-#        name: build-go-dev
-#        run: |
-#          rm -rf ./pkg
-#          mkdir ./pkg
-#
-#          make ci-bootstrap dev
-#      - id: test-ui
-#        name: test-ui
-#        run: |
-#          export PATH="${PWD}/bin:${PATH}"
-#
-#          # Run Ember tests
-#          cd ui
-#          mkdir -p test-results/qunit
-#          yarn test:oss
-#      - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
-#        with:
-#          name: test-results-ui
-#          path: ui/test-results
-#        if: success() || failure()
-#      - uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # TSCCR: no entry for repository "test-summary/action"
-#        with:
-#          paths: "ui/test-results/qunit/results.xml"
-#          show: "fail"
-#        if: always()
+  test-ui:
+    name: Test UI
+    # The test-ui job is only run on:
+    # - pushes to main and branches starting with "release/"
+    # - PRs where the branch starts with "ui/", "backport/ui/", "merge", or when base branch starts with "release/"
+    # - PRs with the "ui" label on GitHub
+    if: |
+      github.ref_name == 'main' ||
+      startsWith(github.ref_name, 'release/') ||
+      startsWith(github.head_ref, 'ui/') ||
+      startsWith(github.head_ref, 'backport/ui/') ||
+      startsWith(github.head_ref, 'merge') ||
+      contains(github.event.pull_request.labels.*.name, 'ui')
+    needs:
+      - setup
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: ./.github/actions/set-up-go
+      # Setup node.js without caching to allow running npm install -g yarn (next step)
+      - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
+        with:
+          node-version-file: "./ui/package.json"
+      - id: install-yarn
+        run: |
+          npm install -g yarn
+      # Setup node.js with caching using the yarn.lock file
+      - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
+        with:
+          node-version-file: "./ui/package.json"
+          cache: yarn
+          cache-dependency-path: ui/yarn.lock
+      - id: install-browser
+        uses: browser-actions/setup-chrome@db1b524c26f20a8d1a10f7fc385c92387e2d0477 # v1.7.1
+      - id: ui-dependencies
+        name: ui-dependencies
+        working-directory: ./ui
+        run: |
+          yarn install --frozen-lockfile
+          npm rebuild node-sass
+      - id: build-go-dev
+        name: build-go-dev
+        run: |
+          rm -rf ./pkg
+          mkdir ./pkg
+
+          make ci-bootstrap dev
+      - id: test-ui
+        name: test-ui
+        run: |
+          export PATH="${PWD}/bin:${PATH}"
+
+          # Run Ember tests
+          cd ui
+          mkdir -p test-results/qunit
+          yarn test:oss
+      - uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
+        with:
+          name: test-results-ui
+          path: ui/test-results
+        if: success() || failure()
+      - uses: test-summary/action@31493c76ec9e7aa675f1585d3ed6f1da69269a86 # TSCCR: no entry for repository "test-summary/action"
+        with:
+          paths: "ui/test-results/qunit/results.xml"
+          show: "fail"
+        if: always()
 
   tests-completed:
     needs:
       - setup
       - test-go
-      # UI testing is currently disabled.
-      # - test-ui
+      - test-ui
     if: always()
     runs-on: ubuntu-latest
     steps:

api/logical.go

@@ -239,6 +239,86 @@ func (c *Logical) ListPageWithContext(ctx context.Context, path string, after st
 	return ParseSecret(resp.Body)
 }
 
+func (c *Logical) Scan(path string) (*Secret, error) {
+	return c.ScanWithContext(context.Background(), path)
+}
+
+func (c *Logical) ScanWithContext(ctx context.Context, path string) (*Secret, error) {
+	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
+	defer cancelFunc()
+
+	r := c.c.NewRequest("SCAN", "/v1/"+path)
+	// Set this for broader compatibility, but we use SCAN above to be able to
+	// handle the wrapping lookup function
+	r.Method = http.MethodGet
+	r.Params.Set("scan", "true")
+
+	resp, err := c.c.rawRequestWithContext(ctx, r)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+	if resp != nil && resp.StatusCode == 404 {
+		secret, parseErr := ParseSecret(resp.Body)
+		switch parseErr {
+		case nil:
+		case io.EOF:
+			return nil, nil
+		default:
+			return nil, parseErr
+		}
+		if secret != nil && (len(secret.Warnings) > 0 || len(secret.Data) > 0) {
+			return secret, nil
+		}
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return ParseSecret(resp.Body)
+}
+
+func (c *Logical) ScanPage(path string, after string, limit int) (*Secret, error) {
+	return c.ScanPageWithContext(context.Background(), path, after, limit)
+}
+
+func (c *Logical) ScanPageWithContext(ctx context.Context, path string, after string, limit int) (*Secret, error) {
+	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
+	defer cancelFunc()
+
+	r := c.c.NewRequest("SCAN", "/v1/"+path)
+	// Set this for broader compatibility, but we use SCAN above to be able to
+	// handle the wrapping lookup function.
+	r.Method = http.MethodGet
+	r.Params.Set("scan", "true")
+	r.Params.Set("after", after)
+	r.Params.Set("limit", fmt.Sprintf("%d", limit))
+
+	resp, err := c.c.rawRequestWithContext(ctx, r)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+	if resp != nil && resp.StatusCode == 404 {
+		secret, parseErr := ParseSecret(resp.Body)
+		switch parseErr {
+		case nil:
+		case io.EOF:
+			return nil, nil
+		default:
+			return nil, parseErr
+		}
+		if secret != nil && (len(secret.Warnings) > 0 || len(secret.Data) > 0) {
+			return secret, nil
+		}
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return ParseSecret(resp.Body)
+}
+
 func (c *Logical) Write(path string, data map[string]interface{}) (*Secret, error) {
 	return c.WriteWithContext(context.Background(), path, data)
 }

builtin/logical/kv/backend.go

@@ -182,6 +182,8 @@ func pathInvalid(b *versionedKVBackend) []*framework.Path {
 			subCommand = "get"
 		case logical.ListOperation:
 			subCommand = "list"
+		case logical.ScanOperation:
+			subCommand = "scan"
 		case logical.DeleteOperation:
 			subCommand = "delete"
 		}
@@ -200,6 +202,7 @@ func pathInvalid(b *versionedKVBackend) []*framework.Path {
 				logical.ReadOperation:   &framework.PathOperation{Callback: handler, Unpublished: true},
 				logical.DeleteOperation: &framework.PathOperation{Callback: handler, Unpublished: true},
 				logical.ListOperation:   &framework.PathOperation{Callback: handler, Unpublished: true},
+				logical.ScanOperation:   &framework.PathOperation{Callback: handler, Unpublished: true},
 			},
 
 			HelpDescription: pathInvalidHelp,

builtin/logical/kv/passthrough.go

@@ -69,6 +69,7 @@ func LeaseSwitchedPassthroughBackend(ctx context.Context, conf *logical.BackendC
 					logical.UpdateOperation: b.handleWrite(),
 					logical.DeleteOperation: b.handleDelete(),
 					logical.ListOperation:   b.handleList(),
+					logical.ScanOperation:   b.handleScan(),
 				},
 
 				ExistenceCheck: b.handleExistenceCheck(),
@@ -256,6 +257,30 @@ func (b *PassthroughBackend) handleList() framework.OperationFunc {
 	}
 }
 
+func (b *PassthroughBackend) handleScan() framework.OperationFunc {
+	return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+		// Right now we only handle directories, so ensure it ends with /; however,
+		// some physical backends may not handle the "/" case properly, so only add
+		// it if we're not listing the root
+		path := data.Get("path").(string)
+		if path != "" && !strings.HasSuffix(path, "/") {
+			path = path + "/"
+		}
+
+		// List the keys at the prefix given by the request
+		var keys []string
+		err := logical.ScanView(ctx, logical.NewStorageView(req.Storage, path), func(p string) {
+			keys = append(keys, p)
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		// Generate the response
+		return logical.ListResponse(keys), nil
+	}
+}
+
 const passthroughHelp = `
 The kv backend reads and writes arbitrary secrets to the backend.
 The secrets are encrypted/decrypted by Vault: they are never stored

builtin/logical/kv/passthrough_test.go

@@ -210,6 +210,39 @@ func TestPassthroughBackend_List(t *testing.T) {
 	test(b)
 }
 
+func TestPassthroughBackend_Scan(t *testing.T) {
+	test := func(b logical.Backend) {
+		req := logical.TestRequest(t, logical.UpdateOperation, "foo")
+		req.Data["raw"] = "test"
+		storage := req.Storage
+
+		if _, err := b.HandleRequest(context.Background(), req); err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		req = logical.TestRequest(t, logical.ScanOperation, "")
+		req.Storage = storage
+		resp, err := b.HandleRequest(context.Background(), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		expected := &logical.Response{
+			Data: map[string]interface{}{
+				"keys": []string{"foo"},
+			},
+		}
+
+		if !reflect.DeepEqual(resp, expected) {
+			t.Fatalf("bad response.\n\nexpected: %#v\n\nGot: %#v", expected, resp)
+		}
+	}
+	b := testPassthroughBackend()
+	test(b)
+	b = testPassthroughLeasedBackend()
+	test(b)
+}
+
 func TestPassthroughBackend_Revoke(t *testing.T) {
 	test := func(b logical.Backend) {
 		req := logical.TestRequest(t, logical.RevokeOperation, "kv")

builtin/logical/kv/path_metadata.go

@@ -71,6 +71,7 @@ version-agnostic information about a secret.
 			logical.ReadOperation:   b.upgradeCheck(b.pathMetadataRead()),
 			logical.DeleteOperation: b.upgradeCheck(b.pathMetadataDelete()),
 			logical.ListOperation:   b.upgradeCheck(b.pathMetadataList()),
+			logical.ScanOperation:   b.upgradeCheck(b.pathMetadataScan()),
 			logical.PatchOperation:  b.upgradeCheck(b.pathMetadataPatch()),
 		},
 
@@ -125,6 +126,32 @@ func (b *versionedKVBackend) pathMetadataList() framework.OperationFunc {
 	}
 }
 
+func (b *versionedKVBackend) pathMetadataScan() framework.OperationFunc {
+	return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+		key := data.Get("path").(string)
+
+		// Get an encrypted key storage object
+		wrapper, err := b.getKeyEncryptor(ctx, req.Storage)
+		if err != nil {
+			return nil, err
+		}
+
+		es := wrapper.Wrap(req.Storage)
+		view := logical.NewStorageView(es, key)
+
+		// Use encrypted key storage to recursively list the keys
+		var keys []string
+		err = logical.ScanView(ctx, view, func(path string) {
+			keys = append(keys, path)
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		return logical.ListResponse(keys), nil
+	}
+}
+
 func (b *versionedKVBackend) pathMetadataRead() framework.OperationFunc {
 	return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
 		// Create a read-only transaction if we can. We do not need to commit

changelog/763.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+**Scanning**: introduce the ability to recursively list (scan) within plugins, adding a separate `scan` ACL capability, operation type, HTTP verb (`SCAN` with `GET` fallback via `?scan=true`), API, and CLI support. This also adds support to the KVv1 and KVv2 engines.
+```

command/commands.go

@@ -578,6 +578,11 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) map[string]cli.Co
 				BaseCommand: getBaseCommand(),
 			}, nil
 		},
+		"scan": func() (cli.Command, error) {
+			return &ScanCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
 		"secrets": func() (cli.Command, error) {
 			return &SecretsCommand{
 				BaseCommand: getBaseCommand(),
@@ -778,6 +783,11 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) map[string]cli.Co
 				BaseCommand: getBaseCommand(),
 			}, nil
 		},
+		"kv scan": func() (cli.Command, error) {
+			return &KVScanCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
 		"monitor": func() (cli.Command, error) {
 			return &MonitorCommand{
 				BaseCommand: getBaseCommand(),