Add scan operation for recursive listing (openbao#763)

* Add SCAN as logical operation This introduces SCAN as a new logical operation type, either via a GET with ?scan=true or via the SCAN HTTP verb. The intent of SCAN is to allow recursive listing of entries, perhaps using logical.ScanView as a helper. We require a separate operation because we want to permission this distinctly from a LIST: operators may wish to deny users a recursive LIST but grant them a shallow LIST, e.g., for resource usage or to segment secrets in various other ways. Notably, with paginated listing, this can still be rather efficient and need not return all results. Signed-off-by: Alexander Scheel <[email protected]> * Add Scan support to KV Using the new operation, this adds recursive listing (scan) support to KVv2, allowing a user to see all entries in a single call. With future support for a paginated form of ScanView, this can be constrained even in large K/V entries. Resolves: openbao#549 Signed-off-by: Alexander Scheel <[email protected]> * Add Scan(...) support to the API Signed-off-by: Alexander Scheel <[email protected]> * Add support for scanning to CLI Signed-off-by: Alexander Scheel <[email protected]> * Add changelog entry Signed-off-by: Alexander Scheel <[email protected]> * Add RFC to website Signed-off-by: Alexander Scheel <[email protected]> * Add scan operation to policy docs Signed-off-by: Alexander Scheel <[email protected]> --------- Signed-off-by: Alexander Scheel <[email protected]>
klaus-sap · Dec 12, 2024 · c3173a0 · c3173a0
1 parent 24f2dbf
commit c3173a0
Show file tree

Hide file tree

Showing 27 changed files with 995 additions and 16 deletions.
diff --git a/api/logical.go b/api/logical.go
@@ -239,6 +239,86 @@ func (c *Logical) ListPageWithContext(ctx context.Context, path string, after st
 	return ParseSecret(resp.Body)
 }
 
+func (c *Logical) Scan(path string) (*Secret, error) {
+	return c.ScanWithContext(context.Background(), path)
+}
+
+func (c *Logical) ScanWithContext(ctx context.Context, path string) (*Secret, error) {
+	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
+	defer cancelFunc()
+
+	r := c.c.NewRequest("SCAN", "/v1/"+path)
+	// Set this for broader compatibility, but we use SCAN above to be able to
+	// handle the wrapping lookup function
+	r.Method = http.MethodGet
+	r.Params.Set("scan", "true")
+
+	resp, err := c.c.rawRequestWithContext(ctx, r)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+	if resp != nil && resp.StatusCode == 404 {
+		secret, parseErr := ParseSecret(resp.Body)
+		switch parseErr {
+		case nil:
+		case io.EOF:
+			return nil, nil
+		default:
+			return nil, parseErr
+		}
+		if secret != nil && (len(secret.Warnings) > 0 || len(secret.Data) > 0) {
+			return secret, nil
+		}
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return ParseSecret(resp.Body)
+}
+
+func (c *Logical) ScanPage(path string, after string, limit int) (*Secret, error) {
+	return c.ScanPageWithContext(context.Background(), path, after, limit)
+}
+
+func (c *Logical) ScanPageWithContext(ctx context.Context, path string, after string, limit int) (*Secret, error) {
+	ctx, cancelFunc := c.c.withConfiguredTimeout(ctx)
+	defer cancelFunc()
+
+	r := c.c.NewRequest("SCAN", "/v1/"+path)
+	// Set this for broader compatibility, but we use SCAN above to be able to
+	// handle the wrapping lookup function.
+	r.Method = http.MethodGet
+	r.Params.Set("scan", "true")
+	r.Params.Set("after", after)
+	r.Params.Set("limit", fmt.Sprintf("%d", limit))
+
+	resp, err := c.c.rawRequestWithContext(ctx, r)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+	if resp != nil && resp.StatusCode == 404 {
+		secret, parseErr := ParseSecret(resp.Body)
+		switch parseErr {
+		case nil:
+		case io.EOF:
+			return nil, nil
+		default:
+			return nil, parseErr
+		}
+		if secret != nil && (len(secret.Warnings) > 0 || len(secret.Data) > 0) {
+			return secret, nil
+		}
+		return nil, nil
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	return ParseSecret(resp.Body)
+}
+
 func (c *Logical) Write(path string, data map[string]interface{}) (*Secret, error) {
 	return c.WriteWithContext(context.Background(), path, data)
 }

diff --git a/builtin/logical/kv/backend.go b/builtin/logical/kv/backend.go
@@ -182,6 +182,8 @@ func pathInvalid(b *versionedKVBackend) []*framework.Path {
 			subCommand = "get"
 		case logical.ListOperation:
 			subCommand = "list"
+		case logical.ScanOperation:
+			subCommand = "scan"
 		case logical.DeleteOperation:
 			subCommand = "delete"
 		}
@@ -200,6 +202,7 @@ func pathInvalid(b *versionedKVBackend) []*framework.Path {
 				logical.ReadOperation:   &framework.PathOperation{Callback: handler, Unpublished: true},
 				logical.DeleteOperation: &framework.PathOperation{Callback: handler, Unpublished: true},
 				logical.ListOperation:   &framework.PathOperation{Callback: handler, Unpublished: true},
+				logical.ScanOperation:   &framework.PathOperation{Callback: handler, Unpublished: true},
 			},
 
 			HelpDescription: pathInvalidHelp,

diff --git a/builtin/logical/kv/passthrough.go b/builtin/logical/kv/passthrough.go
@@ -69,6 +69,7 @@ func LeaseSwitchedPassthroughBackend(ctx context.Context, conf *logical.BackendC
 					logical.UpdateOperation: b.handleWrite(),
 					logical.DeleteOperation: b.handleDelete(),
 					logical.ListOperation:   b.handleList(),
+					logical.ScanOperation:   b.handleScan(),
 				},
 
 				ExistenceCheck: b.handleExistenceCheck(),
@@ -256,6 +257,30 @@ func (b *PassthroughBackend) handleList() framework.OperationFunc {
 	}
 }
 
+func (b *PassthroughBackend) handleScan() framework.OperationFunc {
+	return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+		// Right now we only handle directories, so ensure it ends with /; however,
+		// some physical backends may not handle the "/" case properly, so only add
+		// it if we're not listing the root
+		path := data.Get("path").(string)
+		if path != "" && !strings.HasSuffix(path, "/") {
+			path = path + "/"
+		}
+
+		// List the keys at the prefix given by the request
+		var keys []string
+		err := logical.ScanView(ctx, logical.NewStorageView(req.Storage, path), func(p string) {
+			keys = append(keys, p)
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		// Generate the response
+		return logical.ListResponse(keys), nil
+	}
+}
+
 const passthroughHelp = `
 The kv backend reads and writes arbitrary secrets to the backend.
 The secrets are encrypted/decrypted by Vault: they are never stored

diff --git a/builtin/logical/kv/passthrough_test.go b/builtin/logical/kv/passthrough_test.go
@@ -210,6 +210,39 @@ func TestPassthroughBackend_List(t *testing.T) {
 	test(b)
 }
 
+func TestPassthroughBackend_Scan(t *testing.T) {
+	test := func(b logical.Backend) {
+		req := logical.TestRequest(t, logical.UpdateOperation, "foo")
+		req.Data["raw"] = "test"
+		storage := req.Storage
+
+		if _, err := b.HandleRequest(context.Background(), req); err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		req = logical.TestRequest(t, logical.ScanOperation, "")
+		req.Storage = storage
+		resp, err := b.HandleRequest(context.Background(), req)
+		if err != nil {
+			t.Fatalf("err: %v", err)
+		}
+
+		expected := &logical.Response{
+			Data: map[string]interface{}{
+				"keys": []string{"foo"},
+			},
+		}
+
+		if !reflect.DeepEqual(resp, expected) {
+			t.Fatalf("bad response.\n\nexpected: %#v\n\nGot: %#v", expected, resp)
+		}
+	}
+	b := testPassthroughBackend()
+	test(b)
+	b = testPassthroughLeasedBackend()
+	test(b)
+}
+
 func TestPassthroughBackend_Revoke(t *testing.T) {
 	test := func(b logical.Backend) {
 		req := logical.TestRequest(t, logical.RevokeOperation, "kv")

diff --git a/builtin/logical/kv/path_metadata.go b/builtin/logical/kv/path_metadata.go
@@ -71,6 +71,7 @@ version-agnostic information about a secret.
 			logical.ReadOperation:   b.upgradeCheck(b.pathMetadataRead()),
 			logical.DeleteOperation: b.upgradeCheck(b.pathMetadataDelete()),
 			logical.ListOperation:   b.upgradeCheck(b.pathMetadataList()),
+			logical.ScanOperation:   b.upgradeCheck(b.pathMetadataScan()),
 			logical.PatchOperation:  b.upgradeCheck(b.pathMetadataPatch()),
 		},
 
@@ -125,6 +126,32 @@ func (b *versionedKVBackend) pathMetadataList() framework.OperationFunc {
 	}
 }
 
+func (b *versionedKVBackend) pathMetadataScan() framework.OperationFunc {
+	return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+		key := data.Get("path").(string)
+
+		// Get an encrypted key storage object
+		wrapper, err := b.getKeyEncryptor(ctx, req.Storage)
+		if err != nil {
+			return nil, err
+		}
+
+		es := wrapper.Wrap(req.Storage)
+		view := logical.NewStorageView(es, key)
+
+		// Use encrypted key storage to recursively list the keys
+		var keys []string
+		err = logical.ScanView(ctx, view, func(path string) {
+			keys = append(keys, path)
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		return logical.ListResponse(keys), nil
+	}
+}
+
 func (b *versionedKVBackend) pathMetadataRead() framework.OperationFunc {
 	return func(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
 		// Create a read-only transaction if we can. We do not need to commit

diff --git a/changelog/763.txt b/changelog/763.txt
@@ -0,0 +1,3 @@
+```release-note:feature
+**Scanning**: introduce the ability to recursively list (scan) within plugins, adding a separate `scan` ACL capability, operation type, HTTP verb (`SCAN` with `GET` fallback via `?scan=true`), API, and CLI support. This also adds support to the KVv1 and KVv2 engines.
+```
diff --git a/command/commands.go b/command/commands.go
@@ -578,6 +578,11 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) map[string]cli.Co
 				BaseCommand: getBaseCommand(),
 			}, nil
 		},
+		"scan": func() (cli.Command, error) {
+			return &ScanCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
 		"secrets": func() (cli.Command, error) {
 			return &SecretsCommand{
 				BaseCommand: getBaseCommand(),
@@ -778,6 +783,11 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) map[string]cli.Co
 				BaseCommand: getBaseCommand(),
 			}, nil
 		},
+		"kv scan": func() (cli.Command, error) {
+			return &KVScanCommand{
+				BaseCommand: getBaseCommand(),
+			}, nil
+		},
 		"monitor": func() (cli.Command, error) {
 			return &MonitorCommand{
 				BaseCommand: getBaseCommand(),