From 4b851d84c2092b43e945c02751ded25efb3facf2 Mon Sep 17 00:00:00 2001 From: James Kerr Date: Wed, 27 Mar 2024 10:48:33 -0700 Subject: [PATCH] Remove JS Zed Query Parser (#2972) Removed zed parser. Added domain models for better handler code. Fixed e2e test flakiness. Query is parsed async. Error messages are better. --- .github/actions/setup-zui/action.yml | 6 +- .node-version | 2 +- apps/zui/.node-version | 1 - apps/zui/.swcrc | 1 - apps/zui/CODE.md | 20 +- apps/zui/src/app/commands/open-query.ts | 5 - apps/zui/src/app/commands/queries.ts | 114 -- apps/zui/src/app/core/models/active-query.ts | 5 - apps/zui/src/app/core/models/zed-ast.ts | 29 +- apps/zui/src/app/core/models/zed-expr.ts | 32 + apps/zui/src/app/core/models/zed-script.ts | 7 - .../core/state/create-wait-for-selector.ts | 32 + .../right-pane/history/history-item.tsx | 9 +- .../features/right-pane/history/section.tsx | 8 - .../features/right-pane/versions-section.tsx | 7 +- .../sidebar/flows/get-query-item-ctx-menu.ts | 103 -- .../sidebar/queries-section/index.tsx | 23 +- .../sidebar/queries-section/queries-tree.tsx | 47 +- apps/zui/src/app/menus/header-context-menu.ts | 6 +- apps/zui/src/app/menus/open-query-menu.ts | 3 +- apps/zui/src/app/menus/pool-toolbar-menu.ts | 5 +- apps/zui/src/app/menus/query-context-menu.ts | 4 +- apps/zui/src/app/menus/saved-query-menu.ts | 68 - apps/zui/src/app/menus/value-context-menu.ts | 3 +- .../src/components/subscribe-to-events.tsx | 7 - apps/zui/src/core/domain-model.ts | 25 + apps/zui/src/core/handlers.ts | 4 + apps/zui/src/core/main/main-object.ts | 7 + apps/zui/src/core/query/run.ts | 35 +- apps/zui/src/core/query/use-query.ts | 7 +- apps/zui/src/core/renderer.ts | 3 + apps/zui/src/css/_global.scss | 2 +- apps/zui/src/css/settings/_colors.scss | 2 + apps/zui/src/domain/editor/handlers.ts | 15 +- apps/zui/src/domain/editor/messages.ts | 5 + apps/zui/src/domain/editor/operations.ts | 7 + apps/zui/src/domain/editor/parse.test.ts | 19 + apps/zui/src/domain/handlers.ts | 2 + apps/zui/src/domain/lake/handlers.ts | 45 + apps/zui/src/domain/messages.ts | 9 +- apps/zui/src/domain/named-queries/handlers.ts | 41 + apps/zui/src/domain/named-queries/messages.ts | 5 + apps/zui/src/domain/operations.ts | 1 + apps/zui/src/domain/results/handlers/view.ts | 4 +- apps/zui/src/domain/session/handlers/pins.ts | 5 +- .../src/domain/session/handlers/queries.ts | 55 +- .../domain/session/handlers/submit-search.ts | 56 +- .../src/domain/session/menus/toolbar-menu.ts | 2 +- apps/zui/src/domain/session/messages.ts | 1 - apps/zui/src/domain/snapshots/handlers.ts | 29 + apps/zui/src/domain/window/handlers.ts | 5 +- .../run-main/run-protocol-handlers.ts | 21 +- apps/zui/src/js/api/current/current-api.ts | 3 +- apps/zui/src/js/api/queries/import.ts | 1 - apps/zui/src/js/api/queries/queries-api.ts | 122 +- apps/zui/src/js/api/queries/types.ts | 2 +- apps/zui/src/js/api/zui-api.ts | 13 - .../TabBar/use-query-id-name-map.ts | 8 - .../components/status-bar/query-progress.tsx | 81 -- .../js/components/status-bar/status-bar.tsx | 32 - .../js/components/status-bar/type-count.tsx | 14 - .../src/js/initializers/init-async-tasks.ts | 9 + .../src/js/initializers/init-domain-models.ts | 6 + apps/zui/src/js/initializers/initStore.ts | 10 +- apps/zui/src/js/initializers/initialize.ts | 16 +- apps/zui/src/js/models/ast.ts | 92 -- apps/zui/src/js/models/program.test.ts | 179 +-- apps/zui/src/js/models/program.ts | 82 +- apps/zui/src/js/models/query-model.ts | 20 - apps/zui/src/js/state/Appearance/index.ts | 7 +- apps/zui/src/js/state/Appearance/types.ts | 1 - apps/zui/src/js/state/Current/selectors.ts | 23 +- .../state/Queries/flows/get-query-source.ts | 4 +- apps/zui/src/js/state/Queries/selectors.ts | 10 +- apps/zui/src/js/state/QueryInfo/index.ts | 8 + apps/zui/src/js/state/QueryInfo/reducer.ts | 28 + apps/zui/src/js/state/QueryInfo/selectors.ts | 9 + .../RemoteQueries/flows/remote-queries.ts | 172 --- apps/zui/src/js/state/RemoteQueries/index.ts | 37 - apps/zui/src/js/state/Results/reducer.ts | 25 +- apps/zui/src/js/state/Results/selectors.ts | 16 +- apps/zui/src/js/state/Results/types.ts | 1 - apps/zui/src/js/state/Results/util.ts | 3 +- apps/zui/src/js/state/Tab/reducer.ts | 2 + apps/zui/src/js/state/Table/selectors.ts | 6 +- apps/zui/src/js/state/Tabs/flows.ts | 2 +- .../js/state/stores/create-window-store.ts | 7 +- .../src/js/state/stores/get-persistable.ts | 1 - .../stores/renderer-events-middleware.ts | 11 + apps/zui/src/js/state/stores/root-reducer.ts | 2 - apps/zui/src/js/state/types.ts | 1 - apps/zui/src/js/zed-script/toZedScript.ts | 4 +- apps/zui/src/models/active.ts | 39 + apps/zui/src/models/browser-tab.ts | 56 + apps/zui/src/models/editor-snapshot.ts | 68 + apps/zui/src/models/frame.ts | 8 + apps/zui/src/models/named-query.ts | 29 + apps/zui/src/models/session-history.ts | 34 + apps/zui/src/models/session.ts | 126 ++ .../zui/src/modules/async-tasks/async-task.ts | 30 + .../src/modules/async-tasks/async-tasks.ts | 33 + apps/zui/src/modules/async-tasks/index.ts | 2 + .../zui/src/test/shared/__mocks__/electron.ts | 2 + .../histogram-pane/histogram-pane.module.css | 1 + .../zui/src/views/histogram-pane/run-query.ts | 196 ++- .../views/histogram-pane/settings-form.tsx | 4 +- .../src/views/preview-load-modal/index.tsx | 7 +- .../results-pane/errors/default-error.tsx | 2 +- .../views/results-pane/run-results-query.tsx | 23 +- apps/zui/src/views/session-page/footer.tsx | 11 +- apps/zui/src/views/session-page/grid.tsx | 4 +- apps/zui/src/views/session-page/loader.ts | 79 +- .../src/views/session-page/pins/dialog.tsx | 1 + apps/zui/src/views/session-page/route.tsx | 4 +- .../src/views/session-page/use-title-form.ts | 6 +- packages/zed-js/.swcrc | 31 - packages/zed-js/jest.config.ts | 17 +- packages/zed-js/package.json | 3 +- packages/zed-js/project.json | 12 +- packages/zed-js/src/client/base-client.ts | 17 + packages/zed-node/src/parser.d.ts | 3 - packages/zed-node/src/parser.ts | 5 - packages/zed-wasm/.eslintrc.json | 21 - packages/zed-wasm/.gitignore | 5 + packages/zed-wasm/go.mod | 45 +- packages/zed-wasm/go.sum | 199 +-- packages/zed-wasm/index.html | 70 - packages/zed-wasm/lib/bridge.d.ts | 3 - packages/zed-wasm/lib/bridge.js | 14 + packages/zed-wasm/lib/global-shim.js | 2 - packages/zed-wasm/lib/node-globals.js | 9 + packages/zed-wasm/lib/wasm_exec.d.ts | 0 packages/zed-wasm/lib/wasm_exec.js | 1208 ++++++++--------- packages/zed-wasm/lib/wasm_exec_node.js | 52 - packages/zed-wasm/main.go | 22 +- packages/zed-wasm/package.json | 30 +- packages/zed-wasm/playwright.config.ts | 81 ++ packages/zed-wasm/project.json | 51 +- packages/zed-wasm/src/browser.js | 37 + packages/zed-wasm/src/golang-wasm.d.ts | 3 - packages/zed-wasm/src/index.js | 22 + packages/zed-wasm/src/index.ts | 52 - packages/zed-wasm/src/interface.js | 24 + packages/zed-wasm/test/browser/parse.spec.ts | 12 + packages/zed-wasm/test/browser/zq.test.js | 13 + packages/zed-wasm/test/cjs/parse.test.js | 44 + packages/zed-wasm/test/node/parser.test.js | 37 + packages/zed-wasm/test/node/zq.test.js | 17 + packages/zed-wasm/test/pages/parse.html | 55 + packages/zed-wasm/test/pages/zq.html | 60 + packages/zed-wasm/test/zq.test.js | 51 - packages/zed-wasm/tsconfig.json | 23 - packages/zed-wasm/tsconfig.lib.json | 10 - packages/zui-player/helpers/test-app.ts | 14 + packages/zui-player/tests/queries.spec.ts | 1 + tsconfig.base.json | 1 - yarn.lock | 88 ++ 157 files changed, 2560 insertions(+), 2698 deletions(-) delete mode 100644 apps/zui/.node-version delete mode 100644 apps/zui/src/app/commands/open-query.ts delete mode 100644 apps/zui/src/app/commands/queries.ts create mode 100644 apps/zui/src/app/core/models/zed-expr.ts create mode 100644 apps/zui/src/app/core/state/create-wait-for-selector.ts delete mode 100644 apps/zui/src/app/features/sidebar/flows/get-query-item-ctx-menu.ts delete mode 100644 apps/zui/src/app/menus/saved-query-menu.ts create mode 100644 apps/zui/src/core/domain-model.ts create mode 100644 apps/zui/src/core/renderer.ts create mode 100644 apps/zui/src/domain/editor/operations.ts create mode 100644 apps/zui/src/domain/editor/parse.test.ts create mode 100644 apps/zui/src/domain/lake/handlers.ts create mode 100644 apps/zui/src/domain/named-queries/handlers.ts create mode 100644 apps/zui/src/domain/named-queries/messages.ts create mode 100644 apps/zui/src/domain/snapshots/handlers.ts delete mode 100644 apps/zui/src/js/components/status-bar/query-progress.tsx delete mode 100644 apps/zui/src/js/components/status-bar/status-bar.tsx delete mode 100644 apps/zui/src/js/components/status-bar/type-count.tsx create mode 100644 apps/zui/src/js/initializers/init-async-tasks.ts create mode 100644 apps/zui/src/js/initializers/init-domain-models.ts delete mode 100644 apps/zui/src/js/models/ast.ts create mode 100644 apps/zui/src/js/state/QueryInfo/index.ts create mode 100644 apps/zui/src/js/state/QueryInfo/reducer.ts create mode 100644 apps/zui/src/js/state/QueryInfo/selectors.ts delete mode 100644 apps/zui/src/js/state/RemoteQueries/flows/remote-queries.ts delete mode 100644 apps/zui/src/js/state/RemoteQueries/index.ts create mode 100644 apps/zui/src/js/state/stores/renderer-events-middleware.ts create mode 100644 apps/zui/src/models/active.ts create mode 100644 apps/zui/src/models/browser-tab.ts create mode 100644 apps/zui/src/models/editor-snapshot.ts create mode 100644 apps/zui/src/models/frame.ts create mode 100644 apps/zui/src/models/named-query.ts create mode 100644 apps/zui/src/models/session-history.ts create mode 100644 apps/zui/src/models/session.ts create mode 100644 apps/zui/src/modules/async-tasks/async-task.ts create mode 100644 apps/zui/src/modules/async-tasks/async-tasks.ts create mode 100644 apps/zui/src/modules/async-tasks/index.ts delete mode 100644 packages/zed-js/.swcrc delete mode 100644 packages/zed-node/src/parser.d.ts delete mode 100644 packages/zed-node/src/parser.ts delete mode 100644 packages/zed-wasm/.eslintrc.json create mode 100644 packages/zed-wasm/.gitignore delete mode 100644 packages/zed-wasm/index.html delete mode 100644 packages/zed-wasm/lib/bridge.d.ts create mode 100644 packages/zed-wasm/lib/bridge.js delete mode 100644 packages/zed-wasm/lib/global-shim.js create mode 100644 packages/zed-wasm/lib/node-globals.js delete mode 100644 packages/zed-wasm/lib/wasm_exec.d.ts delete mode 100644 packages/zed-wasm/lib/wasm_exec_node.js create mode 100644 packages/zed-wasm/playwright.config.ts create mode 100644 packages/zed-wasm/src/browser.js delete mode 100644 packages/zed-wasm/src/golang-wasm.d.ts create mode 100644 packages/zed-wasm/src/index.js delete mode 100644 packages/zed-wasm/src/index.ts create mode 100644 packages/zed-wasm/src/interface.js create mode 100644 packages/zed-wasm/test/browser/parse.spec.ts create mode 100644 packages/zed-wasm/test/browser/zq.test.js create mode 100644 packages/zed-wasm/test/cjs/parse.test.js create mode 100644 packages/zed-wasm/test/node/parser.test.js create mode 100644 packages/zed-wasm/test/node/zq.test.js create mode 100644 packages/zed-wasm/test/pages/parse.html create mode 100644 packages/zed-wasm/test/pages/zq.html delete mode 100644 packages/zed-wasm/test/zq.test.js delete mode 100644 packages/zed-wasm/tsconfig.json delete mode 100644 packages/zed-wasm/tsconfig.lib.json diff --git a/.github/actions/setup-zui/action.yml b/.github/actions/setup-zui/action.yml index 99f4bcd69e..387a032669 100644 --- a/.github/actions/setup-zui/action.yml +++ b/.github/actions/setup-zui/action.yml @@ -1,7 +1,7 @@ name: Setup Zui description: Shared steps for setting up Zui in workflows runs: - using: "composite" + using: 'composite' steps: - name: Install Go uses: actions/setup-go@v2 @@ -21,6 +21,10 @@ runs: run: yarn --inline-builds shell: bash + - name: Install Playwright Deps + run: yarn workspace @brimdata/zed-wasm run playwright install + shell: bash + - name: Yarn Build run: yarn run build shell: bash diff --git a/.node-version b/.node-version index 3027af39c1..87ec8842b1 100644 --- a/.node-version +++ b/.node-version @@ -1 +1 @@ -16.10.0 +18.18.2 diff --git a/apps/zui/.node-version b/apps/zui/.node-version deleted file mode 100644 index 56bfee434b..0000000000 --- a/apps/zui/.node-version +++ /dev/null @@ -1 +0,0 @@ -v16.10.0 diff --git a/apps/zui/.swcrc b/apps/zui/.swcrc index d6f0426068..e15c4688b1 100644 --- a/apps/zui/.swcrc +++ b/apps/zui/.swcrc @@ -14,5 +14,4 @@ "type": "commonjs", "ignoreDynamic": true } - } diff --git a/apps/zui/CODE.md b/apps/zui/CODE.md index 389496ce06..bac3b93bbe 100644 --- a/apps/zui/CODE.md +++ b/apps/zui/CODE.md @@ -81,13 +81,29 @@ _Main Process Initializers_ Code that needs to be run one time before the app starts up can be put in an initializer. An initializer is a file that lives in the folder `src/electron/initializers/`. It must export a function named _initialize(main)_ that takes the Main Object as its only argument. See the FAQ for an example of creating a new initializer. +_Query Session_ + +This is a type of page that a tab can hold it the app. The page contains an editor pane, a results pane, and an inspector with various tabs related to querying data. These are session history, global history, data details, columns, and more. + _Query_ -A query in the app is like a container object. It holds the name and id of the query. It does not contain the zed code. Those are stored in a QueryVersion. Each Query has many QueryVersions, showing the history of that query. +A query in the app is like a container object. It holds the name and id of the query. It does not contain the zed code. Those are stored in a EditorSnapshot. Each Query has many EditorSnapshots, showing the history of that query. + +_Editor Snapshot_ - formerly QueryVersion + +This is an object that represents the state of a query editor at a given point in time. It contains fields like: pins, value, createdAt, lastRanAt, and queryId. The editor snapshot object will belong to either a named query, or a session query. + +_Query Text_ + +This is text of the final Zed Query we will send to the backend. _Session Query_ -A session query is like an unnamed Query. Each session (tab) has exactly one SessionQuery associated with it. The SessionQuery has many QueryVersions associated with it. +A session query is like an unnamed Query. Each session (tab) has exactly one SessionQuery associated with it. The SessionQuery has many EditorSnapshots associated with it. + +_Acitve Query_ + +This refers to whatever query is currently being presented in the query session page. This can either be a query or a session query. _Store_ diff --git a/apps/zui/src/app/commands/open-query.ts b/apps/zui/src/app/commands/open-query.ts deleted file mode 100644 index a99eb3a5fd..0000000000 --- a/apps/zui/src/app/commands/open-query.ts +++ /dev/null @@ -1,5 +0,0 @@ -import {createCommand} from "./command" - -export const openQuery = createCommand("openQuery", ({api}, id: string) => { - api.queries.open(id) -}) diff --git a/apps/zui/src/app/commands/queries.ts b/apps/zui/src/app/commands/queries.ts deleted file mode 100644 index 976e778aa2..0000000000 --- a/apps/zui/src/app/commands/queries.ts +++ /dev/null @@ -1,114 +0,0 @@ -import {nanoid} from "@reduxjs/toolkit" -import {QuerySource} from "src/js/api/queries/types" -import Appearance from "src/js/state/Appearance" -import {QueriesView} from "src/js/state/Appearance/types" -import Current from "src/js/state/Current" -import Editor from "src/js/state/Editor" -import Layout from "src/js/state/Layout" -import {createCommand} from "./command" - -export const save = createCommand( - "queries.save", - async ({api, getState}, name: string) => { - const attrs = Editor.getSnapshot(getState()) - const query = await api.queries.create({name, versions: [attrs]}) - api.queries.open(query.id) - } -) - -export const lock = createCommand("queries.lock", ({api, getState}) => { - const active = Current.getActiveQuery(getState()) - if (!active) return - return api.queries.update({ - id: active.id(), - changes: {isReadOnly: true}, - }) -}) - -export const unlock = createCommand("queries.lock", ({api, getState}) => { - const active = Current.getActiveQuery(getState()) - if (!active) return - return api.queries.update({ - id: active.id(), - changes: {isReadOnly: false}, - }) -}) - -export const moveToSource = createCommand( - "queries.moveToSource", - async ({api, getState, dispatch}, type: QuerySource) => { - const active = Current.getActiveQuery(getState()) - if (!active) return - const query = active.query - await api.queries.delete(query.id) - await api.queries.create({ - ...query.serialize(), - type, - versions: query.versions, - }) - dispatch(Appearance.setCurrentSectionName("queries")) - dispatch(Appearance.setQueriesView(type as QueriesView)) - api.queries.open(query.id) - } -) - -export const copyToSource = createCommand( - "queries.copyToSource", - async ({api, getState, dispatch}, type: QueriesView) => { - const active = Current.getActiveQuery(getState()) - if (!active) return - try { - const query = active.query - const newQuery = await api.queries.create({ - ...query.serialize(), - id: nanoid(), - type, - versions: query.versions, - }) - dispatch(Appearance.setQueriesView(type)) - api.queries.open(newQuery.id) - api.toast.success("Query Copied") - } catch (e) { - api.toast.error(`Copy Failed: ${e}`) - } - } -) - -export const duplicate = createCommand( - "queries.duplicate", - async ({api, getState}) => { - const active = Current.getActiveQuery(getState()) - if (!active) return - const query = active.query - const dup = await api.queries.create({ - ...query.serialize(), - id: nanoid(), - name: query.name + " (copy)", - versions: query.versions.map((v) => ({...v, version: nanoid()})), - type: api.queries.getSource(query.id), - }) - api.queries.open(dup.id) - } -) - -export const deleteCmd = createCommand( - "queries.delete", - async ({api, getState}) => { - const active = Current.getActiveQuery(getState()) - if (!active) return - await api.queries.delete(active.id()) - } -) - -export const rename = createCommand("queries.rename", ({dispatch}) => { - dispatch(Layout.showTitleForm()) -}) - -export const openLatestVersion = createCommand( - "queries.openLatestVersion", - ({api, getState}) => { - const active = Current.getActiveQuery(getState()) - if (!active) return - api.queries.open(active.query.id) - } -) diff --git a/apps/zui/src/app/core/models/active-query.ts b/apps/zui/src/app/core/models/active-query.ts index 7d879443c7..933e1a1101 100644 --- a/apps/zui/src/app/core/models/active-query.ts +++ b/apps/zui/src/app/core/models/active-query.ts @@ -1,4 +1,3 @@ -import {ZedAst} from "src/app/core/models/zed-ast" import {QueryModel} from "src/js/models/query-model" import {QueryVersion} from "src/js/state/QueryVersions/types" @@ -63,8 +62,4 @@ export class ActiveQuery { toZed() { return QueryModel.versionToZed(this.version) } - - toAst() { - return new ZedAst(QueryModel.versionToZed(this.version)) - } } diff --git a/apps/zui/src/app/core/models/zed-ast.ts b/apps/zui/src/app/core/models/zed-ast.ts index d8c0274bac..7b22c71737 100644 --- a/apps/zui/src/app/core/models/zed-ast.ts +++ b/apps/zui/src/app/core/models/zed-ast.ts @@ -1,19 +1,8 @@ -import {parse as parseAst} from "zed/compiler/parser/parser" -import {fieldExprToName} from "src/js/models/ast" import {toFieldPath} from "src/js/zed-script/toZedScript" +import {fieldExprToName} from "./zed-expr" export class ZedAst { - public tree: any - public error: Error | null - - constructor(public script: string) { - try { - this.tree = parseAst(script) - } catch (e) { - this.tree = null - this.error = e - } - } + constructor(public tree: any, public error: Error | null) {} get poolName() { const from = this.from @@ -34,6 +23,12 @@ export class ZedAst { return trunks.filter((t) => t.source.kind === "Pool").map((t) => t.source) } + get groupByKeys() { + const g = this.ops.find((op) => op.kind === "Summarize") + const keys = g ? g.keys : [] + return keys.map((k) => fieldExprToName(k.lhs || k.rhs)) + } + private _ops: any[] get ops() { if (this._ops) return this._ops @@ -77,3 +72,11 @@ export class ZedAst { export const OP_EXPR_PROC = "OpExpr" export const PARALLEL_PROC = "Parallel" +// are all these needed? +export const HEAD_PROC = "Head" +export const TAIL_PROC = "Tail" +export const SORT_PROC = "Sort" +export const FILTER_PROC = "Filter" +export const PRIMITIVE_PROC = "Primitive" +export const REGEXP_SEARCH_PROC = "RegexpSearch" +export const ANALYTIC_PROCS = ["Summarize"] diff --git a/apps/zui/src/app/core/models/zed-expr.ts b/apps/zui/src/app/core/models/zed-expr.ts new file mode 100644 index 0000000000..bbf6df9d2d --- /dev/null +++ b/apps/zui/src/app/core/models/zed-expr.ts @@ -0,0 +1,32 @@ +import {toFieldPath} from "src/js/zed-script/toZedScript" + +export function fieldExprToName(expr) { + let s = _fieldExprToName(expr) + // const r = toFieldPath(s) + return s +} + +function _fieldExprToName(expr): string | string[] { + switch (expr.kind) { + case "BinaryExpr": + if (expr.op == "." || expr.op == "[") { + return [] + .concat(_fieldExprToName(expr.lhs), _fieldExprToName(expr.rhs)) + .filter((n) => n !== "this") + } + return "" + case "ID": + return expr.name + case "This": + return "this" + case "Primitive": + return expr.text + case "Call": + var args = expr.args + .map((e) => toFieldPath(_fieldExprToName(e))) + .join(",") + return `${expr.name}(${args})` + default: + return "" + } +} diff --git a/apps/zui/src/app/core/models/zed-script.ts b/apps/zui/src/app/core/models/zed-script.ts index b98a512e18..8b1189a00f 100644 --- a/apps/zui/src/app/core/models/zed-script.ts +++ b/apps/zui/src/app/core/models/zed-script.ts @@ -1,13 +1,6 @@ -import {ZedAst} from "./zed-ast" - export class ZedScript { constructor(public script: string) {} - private _ast: ZedAst - get ast() { - return this._ast || (this._ast = new ZedAst(this.script)) - } - isEmpty() { const lines = this.script.split("\n") const comment = /^\s*\/\/.*$/ diff --git a/apps/zui/src/app/core/state/create-wait-for-selector.ts b/apps/zui/src/app/core/state/create-wait-for-selector.ts new file mode 100644 index 0000000000..3533f6bb77 --- /dev/null +++ b/apps/zui/src/app/core/state/create-wait-for-selector.ts @@ -0,0 +1,32 @@ +export function createWaitForSelector(store) { + return function waitForSelector( + select, + options: {signal?: AbortSignal} = {} + ) { + let resolve + let targetValue + + const unsubscribe = store.subscribe(() => { + const state = store.getState() + if (select(state) === targetValue) { + unsubscribe() + resolve() + } + }) + + let promise = new Promise((res, reject) => { + resolve = res + options.signal?.addEventListener("abort", () => { + unsubscribe() + reject(new DOMException("AbortError")) + }) + }) + + return { + toReturn(value) { + targetValue = value + return promise + }, + } + } +} diff --git a/apps/zui/src/app/features/right-pane/history/history-item.tsx b/apps/zui/src/app/features/right-pane/history/history-item.tsx index 89db93d39b..ee18fa64cb 100644 --- a/apps/zui/src/app/features/right-pane/history/history-item.tsx +++ b/apps/zui/src/app/features/right-pane/history/history-item.tsx @@ -1,7 +1,6 @@ import {formatDistanceToNowStrict} from "date-fns" import React, {useMemo} from "react" import {useSelector} from "react-redux" -import {useZuiApi} from "src/app/core/context" import Current from "src/js/state/Current" import Queries from "src/js/state/Queries" import QueryVersions from "src/js/state/QueryVersions" @@ -10,6 +9,7 @@ import {useEntryMenu} from "./use-entry-menu" import {State} from "src/js/state/types" import {ActiveQuery} from "src/app/core/models/active-query" import {NodeRendererProps} from "react-arborist" +import {Snapshots} from "src/domain/handlers" const Wrap = styled.div` height: 100%; @@ -101,7 +101,6 @@ function getTimestamp(active: ActiveQuery) { } export function HistoryItem({node}: NodeRendererProps) { - const api = useZuiApi() const {index, queryId, version} = node.data const onContextMenu = useEntryMenu(index) const sessionId = useSelector(Current.getSessionId) @@ -118,7 +117,11 @@ export function HistoryItem({node}: NodeRendererProps) { const active = new ActiveQuery(session, query, versionObj) const onClick = () => { if (active.isDeleted()) return - api.queries.open(active.id(), {version: active.versionId(), history: false}) + Snapshots.show({ + sessionId, + namedQueryId: queryId, + snapshotId: version, + }) } const type = getType(active) const value = getValue(active) diff --git a/apps/zui/src/app/features/right-pane/history/section.tsx b/apps/zui/src/app/features/right-pane/history/section.tsx index 49ab74dc23..b1a16c6e93 100644 --- a/apps/zui/src/app/features/right-pane/history/section.tsx +++ b/apps/zui/src/app/features/right-pane/history/section.tsx @@ -7,7 +7,6 @@ import {isEmpty} from "lodash" import {EmptyText} from "../common" import {FillFlexParent} from "src/components/fill-flex-parent" import {Tree} from "react-arborist" -import {useZuiApi} from "src/app/core/context" import {TREE_ITEM_HEIGHT} from "../../sidebar/item" const BG = styled.div` @@ -18,7 +17,6 @@ const BG = styled.div` ` export function HistorySection() { - const api = useZuiApi() const sessionHistory = useSelector(Current.getSessionHistory) || [] const history = useMemo( () => @@ -46,12 +44,6 @@ export function HistorySection() { indent={8} disableDrag disableDrop - onActivate={(node) => { - api.queries.open(node.data.queryId, { - version: node.data.version, - history: false, - }) - }} > {HistoryItem} diff --git a/apps/zui/src/app/features/right-pane/versions-section.tsx b/apps/zui/src/app/features/right-pane/versions-section.tsx index 60af7c71d7..71803317a4 100644 --- a/apps/zui/src/app/features/right-pane/versions-section.tsx +++ b/apps/zui/src/app/features/right-pane/versions-section.tsx @@ -6,8 +6,8 @@ import Current from "src/js/state/Current" import {QueryModel} from "src/js/models/query-model" import {EmptyText} from "./common" import {FillFlexParent} from "src/components/fill-flex-parent" -import {useZuiApi} from "src/app/core/context" import {TREE_ITEM_HEIGHT} from "../sidebar/item" +import {NamedQueries} from "src/domain/handlers" const EmptyMessage = () => { return Open a saved query to see the previous versions. @@ -23,7 +23,6 @@ const VersionsSection = () => { } const VersionsList = ({query}: {query: QueryModel}) => { - const api = useZuiApi() const data = useMemo(() => { return query.versions .map((v) => ({...v, id: v.version})) @@ -43,9 +42,7 @@ const VersionsList = ({query}: {query: QueryModel}) => { padding={8} data={data} selection={currentId} - onActivate={(node) => - api.queries.open(query.id, {version: node.id}) - } + onActivate={(node) => NamedQueries.show(query.id, node.id)} > {VersionItem} diff --git a/apps/zui/src/app/features/sidebar/flows/get-query-item-ctx-menu.ts b/apps/zui/src/app/features/sidebar/flows/get-query-item-ctx-menu.ts deleted file mode 100644 index 4e00bce91b..0000000000 --- a/apps/zui/src/app/features/sidebar/flows/get-query-item-ctx-menu.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { - deleteRemoteQueries, - isRemoteLib, -} from "src/js/state/RemoteQueries/flows/remote-queries" -import lib from "src/js/lib" -import toast from "react-hot-toast" -import {ipcRenderer, MenuItemConstructorOptions} from "electron" -import exportQueryLib from "src/js/flows/exportQueryLib" -import Queries from "src/js/state/Queries" -import QueryVersions from "src/js/state/QueryVersions" -import {invoke} from "src/core/invoke" - -const getQueryItemCtxMenu = - ({data, tree, handlers}) => - (dispatch, getState, {api}) => { - const {id, isReadOnly} = data - const isGroup = "items" in data - const hasMultiSelected = - tree.getSelectedIds().length > 1 && - !!tree.getSelectedIds().find((id) => id === data.id) - const isRemoteItem = dispatch(isRemoteLib([id])) - const query = Queries.build(getState(), id) - - const handleDelete = () => { - const selected = Array.from(new Set([...tree.getSelectedIds(), data.id])) - return invoke("showMessageBoxOp", { - type: "warning", - title: "Confirm Delete Query Window", - message: `Are you sure you want to delete the ${ - hasMultiSelected ? selected.length : "" - } selected item${hasMultiSelected ? "s" : ""}?`, - buttons: ["OK", "Cancel"], - }).then(({response}) => { - if (response === 0) { - selected.forEach((id) => dispatch(QueryVersions.at(id).deleteAll())) - if (isRemoteItem) dispatch(deleteRemoteQueries(selected)) - else dispatch(Queries.removeItems(selected)) - } - }) - } - - if (hasMultiSelected) - return [ - { - label: "Delete Selected", - click: handleDelete, - }, - ] - - return [ - { - label: "Run Query", - visible: !isGroup, - click: () => api.queries.open(id), - }, - { - label: "Copy Query Value", - visible: !isGroup, - click: () => { - lib.doc.copyToClipboard(query?.latestVersion()?.value) - toast("Query value copied to clipboard") - }, - }, - { - label: "Export Folder as JSON", - visible: isGroup, - click: async () => { - const {canceled, filePath} = await ipcRenderer.invoke( - "windows:showSaveDialog", - { - title: `Save Queries Folder as JSON`, - buttonLabel: "Export", - defaultPath: `${data.name}.json`, - properties: ["createDirectory"], - showsTagField: false, - } - ) - if (canceled) return - toast.promise( - dispatch(exportQueryLib(filePath, api.exportQueries(id))), - { - loading: "Exporting Queries...", - success: "Export Complete", - error: "Error Exporting Queries", - } - ) - }, - }, - {type: "separator"}, - { - label: "Rename", - enabled: !isReadOnly, - click: () => handlers.edit(), - }, - {type: "separator"}, - { - label: "Delete", - click: handleDelete, - }, - ] as MenuItemConstructorOptions[] - } - -export default getQueryItemCtxMenu diff --git a/apps/zui/src/app/features/sidebar/queries-section/index.tsx b/apps/zui/src/app/features/sidebar/queries-section/index.tsx index aab06dfca0..1c6e4f25a1 100644 --- a/apps/zui/src/app/features/sidebar/queries-section/index.tsx +++ b/apps/zui/src/app/features/sidebar/queries-section/index.tsx @@ -1,38 +1,17 @@ import React, {useState} from "react" -import {useDispatch, useSelector} from "react-redux" -import Appearance from "src/js/state/Appearance" import {Content} from "../content" import {SearchBar} from "../search-bar" import {Toolbar} from "../toolbar" import {QueriesTree} from "./queries-tree" -import {ToolbarTabs} from "src/components/toolbar-tabs" export function QueriesSection() { - const dispatch = useDispatch() - const view = useSelector(Appearance.getQueriesView) - const [searchTerm, setSearchTerm] = useState("") return ( <> - + - dispatch(Appearance.setQueriesView("local")), - checked: "local" === view, - }, - { - label: "Remote", - click: () => dispatch(Appearance.setQueriesView("remote")), - checked: "remote" === view, - }, - ]} - /> - case "remote": - return - default: - return null - } -} - -function LocalQueriesTree({searchTerm}: Props) { +export function QueriesTree({searchTerm}: Props) { const queries = useSelector(Queries.raw).items if (queries.length) { - return + return } else { return ( @@ -47,29 +34,13 @@ function LocalQueriesTree({searchTerm}: Props) { } } -function RemoteQueriesTree({searchTerm}) { - const dispatch = useDispatch() - const queries = useSelector(RemoteQueries.raw).items - useEffect(() => { - dispatch(refreshRemoteQueries()) - }, []) - if (queries.length) { - return - } else { - return ( - - ) - } -} - -function QueryTree(props: { +function TreeOfQueries(props: { queries: (Query | Group)[] searchTerm: string - type: "local" | "remote" }) { const dispatch = useDispatch() const api = useZuiApi() - const id = useSelector(Current.getQueryId) + const id = useSelector(Current.getSessionRouteParentId) const tree = useRef>() const [{isOver}, drop] = useQueryImportOnDrop() const initialOpenState = useSelector(Appearance.getQueriesOpenState) @@ -90,7 +61,6 @@ function QueryTree(props: { initialOpenState={initialOpenState} openByDefault={false} padding={8} - disableDrag={props.type === "remote"} ref={tree} selection={id} className="sidebar-tree" @@ -103,7 +73,9 @@ function QueryTree(props: { data={props.queries} childrenAccessor="items" onActivate={(node) => { - if (node.isLeaf && id !== node.id) api.queries.open(node.id) + if (node.isLeaf && id !== node.id) { + NamedQueries.show(node.id) + } }} onMove={(args) => { dispatch( @@ -121,7 +93,6 @@ function QueryTree(props: { return api.queries.create({ name: "", parentId, - type: props.type, }) } else { return api.queries.createGroup("", parentId) diff --git a/apps/zui/src/app/menus/header-context-menu.ts b/apps/zui/src/app/menus/header-context-menu.ts index 5252e05393..6fc2bbaf67 100644 --- a/apps/zui/src/app/menus/header-context-menu.ts +++ b/apps/zui/src/app/menus/header-context-menu.ts @@ -7,16 +7,16 @@ import { } from "src/js/flows/searchBar/actions" import {createMenu} from "src/core/menu" import {submitSearch} from "src/domain/session/handlers" +import QueryInfo from "src/js/state/QueryInfo" function getWhenContext(api: ZuiApi, column: ZedColumn) { - const query = api.current.query - const ast = query.toAst() + const {isSummarized} = api.select(QueryInfo.get) return { isRecord: column.isRecordType, isGrouped: column.isGrouped, isSortedAsc: column.isSortedAsc, isSortedDesc: column.isSortedDesc, - isSummarized: ast.isSummarized, + isSummarized, } } diff --git a/apps/zui/src/app/menus/open-query-menu.ts b/apps/zui/src/app/menus/open-query-menu.ts index 3cd2f0ab79..3e18fbf30c 100644 --- a/apps/zui/src/app/menus/open-query-menu.ts +++ b/apps/zui/src/app/menus/open-query-menu.ts @@ -1,6 +1,7 @@ import {MenuItemConstructorOptions} from "electron" import {Item} from "src/js/state/Queries/types" import {createMenu} from "src/core/menu" +import {NamedQueries} from "src/domain/handlers" export const openQueryMenu = createMenu(({api}) => { function createMenuItems(items: Item[]) { @@ -13,7 +14,7 @@ export const openQueryMenu = createMenu(({api}) => { } else { return { label: query.name, - click: () => api.queries.open(query.id), + click: () => NamedQueries.show(query.id), } } }) diff --git a/apps/zui/src/app/menus/pool-toolbar-menu.ts b/apps/zui/src/app/menus/pool-toolbar-menu.ts index 18c5bc6a33..706c047083 100644 --- a/apps/zui/src/app/menus/pool-toolbar-menu.ts +++ b/apps/zui/src/app/menus/pool-toolbar-menu.ts @@ -1,8 +1,9 @@ import {Pool} from "../core/pools/pool" import {createMenu} from "src/core/menu" +import {Snapshots} from "src/domain/handlers" import {chooseFiles} from "src/domain/loads/handlers" -export const poolToolbarMenu = createMenu(({api}, pool: Pool) => { +export const poolToolbarMenu = createMenu((_, pool: Pool) => { return [ { display: "icon-label", @@ -18,7 +19,7 @@ export const poolToolbarMenu = createMenu(({api}, pool: Pool) => { label: "Query Pool", iconName: "query", click: () => { - api.queries.open({ + Snapshots.createAndShow({ pins: [{type: "from", value: pool.name}], value: "", }) diff --git a/apps/zui/src/app/menus/query-context-menu.ts b/apps/zui/src/app/menus/query-context-menu.ts index c699dae0a4..8a73141bcb 100644 --- a/apps/zui/src/app/menus/query-context-menu.ts +++ b/apps/zui/src/app/menus/query-context-menu.ts @@ -3,8 +3,8 @@ import {Group, Query} from "src/js/state/Queries/types" import {copyQueryToClipboard} from "../commands/copy-query-to-clipboard" import {deleteQueries} from "../commands/delete-queries" import {exportQueryGroup} from "../commands/export-query-group" -import {openQuery} from "../commands/open-query" import {createMenu} from "src/core/menu" +import {NamedQueries} from "src/domain/handlers" export const queryContextMenu = createMenu( (_, tree: TreeApi, node: NodeApi) => { @@ -32,7 +32,7 @@ export const queryContextMenu = createMenu( { label: "Open Query", visible: node.isLeaf, - click: () => openQuery.run(node.id), + click: () => NamedQueries.show(node.id), }, {type: "separator"}, { diff --git a/apps/zui/src/app/menus/saved-query-menu.ts b/apps/zui/src/app/menus/saved-query-menu.ts deleted file mode 100644 index d19b04d965..0000000000 --- a/apps/zui/src/app/menus/saved-query-menu.ts +++ /dev/null @@ -1,68 +0,0 @@ -import * as queries from "src/app/commands/queries" -import {ActiveQuery} from "../core/models/active-query" -import {createMenu} from "src/core/menu" -import {openQueryMenu} from "./open-query-menu" - -export const savedQueryMenu = createMenu((_, active: ActiveQuery) => { - const query = active.query - return [ - { - label: "Go to Latest Version", - click: () => queries.openLatestVersion.run(), - visible: active.isOutdated(), - }, - {label: "Switch Query", nestedMenu: openQueryMenu()}, - {type: "separator"}, - { - label: "Duplicate", - click: () => queries.duplicate.run(), - enabled: !query.isReadOnly, - }, - { - label: "Move To Remote", - click: () => queries.moveToSource.run("remote"), - enabled: !query.isReadOnly, - visible: query.isLocal, - }, - { - label: "Move To Local", - click: () => queries.moveToSource.run("local"), - enabled: !query.isReadOnly, - visible: query.isRemote, - }, - { - label: "Copy To Remote", - click: () => queries.copyToSource.run("remote"), - enabled: !query.isReadOnly, - visible: query.isLocal, - }, - { - label: "Copy To Local", - click: () => queries.copyToSource.run("local"), - enabled: !query.isReadOnly, - visible: query.isRemote, - }, - { - label: "Lock Query", - click: () => queries.lock.run(), - visible: !query.isReadOnly, - }, - - {type: "separator"}, - { - label: "Unlock Query", - click: () => queries.unlock.run(), - visible: query.isReadOnly, - }, - { - label: "Rename...", - click: () => queries.rename.run(), - enabled: !query.isReadOnly, - }, - { - label: "Delete", - click: () => queries.deleteCmd.run(), - enabled: !query.isReadOnly, - }, - ] -}) diff --git a/apps/zui/src/app/menus/value-context-menu.ts b/apps/zui/src/app/menus/value-context-menu.ts index e1cdb50e80..aefe025d95 100644 --- a/apps/zui/src/app/menus/value-context-menu.ts +++ b/apps/zui/src/app/menus/value-context-menu.ts @@ -1,12 +1,13 @@ import * as zed from "@brimdata/zed-js" import {createMenu} from "src/core/menu" import ZuiApi from "src/js/api/zui-api" +import QueryInfo from "src/js/state/QueryInfo" function getWhenContext(api: ZuiApi, value: zed.Any) { return { isPrimitive: zed.isPrimitive(value), isIterable: zed.isIterable(value), - isGroupBy: api.current.query.toAst().isSummarized, + isGroupBy: api.select(QueryInfo.get).isSummarized, selectedText: document.getSelection().toString() || null, isIp: value instanceof zed.Ip, } diff --git a/apps/zui/src/components/subscribe-to-events.tsx b/apps/zui/src/components/subscribe-to-events.tsx index 2ea5b20042..04f8ed8d06 100644 --- a/apps/zui/src/components/subscribe-to-events.tsx +++ b/apps/zui/src/components/subscribe-to-events.tsx @@ -7,10 +7,6 @@ import {useDispatch} from "src/app/core/state" import {subscribeEvents} from "src/js/flows/subscribeEvents" import lake from "src/js/models/lake" import Lakes from "src/js/state/Lakes" -import { - getRemotePoolForLake, - refreshRemoteQueries, -} from "src/js/state/RemoteQueries/flows/remote-queries" type LakeSourceMap = {[lakeId: string]: EventSource} const lakeSourceMap: LakeSourceMap = {} @@ -53,9 +49,6 @@ export function SubscribeToEvents() { new Error("No 'pool_id' from branch-commit event") ) - const remotePool = dispatch(getRemotePoolForLake(l.id)) - if (poolId === remotePool?.id) - dispatch(refreshRemoteQueries(lake(l))) dispatch(syncPool(poolId, l.id)).catch((e) => { console.error("branch-commit update failed: ", e) }) diff --git a/apps/zui/src/core/domain-model.ts b/apps/zui/src/core/domain-model.ts new file mode 100644 index 0000000000..21375e2c54 --- /dev/null +++ b/apps/zui/src/core/domain-model.ts @@ -0,0 +1,25 @@ +import {Dispatch, State, Store} from "src/js/state/types" + +type Selector = (state: State, ...args: any) => any + +export class DomainModel { + static store: Store + + static select(selector: T): ReturnType { + return selector(this.store.getState()) + } + + static dispatch(action: Parameters[0]) { + return this.store.dispatch(action) + } + + constructor(public attrs: Attrs) {} + + protected dispatch(action: Parameters[0]) { + return DomainModel.dispatch(action) + } + + protected select(selector: T): ReturnType { + return DomainModel.select(selector) + } +} diff --git a/apps/zui/src/core/handlers.ts b/apps/zui/src/core/handlers.ts index 7ea7de99bf..17ddc275b4 100644 --- a/apps/zui/src/core/handlers.ts +++ b/apps/zui/src/core/handlers.ts @@ -5,14 +5,18 @@ import toast from "react-hot-toast" import {isFunction, isString} from "lodash" import ZuiApi from "src/js/api/zui-api" import {startTransition} from "react" +import {createWaitForSelector} from "src/app/core/state/create-wait-for-selector" +import {AsyncTasks} from "src/modules/async-tasks" export type HandlerContext = { dispatch: AppDispatch select any>(selector: Fn): ReturnType + waitForSelector: ReturnType invoke: typeof invoke toast: typeof toast oldApi: ZuiApi transition: typeof startTransition + asyncTasks: AsyncTasks } let context: HandlerContext | null = null diff --git a/apps/zui/src/core/main/main-object.ts b/apps/zui/src/core/main/main-object.ts index 515a5135ca..1392194e25 100644 --- a/apps/zui/src/core/main/main-object.ts +++ b/apps/zui/src/core/main/main-object.ts @@ -137,4 +137,11 @@ export class MainObject { const auth = await this.dispatch(getAuthToken(lake)) return new Client(lake.getAddress(), {auth}) } + + async createDefaultClient() { + const port = this.args.lakePort + const user = this.appMeta.userName + const lake = Lakes.getDefaultLake(port, user) + return this.createClient(lake.id) + } } diff --git a/apps/zui/src/core/query/run.ts b/apps/zui/src/core/query/run.ts index 90a0606071..f04359f5a7 100644 --- a/apps/zui/src/core/query/run.ts +++ b/apps/zui/src/core/query/run.ts @@ -1,40 +1,45 @@ -import {ResultStream} from "@brimdata/zed-js" import ErrorFactory from "src/js/models/ErrorFactory" import Current from "src/js/state/Current" import Results from "src/js/state/Results" import {Thunk} from "src/js/state/types" import {isAbortError} from "src/util/is-abort-error" +import {createHandler} from "../handlers" +import {query} from "src/domain/lake/handlers" +// Add a signal param here export function nextPage(id: string): Thunk { return async (dispatch, getState) => { if (Results.isFetching(id)(getState())) return if (Results.isComplete(id)(getState())) return - if (Results.isLimited(id)(getState())) return - dispatch(Results.nextPage({id})) - dispatch(run(id)) + if (Results.canPaginate(id)(getState())) { + dispatch(Results.nextPage({id})) + run(id) + } } } +// Add a signal param here export function firstPage(opts: {id: string; query: string}): Thunk { return async (dispatch, getState, {api}) => { const {id, query} = opts const key = Current.getLocation(getState()).key const tabId = api.current.tabId dispatch(Results.init({query, key, id, tabId})) - dispatch(run(id)) + run(id) } } -function run(id: string): Thunk> { - return async (dispatch, getState, {api}) => { - const tabId = api.current.tabId - const isFirstPage = Results.getPage(id)(getState()) === 1 - const prevVals = Results.getValues(id)(getState()) - const prevShapes = Results.getShapes(id)(getState()) - const paginatedQuery = Results.getPaginatedQuery(id)(getState()) - +// Add a signal param here +const run = createHandler( + async ({select, dispatch, asyncTasks}, id: string) => { + const tabId = select(Current.getTabId) + const isFirstPage = select(Results.getPage(id)) === 1 + const prevVals = select(Results.getValues(id)) + const prevShapes = select(Results.getShapes(id)) + const paginatedQuery = select(Results.getPaginatedQuery(id)) + const {signal} = asyncTasks.createOrReplace([tabId, id]) try { - const res = await api.query(paginatedQuery, {id, tabId}) + const res = await query(paginatedQuery, {signal}) await res.collect(({rows, shapesMap}) => { const values = isFirstPage ? rows : [...prevVals, ...rows] const shapes = isFirstPage ? shapesMap : {...prevShapes, ...shapesMap} @@ -54,4 +59,4 @@ function run(id: string): Thunk> { return null } } -} +) diff --git a/apps/zui/src/core/query/use-query.ts b/apps/zui/src/core/query/use-query.ts index 77e9325599..a250cb84fd 100644 --- a/apps/zui/src/core/query/use-query.ts +++ b/apps/zui/src/core/query/use-query.ts @@ -1,12 +1,7 @@ import {useDispatch} from "src/app/core/state" -import {firstPage, nextPage} from "./run" +import {nextPage} from "./run" import {useCallback} from "react" -export function useRun(opts: {id: string; query: string}) { - const dispatch = useDispatch() - return useCallback(() => dispatch(firstPage(opts)), [opts.id, opts.query]) -} - export function useNextPage(id: string) { const dispatch = useDispatch() return useCallback(() => dispatch(nextPage(id)), [id]) diff --git a/apps/zui/src/core/renderer.ts b/apps/zui/src/core/renderer.ts new file mode 100644 index 0000000000..e4e56edece --- /dev/null +++ b/apps/zui/src/core/renderer.ts @@ -0,0 +1,3 @@ +import EventEmitter from "events" + +export class Renderer extends EventEmitter {} diff --git a/apps/zui/src/css/_global.scss b/apps/zui/src/css/_global.scss index f35091db39..54bca5946d 100644 --- a/apps/zui/src/css/_global.scss +++ b/apps/zui/src/css/_global.scss @@ -77,7 +77,7 @@ code { } pre { - overflow-x: scroll; + overflow-x: auto; } code { diff --git a/apps/zui/src/css/settings/_colors.scss b/apps/zui/src/css/settings/_colors.scss index 65d83967a3..8ac3e72b4c 100644 --- a/apps/zui/src/css/settings/_colors.scss +++ b/apps/zui/src/css/settings/_colors.scss @@ -75,6 +75,8 @@ --medium: 300ms; --slow: 450ms; --measure: 45ch; + + --gutter: var(--space-s); } /** diff --git a/apps/zui/src/domain/editor/handlers.ts b/apps/zui/src/domain/editor/handlers.ts index b4c19f5690..055f9220b8 100644 --- a/apps/zui/src/domain/editor/handlers.ts +++ b/apps/zui/src/domain/editor/handlers.ts @@ -1,5 +1,5 @@ import * as zed from "@brimdata/zed-js" -import program from "src/js/models/program" +import {drillDown} from "src/js/models/program" import { appendQueryCountBy, appendQueryExclude, @@ -14,6 +14,7 @@ import {toZedScript} from "src/js/zed-script/toZedScript" import {submitSearch} from "src/domain/session/handlers" import {createHandler} from "src/core/handlers" import Selection from "src/js/state/Selection" +import QueryInfo from "src/js/state/QueryInfo" export const copyValueToClipboard = createHandler( "editor.copyValueToClipboard", @@ -88,14 +89,18 @@ export const newSearchWithValue = createHandler( export const pivotToValues = createHandler( "editor.pivotToValues", - ({select, dispatch}) => { + async ({select, dispatch}) => { const field = select(Selection.getField) const query = select(Editor.getValue) + const info = select(QueryInfo.get) // So this only works if the count() by field is in the editor, not in a pin. const record = field.rootRecord - const newProgram = program(query) - .drillDown(record as zed.Record) - .string() + const newProgram = await drillDown( + query, + record as zed.Record, + info.isSummarized, + info.groupByKeys + ) if (newProgram) { dispatch(Editor.setValue(newProgram)) diff --git a/apps/zui/src/domain/editor/messages.ts b/apps/zui/src/domain/editor/messages.ts index 9dbb3e61a6..196823261f 100644 --- a/apps/zui/src/domain/editor/messages.ts +++ b/apps/zui/src/domain/editor/messages.ts @@ -1,4 +1,5 @@ import * as hds from "./handlers" +import * as ops from "./operations" export type EditorHandlers = { "editor.copyValueToClipboard": typeof hds.copyValueToClipboard @@ -13,3 +14,7 @@ export type EditorHandlers = { "editor.sortDesc": typeof hds.sortDesc "editor.fuse": typeof hds.fuse } + +export type EditorOperations = { + "editor.parse": typeof ops.parse +} diff --git a/apps/zui/src/domain/editor/operations.ts b/apps/zui/src/domain/editor/operations.ts new file mode 100644 index 0000000000..47d23eb6b3 --- /dev/null +++ b/apps/zui/src/domain/editor/operations.ts @@ -0,0 +1,7 @@ +import {createOperation} from "src/core/operations" +import {lake} from "src/zui" + +export const parse = createOperation("editor.parse", async (ctx, string) => { + const resp = await lake.client.compile(string) + return resp.toJS() +}) diff --git a/apps/zui/src/domain/editor/parse.test.ts b/apps/zui/src/domain/editor/parse.test.ts new file mode 100644 index 0000000000..c7e7a438ae --- /dev/null +++ b/apps/zui/src/domain/editor/parse.test.ts @@ -0,0 +1,19 @@ +/** + * @jest-environment jsdom + */ +import {SystemTest} from "src/test/system" +import {parse} from "./operations" + +new SystemTest("editor.parse op") + +test("editor.parse", async () => { + const result = await parse("from source | count() by name") + expect(result.map((o) => o.kind)).toEqual(["From", "Summarize"]) +}) + +test("editor.parse error", async () => { + await expect(parse("from source | ;;;(")).rejects.toHaveProperty( + "error", + expect.stringContaining("error parsing Zed at column 15") + ) +}) diff --git a/apps/zui/src/domain/handlers.ts b/apps/zui/src/domain/handlers.ts index 5d1a46bbf9..c223e61f9a 100644 --- a/apps/zui/src/domain/handlers.ts +++ b/apps/zui/src/domain/handlers.ts @@ -7,3 +7,5 @@ import "./window/handlers" import "./session/handlers/navigation" import "./loads/handlers" import "./pools/handlers" +export * as NamedQueries from "./named-queries/handlers" +export * as Snapshots from "./snapshots/handlers" diff --git a/apps/zui/src/domain/lake/handlers.ts b/apps/zui/src/domain/lake/handlers.ts new file mode 100644 index 0000000000..f583aea7d0 --- /dev/null +++ b/apps/zui/src/domain/lake/handlers.ts @@ -0,0 +1,45 @@ +import {Client} from "@brimdata/zed-js" +import {createHandler} from "src/core/handlers" +import {LakeModel} from "src/js/models/lake" +import Current from "src/js/state/Current" +import {validateToken} from "src/js/auth0/utils" +import {getAuthCredentials} from "src/js/flows/lake/getAuthCredentials" +import Lakes from "src/js/state/Lakes" +import LakeStatuses from "src/js/state/LakeStatuses" + +export const getAuthToken = createHandler( + async ({dispatch}, lake: LakeModel) => { + if (!lake.authType) return null + if (lake.authType === "none") return null + const token = lake.authData.accessToken + if (validateToken(token)) { + return token + } else { + const newToken = await dispatch(getAuthCredentials(lake)) + if (newToken) { + dispatch(Lakes.setAccessToken({lakeId: lake.id, accessToken: newToken})) + return newToken + } else { + dispatch(LakeStatuses.set(lake.id, "login-required")) + throw new Error("Login Required") + } + } + } +) + +export const createClient = createHandler(async ({select}) => { + const lake = select(Current.mustGetLake) + const auth = await getAuthToken(lake) + return new Client(lake.getAddress(), {auth}) +}) + +type Options = { + signal?: AbortSignal +} + +export const query = createHandler( + async (ctx, text: string, options: Options = {}) => { + const client = await createClient() + return await client.query(text, {signal: options.signal}) + } +) diff --git a/apps/zui/src/domain/messages.ts b/apps/zui/src/domain/messages.ts index 6a25ab059a..10d3069ba3 100644 --- a/apps/zui/src/domain/messages.ts +++ b/apps/zui/src/domain/messages.ts @@ -10,7 +10,8 @@ import {EnvOperations} from "./env/messages" import {UpdatesOperations} from "./updates/messages" import {LoadsHandlers, LoadsOperations} from "./loads/messages" import {CommandsOperations} from "./commands/messages" -import {EditorHandlers} from "./editor/messages" +import {EditorHandlers, EditorOperations} from "./editor/messages" +import {NamedQueriesHandlers} from "./named-queries/messages" export type Handlers = ResultsHandlers & MenusHandlers & @@ -19,7 +20,8 @@ export type Handlers = ResultsHandlers & SessionHandlers & LoadsHandlers & PoolsHandlers & - EditorHandlers + EditorHandlers & + NamedQueriesHandlers export type Operations = PoolsOperations & LegacyOperations & @@ -31,7 +33,8 @@ export type Operations = PoolsOperations & LoadsOperations & WindowOperations & MenusOperations & - CommandsOperations + CommandsOperations & + EditorOperations export type OperationName = keyof Operations export type HandlerName = keyof Handlers diff --git a/apps/zui/src/domain/named-queries/handlers.ts b/apps/zui/src/domain/named-queries/handlers.ts new file mode 100644 index 0000000000..f35d463bb3 --- /dev/null +++ b/apps/zui/src/domain/named-queries/handlers.ts @@ -0,0 +1,41 @@ +import {createHandler} from "src/core/handlers" +import {Active} from "src/models/active" +import {EditorSnapshot} from "src/models/editor-snapshot" +import {NamedQuery} from "src/models/named-query" +import {Session} from "src/models/session" + +/** + * This handler is called when the user submits the form to name their + * query for the first time. + */ +export const create = createHandler(async ({oldApi}, name: string) => { + const {parentId: _, ...attrs} = Active.snapshot.attrs + const query = await oldApi.queries.create({name, versions: [attrs]}) + const namedQuery = new NamedQuery({ + id: query.id, + name: query.name, + }) + Active.session.navigate(namedQuery.lastSnapshot, namedQuery) +}) + +/** + * This handler is called when the user updates a query to a new version. + */ +export const update = createHandler("namedQueries.update", () => { + const {session, snapshot} = Active + const {namedQuery} = session + const newSnapshot = snapshot.clone({parentId: namedQuery.id}) + newSnapshot.save() + session.navigate(newSnapshot, namedQuery) +}) + +/* This handler is called when you want to display a named query in a session */ +export const show = createHandler((_, id: string, snapshotId?: string) => { + const query = NamedQuery.find(id) + const snapshot = snapshotId + ? EditorSnapshot.find(query.id, snapshotId) + : query.lastSnapshot + + Session.activateLastFocused() + Active.session.navigate(snapshot, query) +}) diff --git a/apps/zui/src/domain/named-queries/messages.ts b/apps/zui/src/domain/named-queries/messages.ts new file mode 100644 index 0000000000..907768cbcd --- /dev/null +++ b/apps/zui/src/domain/named-queries/messages.ts @@ -0,0 +1,5 @@ +import * as handlers from "./handlers" + +export type NamedQueriesHandlers = { + "namedQueries.update": typeof handlers.update +} diff --git a/apps/zui/src/domain/operations.ts b/apps/zui/src/domain/operations.ts index c2361e1b73..1c02a64711 100644 --- a/apps/zui/src/domain/operations.ts +++ b/apps/zui/src/domain/operations.ts @@ -6,3 +6,4 @@ export * as loadersOperations from "./loads/operations" export * as windowOperations from "./window/operations" export * as menusOperations from "./menus/operations" export * as commandsOperations from "./commands/operations" +export * as editorOperations from "./editor/operations" diff --git a/apps/zui/src/domain/results/handlers/view.ts b/apps/zui/src/domain/results/handlers/view.ts index 94364680e1..45a0abdc72 100644 --- a/apps/zui/src/domain/results/handlers/view.ts +++ b/apps/zui/src/domain/results/handlers/view.ts @@ -38,9 +38,9 @@ export const showExportDialog = createHandler( export const toggleHistogram = createHandler( "results.toggleHistogram", - ({dispatch, select, oldApi}) => { + ({dispatch, select}) => { const isShown = select(Layout.getShowHistogram) - if (!isShown) runHistogramQuery(oldApi) + if (!isShown) runHistogramQuery() dispatch(Layout.toggleHistogram()) } ) diff --git a/apps/zui/src/domain/session/handlers/pins.ts b/apps/zui/src/domain/session/handlers/pins.ts index 0b9c4135dd..68ab000d48 100644 --- a/apps/zui/src/domain/session/handlers/pins.ts +++ b/apps/zui/src/domain/session/handlers/pins.ts @@ -10,6 +10,7 @@ import {submitSearch} from "src/domain/session/handlers" import {createHandler} from "src/core/handlers" import ZuiApi from "src/js/api/zui-api" import Selection from "src/js/state/Selection" +import {Snapshots} from "src/domain/handlers" export const createPinFromEditor = createHandler( "session.createPinFromEditor", @@ -42,9 +43,9 @@ export const createFromPin = createHandler( export const setFromPin = createHandler( "session.setFromPin", - ({dispatch, select, oldApi}, value: string) => { + ({dispatch, select}, value: string) => { if (select(Tabs.none)) { - oldApi.queries.open({pins: [{type: "from", value}], value: ""}) + Snapshots.createAndShow({pins: [{type: "from", value}], value: ""}) } else { dispatch(Editor.setFrom(value)) submitSearch() diff --git a/apps/zui/src/domain/session/handlers/queries.ts b/apps/zui/src/domain/session/handlers/queries.ts index b1ec16040c..565ef6b5a5 100644 --- a/apps/zui/src/domain/session/handlers/queries.ts +++ b/apps/zui/src/domain/session/handlers/queries.ts @@ -1,50 +1,55 @@ import {createHandler} from "src/core/handlers" import Current from "src/js/state/Current" -import Editor from "src/js/state/Editor" import Layout from "src/js/state/Layout" import {plusOne} from "src/util/plus-one" import {submitSearch} from "./submit-search" +import {ZedAst} from "src/app/core/models/zed-ast" +import {Active} from "src/models/active" +import {create} from "src/domain/named-queries/handlers" export const editQuery = createHandler("session.editQuery", ({dispatch}) => { dispatch(Layout.showTitleForm()) }) -export const updateQuery = createHandler( - "session.updateQuery", - ({select, oldApi}) => { - const snapshot = select(Editor.getSnapshot) - const active = select(Current.getActiveQuery) - const id = active.query.id - oldApi.queries.addVersion(id, snapshot) - oldApi.queries.open(id, {history: "replace"}) - } -) - export const runQuery = createHandler("session.runQuery", () => { submitSearch() }) export const saveAsNewQuery = createHandler( "session.saveAsNewQuery", - async ({select, oldApi, dispatch}) => { + async ({select, dispatch}) => { const name = select(Current.getActiveQuery).name() - const attrs = select(Editor.getSnapshot) const newName = plusOne(name) - const query = await oldApi.queries.create({ - name: newName, - versions: [attrs], - }) - oldApi.queries.open(query.id) + await create(newName) setTimeout(() => { dispatch(Layout.showTitleForm()) }) } ) -export const resetQuery = createHandler( - "session.resetQuery", - ({select, oldApi}) => { - const snapshot = select(Editor.getSnapshot) - oldApi.queries.open(snapshot) +export const resetQuery = createHandler("session.resetQuery", () => { + const {session} = Active + session.navigate(session.snapshot) +}) + +const fetchAst = createHandler(async ({invoke}, string) => { + let tree + try { + tree = await invoke("editor.parse", string) + } catch (error) { + tree = {error} } -) + return tree +}) + +export const fetchQueryInfo = createHandler(async (_, query: string) => { + const tree = await fetchAst(query) + const ast = new ZedAst(tree, tree.error) + return { + isSummarized: ast.isSummarized, + poolName: ast.poolName, + sorts: ast.sorts, + error: ast.error, + groupByKeys: ast.groupByKeys, + } +}) diff --git a/apps/zui/src/domain/session/handlers/submit-search.ts b/apps/zui/src/domain/session/handlers/submit-search.ts index fbb2e57bbf..573a8d2fcf 100644 --- a/apps/zui/src/domain/session/handlers/submit-search.ts +++ b/apps/zui/src/domain/session/handlers/submit-search.ts @@ -1,42 +1,24 @@ -import Current from "src/js/state/Current" -import Editor from "src/js/state/Editor" -import Results from "src/js/state/Results" -import QueryVersions from "../../../js/state/QueryVersions" -import {QueryModel} from "../../../js/models/query-model" -import {RESULTS_QUERY} from "src/views/results-pane/run-results-query" -import Table from "src/js/state/Table" -import Inspector from "src/js/state/Inspector" import {createHandler} from "src/core/handlers" -import Selection from "src/js/state/Selection" +import {Active} from "src/models/active" -export const submitSearch = createHandler((ctx) => { - const {dispatch, select, oldApi} = ctx - const api = oldApi +/** + * Save the active snapshot under the session id. + * + * Redirect the app to the previous parent id, + * but the active snapshot id that was just saved. + * + * The session page is essentially a form to create + * a new editor snapshot under that session id. + * + * It's should be thought of as POST /session/:id/snapshots + */ +export const submitSearch = createHandler(async () => { + const {session} = Active + const nextSnapshot = Active.snapshot - dispatch(Selection.reset()) - dispatch(Table.setScrollPosition({top: 0, left: 0})) - dispatch(Inspector.setScrollPosition({top: 0, left: 0})) - - const nextVersion = select(Editor.getSnapshot) - const active = select(Current.getActiveQuery) - const error = QueryModel.checkSyntax(nextVersion) - - // An error with the syntax - if (error) { - const tabId = select(Current.getTabId) - dispatch(Results.error({id: RESULTS_QUERY, error, tabId})) - return + if (nextSnapshot.equals(session.snapshot)) { + session.load() + } else { + session.navigate(Active.snapshot, session.namedQuery) } - - // Reuse the version url if the next version is the same as the latest - // of this query, either session or saved. - if (QueryVersions.areEqual(active.version, nextVersion)) { - api.queries.open(active.id(), {version: active.versionId()}) - return - } - - // This is a new query, add a new version to the session, - // And open the current active query with the version set to the new one. - api.queries.addVersion(active.session.id, nextVersion) - api.queries.open(active.id(), {version: nextVersion.version}) }) diff --git a/apps/zui/src/domain/session/menus/toolbar-menu.ts b/apps/zui/src/domain/session/menus/toolbar-menu.ts index 564df25372..130e9bc1e5 100644 --- a/apps/zui/src/domain/session/menus/toolbar-menu.ts +++ b/apps/zui/src/domain/session/menus/toolbar-menu.ts @@ -5,7 +5,7 @@ export const sessionToolbarMenu = createMenu((_, query: ActiveQuery) => { return [ { label: "Update Query", - command: "session.updateQuery", + command: "namedQueries.update", iconName: "check", visible: query.isModified(), }, diff --git a/apps/zui/src/domain/session/messages.ts b/apps/zui/src/domain/session/messages.ts index bd59b57eba..24d5cbc185 100644 --- a/apps/zui/src/domain/session/messages.ts +++ b/apps/zui/src/domain/session/messages.ts @@ -7,7 +7,6 @@ export type SessionHandlers = { "session.canGoForward": typeof handlers.canGoForward "session.createPinFromEditor": typeof handlers.createPinFromEditor "session.editQuery": typeof handlers.editQuery - "session.updateQuery": typeof handlers.updateQuery "session.runQuery": typeof handlers.runQuery "session.saveAsNewQuery": typeof handlers.saveAsNewQuery "session.resetQuery": typeof handlers.resetQuery diff --git a/apps/zui/src/domain/snapshots/handlers.ts b/apps/zui/src/domain/snapshots/handlers.ts new file mode 100644 index 0000000000..d89887fbb4 --- /dev/null +++ b/apps/zui/src/domain/snapshots/handlers.ts @@ -0,0 +1,29 @@ +import {createHandler} from "src/core/handlers" +import {Active} from "src/models/active" +import {EditorSnapshot} from "src/models/editor-snapshot" +import {NamedQuery} from "src/models/named-query" +import {Session} from "src/models/session" + +/** + * This is called when you click on a session history entry. + */ + +type Args = { + sessionId: string + namedQueryId?: string + snapshotId: string +} + +export const show = createHandler((ctx, args: Args) => { + const {session} = Active + const namedQuery = NamedQuery.find(args.namedQueryId) + const snapshot = EditorSnapshot.find(session.id, args.snapshotId) + Active.session.navigate(snapshot, namedQuery) +}) + +export const createAndShow = createHandler( + (ctx, args: Partial) => { + Session.activateLastFocused() + Active.session.navigate(new EditorSnapshot(args)) + } +) diff --git a/apps/zui/src/domain/window/handlers.ts b/apps/zui/src/domain/window/handlers.ts index 1b12c3675b..cdb589c850 100644 --- a/apps/zui/src/domain/window/handlers.ts +++ b/apps/zui/src/domain/window/handlers.ts @@ -7,6 +7,7 @@ import {PaneName} from "src/js/state/Layout/types" import Appearance from "src/js/state/Appearance" import Layout from "src/js/state/Layout" import Modal from "src/js/state/Modal" +import {Snapshots} from "../handlers" export const showErrorMessage = createHandler( "window.showErrorMessage", @@ -39,9 +40,7 @@ export const showWelcomePage = createHandler( export const query = createHandler( "window.query", (ctx, params: QueryParams) => { - ctx.dispatch((d, getState, {api}) => { - api.queries.open(params) - }) + Snapshots.createAndShow(params) } ) diff --git a/apps/zui/src/electron/run-main/run-protocol-handlers.ts b/apps/zui/src/electron/run-main/run-protocol-handlers.ts index d7d18037f6..96c021fb79 100644 --- a/apps/zui/src/electron/run-main/run-protocol-handlers.ts +++ b/apps/zui/src/electron/run-main/run-protocol-handlers.ts @@ -1,5 +1,13 @@ -import {app, protocol} from "electron" +import {app, protocol, net} from "electron" import path from "path" +import {pathToFileURL} from "url" + +protocol.registerSchemesAsPrivileged([ + { + scheme: "app-asset", + privileges: {standard: true, supportFetchAPI: true, bypassCSP: true}, + }, +]) export function runProtocolHandlers() { app.whenReady().then(() => { @@ -10,5 +18,16 @@ export function runProtocolHandlers() { const absPath = path.join(rootPath, relPath) callback(absPath) }) + + protocol.handle("app-asset", (request) => { + const {host, pathname, href} = new URL(request.url) + if (host === "node_modules") { + const path = pathname.slice(1) + const file = require.resolve(path) + const url = pathToFileURL(file).toString() + return net.fetch(url, {bypassCustomProtocolHandlers: true}) + } + throw new Error("Unknown App Asset " + href) + }) }) } diff --git a/apps/zui/src/js/api/current/current-api.ts b/apps/zui/src/js/api/current/current-api.ts index f6bec7c498..cb1d0fb515 100644 --- a/apps/zui/src/js/api/current/current-api.ts +++ b/apps/zui/src/js/api/current/current-api.ts @@ -1,6 +1,7 @@ import * as zed from "@brimdata/zed-js" import Current from "src/js/state/Current" import LogDetails from "src/js/state/LogDetails" +import QueryInfo from "src/js/state/QueryInfo" import {GetState} from "src/js/state/types" export class CurrentApi { @@ -21,7 +22,7 @@ export class CurrentApi { } get poolName() { - return Current.getActiveQuery(this.getState()).toAst().poolName + return QueryInfo.get(this.getState()).poolName } get value() { diff --git a/apps/zui/src/js/api/queries/import.ts b/apps/zui/src/js/api/queries/import.ts index 49f254aa31..c5273fe44b 100644 --- a/apps/zui/src/js/api/queries/import.ts +++ b/apps/zui/src/js/api/queries/import.ts @@ -17,7 +17,6 @@ export const queriesImport = } if (id) { dispatch(Appearance.setCurrentSectionName("queries")) - dispatch(Appearance.setQueriesView("local")) setTimeout(() => { selectQuery.trigger(resp.id) }) diff --git a/apps/zui/src/js/api/queries/queries-api.ts b/apps/zui/src/js/api/queries/queries-api.ts index 070d274f45..3718979d6f 100644 --- a/apps/zui/src/js/api/queries/queries-api.ts +++ b/apps/zui/src/js/api/queries/queries-api.ts @@ -1,22 +1,11 @@ import {nanoid} from "@reduxjs/toolkit" -import tabHistory from "src/app/router/tab-history" -import {queryPath} from "src/app/router/utils/paths" -import Current from "src/js/state/Current" import Queries from "src/js/state/Queries" import QueryVersions from "src/js/state/QueryVersions" import {QueryVersion} from "src/js/state/QueryVersions/types" -import { - deleteRemoteQueries, - isRemoteLib, - appendRemoteQueries, -} from "src/js/state/RemoteQueries/flows/remote-queries" -import SessionHistories from "src/js/state/SessionHistories" -import Tabs from "src/js/state/Tabs" import {AppDispatch, GetState} from "../../state/types" import {queriesImport} from "./import" -import {CreateQueryParams, OpenQueryOptions, QueryParams} from "./types" +import {CreateQueryParams, QueryParams} from "./types" import {Query} from "src/js/state/Queries/types" -import RemoteQueries from "src/js/state/RemoteQueries" import SessionQueries from "src/js/state/SessionQueries" import {invoke} from "src/core/invoke" @@ -40,19 +29,11 @@ export class QueriesApi { } async create(params: CreateQueryParams) { - const type = params.type ?? "local" const query = {id: params.id ?? nanoid(), name: params.name ?? ""} const versions = params.versions ?? [QueryVersions.initial()] - - if (type === "local") { - this.dispatch(Queries.addItem(query, params.parentId)) - versions.forEach((version) => this.addVersion(query.id, version)) - return this.find(query.id) - } else if (type === "remote") { - const records = versions.map((version) => ({...query, ...version})) - await this.dispatch(appendRemoteQueries(records)) - return this.find(query.id) - } + this.dispatch(Queries.addItem(query, params.parentId)) + versions.forEach((version) => this.createEditorSnapshot(query.id, version)) + return this.find(query.id) } createGroup(name: string, parentId: string) { @@ -62,24 +43,7 @@ export class QueriesApi { } async update(args: {id: string; changes: Partial}) { - switch (this.getSource(args.id)) { - case "local": - this.dispatch(Queries.editItem(args)) - return true - case "remote": - var query = Queries.build(this.getState(), args.id) - if (!query) return false - var serialized = query.serialize() - var version = query.latestVersion() - var insert = { - ...serialized, - ...args.changes, - ...version, - ts: new Date().toISOString(), - } - await this.dispatch(appendRemoteQueries([insert])) - return true - } + this.dispatch(Queries.editItem(args)) } async delete(id: string | string[]) { @@ -87,11 +51,7 @@ export class QueriesApi { await Promise.all( ids.map(async (id) => { this.dispatch(QueryVersions.at(id).deleteAll()) - if (this.dispatch(isRemoteLib([id]))) { - await this.dispatch(deleteRemoteQueries([id])) - } else { - this.dispatch(Queries.removeItems([id])) - } + this.dispatch(Queries.removeItems([id])) }) ) } @@ -100,7 +60,7 @@ export class QueriesApi { this.update({id, changes: {name}}) } - addVersion(queryId: string, params: QueryVersion | QueryParams) { + createEditorSnapshot(queryId: string, params: QueryVersion | QueryParams) { const ts = new Date().toISOString() const id = nanoid() const version = {ts, version: id, ...params} @@ -111,74 +71,6 @@ export class QueriesApi { getSource(id: string) { if (SessionQueries.find(this.getState(), id)) return "session" if (Queries.find(this.getState(), id)) return "local" - if (RemoteQueries.find(this.getState(), id)) return "remote" return null } - - /** - * When you open a query, find the nearest query session tab. - * If one doesn't exist, create it. Next, check the history - * location.pathname for that tab. If it's the same - * as the url you are about to open, reload it. Don't push - * to the session history or the tab history. - * - * If it's not the same, push that url to the tab history - * and optionally to the session history. - * This is a candidate for a refactor - * - * TODO: This should be a command, not part of the api like this - */ - open(id: string | QueryParams, options: Partial = {}) { - const opts = openQueryOptions(options) - const tab = this.select(Tabs.findFirstQuerySession) - const tabId = tab ? tab.id : nanoid() - - let queryId: string, versionId: string - if (typeof id === "string") { - const q = this.select((state) => Queries.build(state, id)) - - queryId = id - versionId = opts.version || q?.latestVersionId() || "0" - } else { - queryId = tabId - versionId = nanoid() - this.addVersion(queryId, { - ...id, - version: versionId, - ts: new Date().toISOString(), - }) - } - const url = queryPath(queryId, versionId) - if (tab) { - this.dispatch(Tabs.activate(tabId)) - } else { - this.dispatch(Tabs.create(url, tabId)) - } - - const history = this.select(Current.getHistory) - if (history.location.pathname === url) { - this.dispatch(tabHistory.reload()) - } else { - if (opts.history === "replace") { - this.dispatch(tabHistory.replace(url)) - this.dispatch(SessionHistories.replace(queryId, versionId)) - } else if (opts.history) { - this.dispatch(tabHistory.push(url)) - this.dispatch(SessionHistories.push(queryId, versionId)) - } else { - this.dispatch(tabHistory.push(url)) - } - } - } - - private select any>(selector: T) { - return selector(this.getState()) - } } - -const openQueryOptions = ( - user: Partial -): OpenQueryOptions => ({ - history: true, - ...user, -}) diff --git a/apps/zui/src/js/api/queries/types.ts b/apps/zui/src/js/api/queries/types.ts index 75970819d1..1f1499a4d2 100644 --- a/apps/zui/src/js/api/queries/types.ts +++ b/apps/zui/src/js/api/queries/types.ts @@ -23,4 +23,4 @@ export type Select = any>( selector: T ) => ReturnType -export type QuerySource = "local" | "remote" | "session" +export type QuerySource = "local" | "session" diff --git a/apps/zui/src/js/api/zui-api.ts b/apps/zui/src/js/api/zui-api.ts index d40cda9bf8..e2d92d9232 100644 --- a/apps/zui/src/js/api/zui-api.ts +++ b/apps/zui/src/js/api/zui-api.ts @@ -65,19 +65,6 @@ export default class ZuiApi { return [ctl.signal, cleanup] as const } - async query(body: string, opts: {id?: string; tabId?: string} = {}) { - const zealot = await this.getZealot() - const [signal, cleanup] = this.createAbortable(opts.tabId, opts.id) - try { - const resp = await zealot.query(body, {signal}) - resp.on("success", cleanup) - return resp - } catch (e) { - cleanup() - throw e - } - } - select ReturnType>(fn: T) { return fn(this.getState()) } diff --git a/apps/zui/src/js/components/TabBar/use-query-id-name-map.ts b/apps/zui/src/js/components/TabBar/use-query-id-name-map.ts index 5bce54dca3..04c2248dcf 100644 --- a/apps/zui/src/js/components/TabBar/use-query-id-name-map.ts +++ b/apps/zui/src/js/components/TabBar/use-query-id-name-map.ts @@ -1,13 +1,11 @@ import {useSelector} from "react-redux" import Queries from "src/js/state/Queries" -import RemoteQueries from "src/js/state/RemoteQueries" import TreeModel from "tree-model" import {Query} from "src/js/state/Queries/types" import SessionQueries from "src/js/state/SessionQueries" export const useQueryIdNameMap = () => { const localRaw = useSelector(Queries.raw) - const remoteRaw = useSelector(RemoteQueries.raw) const sessionRaw = useSelector(SessionQueries.raw) const idNameMap = {} @@ -20,12 +18,6 @@ export const useQueryIdNameMap = () => { } return true }) - new TreeModel({childrenPropertyName: "items"}).parse(remoteRaw).walk((n) => { - if (!("items" in n.model)) { - idNameMap[n.model.id] = n.model.name - } - return true - }) return idNameMap } diff --git a/apps/zui/src/js/components/status-bar/query-progress.tsx b/apps/zui/src/js/components/status-bar/query-progress.tsx deleted file mode 100644 index 515e9166f9..0000000000 --- a/apps/zui/src/js/components/status-bar/query-progress.tsx +++ /dev/null @@ -1,81 +0,0 @@ -import React from "react" -import {useSelector} from "react-redux" -import Results from "src/js/state/Results" -import {RESULTS_QUERY} from "src/views/results-pane/run-results-query" -import styled from "styled-components" - -const Loader = styled.div` - &, - &:after { - border-radius: 50%; - width: 1em; - height: 1em; - } - & { - position: relative; - text-indent: -9999em; - border-top: 0.2em solid rgba(0, 0, 0, 0.1); - border-right: 0.2em solid rgba(0, 0, 0, 0.1); - border-bottom: 0.2em solid rgba(0, 0, 0, 0.1); - border-left: 0.2em solid var(--fg-color); - transform: translateZ(0); - animation: load8 1.1s infinite linear; - } - @-webkit-keyframes load8 { - 0% { - -webkit-transform: rotate(0deg); - transform: rotate(0deg); - } - 100% { - -webkit-transform: rotate(360deg); - transform: rotate(360deg); - } - } - @keyframes load8 { - 0% { - -webkit-transform: rotate(0deg); - transform: rotate(0deg); - } - 100% { - -webkit-transform: rotate(360deg); - transform: rotate(360deg); - } - } -` - -const Span = styled.span` - display: inline-flex; - align-items: center; - gap: 10px; -` - -export function QueryProgress() { - const status = useSelector(Results.getStatus(RESULTS_QUERY)) - const count = useSelector(Results.getCount(RESULTS_QUERY)) - if (status === "FETCHING") { - return ( - - Fetching - - - ) - } else if (status === "COMPLETE") { - return ( - - Results: {count} - - ) - } else if (status === "INCOMPLETE") { - return ( - - Results: First {count} - - ) - } else if (status === "LIMIT") { - return ( - - Results: Limited to first {count} - - ) - } -} diff --git a/apps/zui/src/js/components/status-bar/status-bar.tsx b/apps/zui/src/js/components/status-bar/status-bar.tsx deleted file mode 100644 index 69a9edc1ab..0000000000 --- a/apps/zui/src/js/components/status-bar/status-bar.tsx +++ /dev/null @@ -1,32 +0,0 @@ -import React from "react" -import styled from "styled-components" -import {QueryProgress} from "./query-progress" -import {TypeCount} from "./type-count" -import Tabs from "src/js/state/Tabs" -import {useSelector} from "react-redux" - -const BG = styled.footer` - grid-area: status; - user-select: none; - background: var(--chrome-color); - border-top: 1px solid var(--border-color); - position: relative; - overflow: hidden; - padding: 0 22px; - display: flex; - align-items: center; - gap: 24px; - font-size: 13px; - line-height: 13px; - opacity: 1; -` - -export default function StatusBar() { - if (useSelector(Tabs.none)) return - return ( - - - - - ) -} diff --git a/apps/zui/src/js/components/status-bar/type-count.tsx b/apps/zui/src/js/components/status-bar/type-count.tsx deleted file mode 100644 index fffb042bbc..0000000000 --- a/apps/zui/src/js/components/status-bar/type-count.tsx +++ /dev/null @@ -1,14 +0,0 @@ -import React from "react" -import {useSelector} from "react-redux" -import Results from "src/js/state/Results" -import {RESULTS_QUERY} from "src/views/results-pane/run-results-query" - -export function TypeCount() { - const shapes = useSelector(Results.getShapes(RESULTS_QUERY)) - const status = useSelector(Results.getStatus(RESULTS_QUERY)) - if (["COMPLETE", "LIMIT", "INCOMPLETE"].includes(status)) { - return Shapes: {Object.keys(shapes).length} - } else { - return null - } -} diff --git a/apps/zui/src/js/initializers/init-async-tasks.ts b/apps/zui/src/js/initializers/init-async-tasks.ts new file mode 100644 index 0000000000..9e875c9d94 --- /dev/null +++ b/apps/zui/src/js/initializers/init-async-tasks.ts @@ -0,0 +1,9 @@ +import {Renderer} from "src/core/renderer" +import {AsyncTasks} from "src/modules/async-tasks" + +export function initAsyncTasks(renderer: Renderer) { + const tasks = new AsyncTasks() + renderer.on("close", () => tasks.abortAll()) + renderer.on("tab-close", (tabId) => tasks.abort([tabId])) + return tasks +} diff --git a/apps/zui/src/js/initializers/init-domain-models.ts b/apps/zui/src/js/initializers/init-domain-models.ts new file mode 100644 index 0000000000..cd29ca2296 --- /dev/null +++ b/apps/zui/src/js/initializers/init-domain-models.ts @@ -0,0 +1,6 @@ +import {DomainModel} from "src/core/domain-model" +import {Store} from "../state/types" + +export function initDomainModels(args: {store: Store}) { + DomainModel.store = args.store +} diff --git a/apps/zui/src/js/initializers/initStore.ts b/apps/zui/src/js/initializers/initStore.ts index 6acec5b81e..76978c938a 100644 --- a/apps/zui/src/js/initializers/initStore.ts +++ b/apps/zui/src/js/initializers/initStore.ts @@ -2,6 +2,8 @@ import {enableMapSet} from "immer" import ZuiApi from "../api/zui-api" import {createWindowStore} from "../state/stores/create-window-store" import {invoke} from "src/core/invoke" +import {ipcRendererReduxMiddleware} from "../state/stores/ipc-redux-middleware" +import {createRendererEventsMiddleware} from "../state/stores/renderer-events-middleware" enableMapSet() @@ -15,9 +17,13 @@ function getInitialState(windowId) { }) } -export default async (api: ZuiApi) => { +export default async (api: ZuiApi, renderer) => { const windowId = global.windowId const initialState = await getInitialState(windowId) const extraArgument = {api} - return createWindowStore(initialState, extraArgument) + const middleware = [ + ipcRendererReduxMiddleware, + createRendererEventsMiddleware(renderer), + ] + return createWindowStore(initialState, extraArgument, middleware) } diff --git a/apps/zui/src/js/initializers/initialize.ts b/apps/zui/src/js/initializers/initialize.ts index 0304ffd058..9683aa975a 100644 --- a/apps/zui/src/js/initializers/initialize.ts +++ b/apps/zui/src/js/initializers/initialize.ts @@ -17,6 +17,10 @@ import toast from "react-hot-toast" import {startTransition} from "react" import {initResizeListener} from "./init-resize-listener" import {setMenuContext} from "src/core/menu" +import {createWaitForSelector} from "src/app/core/state/create-wait-for-selector" +import {initAsyncTasks} from "./init-async-tasks" +import {Renderer} from "src/core/renderer" +import {initDomainModels} from "./init-domain-models" const getWindowId = () => { const params = new URLSearchParams(window.location.search) @@ -32,12 +36,14 @@ export default async function initialize( windowId: string = getWindowId(), windowName: WindowName = getWindowName() ) { + const renderer = new Renderer() global.featureFlags = globalThis.zui.featureFlags global.windowId = windowId global.windowName = windowName const api = new ZuiApi() - const store = await initStore(api) + const store = await initStore(api, renderer) + const asyncTasks = initAsyncTasks(renderer) await initGlobals(store) await initLake(store) api.init(store.dispatch, store.getState) @@ -47,9 +53,14 @@ export default async function initialize( transition: startTransition, oldApi: api, dispatch: store.dispatch, + waitForSelector: createWaitForSelector(store), select: (fn) => fn(store.getState()), invoke: invoke, - toast: toast, + toast, + asyncTasks, + }) + initDomainModels({ + store, }) setMenuContext({select: (fn) => fn(store.getState()), api}) initDebugGlobals(store, api) @@ -60,5 +71,6 @@ export default async function initialize( initializeMonaco() initializePluginContextSync(store) initResizeListener() + return {store, api} } diff --git a/apps/zui/src/js/models/ast.ts b/apps/zui/src/js/models/ast.ts deleted file mode 100644 index 441f5abc38..0000000000 --- a/apps/zui/src/js/models/ast.ts +++ /dev/null @@ -1,92 +0,0 @@ -import {toFieldPath} from "../zed-script/toZedScript" - -type ColumnName = string | string[] - -export default function ast(tree: any) { - return { - valid() { - return !tree.error - }, - error() { - return tree.error || null - }, - groupByKeys(): ColumnName[] { - const g = this.proc("Summarize") - const keys = g ? g.keys : [] - return keys.map((k) => fieldExprToName(k.lhs || k.rhs)) - }, - proc(name: string) { - return getOps(tree).find((p) => p.kind === name) - }, - procs(name: string): any[] { - return getOps(tree).filter((p) => p.kind === name) - }, - getProcs() { - return getOps(tree) - }, - self() { - return tree - }, - } -} - -export function fieldExprToName(expr) { - let s = _fieldExprToName(expr) - // const r = toFieldPath(s) - return s -} - -function _fieldExprToName(expr): string | string[] { - switch (expr.kind) { - case "BinaryExpr": - if (expr.op == "." || expr.op == "[") { - return [] - .concat(_fieldExprToName(expr.lhs), _fieldExprToName(expr.rhs)) - .filter((n) => n !== "this") - } - return "" - case "ID": - return expr.name - case "This": - return "this" - case "Primitive": - return expr.text - case "Call": - var args = expr.args - .map((e) => toFieldPath(_fieldExprToName(e))) - .join(",") - return `${expr.name}(${args})` - default: - return "" - } -} - -function getOps(ast) { - if (!ast || ast.error) return [] - const list = [] - collectOps(ast, list) - return list -} - -function collectOps(op, list) { - if (Array.isArray(op)) { - for (const o of op) collectOps(o, list) - return - } - list.push(op) - if (op.kind === PARALLEL_PROC) { - for (const p of op.paths) collectOps(p, list) - } else if (op.kind === OP_EXPR_PROC) { - collectOps(op.expr, list) - } -} - -export const HEAD_PROC = "Head" -export const TAIL_PROC = "Tail" -export const SORT_PROC = "Sort" -export const FILTER_PROC = "Filter" -export const PRIMITIVE_PROC = "Primitive" -export const PARALLEL_PROC = "Parallel" -export const OP_EXPR_PROC = "OpExpr" -export const REGEXP_SEARCH_PROC = "RegexpSearch" -export const ANALYTIC_PROCS = ["Summarize"] diff --git a/apps/zui/src/js/models/program.test.ts b/apps/zui/src/js/models/program.test.ts index 5cd003db9d..0f503ebb21 100644 --- a/apps/zui/src/js/models/program.test.ts +++ b/apps/zui/src/js/models/program.test.ts @@ -1,12 +1,21 @@ +/** + * @jest-environment jsdom + */ + import {createField, createRecord} from "@brimdata/zed-js" -import program from "./program" +import program, {drillDown, getFilter} from "./program" +import {SystemTest} from "src/test/system" +import {fetchQueryInfo} from "src/domain/session/handlers" + +new SystemTest("program.test") + +const ast = (string) => fetchQueryInfo(string) describe("excluding and including", () => { const field = createField("uid", "123") test("excluding a field", () => { const script = program('_path=="weird"').exclude(field).string() - expect(script).toEqual('_path=="weird" | uid!="123"') }) @@ -48,83 +57,81 @@ describe("drill down", () => { count: 24, }) - test("when there is no leading filter", () => { - const script = program('count() by this["i d"]["orig h"]') - .drillDown(result) - .string() + async function run(value: any, text: string) { + const info = await fetchQueryInfo(text) + return drillDown(text, value, info.isSummarized, info.groupByKeys) + } + + test("when there is no leading filter", async () => { + const script = await run(result, 'count() by this["i d"]["orig h"]') expect(script).toBe('this["i d"]["orig h"]==192.168.0.54') }) - test("when there is a sort on there", () => { - const script = program('name=="james" | count() by proto | sort -r count') - .drillDown(result) - .string() + test("when there is a sort on there", async () => { + const script = await run( + result, + 'name=="james" | count() by proto | sort -r count' + ) expect(script).toBe('name=="james" | proto=="udp"') }) - test("when there is a grep with a star", () => { - const script = program( + test("when there is a grep with a star", async () => { + const script = await run( + result, 'grep(/(*|Elm)/) Category=="Furnishings" | count() by proto' ) - .drillDown(result) - .string() expect(script).toBe( 'grep(/(*|Elm)/) Category=="Furnishings" | proto=="udp"' ) }) - test("combines keys in the group by proc", () => { - const script = program( + test("combines keys in the group by proc", async () => { + const script = await run( + result, '_path=="dns" | count() by id.orig_h, proto, query | sort -r' ) - .drillDown(result) - .string() expect(script).toBe( '_path=="dns" | id.orig_h==192.168.0.54 proto=="udp" query=="WPAD"' ) }) - test("removes *", () => { - const script = program("* | count() by id.orig_h") - .drillDown(result) - .string() + test("removes *", async () => { + const script = await run(result, "* | count() by id.orig_h") expect(script).toBe("id.orig_h==192.168.0.54") }) - test("easy peasy", () => { - const script = program("names james | count() by proto") - .drillDown(result) - .string() + test("easy peasy", async () => { + const script = await run(result, "names james | count() by proto") expect(script).toBe('names james | proto=="udp"') }) - test("count by and filter the same", () => { + test("count by and filter the same", async () => { const result = createRecord({md5: "123", count: 1}) - const script = program('md5=="123" | count() by md5 | sort -r | head 5') - .drillDown(result) - .string() + const script = await run( + result, + 'md5=="123" | count() by md5 | sort -r | head 5' + ) expect(script).toEqual('md5=="123"') }) - test("filter query", () => { + test("filter query", async () => { const result = createRecord({ md5: "9f51ef98c42df4430a978e4157c43dd5", count: 21, }) - const script = program( + const script = await run( + result, '_path=="files" filename!="-" | count() by md5,filename | count() by md5 | sort -r | filter count > 1' ) - .drillDown(result) - .string() expect(script).toEqual( '_path=="files" filename!="-" | md5=="9f51ef98c42df4430a978e4157c43dd5"' @@ -178,91 +185,97 @@ describe("sort by", () => { }) }) -describe("#hasAnalytics()", () => { - test("head proc does not have analytics", () => { - expect(program("* | head 2").hasAnalytics()).toBe(false) +jest.setTimeout(30_000) +describe("#isSummarized", () => { + test("head proc does not have analytics", async () => { + const tree = await ast("* | head 2") + expect(tree.isSummarized).toBe(false) }) - test("sort proc does not have analytics", () => { - expect(program("* | sort -r id.resp_p").hasAnalytics()).toBe(false) + test("sort proc does not have analytics", async () => { + const tree = await ast("* | sort -r id.resp_p") + expect(tree.isSummarized).toBe(false) }) - test("every proc does contain analytics", () => { - expect(program("* | count() by every(1h)").hasAnalytics()).toBe(true) + test("every proc does contain analytics", async () => { + const tree = await ast("* | count() by every(1h)") + expect(tree.isSummarized).toBe(true) }) - test("parallel procs when one does have analytics", () => { - expect( - program( - "* | fork ( => count() by every(1h) => count() by id.resp_h )" - ).hasAnalytics() - ).toBe(true) + test("parallel procs when one does have analytics", async () => { + const tree = await ast( + "* | fork ( => count() by every(1h) => count() by id.resp_h )" + ) + expect(tree.isSummarized).toBe(true) }) - test("parallel procs when both do not have analytics", () => { - expect(program("* | head 100; head 200").hasAnalytics()).toBe(false) + test("parallel procs when both do not have analytics", async () => { + jest.spyOn(global.console, "error").mockImplementation(() => {}) + const tree = await ast("* | head 100; head 200") + expect(tree.isSummarized).toBe(false) }) - test("when there are no procs", () => { - expect(program("*").hasAnalytics()).toBe(false) + test("when there are no procs", async () => { + const tree = await ast("*") + expect(tree.isSummarized).toBe(false) }) - test("for a crappy string", () => { - expect(program("-r").hasAnalytics()).toBe(false) + test("for a crappy string", async () => { + const tree = await ast("-r") + expect(tree.isSummarized).toBe(false) }) - test("for sequential proc", () => { - expect( - program("*google* | head 3 | sort -r id.resp_p").hasAnalytics() - ).toBe(false) + test("for sequential proc", async () => { + const tree = await ast("*google* | head 3 | sort -r id.resp_p") + expect(tree.isSummarized).toBe(false) }) - test("for cut proc", () => { - expect( - program( - "* | fork ( => cut uid, _path => cut uid ) | tail 1" - ).hasAnalytics() - ).toBe(false) + test("for cut proc", async () => { + const tree = await ast("* | fork ( => cut uid, _path => cut uid ) | tail 1") + expect(tree.isSummarized).toBe(false) }) - test("for filter proc", () => { - expect(program('* | filter _path=="conn"').hasAnalytics()).toBe(false) + test("for filter proc", async () => { + const tree = await ast('* | filter _path=="conn"') + expect(tree.isSummarized).toBe(false) }) }) describe("extracting the first filter", () => { - test("*", () => { - expect(program("*").filter()).toEqual("*") + async function run(text) { + const info = await fetchQueryInfo(text) + return getFilter(text, info.isSummarized) + } + + test("*", async () => { + expect(await run("*")).toEqual("*") }) - test('_path=="conn"', () => { - expect(program('_path=="conn"').filter()).toEqual('_path=="conn"') + test('_path=="conn"', async () => { + expect(await run('_path=="conn"')).toEqual('_path=="conn"') }) - test('_path=="conn" | sum(duration)', () => { - expect(program('_path=="conn" | sum(duration)').filter()).toEqual( - '_path=="conn"' - ) + test('_path=="conn" | sum(duration)', async () => { + expect(await run('_path=="conn" | sum(duration)')).toEqual('_path=="conn"') }) - test('_path=="conn" | filter a', () => { - expect(program('_path=="conn" | filter a').filter()).toEqual( + test('_path=="conn" | filter a', async () => { + expect(await run('_path=="conn" | filter a')).toEqual( '_path=="conn" | filter a' ) }) - test("count()", () => { - expect(program("count()").filter()).toEqual("*") + test("count()", async () => { + expect(await run("count()")).toEqual("*") }) - test("dns | count() | filter num > 1", () => { - expect(program("dns | count() | filter num > 1").filter()).toEqual("dns") + test("dns | count() | filter num > 1", async () => { + expect(await run("dns | count() | filter num > 1")).toEqual("dns") }) -}) -describe("cut", () => { - test("cut some fields", () => { - expect(program("my filter").cut("ts", "_path").string()).toBe( + test("cut some fields", async () => { + const filter = await run("my filter") + expect(program(filter).cut("ts", "_path").string()).toBe( "my filter | cut ts, _path" ) }) diff --git a/apps/zui/src/js/models/program.ts b/apps/zui/src/js/models/program.ts index bc25202a8e..e0e22abfbf 100644 --- a/apps/zui/src/js/models/program.ts +++ b/apps/zui/src/js/models/program.ts @@ -1,11 +1,9 @@ import * as zed from "@brimdata/zed-js" -import {parse as parseAst} from "zed/compiler/parser/parser" import {isEmpty, last} from "lodash" import {trim} from "../lib/Str" -import ast, {ANALYTIC_PROCS} from "./ast" import syntax from "./syntax" -export default function (p = "") { +export default function program(p = "") { return { exclude(field: zed.Field) { p = appendWithPipe(p, syntax.exclude(field)) @@ -36,23 +34,6 @@ export default function (p = "") { return this.cut(fields.map((fieldName) => "quiet(" + fieldName + ")")) }, - drillDown(log: zed.Record) { - let filter = this.filter() - - const newFilters = this.ast() - .groupByKeys() - .map((name) => log.tryField(name)) - .filter((f) => !!f) - .map(syntax.include) - .join(" ") - - if (/^\s*\*\s*$/.test(filter)) filter = "" - if (newFilters.includes(filter)) filter = "" - - p = appendWithPipe(filter, newFilters) - return this - }, - countBy(name: string | string[]) { p = appendWithPipe(p, syntax.countBy(name)) return this @@ -64,29 +45,6 @@ export default function (p = "") { return this }, - ast() { - let tree - try { - tree = parseAst(p) - } catch (error) { - tree = {error} - } - return ast(tree) - }, - - filter() { - const [head, ...tail] = p.split( - /\|?\s*(summarize|count|countdistinct|sum)/i - ) - - if (isEmpty(tail) && this.hasAnalytics()) { - return "*" - } else { - if (isEmpty(trim(head))) return "*" - return trim(head) - } - }, - procs() { const [_, ...procs] = p.split("|") return procs.join("|") @@ -95,16 +53,42 @@ export default function (p = "") { string() { return p.trim() === "" ? "*" : p }, + } +} - hasAnalytics() { - for (const proc of this.ast().getProcs()) { - if (ANALYTIC_PROCS.includes(proc.kind)) return true - } - return false - }, +export function getFilter(string: string, isSummarized: boolean) { + const [head, ...tail] = string.split( + /\|?\s*(summarize|count|countdistinct|sum)/i + ) + if (isEmpty(tail) && isSummarized) { + return "*" + } else { + if (isEmpty(trim(head))) return "*" + return trim(head) } } +export async function drillDown( + script: string, + value: zed.Record, + isSummarized: boolean, + groupByKeys: string[] +) { + let filter = await getFilter(script, isSummarized) + + const newFilters = groupByKeys + .map((name) => value.tryField(name)) + .filter((f) => !!f) + .map(syntax.include) + .join(" ") + + if (/^\s*\*\s*$/.test(filter)) filter = "" + if (newFilters.includes(filter)) filter = "" + + script = appendWithPipe(filter, newFilters) + return script +} + function appendWithPipe(program, filter) { if (isEmpty(program)) return filter if (!isWhitespace(last(program))) program += " " diff --git a/apps/zui/src/js/models/query-model.ts b/apps/zui/src/js/models/query-model.ts index 20d3b82c5a..b6b72a7f86 100644 --- a/apps/zui/src/js/models/query-model.ts +++ b/apps/zui/src/js/models/query-model.ts @@ -1,7 +1,6 @@ import {Query} from "src/js/state/Queries/types" import {isEmpty, last} from "lodash" import {QueryPinInterface} from "../state/Editor/types" -import {parse as parseAst} from "zed/compiler/parser/parser" import buildPin from "src/js/state/Editor/models/build-pin" import {QueryVersion} from "src/js/state/QueryVersions/types" import {QuerySource} from "src/js/api/queries/types" @@ -35,14 +34,6 @@ export class QueryModel implements Query { return this.current?.pins ?? [] } - get isLocal() { - return this.source === "local" - } - - get isRemote() { - return this.source === "remote" - } - hasVersion(version: string): boolean { return !!this.versions?.map((v) => v.version).includes(version) } @@ -69,17 +60,6 @@ export class QueryModel implements Query { } } - static checkSyntax(version: QueryVersion) { - const zed = this.versionToZed(version) - let error = null - try { - parseAst(zed) - } catch (e) { - error = e - } - return error - } - static versionToZed(version: QueryVersion): string { let pinS = [] if (!isEmpty(version?.pins)) diff --git a/apps/zui/src/js/state/Appearance/index.ts b/apps/zui/src/js/state/Appearance/index.ts index 13dc9c6602..784e71d304 100644 --- a/apps/zui/src/js/state/Appearance/index.ts +++ b/apps/zui/src/js/state/Appearance/index.ts @@ -1,6 +1,6 @@ import {createSlice, PayloadAction} from "@reduxjs/toolkit" import {State} from "../types" -import {HistoryView, QueriesView, SectionName, OpenMap} from "./types" +import {HistoryView, SectionName, OpenMap} from "./types" const init = () => ({ sidebarIsOpen: true, @@ -8,7 +8,6 @@ const init = () => ({ secondarySidebarIsOpen: true, secondarySidebarWidth: 250, currentSectionName: "pools" as SectionName, - queriesView: "local" as QueriesView, historyView: "linear" as HistoryView, poolsOpenState: {} as OpenMap, queriesOpenState: {} as OpenMap, @@ -22,7 +21,6 @@ const select = { secondarySidebarWidth: (state: State) => state.appearance.secondarySidebarWidth, getCurrentSectionName: (state: State) => state.appearance.currentSectionName, - getQueriesView: (state: State) => state.appearance.queriesView, getHistoryView: (state: State) => state.appearance.historyView, getPoolsOpenState: (state: State) => state.appearance.poolsOpenState, getQueriesOpenState: (state: State) => state.appearance.queriesOpenState, @@ -51,9 +49,6 @@ const slice = createSlice({ setCurrentSectionName(s, action: PayloadAction) { s.currentSectionName = action.payload }, - setQueriesView(s, action: PayloadAction) { - s.queriesView = action.payload - }, setHistoryView: (s, a: PayloadAction) => { s.historyView = a.payload }, diff --git a/apps/zui/src/js/state/Appearance/types.ts b/apps/zui/src/js/state/Appearance/types.ts index b0b7a16c6e..b1d25e4435 100644 --- a/apps/zui/src/js/state/Appearance/types.ts +++ b/apps/zui/src/js/state/Appearance/types.ts @@ -1,4 +1,3 @@ export type HistoryView = "tree" | "linear" -export type QueriesView = "local" | "remote" export type SectionName = "pools" | "queries" | "history" export type OpenMap = {[id: string]: boolean} diff --git a/apps/zui/src/js/state/Current/selectors.ts b/apps/zui/src/js/state/Current/selectors.ts index caaf7c89da..1cc2c07a83 100644 --- a/apps/zui/src/js/state/Current/selectors.ts +++ b/apps/zui/src/js/state/Current/selectors.ts @@ -18,6 +18,7 @@ import {entitiesToArray} from "../utils" import lake from "src/js/models/lake" import {defaultLake} from "src/js/initializers/initLakeParams" import {getActive} from "../Tabs/selectors" +import QueryInfo from "../QueryInfo" export const getHistory = ( state, @@ -36,7 +37,7 @@ export const getLocation = (state: State) => { return getHistory(state)?.location } -const getQueryUrlParams = createSelector(getLocation, (location) => { +export const getQueryUrlParams = createSelector(getLocation, (location) => { const path = location.pathname const routes = [queryVersion.path, query.path] const match = matchPath<{queryId: string; version: string}>(path, routes) @@ -52,6 +53,10 @@ export const getVersion = (state: State): QueryVersion => { ) } +export const getQueryText = createSelector(getVersion, (version) => { + return QueryModel.versionToZed(version) +}) + const getRawSession = (state: State) => { const id = getSessionId(state) return SessionQueries.find(state, id) @@ -67,11 +72,11 @@ const getSessionVersions = (state: State) => { } export const getNamedQuery = (state: State) => { - const queryId = getQueryId(state) + const queryId = getSessionRouteParentId(state) return Queries.build(state, queryId) } -export const getQueryId = (state: State) => { +export const getSessionRouteParentId = (state: State) => { const {queryId} = getQueryUrlParams(state) return queryId } @@ -166,21 +171,17 @@ export const getSessionId = getTabId export function getOpEventContext(state: State) { return { lakeId: getLakeId(state), - poolName: getActiveQuery(state).toAst().poolName as string | null, + poolName: QueryInfo.get(state).poolName, } } export type OpEventContext = ReturnType -export const getPoolNameFromQuery = createSelector(getActiveQuery, (q) => { - return q.toAst().poolName -}) - export const getPoolFromQuery = createSelector( - getPoolNameFromQuery, + QueryInfo.get, getPools, - (name, pools) => { - return pools.find((p) => p.data.name === name) ?? null + (info, pools) => { + return pools.find((p) => p.data.name === info.poolName) ?? null } ) diff --git a/apps/zui/src/js/state/Queries/flows/get-query-source.ts b/apps/zui/src/js/state/Queries/flows/get-query-source.ts index 6ce35901c3..252342c9da 100644 --- a/apps/zui/src/js/state/Queries/flows/get-query-source.ts +++ b/apps/zui/src/js/state/Queries/flows/get-query-source.ts @@ -1,14 +1,12 @@ import Queries from "src/js/state/Queries/index" -import RemoteQueries from "src/js/state/RemoteQueries" import SessionQueries from "src/js/state/SessionQueries" import {Thunk} from "../../types" -export type QuerySource = "local" | "remote" | "session" +export type QuerySource = "local" | "session" export const getQuerySource = (id?: string): Thunk => (_d, getState): QuerySource => { if (SessionQueries.find(getState(), id)) return "session" if (Queries.find(getState(), id)) return "local" - if (RemoteQueries.find(getState(), id)) return "remote" return null } diff --git a/apps/zui/src/js/state/Queries/selectors.ts b/apps/zui/src/js/state/Queries/selectors.ts index c9e24cbf0e..95d373aaea 100644 --- a/apps/zui/src/js/state/Queries/selectors.ts +++ b/apps/zui/src/js/state/Queries/selectors.ts @@ -15,12 +15,6 @@ export const find = (state: State, id: string): Query | null => { .first((n) => n.model.id === id)?.model } -export const findRemoteQuery = (state: State, id: string): Query | null => { - return new TreeModel({childrenPropertyName: "items"}) - .parse(state.remoteQueries) - .first((n) => n.model.id === id && !("items" in n.model))?.model -} - export const findSessionQuery = (state: State, id: string): Query | null => { return state.sessionQueries[id] } @@ -35,12 +29,10 @@ const getQueryVersions = (state: State, id: string) => { export const build = createSelector( find, - findRemoteQuery, findSessionQuery, getQueryVersions, - (localMeta, remoteMeta, sessionMeta, versions) => { + (localMeta, sessionMeta, versions) => { if (localMeta) return new QueryModel(localMeta, versions, "local") - if (remoteMeta) return new QueryModel(remoteMeta, versions, "remote") if (sessionMeta) return new QueryModel(sessionMeta, versions, "session") return null } diff --git a/apps/zui/src/js/state/QueryInfo/index.ts b/apps/zui/src/js/state/QueryInfo/index.ts new file mode 100644 index 0000000000..7864010c8a --- /dev/null +++ b/apps/zui/src/js/state/QueryInfo/index.ts @@ -0,0 +1,8 @@ +import {reducer, actions} from "./reducer" +import * as selectors from "./selectors" + +export default { + reducer, + ...actions, + ...selectors, +} diff --git a/apps/zui/src/js/state/QueryInfo/reducer.ts b/apps/zui/src/js/state/QueryInfo/reducer.ts new file mode 100644 index 0000000000..1b006db60b --- /dev/null +++ b/apps/zui/src/js/state/QueryInfo/reducer.ts @@ -0,0 +1,28 @@ +import {createSlice} from "@reduxjs/toolkit" + +const getInitialState = () => { + return { + isParsed: false, + isSummarized: false, + poolName: null, + sorts: [], + error: null, + groupByKeys: [], + } +} + +const slice = createSlice({ + name: "TAB_QUERY_INFO", + initialState: getInitialState(), + reducers: { + set(_, action) { + return action.payload + }, + reset(_) { + return getInitialState() + }, + }, +}) + +export const reducer = slice.reducer +export const actions = slice.actions diff --git a/apps/zui/src/js/state/QueryInfo/selectors.ts b/apps/zui/src/js/state/QueryInfo/selectors.ts new file mode 100644 index 0000000000..a0565cac7d --- /dev/null +++ b/apps/zui/src/js/state/QueryInfo/selectors.ts @@ -0,0 +1,9 @@ +import {createSelector} from "reselect" +import activeTabSelect from "../Tab/activeTabSelect" + +export const get = activeTabSelect((tab) => { + return tab.queryInfo +}) + +export const getIsParsed = createSelector(get, (info) => info.isParsed) +export const getIsSummarized = createSelector(get, (info) => info.isSummarized) diff --git a/apps/zui/src/js/state/RemoteQueries/flows/remote-queries.ts b/apps/zui/src/js/state/RemoteQueries/flows/remote-queries.ts deleted file mode 100644 index 1425065e9c..0000000000 --- a/apps/zui/src/js/state/RemoteQueries/flows/remote-queries.ts +++ /dev/null @@ -1,172 +0,0 @@ -import {Pool} from "src/app/core/pools/pool" -import {compact, forEach, intersection} from "lodash" -import {Query} from "src/js/state/Queries/types" -import Pools from "src/js/state/Pools" -import Current from "src/js/state/Current" -import RemoteQueries from "src/js/state/RemoteQueries" -import {Thunk} from "src/js/state/types" -import QueryVersions from "src/js/state/QueryVersions" -import {QueryVersion} from "src/js/state/QueryVersions/types" -import {LakeModel} from "src/js/models/lake" -import {invoke} from "src/core/invoke" - -export const remoteQueriesPoolName = "_remote-queries" - -type RemoteQueryRecord = Query & QueryVersion & {tombstone?: boolean} - -const queriesToRemoteQueries = ( - qs: (Query & QueryVersion)[], - isTombstone = false -): RemoteQueryRecord[] => { - return qs.map((q) => ({ - ...q, - tombstone: isTombstone, - })) -} - -/* -remoteQueriesToQueries groups an array of raw query records into -an array of Query metadata and a map of QueryVersion arrays, keyed -by the queryId. It does not include queries if their most recent entry -has a tombstone, and if there are duplicate entries with the same version -then only the most recent will be used. To manage this, the provided array -of raw records must already be sorted by ts. - */ -const remoteQueriesToQueries = ( - remoteRecords: RemoteQueryRecord[] -): {queries: Query[]; versions: {[queryId: string]: QueryVersion[]}} => { - const seenQuerySet = new Set() - const versions = {} - const queries = compact( - remoteRecords.map((r) => { - if (seenQuerySet.has(r.id)) return - seenQuerySet.add(r.id) - if (r.tombstone) return - versions[r.id] = [] - const {id, name, description = "", isReadOnly = false} = r - return {id, name, description, isReadOnly} - }) - ) - - const seenVersionSet = new Set() - remoteRecords.forEach((r) => { - if (!versions[r.id]) return - if (seenVersionSet.has(r.version)) return - seenVersionSet.add(r.version) - const {version, value = "", pins = [], ts} = r - versions[r.id].push({version, ts, value, pins}) - }) - - return {queries, versions} -} - -export const isRemoteLib = (ids: string[]) => (_d, getState) => { - const remoteIds = RemoteQueries.raw(getState())?.items.map((i) => i.id) - return intersection(ids, remoteIds).length > 0 -} - -export const getRemotePoolForLake = - (lakeId: string): Thunk => - (_d, getState) => { - return Pools.getByName(lakeId, remoteQueriesPoolName)(getState()) - } - -export const refreshRemoteQueries = - (lake?: LakeModel): Thunk> => - async (dispatch, gs, {api}) => { - const zealot = await api.getZealot(lake) - try { - const queryReq = await zealot.query( - `from '${remoteQueriesPoolName}' - | sort -r ts - | cut name, tombstone, value, description, id, ts, version, pins, quiet(isReadOnly)` - ) - - const remoteRecords = (await queryReq.js()) as (Query & - QueryVersion & {tombstone?: boolean})[] - - const {queries, versions} = remoteQueriesToQueries(remoteRecords) - - dispatch(RemoteQueries.set(queries)) - forEach(versions, (versions, queryId) => { - dispatch(QueryVersions.at(queryId).sync(versions)) - }) - } catch (e) { - if (/pool not found/.test(e.message)) { - dispatch(RemoteQueries.set([])) - return - } else throw e - } - } - -/* - setRemoteQueries will create or update Remote queries stored in a special - pool defined in the 'remoteQueriesPoolName' constant, and this function will - create that pool if it does not exist. To determine that existence, we rely - on redux's list of existing pools which means that this thunk depends on - that state being populated - */ -export const appendRemoteQueries = - (queries: (Query & QueryVersion)[]): Thunk> => - async (dispatch) => { - const remote = queriesToRemoteQueries(queries) - await dispatch(loadRemoteQueries(remote)) - await dispatch(refreshRemoteQueries()) - } - -const loadRemoteQueries = - (queries: RemoteQueryRecord[]): Thunk> => - async (dispatch, _gs) => { - const poolId = await dispatch(getOrCreateRemotePoolId()) - const data = queries.map((d) => JSON.stringify(d)).join("\n") - try { - await invoke("pools.load", poolId, data, { - branch: "main", - message: { - author: "zui", - body: - "automatic remote query load for id(s): " + - queries.map((q) => q.id).join(", "), - }, - }) - } catch (e) { - console.error(e) - throw new Error("error loading remote queries: " + e) - } - } - -// eslint-disable-next-line -const getOrCreateRemotePoolId = - (): Thunk> => - async (dispatch, getState, {api}) => { - let rqPoolId = Pools.getByName( - Current.getLakeId(getState()), - remoteQueriesPoolName - )(getState())?.id - if (!rqPoolId) { - // create remote-queries pool if it doesn't already exist - rqPoolId = await api.pools.create(remoteQueriesPoolName) - } - - return rqPoolId - } - -/* -deleteRemoteQueries handles deletion by flipping a 'tombstone' boolean column -for a given query's record. Since the intent is to delete the query, the value -and metadata for this tombstone record will be empty - */ -export const deleteRemoteQueries = - (queryIds: string[]): Thunk> => - async (dispatch) => { - const queryDefaults = { - name: "", - version: "", - ts: new Date().toISOString(), - value: "", - pins: [], - } - const queries = queryIds.map((id) => ({...queryDefaults, id})) - await dispatch(loadRemoteQueries(queriesToRemoteQueries(queries, true))) - await dispatch(refreshRemoteQueries()) - } diff --git a/apps/zui/src/js/state/RemoteQueries/index.ts b/apps/zui/src/js/state/RemoteQueries/index.ts deleted file mode 100644 index bd6b0a09e9..0000000000 --- a/apps/zui/src/js/state/RemoteQueries/index.ts +++ /dev/null @@ -1,37 +0,0 @@ -import {createSlice} from "@reduxjs/toolkit" -import {State} from "../types" -import {Group, Query} from "../Queries/types" -import TreeModel from "tree-model" - -const slice = createSlice({ - name: "$remoteQueries", - initialState: { - id: "root", - name: "root", - isOpen: true, - items: [], - }, - reducers: { - set(s, a) { - s.items = a.payload - }, - }, -}) - -export default { - reducer: slice.reducer, - ...slice.actions, - raw: (state: State) => state.remoteQueries, - find: (state: State, id: string): Query => { - return new TreeModel({childrenPropertyName: "items"}) - .parse(state.remoteQueries) - .first((n) => n.model.id === id && !("items" in n.model))?.model - }, - getGroupById: - (groupId: string) => - (state: State): Group => { - return new TreeModel({childrenPropertyName: "items"}) - .parse(state.remoteQueries) - .first((n) => n.model.id === groupId && "items" in n.model)?.model - }, -} diff --git a/apps/zui/src/js/state/Results/reducer.ts b/apps/zui/src/js/state/Results/reducer.ts index 4e6f1bfe30..9a85a21f7c 100644 --- a/apps/zui/src/js/state/Results/reducer.ts +++ b/apps/zui/src/js/state/Results/reducer.ts @@ -1,6 +1,5 @@ import {createSlice, PayloadAction as Pay} from "@reduxjs/toolkit" import * as zed from "@brimdata/zed-js" -import program from "src/js/models/program" import {ResultsState} from "./types" import {initialResultData} from "./util" @@ -18,11 +17,19 @@ const slice = createSlice({ name: "TAB_RESULTS", initialState: {} as ResultsState, reducers: { - init(s, a: Pay<{id: string; query: string; key: string; tabId: string}>) { + init( + s, + a: Pay<{ + id: string + query: string + key: string + tabId: string + }> + ) { const r = access(s, a.payload.id) r.query = a.payload.query - r.aggregation = program(a.payload.query).hasAnalytics() r.key = a.payload.key + r.canPaginate = false r.page = 1 r.status = "FETCHING" r.values = [] @@ -48,11 +55,17 @@ const slice = createSlice({ r.shapes = a.payload.shapes }, + setCanPaginate( + s, + a: Pay<{id: string; canPaginate: boolean; tabId: string}> + ) { + const r = access(s, a.payload.id) + r.canPaginate = a.payload.canPaginate + }, + success(s, a: Pay<{id: string; count?: number; tabId: string}>) { const r = access(s, a.payload.id) - if (r.aggregation && a.payload.count === r.aggregationLimit) { - r.status = "LIMIT" - } else if (a.payload.count === r.perPage) { + if (a.payload.count === r.perPage) { r.status = "INCOMPLETE" } else { r.status = "COMPLETE" diff --git a/apps/zui/src/js/state/Results/selectors.ts b/apps/zui/src/js/state/Results/selectors.ts index 7a917245e1..a9e257a1c6 100644 --- a/apps/zui/src/js/state/Results/selectors.ts +++ b/apps/zui/src/js/state/Results/selectors.ts @@ -30,11 +30,7 @@ export const getStatus = resultsSelect((results) => { }) export const getPaginatedQuery = resultsSelect((results) => { - if (results.aggregation) { - return paginate(results.query, results.aggregationLimit, 1) - } else { - return paginate(results.query, results.perPage, results.page) - } + return paginate(results.query, results.perPage, results.page) }) export const getQuery = resultsSelect((results) => { @@ -45,8 +41,8 @@ export const isFetching = resultsSelect((results) => { return results.status === "FETCHING" }) -export const isLimited = resultsSelect((results) => { - return results.status === "LIMIT" +export const canPaginate = resultsSelect((results) => { + return results.canPaginate }) export const isComplete = resultsSelect( @@ -57,16 +53,10 @@ export const isIncomplete = resultsSelect( (results) => results.status === "INCOMPLETE" ) -export const isAggregation = resultsSelect((results) => results.aggregation) - export const getKey = resultsSelect((results) => { return results.key }) -export const getAggregationLimit = resultsSelect((results) => { - return results.aggregationLimit -}) - export const getError = resultsSelect((results) => results.error) export const getPage = resultsSelect((results) => results.page) export const getPerPage = resultsSelect((results) => results.perPage) diff --git a/apps/zui/src/js/state/Results/types.ts b/apps/zui/src/js/state/Results/types.ts index 80b8e0e816..c26ec72eb9 100644 --- a/apps/zui/src/js/state/Results/types.ts +++ b/apps/zui/src/js/state/Results/types.ts @@ -5,7 +5,6 @@ export type ResultsStatus = | "FETCHING" | "INCOMPLETE" | "COMPLETE" - | "LIMIT" | "ERROR" export type ResultData = ReturnType diff --git a/apps/zui/src/js/state/Results/util.ts b/apps/zui/src/js/state/Results/util.ts index b72f7b6dc6..8cc119da02 100644 --- a/apps/zui/src/js/state/Results/util.ts +++ b/apps/zui/src/js/state/Results/util.ts @@ -7,8 +7,7 @@ export const initialResultData = () => ({ status: "INIT" as ResultsStatus, page: 1, perPage: 500, - aggregationLimit: 2000, - aggregation: false, + canPaginate: false, key: "", query: "*", error: null as null | any, diff --git a/apps/zui/src/js/state/Tab/reducer.ts b/apps/zui/src/js/state/Tab/reducer.ts index ffb69a4dcd..19c27303b5 100644 --- a/apps/zui/src/js/state/Tab/reducer.ts +++ b/apps/zui/src/js/state/Tab/reducer.ts @@ -7,6 +7,7 @@ import logDetails from "../LogDetails/reducer" import {reducer as results} from "../Results/reducer" import {reducer as histogram} from "../Histogram/reducer" import {reducer as selection} from "../Selection/reducer" +import {reducer as queryInfo} from "../QueryInfo/reducer" import {nanoid} from "@reduxjs/toolkit" const tabReducer = combineReducers({ @@ -22,6 +23,7 @@ const tabReducer = combineReducers({ histogram, table, selection, + queryInfo, }) export type TabReducer = typeof tabReducer diff --git a/apps/zui/src/js/state/Table/selectors.ts b/apps/zui/src/js/state/Table/selectors.ts index 2ccfdca032..4682a038dc 100644 --- a/apps/zui/src/js/state/Table/selectors.ts +++ b/apps/zui/src/js/state/Table/selectors.ts @@ -1,9 +1,9 @@ -import {getActiveQuery} from "../Current/selectors" import activeTabSelect from "../Tab/activeTabSelect" import {createSelector} from "@reduxjs/toolkit" import {initialSettings} from "./reducer" import * as zed from "@brimdata/zed-js" import {TableSettingsState} from "./types" +import QueryInfo from "../QueryInfo" export const getShape = activeTabSelect((tab) => tab.table.shape) export const getSettings = activeTabSelect((tab) => tab.table.settings) @@ -23,8 +23,8 @@ export const getColumnExpandedDefault = activeTabSelect( ) export const getColumnSorts = createSelector( - getActiveQuery, - (query) => query.toAst().sorts + QueryInfo.get, + (info) => info.sorts ) export const getColumnVisible = createSelector( diff --git a/apps/zui/src/js/state/Tabs/flows.ts b/apps/zui/src/js/state/Tabs/flows.ts index 9e6a921318..c781d0e8a6 100644 --- a/apps/zui/src/js/state/Tabs/flows.ts +++ b/apps/zui/src/js/state/Tabs/flows.ts @@ -21,7 +21,7 @@ export const createQuerySession = (dispatch, getState, {api}) => { const sessionId = nanoid() const version = "0" - api.queries.addVersion(sessionId, {version, value: "", pins: []}) + api.queries.createEditorSnapshot(sessionId, {version, value: "", pins: []}) const url = queryPath(sessionId, version) return dispatch(create(url, sessionId)) } diff --git a/apps/zui/src/js/state/stores/create-window-store.ts b/apps/zui/src/js/state/stores/create-window-store.ts index 7ae9e1a4a5..fc9027f440 100644 --- a/apps/zui/src/js/state/stores/create-window-store.ts +++ b/apps/zui/src/js/state/stores/create-window-store.ts @@ -1,11 +1,12 @@ import {configureStore} from "@reduxjs/toolkit" import {State, ThunkExtraArg} from "../types" -import {ipcRendererReduxMiddleware} from "./ipc-redux-middleware" + import rootReducer from "./root-reducer" export function createWindowStore( initialState: State, - extraArgument: ThunkExtraArg + extraArgument: ThunkExtraArg, + middleware = [] ) { return configureStore({ reducer: rootReducer, @@ -16,7 +17,7 @@ export function createWindowStore( serializableCheck: false, immutableCheck: false, }) - return defaults.concat(ipcRendererReduxMiddleware) + return defaults.concat(middleware) }, }) } diff --git a/apps/zui/src/js/state/stores/get-persistable.ts b/apps/zui/src/js/state/stores/get-persistable.ts index 984cbefe97..10783ad9f7 100644 --- a/apps/zui/src/js/state/stores/get-persistable.ts +++ b/apps/zui/src/js/state/stores/get-persistable.ts @@ -12,7 +12,6 @@ export const GLOBAL_PERSIST: StateKey[] = [ "launches", "queries", "queryVersions", - "remoteQueries", "sessionQueries", "poolSettings", ] diff --git a/apps/zui/src/js/state/stores/renderer-events-middleware.ts b/apps/zui/src/js/state/stores/renderer-events-middleware.ts new file mode 100644 index 0000000000..2527a10d1a --- /dev/null +++ b/apps/zui/src/js/state/stores/renderer-events-middleware.ts @@ -0,0 +1,11 @@ +import {Renderer} from "src/core/renderer" +import Tabs from "../Tabs" + +export function createRendererEventsMiddleware(renderer: Renderer) { + return (_store) => (next) => (action) => { + if (action.type === Tabs.remove.toString()) { + renderer.emit("tab-close", action.payload) + } + return next(action) + } +} diff --git a/apps/zui/src/js/state/stores/root-reducer.ts b/apps/zui/src/js/state/stores/root-reducer.ts index f1aa4d40d3..584df73646 100644 --- a/apps/zui/src/js/state/stores/root-reducer.ts +++ b/apps/zui/src/js/state/stores/root-reducer.ts @@ -13,7 +13,6 @@ import Toolbars from "../Toolbars" import ConfigPropValues from "../ConfigPropValues" import Launches from "../Launches" import Appearance from "../Appearance" -import RemoteQueries from "../RemoteQueries" import Loads from "../Loads" import QueryVersions from "../QueryVersions" import SessionQueries from "../SessionQueries" @@ -38,7 +37,6 @@ const rootReducer = combineReducers({ poolSettings: PoolSettings.reducer, queries: Queries.reducer, queryVersions: QueryVersions.reducer, - remoteQueries: RemoteQueries.reducer, sessionHistories: SessionHistories.reducer, sessionQueries: SessionQueries.reducer, tabHistories: TabHistories.reducer, diff --git a/apps/zui/src/js/state/types.ts b/apps/zui/src/js/state/types.ts index da41e400a5..7289e3f6a9 100644 --- a/apps/zui/src/js/state/types.ts +++ b/apps/zui/src/js/state/types.ts @@ -49,7 +49,6 @@ export type State = { poolSettings: PoolSettingsState queries: QueriesState queryVersions: QueryVersionsState - remoteQueries: QueriesState sessionHistories: SessionHistoriesState sessionQueries: SessionQueriesState tabHistories: TabHistoriesState diff --git a/apps/zui/src/js/zed-script/toZedScript.ts b/apps/zui/src/js/zed-script/toZedScript.ts index d37b8560c2..7422fa0215 100644 --- a/apps/zui/src/js/zed-script/toZedScript.ts +++ b/apps/zui/src/js/zed-script/toZedScript.ts @@ -29,7 +29,9 @@ export function toZedScript(object: unknown): string { if (object instanceof Date) return toZedScriptDate(object) if (typeof object === "boolean") return toZedScriptBool(object) if (object === null) return toZedScriptNull() - throw new Error(`Can't convert object to Zed script: ${object}`) + throw new Error( + `Can't convert object to Zed script: ${JSON.stringify(object)}` + ) } const DOUBLE_QUOTE = /"/g diff --git a/apps/zui/src/models/active.ts b/apps/zui/src/models/active.ts new file mode 100644 index 0000000000..9ac550aa16 --- /dev/null +++ b/apps/zui/src/models/active.ts @@ -0,0 +1,39 @@ +import {DomainModel} from "src/core/domain-model" +import {Session} from "./session" +import Current from "src/js/state/Current" +import {EditorSnapshot} from "./editor-snapshot" +import {BrowserTab} from "./browser-tab" +import Editor from "src/js/state/Editor" +import {Frame} from "./frame" +import {getActiveTab} from "src/js/state/Tabs/selectors" + +export class Active extends DomainModel { + static get tab() { + const {id, lastFocused} = this.select(getActiveTab) + return new BrowserTab({id, lastFocused}) + } + + static get session() { + const params = this.select(Current.getQueryUrlParams) + return new Session({ + id: this.tab.attrs.id, + parentId: params.queryId, + snapshotId: params.version, + }) + } + + static get snapshot() { + const params = this.select(Current.getQueryUrlParams) + + return new EditorSnapshot({ + parentId: params.queryId, + ...this.select(Editor.getSnapshot), + }) + } + + static get frame() { + return new Frame({ + id: globalThis.windowId, + }) + } +} diff --git a/apps/zui/src/models/browser-tab.ts b/apps/zui/src/models/browser-tab.ts new file mode 100644 index 0000000000..bb0b0af1c4 --- /dev/null +++ b/apps/zui/src/models/browser-tab.ts @@ -0,0 +1,56 @@ +import {nanoid} from "@reduxjs/toolkit" +import {orderBy} from "lodash" +import {matchPath} from "react-router" +import {DomainModel} from "src/core/domain-model" +import Tabs from "src/js/state/Tabs" + +type Attrs = { + id: string + lastFocused: string +} + +export class BrowserTab extends DomainModel { + static find(id: string) { + const attrs = this.select(Tabs.findById(id)) + return attrs ? new BrowserTab(attrs) : null + } + + static get all() { + return this.select(Tabs.getData).map((data) => { + const {id, lastFocused} = data + return new BrowserTab({id, lastFocused}) + }) + } + + static orderBy(attr: keyof Attrs, direction: "asc" | "desc") { + return orderBy(this.all, [(tab) => tab.attrs[attr]], [direction]) + } + + static create(attrs: Partial = {}) { + const id = attrs.id || nanoid() + const lastFocused = attrs.lastFocused || new Date().toISOString() + globalThis.tabHistories.create(id, [], -1) + this.dispatch(Tabs.add(id)) + return new BrowserTab({id, lastFocused}) + } + + load(pathname: string) { + if (this.history.location.pathname === pathname) { + this.history.replace(pathname) + } else { + this.history.push(pathname) + } + } + + activate() { + this.dispatch(Tabs.activate(this.attrs.id)) + } + + matchesPath(path) { + return !!matchPath(this.history.location.pathname, {path, exact: true}) + } + + get history() { + return globalThis.tabHistories.getOrCreate(this.attrs.id) + } +} diff --git a/apps/zui/src/models/editor-snapshot.ts b/apps/zui/src/models/editor-snapshot.ts new file mode 100644 index 0000000000..a9b55639be --- /dev/null +++ b/apps/zui/src/models/editor-snapshot.ts @@ -0,0 +1,68 @@ +import {nanoid} from "@reduxjs/toolkit" +import {isEqual} from "lodash" +import {queryPath} from "src/app/router/utils/paths" +import {DomainModel} from "src/core/domain-model" +import {QueryPin} from "src/js/state/Editor/types" +import QueryVersions from "src/js/state/QueryVersions" + +type Attrs = { + version: string + ts: string // aka lastRanAt + parentId: string | null + value: string + pins: QueryPin[] +} + +export class EditorSnapshot extends DomainModel { + constructor(attrs: Partial = {}) { + super({ + version: nanoid(), + ts: new Date().toISOString(), + value: "", + pins: [], + parentId: null, + ...attrs, + }) + } + + static find(parentId: string, id: string) { + const attrs = this.select((state) => + QueryVersions.at(parentId).find(state, id) + ) + return attrs ? new EditorSnapshot({...attrs, parentId}) : null + } + + static where(args: {parentId: string}) { + return this.select(QueryVersions.at(args.parentId).all).map( + (attrs) => new EditorSnapshot({...attrs, parentId: args.parentId}) + ) + } + + get id() { + return this.attrs.version + } + + get pathname() { + return queryPath(this.attrs.parentId, this.attrs.version) + } + + get parentId() { + return this.attrs.parentId + } + + equals(other: EditorSnapshot) { + return ( + isEqual(this.attrs.pins, other.attrs.pins) && + isEqual(this.attrs.value, other.attrs.value) + ) + } + + save() { + const {parentId, ...rest} = this.attrs + this.dispatch(QueryVersions.at(parentId).create({...rest})) + } + + clone(attrs: Partial) { + return new EditorSnapshot({...this.attrs, ...attrs}) + } +} diff --git a/apps/zui/src/models/frame.ts b/apps/zui/src/models/frame.ts new file mode 100644 index 0000000000..1b93428380 --- /dev/null +++ b/apps/zui/src/models/frame.ts @@ -0,0 +1,8 @@ +import {DomainModel} from "src/core/domain-model" + +/** + * A frame is a model for the browser window. + */ +export class Frame extends DomainModel { + static id = globalThis.windowId +} diff --git a/apps/zui/src/models/named-query.ts b/apps/zui/src/models/named-query.ts new file mode 100644 index 0000000000..44fdef3715 --- /dev/null +++ b/apps/zui/src/models/named-query.ts @@ -0,0 +1,29 @@ +import {DomainModel} from "src/core/domain-model" +import Queries from "src/js/state/Queries" +import {EditorSnapshot} from "./editor-snapshot" + +type Attrs = { + name: string + id: string +} + +export class NamedQuery extends DomainModel { + static find(id: string) { + const attrs = this.select((state) => Queries.find(state, id)) + if (!attrs) return null + const {name} = attrs + return new NamedQuery({id, name}) + } + + get id() { + return this.attrs.id + } + + get lastSnapshot() { + return this.snapshots[this.snapshots.length - 1] + } + + get snapshots() { + return EditorSnapshot.where({parentId: this.attrs.id}) + } +} diff --git a/apps/zui/src/models/session-history.ts b/apps/zui/src/models/session-history.ts new file mode 100644 index 0000000000..aca03678af --- /dev/null +++ b/apps/zui/src/models/session-history.ts @@ -0,0 +1,34 @@ +import {DomainModel} from "src/core/domain-model" +import SessionHistories from "src/js/state/SessionHistories" + +type Attrs = { + id: string +} + +export class SessionHistory extends DomainModel { + get id() { + return this.attrs.id + } + + push(parentId: string, snapshotId: string) { + this.dispatch( + SessionHistories.pushById({ + sessionId: this.attrs.id, + entry: { + queryId: parentId, + version: snapshotId, + }, + }) + ) + } + + get entries() { + return this.select(SessionHistories.getById(this.id)) || [] + } + + contains(parentId: string, snapshotId: string) { + return !!this.entries.find( + (item) => item.queryId === parentId && item.version === snapshotId + ) + } +} diff --git a/apps/zui/src/models/session.ts b/apps/zui/src/models/session.ts new file mode 100644 index 0000000000..451de7f676 --- /dev/null +++ b/apps/zui/src/models/session.ts @@ -0,0 +1,126 @@ +import {queryPath} from "src/app/router/utils/paths" +import {DomainModel} from "src/core/domain-model" +import Inspector from "src/js/state/Inspector" +import Table from "src/js/state/Table" +import Selection from "src/js/state/Selection" +import {EditorSnapshot} from "./editor-snapshot" +import {SessionHistory} from "./session-history" +import {NamedQuery} from "./named-query" +import {BrowserTab} from "./browser-tab" +import SessionQueries from "src/js/state/SessionQueries" +import {nanoid} from "@reduxjs/toolkit" +import {queryVersion} from "src/app/router/routes" + +type Attrs = { + id: string + parentId?: string + snapshotId?: string +} + +export class Session extends DomainModel { + static activateLastFocused() { + const tab = BrowserTab.orderBy("lastFocused", "desc").find((tab) => + tab.matchesPath(queryVersion.path) + ) + if (tab) { + tab.activate() + } else { + Session.create().tab.activate() + } + } + + static create() { + const id = nanoid() + const now = new Date().toISOString() + this.dispatch(SessionQueries.init(id)) + BrowserTab.create({id, lastFocused: now}) + return new Session({id}) + } + + get id() { + return this.attrs.id + } + + get parentId() { + if (!this.attrs.parentId) + throw new Error("Session has not yet navigated to a url") + else return this.attrs.parentId + } + + get snapshotId() { + if (!this.attrs.snapshotId) + throw new Error("Session has not yet navigated to a url") + else return this.attrs.snapshotId + } + + get pathname() { + return queryPath(this.parentId, this.snapshotId) + } + + get snapshot() { + let snapshot = EditorSnapshot.find(this.id, this.snapshotId) + if (snapshot) { + return snapshot + } else { + console.warn( + "Did not find snapshot on the session, falling back to named query snapshot" + ) + return EditorSnapshot.find(this.attrs.parentId, this.attrs.snapshotId) // remove after some time has gone by + } + } + + get snapshots() { + return EditorSnapshot.where({parentId: this.id}) + } + + get history() { + return new SessionHistory({id: this.id}) + } + + get hasNamedQuery() { + return this.id !== this.parentId + } + + get namedQuery() { + return this.hasNamedQuery ? NamedQuery.find(this.parentId) : null + } + + get isModified() { + return ( + this.hasNamedQuery && + !!this.namedQuery && + this.snapshot.equals(this.namedQuery.lastSnapshot) + ) + } + + get tab() { + return BrowserTab.find(this.id) + } + + navigate(snapshot: EditorSnapshot, namedQuery?: NamedQuery) { + const sessionSnapshot = snapshot.clone({parentId: this.id}) + sessionSnapshot.save() + new Session({ + id: this.id, + parentId: namedQuery ? namedQuery.id : this.id, + snapshotId: sessionSnapshot.id, + }).load() + } + + load() { + this.reset() + this.tab.load(this.pathname) + } + + pushHistory() { + if (!this.history.contains(this.parentId, this.snapshotId)) { + this.history.push(this.parentId, this.snapshotId) + } + } + + reset() { + this.dispatch(Selection.reset()) + this.dispatch(Table.setScrollPosition({top: 0, left: 0})) + this.dispatch(Inspector.setScrollPosition({top: 0, left: 0})) + } +} diff --git a/apps/zui/src/modules/async-tasks/async-task.ts b/apps/zui/src/modules/async-tasks/async-task.ts new file mode 100644 index 0000000000..041245b36d --- /dev/null +++ b/apps/zui/src/modules/async-tasks/async-task.ts @@ -0,0 +1,30 @@ +import {isAbortError} from "src/util/is-abort-error" + +export class AsyncTask { + private controller = new AbortController() + + constructor(public tags: string[], private destroy: () => void) {} + + containsAll(targetTags: string[]) { + return targetTags.every((targetTag) => this.tags.includes(targetTag)) + } + + abort() { + this.controller.abort() + } + + get signal() { + return this.controller.signal + } + + async run(taskBody: (signal: AbortSignal) => T) { + try { + return await taskBody(this.signal) + } catch (error) { + if (isAbortError(error)) return + throw error + } finally { + this.destroy() + } + } +} diff --git a/apps/zui/src/modules/async-tasks/async-tasks.ts b/apps/zui/src/modules/async-tasks/async-tasks.ts new file mode 100644 index 0000000000..a814ba1b7b --- /dev/null +++ b/apps/zui/src/modules/async-tasks/async-tasks.ts @@ -0,0 +1,33 @@ +import {AsyncTask} from "./async-task" + +export class AsyncTasks { + tasks: AsyncTask[] = [] + + create(tags: string[]): AsyncTask { + const task = new AsyncTask(tags, () => this.remove(task)) + this.tasks.push(task) + return task + } + + remove(task: AsyncTask) { + const index = this.tasks.indexOf(task) + if (index >= 0) this.tasks.splice(index, 1) + } + + createOrReplace(tags: string[]) { + this.abort(tags) + return this.create(tags) + } + + abort(tags: string[]) { + for (const task of this.where(tags)) task.abort() + } + + abortAll() { + for (const task of this.tasks) task.abort() + } + + private where(tags: string[]) { + return this.tasks.filter((task) => task.containsAll(tags)) + } +} diff --git a/apps/zui/src/modules/async-tasks/index.ts b/apps/zui/src/modules/async-tasks/index.ts new file mode 100644 index 0000000000..3e6b726d54 --- /dev/null +++ b/apps/zui/src/modules/async-tasks/index.ts @@ -0,0 +1,2 @@ +export {AsyncTasks} from "./async-tasks" +export {AsyncTask} from "./async-task" diff --git a/apps/zui/src/test/shared/__mocks__/electron.ts b/apps/zui/src/test/shared/__mocks__/electron.ts index d2f62694a4..be1a8546f0 100644 --- a/apps/zui/src/test/shared/__mocks__/electron.ts +++ b/apps/zui/src/test/shared/__mocks__/electron.ts @@ -144,4 +144,6 @@ export const contextBridge = { export const protocol = { interceptFileProtocol: jest.fn(), + registerSchemesAsPrivileged: jest.fn(), + handle: jest.fn(), } diff --git a/apps/zui/src/views/histogram-pane/histogram-pane.module.css b/apps/zui/src/views/histogram-pane/histogram-pane.module.css index 459821fcb0..ed8acea697 100644 --- a/apps/zui/src/views/histogram-pane/histogram-pane.module.css +++ b/apps/zui/src/views/histogram-pane/histogram-pane.module.css @@ -5,6 +5,7 @@ } .dialog { + padding: var(--gutter); z-index: 99999; position: fixed; width: 260px; diff --git a/apps/zui/src/views/histogram-pane/run-query.ts b/apps/zui/src/views/histogram-pane/run-query.ts index d3aec421e6..54dd85afbd 100644 --- a/apps/zui/src/views/histogram-pane/run-query.ts +++ b/apps/zui/src/views/histogram-pane/run-query.ts @@ -1,122 +1,106 @@ import Current from "src/js/state/Current" import PoolSettings from "src/js/state/PoolSettings" -import {QueryModel} from "src/js/models/query-model" import {getInterval, timeUnits} from "./get-interval" import Histogram from "src/js/state/Histogram" import {QueryPin, TimeRangeQueryPin} from "src/js/state/Editor/types" import Results from "src/js/state/Results" -import ZuiApi from "src/js/api/zui-api" import {isAbortError} from "src/util/is-abort-error" +import {createHandler} from "src/core/handlers" +import QueryInfo from "src/js/state/QueryInfo" +import {query} from "src/domain/lake/handlers" export const HISTOGRAM_RESULTS = "histogram" -const POOL_RANGE = "pool-range" -const NULL_TIME_COUNT = "null-time-count" -const MISSING_TIME_COUNT = "missing-time-count" -export async function runHistogramQuery(api: ZuiApi) { - // all these queries should maybe be attached to the same abort signal - // this would change the abortables api a bit - api.abortables.abort({tag: POOL_RANGE}) - api.abortables.abort({tag: NULL_TIME_COUNT}) - api.abortables.abort({tag: MISSING_TIME_COUNT}) - api.abortables.abort({tag: HISTOGRAM_RESULTS}) +export const runHistogramQuery = createHandler( + async ({select, dispatch, waitForSelector, asyncTasks}) => { + const tabId = select(Current.getTabId) + const taskId = "run-histogram-query-task" - const id = HISTOGRAM_RESULTS - const tabId = api.current.tabId - const key = api.current.location.key - const version = api.select(Current.getVersion) - const poolId = api.select(Current.getPoolFromQuery)?.id - const baseQuery = QueryModel.versionToZed(version) - const {timeField, colorField} = api.select((s) => - PoolSettings.findWithDefaults(s, poolId) - ) + asyncTasks.createOrReplace([tabId, taskId]).run(async (signal) => { + await waitForSelector(QueryInfo.getIsParsed, {signal}).toReturn(true) + const id = HISTOGRAM_RESULTS + const key = select(Current.getLocation).key + const version = select(Current.getVersion) + const poolId = select(Current.getPoolFromQuery)?.id + const baseQuery = select(Current.getQueryText) + const {timeField, colorField} = select((s) => + PoolSettings.findWithDefaults(s, poolId) + ) + function getPinRange() { + const rangePin = version.pins.find( + (pin: QueryPin) => + pin.type === "time-range" && + !pin.disabled && + pin.field === timeField + ) as TimeRangeQueryPin + return rangePin + ? ([new Date(rangePin.from), new Date(rangePin.to)] as [Date, Date]) + : null + } + async function getPoolRange() { + const queryText = `from ${poolId} | min(${timeField}), max(${timeField})` + const resp = await query(queryText, {signal}) + const [{min, max}] = await resp.js() + if (!(min instanceof Date && max instanceof Date)) return null + return [min, max] as [Date, Date] + } - function setup() { - api.dispatch(Results.init({id, tabId, key, query: ""})) - api.dispatch(Histogram.init()) - } - - function collect({rows}) { - api.dispatch(Results.setValues({id, tabId, values: rows})) - } - - function error(error: Error) { - if (isAbortError(error)) return - api.dispatch(Results.error({id, tabId, error: error.message})) - } - - function success() { - api.dispatch(Results.success({id, tabId})) - } - - function isRangePin(p: QueryPin) { - return p.type === "time-range" && !p.disabled && p.field === timeField - } + async function getNullTimeCount() { + // Newline after baseQuery in case it ends with a comment. + const queryText = `${baseQuery}\n | ${timeField} == null | count()` + try { + const resp = await query(queryText, {signal}) + const [count] = await resp.js() + dispatch(Histogram.setNullXCount(count ?? 0)) + } catch (e) { + if (isAbortError(e)) return + throw e + } + } - function getPinRange() { - const rangePin = version.pins.find(isRangePin) as TimeRangeQueryPin - return rangePin - ? ([new Date(rangePin.from), new Date(rangePin.to)] as [Date, Date]) - : null - } - - async function getPoolRange() { - const query = `from ${poolId} | min(${timeField}), max(${timeField})` - const resp = await api.query(query, {id: POOL_RANGE, tabId}) - const [{min, max}] = await resp.js() - if (!(min instanceof Date && max instanceof Date)) return null - return [min, max] as [Date, Date] - } - - async function getNullTimeCount() { - // Newline after baseQuery in case it ends with a comment. - const query = `${baseQuery}\n | ${timeField} == null | count()` - try { - const resp = await api.query(query, {id: NULL_TIME_COUNT, tabId}) - const [count] = await resp.js() - api.dispatch(Histogram.setNullXCount(count ?? 0)) - } catch (e) { - if (isAbortError(e)) return - throw e - } - } + async function getMissingTimeCount() { + // Newline after baseQuery in case it ends with a comment. + const queryText = `${baseQuery}\n | !has(${timeField}) | count()` + try { + const resp = await query(queryText, {signal}) + const [count] = await resp.js() + dispatch(Histogram.setMissingXCount(count ?? 0)) + } catch (e) { + if (isAbortError(e)) return + throw e + } + } - async function getMissingTimeCount() { - // Newline after baseQuery in case it ends with a comment. - const query = `${baseQuery}\n | !has(${timeField}) | count()` - try { - const resp = await api.query(query, {id: MISSING_TIME_COUNT, tabId}) - const [count] = await resp.js() - api.dispatch(Histogram.setMissingXCount(count ?? 0)) - } catch (e) { - if (isAbortError(e)) return - throw e - } - } - - async function run() { - const range = getPinRange() || (await getPoolRange()) - if (!range) - throw new Error(`Unable to determine date range using '${timeField}'.`) - - const {unit, number, fn} = getInterval(range) - const interval = `${number}${timeUnits[unit]}` - // Newline after baseQuery in case it ends with a comment. - const query = `${baseQuery}\n | ${timeField} != null | count() by time := bucket(${timeField}, ${interval}), group := ${colorField} | sort time` - const resp = await api.query(query, {id: HISTOGRAM_RESULTS, tabId}) - api.dispatch(Histogram.setInterval({unit, number, fn})) - api.dispatch(Histogram.setRange(range)) - resp.collect(collect) - getNullTimeCount() - getMissingTimeCount() - await resp.promise - } + try { + //setup + dispatch(Results.init({id, tabId, key, query: ""})) + dispatch(Histogram.init()) + // run + const range = getPinRange() || (await getPoolRange()) + if (!range) + throw new Error( + `Unable to determine date range using '${timeField}'.` + ) - try { - setup() - await run() - success() - } catch (e) { - error(e) + const {unit, number, fn} = getInterval(range) + const interval = `${number}${timeUnits[unit]}` + // Newline after baseQuery in case it ends with a comment. + const queryText = `${baseQuery}\n | ${timeField} != null | count() by time := bucket(${timeField}, ${interval}), group := ${colorField} | sort time` + const resp = await query(queryText, {signal}) + dispatch(Histogram.setInterval({unit, number, fn})) + dispatch(Histogram.setRange(range)) + resp.collect(({rows}) => { + dispatch(Results.setValues({id, tabId, values: rows})) + }) + getNullTimeCount() + getMissingTimeCount() + await resp.promise + // success + dispatch(Results.success({id, tabId})) + } catch (error) { + if (isAbortError(error)) return + dispatch(Results.error({id, tabId, error: error.message})) + } + }) } -} +) diff --git a/apps/zui/src/views/histogram-pane/settings-form.tsx b/apps/zui/src/views/histogram-pane/settings-form.tsx index f26369e7c9..22bee4dafc 100644 --- a/apps/zui/src/views/histogram-pane/settings-form.tsx +++ b/apps/zui/src/views/histogram-pane/settings-form.tsx @@ -6,7 +6,6 @@ import {State} from "src/js/state/types" import styles from "./histogram-pane.module.css" import {runHistogramQuery} from "./run-query" import {getDefaults} from "src/js/state/PoolSettings/selectors" -import {useZuiApi} from "src/app/core/context" import forms from "src/components/forms.module.css" import classNames from "classnames" @@ -25,7 +24,6 @@ const defaults = getDefaults() export function SettingsForm(props: Props) { const settings = useSelector((s: State) => PoolSettings.find(s, props.poolId)) const dispatch = useDispatch() - const api = useZuiApi() const form = useForm({defaultValues: settings}) function onSubmit(data: Inputs) { @@ -34,7 +32,7 @@ export function SettingsForm(props: Props) { const id = props.poolId dispatch(PoolSettings.upsert({id, timeField, colorField})) props.close() - runHistogramQuery(api) + runHistogramQuery() } return ( diff --git a/apps/zui/src/views/preview-load-modal/index.tsx b/apps/zui/src/views/preview-load-modal/index.tsx index 9079125cdd..92ab5526c0 100644 --- a/apps/zui/src/views/preview-load-modal/index.tsx +++ b/apps/zui/src/views/preview-load-modal/index.tsx @@ -16,6 +16,7 @@ import {errorToString} from "src/util/error-to-string" import {call} from "src/util/call" import {invoke} from "src/core/invoke" import {FullModal, useFullModal} from "src/components/full-modal" +import {useDispatch} from "src/app/core/state" function Main(props: { original: ResultsControl @@ -79,13 +80,17 @@ export function PreviewLoadModal() { const abort = original.queryAll("*") return abort }, [files, format]) + const dispatch = useDispatch() return (
modal.close()} + onClose={() => { + dispatch(LoadDataForm.reset()) + modal.close() + }} onCancel={onCancel} isValid={!original.error && !preview.error} /> diff --git a/apps/zui/src/views/results-pane/errors/default-error.tsx b/apps/zui/src/views/results-pane/errors/default-error.tsx index df05be0d75..510707c3d5 100644 --- a/apps/zui/src/views/results-pane/errors/default-error.tsx +++ b/apps/zui/src/views/results-pane/errors/default-error.tsx @@ -15,7 +15,7 @@ export function DefaultError(props: {error: unknown}) { return (

Error

-

{props.error.toString()}

+ 
{props.error.toString()}
 ) } diff --git a/apps/zui/src/views/results-pane/run-results-query.tsx b/apps/zui/src/views/results-pane/run-results-query.tsx index 213cacc96e..accab2b316 100644 --- a/apps/zui/src/views/results-pane/run-results-query.tsx +++ b/apps/zui/src/views/results-pane/run-results-query.tsx @@ -1,21 +1,26 @@ import {createHandler} from "src/core/handlers" import {firstPage} from "src/core/query/run" -import {QueryModel} from "src/js/models/query-model" import Current from "src/js/state/Current" +import QueryInfo from "src/js/state/QueryInfo" +import Results from "src/js/state/Results" export const RESULTS_QUERY = "zui-results/main" export const RESULTS_QUERY_COUNT = "zui-results/main-count" -export const runResultsMain = createHandler(({select, dispatch}) => { - const version = select(Current.getVersion) - const query = QueryModel.versionToZed(version) +export const runResultsMain = createHandler( + async ({select, dispatch, waitForSelector}) => { + const query = select(Current.getQueryText) + const tabId = select(Current.getTabId) + dispatch(firstPage({id: RESULTS_QUERY, query})) - dispatch(firstPage({id: RESULTS_QUERY, query})) -}) + // See if we can paginate this query + await waitForSelector(QueryInfo.getIsParsed).toReturn(true) + const canPaginate = !select(QueryInfo.getIsSummarized) + dispatch(Results.setCanPaginate({id: RESULTS_QUERY, canPaginate, tabId})) + } +) export const runResultsCount = createHandler(({select, dispatch}) => { - const version = select(Current.getVersion) - const query = QueryModel.versionToZed(version) + "\n | count()" - + const query = select(Current.getQueryText) + "\n | count()" dispatch(firstPage({id: RESULTS_QUERY_COUNT, query})) }) diff --git a/apps/zui/src/views/session-page/footer.tsx b/apps/zui/src/views/session-page/footer.tsx index 578d77c221..44ba168119 100644 --- a/apps/zui/src/views/session-page/footer.tsx +++ b/apps/zui/src/views/session-page/footer.tsx @@ -87,6 +87,7 @@ const Span = styled.span` export function RowCount() { const status = useSelector(Results.getStatus(RESULTS_QUERY)) const count = useSelector(Results.getCount(RESULTS_QUERY)) + const canPaginate = useSelector(Results.canPaginate(RESULTS_QUERY)) if (status === "FETCHING") { return ( @@ -99,13 +100,13 @@ export function RowCount() { {count?.toLocaleString()} {pluralize("Row", count)} ) - } else if (status === "INCOMPLETE") { + } else if (status === "INCOMPLETE" && canPaginate) { return ( First {count?.toLocaleString()} {pluralize("Row", count)} ) - } else if (status === "LIMIT") { + } else if (status === "INCOMPLETE" && !canPaginate) { return ( Limited to {count?.toLocaleString()} {pluralize("Row", count)} @@ -118,7 +119,7 @@ function ShapeCount() { const shapes = useSelector(Results.getShapes(RESULTS_QUERY)) const status = useSelector(Results.getStatus(RESULTS_QUERY)) const count = Object.keys(shapes).length - if (["COMPLETE", "LIMIT", "INCOMPLETE"].includes(status)) { + if (["COMPLETE", "INCOMPLETE"].includes(status)) { return ( {count?.toLocaleString()} {pluralize("Shape", count)} @@ -133,13 +134,13 @@ function TotalCount() { const status = useSelector(Results.getStatus(RESULTS_QUERY_COUNT)) const values = useSelector(Results.getValues(RESULTS_QUERY_COUNT)) const count = values[0]?.toJS() - if (["COMPLETE", "LIMIT", "INCOMPLETE"].includes(status)) { + if (["COMPLETE", "INCOMPLETE"].includes(status)) { return ( {count?.toLocaleString()} Total {pluralize("Row", count)} ) - } else { + } else if (status === "FETCHING") { return Fetching Total Rows... } } diff --git a/apps/zui/src/views/session-page/grid.tsx b/apps/zui/src/views/session-page/grid.tsx index 49c7dd3beb..dc4b2e7dd8 100644 --- a/apps/zui/src/views/session-page/grid.tsx +++ b/apps/zui/src/views/session-page/grid.tsx @@ -1,11 +1,13 @@ import {useSelector} from "react-redux" import styles from "./grid.module.css" import Layout from "src/js/state/Layout" +import Tab from "src/js/state/Tab" export function Grid({children}) { const title = "min-content" const pins = "min-content" const editorPx = useSelector(Layout.getEditorHeight) + "px" + const key = useSelector(Tab.getLastLocationKey) const editor = `minmax(10vh, min(${editorPx}, 65vh))` const results = "minmax(0, 1fr)" const footer = "min-content" @@ -13,7 +15,7 @@ export function Grid({children}) { const style = {gridTemplateRows: rows.join(" ")} return ( -
+
{children}
) diff --git a/apps/zui/src/views/session-page/loader.ts b/apps/zui/src/views/session-page/loader.ts index 946d452e00..458c35a08f 100644 --- a/apps/zui/src/views/session-page/loader.ts +++ b/apps/zui/src/views/session-page/loader.ts @@ -1,10 +1,8 @@ import Current from "src/js/state/Current" import Editor from "src/js/state/Editor" import {startTransition} from "react" -import {QueryModel} from "../../js/models/query-model" import Notice from "src/js/state/Notice" import Tabs from "src/js/state/Tabs" -import {Thunk} from "src/js/state/types" import {Location} from "history" import Pools from "src/js/state/Pools" import {invoke} from "src/core/invoke" @@ -15,51 +13,56 @@ import { } from "src/views/results-pane/run-results-query" import Layout from "src/js/state/Layout" import {syncPool} from "src/app/core/pools/sync-pool" +import {fetchQueryInfo} from "src/domain/session/handlers" +import QueryInfo from "src/js/state/QueryInfo" +import {createHandler} from "src/core/handlers" +import {Active} from "src/models/active" -export function loadRoute(location: Location): Thunk { - return (dispatch) => { - dispatch(syncPluginContext) +export const loadRoute = createHandler( + async ({select, dispatch}, location: Location) => { + const history = select(Current.getHistory) + const lakeId = select(Current.getLakeId) + const version = select(Current.getVersion) + const program = select(Current.getQueryText) + const histogramVisible = select(Layout.getShowHistogram) + + dispatch(QueryInfo.reset()) dispatch(Tabs.loaded(location.key)) dispatch(Notice.dismiss()) - dispatch(syncEditor) - dispatch(fetchData()) - } -} - -function syncPluginContext(dispatch, getState) { - const poolName = Current.getActiveQuery(getState()).toAst().poolName - const program = QueryModel.versionToZed(Current.getVersion(getState())) - invoke("updatePluginSessionOp", {poolName, program}) -} - -function syncEditor(dispatch, getState) { - const lakeId = Current.getLakeId(getState()) - const version = Current.getVersion(getState()) - const poolName = Current.getActiveQuery(getState()).toAst().poolName - const pool = Pools.getByName(lakeId, poolName)(getState()) - if (pool && !pool.hasSpan()) dispatch(syncPool(pool.id, lakeId)) - - // Give codemirror a chance to update by scheduling this update - setTimeout(() => { - dispatch(Editor.setValue(version?.value ?? "")) - dispatch(Editor.setPins(version?.pins || [])) - }) -} - -function fetchData() { - return (dispatch, getState, {api}) => { - const version = Current.getVersion(getState()) - const histogramVisible = Layout.getShowHistogram(getState()) + // Give editor a chance to update by scheduling this update + setTimeout(() => { + dispatch(Editor.setValue(version?.value ?? "")) + dispatch(Editor.setPins(version?.pins || [])) + }) startTransition(() => { if (version) { runResultsMain() runResultsCount() - if (histogramVisible) { - runHistogramQuery(api) - } + if (histogramVisible) runHistogramQuery() + } + }) + + // We parse the query text on the server. In order to minimize + // latency, we run the query first, then get the query info. + // If you need to wait for the query info, use the waitForSelector + // function and look for QueryInfo.getIsParsed to be true. + const {session} = Active + + fetchQueryInfo(program).then((info) => { + const {poolName, error} = info + const pool = select(Pools.getByName(lakeId, poolName)) + + dispatch(QueryInfo.set({isParsed: true, ...info})) + invoke("updatePluginSessionOp", {poolName, program}) + if (pool && !pool.hasSpan()) { + dispatch(syncPool(pool.id, lakeId)) + } + + if (!error && history.action === "PUSH") { + session.pushHistory() } }) } -} +) diff --git a/apps/zui/src/views/session-page/pins/dialog.tsx b/apps/zui/src/views/session-page/pins/dialog.tsx index 7271cee854..e98cb9186a 100644 --- a/apps/zui/src/views/session-page/pins/dialog.tsx +++ b/apps/zui/src/views/session-page/pins/dialog.tsx @@ -22,6 +22,7 @@ export type DialogProps = { } const BG = styled.dialog` + padding: var(--gutter); border: none; box-shadow: var(--shadow-elevation-medium); border-radius: 6px; diff --git a/apps/zui/src/views/session-page/route.tsx b/apps/zui/src/views/session-page/route.tsx index 722c25d213..5e39d55e87 100644 --- a/apps/zui/src/views/session-page/route.tsx +++ b/apps/zui/src/views/session-page/route.tsx @@ -2,7 +2,6 @@ import React, {useLayoutEffect} from "react" import {useSelector} from "react-redux" import Current from "src/js/state/Current" import Tab from "src/js/state/Tab" -import {useDispatch} from "src/app/core/state" import {loadRoute} from "./loader" import {SessionPage} from "." @@ -12,11 +11,10 @@ import {SessionPage} from "." export function SessionRoute() { const location = useSelector(Current.getLocation) const lastKey = useSelector(Tab.getLastLocationKey) - const dispatch = useDispatch() useLayoutEffect(() => { if (lastKey !== location.key) { - dispatch(loadRoute(location)) + loadRoute(location) } }, [location.key, lastKey]) diff --git a/apps/zui/src/views/session-page/use-title-form.ts b/apps/zui/src/views/session-page/use-title-form.ts index 63b8316e3e..4ed1167b86 100644 --- a/apps/zui/src/views/session-page/use-title-form.ts +++ b/apps/zui/src/views/session-page/use-title-form.ts @@ -1,10 +1,10 @@ import {FormEvent} from "react" import {useSelector} from "react-redux" -import * as queries from "src/app/commands/queries" import {useZuiApi} from "src/app/core/context" import {useDispatch} from "src/app/core/state" import Layout from "src/js/state/Layout" import Current from "src/js/state/Current" +import {create} from "src/domain/named-queries/handlers" export function useTitleForm() { const active = useSelector(Current.getActiveQuery) @@ -18,10 +18,10 @@ export function useTitleForm() { const name = input.value.trim() || "" if (name.length) { - if (active.isSaved()) { + if (active.isSaved() && !active.isModified()) { api.queries.rename(active.query.id, name) } else { - queries.save.run(name) + create(name) } } dispatch(Layout.hideTitleForm()) diff --git a/packages/zed-js/.swcrc b/packages/zed-js/.swcrc deleted file mode 100644 index fb42f4b043..0000000000 --- a/packages/zed-js/.swcrc +++ /dev/null @@ -1,31 +0,0 @@ -{ - "jsc": { - "target": "es2017", - "parser": { - "syntax": "typescript", - "decorators": true, - "dynamicImport": true - }, - "transform": { - "decoratorMetadata": true, - "legacyDecorator": true - }, - "keepClassNames": true, - "externalHelpers": true, - "loose": true - }, - "module": { - "type": "commonjs", - "strict": true, - "noInterop": true - }, - "sourceMaps": true, - "exclude": [ - "jest.config.ts", - ".*\\.spec.tsx?$", - ".*\\.test.tsx?$", - "./src/jest-setup.ts$", - "./**/jest-setup.ts$", - ".*.js$" - ] -} diff --git a/packages/zed-js/jest.config.ts b/packages/zed-js/jest.config.ts index d53165478b..026e39eed7 100644 --- a/packages/zed-js/jest.config.ts +++ b/packages/zed-js/jest.config.ts @@ -1,23 +1,8 @@ -/* eslint-disable */ -import { readFileSync } from 'fs'; - -// Reading the SWC compilation config and remove the "exclude" -// for the test files to be compiled by SWC -const { exclude: _, ...swcJestConfig } = JSON.parse( - readFileSync(`${__dirname}/.swcrc`, 'utf-8') -); - -// disable .swcrc look-up by SWC core because we're passing in swcJestConfig ourselves. -// If we do not disable this, SWC Core will read .swcrc and won't transform our test files due to "exclude" -if (swcJestConfig.swcrc === undefined) { - swcJestConfig.swcrc = false; -} - export default { displayName: 'zed-js', preset: '../../jest.preset.js', transform: { - '^.+\\.[tj]s$': ['@swc/jest', swcJestConfig], + '^.+\\.[tj]s$': ['@swc/jest'], }, moduleFileExtensions: ['ts', 'js', 'html'], }; diff --git a/packages/zed-js/package.json b/packages/zed-js/package.json index 4ccc33af26..33a3ae1a4d 100644 --- a/packages/zed-js/package.json +++ b/packages/zed-js/package.json @@ -2,7 +2,8 @@ "name": "@brimdata/zed-js", "version": "0.0.17", "type": "commonjs", - "exports": "./src/index.js", + "main": "./dist/cjs/src/index.js", + "jsdelivr": "./dist/browser/index.js", "devDependencies": { "@types/event-source-polyfill": "^1.0.1" }, diff --git a/packages/zed-js/project.json b/packages/zed-js/project.json index f5d8e84447..8be5749645 100644 --- a/packages/zed-js/project.json +++ b/packages/zed-js/project.json @@ -6,21 +6,17 @@ "targets": { "build": { "executor": "nx:run-commands", - "options": { - "commands": [] - }, - "dependsOn": ["build-node", "build-browser"], - "outputs": ["{workspaceRoot}/dist/packages/zed-js"] + "options": { "commands": [] }, + "dependsOn": ["build-node", "build-browser"] }, "build-node": { "executor": "@nrwl/js:tsc", "outputs": ["{options.outputPath}"], "options": { - "outputPath": "dist/packages/zed-js", + "outputPath": "packages/zed-js/dist/cjs", "main": "packages/zed-js/src/index.ts", "tsConfig": "packages/zed-js/tsconfig.lib.json", "assets": ["packages/zed-js/*.md"], - "buildableProjectDepsInPackageJsonType": "dependencies", "clean": false } }, @@ -28,7 +24,7 @@ "executor": "@nrwl/esbuild:esbuild", "outputs": ["{options.outputPath}"], "options": { - "outputPath": "dist/packages/zed-js", + "outputPath": "packages/zed-js/dist/browser", "bundle": true, "minify": true, "format": ["esm"], diff --git a/packages/zed-js/src/client/base-client.ts b/packages/zed-js/src/client/base-client.ts index ecb45f4de6..18b33aa361 100644 --- a/packages/zed-js/src/client/base-client.ts +++ b/packages/zed-js/src/client/base-client.ts @@ -4,6 +4,7 @@ import { ResultStream } from '../query/result-stream'; import { createError } from '../util/error'; import * as Types from './types'; import { accept, defaults, parseContent, toJS, wrapAbort } from './utils'; +import { decode } from '../encoder'; export abstract class BaseClient { public abstract fetch: Types.IsoFetch; @@ -50,6 +51,22 @@ export abstract class BaseClient { return new ResultStream(result, abortCtl); } + async compile( + query: string, + options: { signal?: AbortSignal; timeout?: number } = {} + ) { + const abortCtl = wrapAbort(options.signal); + const result = await this.send({ + method: 'POST', + path: `/compile`, + body: JSON.stringify({ query }), + contentType: 'application/json', + signal: abortCtl.signal, + timeout: options.timeout, + }); + return decode(await result.json()); + } + async createPool(name: string, opts: Partial = {}) { const options = defaults(opts, { order: 'desc', diff --git a/packages/zed-node/src/parser.d.ts b/packages/zed-node/src/parser.d.ts deleted file mode 100644 index 5cc2b2c844..0000000000 --- a/packages/zed-node/src/parser.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -declare module 'zed/compiler/parser/parser' { - export default function (script: string): object; -} diff --git a/packages/zed-node/src/parser.ts b/packages/zed-node/src/parser.ts deleted file mode 100644 index a2895ab7e3..0000000000 --- a/packages/zed-node/src/parser.ts +++ /dev/null @@ -1,5 +0,0 @@ -// eslint-disable-next-line -// @ts-ignore -// eslint-disable-next-line -import parser from 'zed/compiler/parser/parser'; -export const parseAst = parser.parse; diff --git a/packages/zed-wasm/.eslintrc.json b/packages/zed-wasm/.eslintrc.json deleted file mode 100644 index c6f0c80ab9..0000000000 --- a/packages/zed-wasm/.eslintrc.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "extends": ["../../.eslintrc.json"], - "ignorePatterns": ["!**/*"], - "overrides": [ - { - "files": ["*.ts", "*.tsx", "*.js", "*.jsx"], - "rules": {} - }, - { - "files": ["*.ts", "*.tsx"], - "rules": { - "@typescript-eslint/ban-ts-comment": "off", - "@typescript-eslint/no-explicit-any": "off" - } - }, - { - "files": ["*.js", "*.jsx"], - "rules": {} - } - ] -} diff --git a/packages/zed-wasm/.gitignore b/packages/zed-wasm/.gitignore new file mode 100644 index 0000000000..68c5d18f00 --- /dev/null +++ b/packages/zed-wasm/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ diff --git a/packages/zed-wasm/go.mod b/packages/zed-wasm/go.mod index 5d4ba66af1..f539f78386 100644 --- a/packages/zed-wasm/go.mod +++ b/packages/zed-wasm/go.mod @@ -3,7 +3,7 @@ module zqjs go 1.21 require ( - github.com/brimdata/zed v1.6.0 + github.com/brimdata/zed v1.12.0 github.com/teamortix/golang-wasm/wasm v0.0.0-20230308073412-915550b3b9ac ) @@ -11,43 +11,42 @@ require ( github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/agnivade/levenshtein v1.1.1 // indirect github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect - github.com/andybalholm/brotli v1.0.4 // indirect - github.com/apache/arrow/go/v11 v11.0.0-20221214174703-0dfec8e98f4f // indirect - github.com/apache/thrift v0.16.0 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect + github.com/apache/arrow/go/v14 v14.0.0 // indirect + github.com/apache/thrift v0.17.0 // indirect github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de // indirect github.com/aws/aws-sdk-go v1.36.17 // indirect github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f // indirect github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc // indirect - github.com/fraugster/parquet-go v0.10.1-0.20220222153523-e6b70a8a7212 // indirect - github.com/goccy/go-json v0.9.11 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/goccy/go-json v0.10.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/google/flatbuffers v2.0.8+incompatible // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect - github.com/klauspost/compress v1.15.9 // indirect - github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kr/text v0.2.0 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect - github.com/pierrec/lz4/v4 v4.1.17 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/segmentio/ksuid v1.0.2 // indirect github.com/x448/float16 v0.8.4 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.23.0 // indirect - golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 // indirect - golang.org/x/mod v0.6.0 // indirect - golang.org/x/net v0.7.0 // indirect - golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect - golang.org/x/tools v0.2.0 // indirect - golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect - google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 // indirect - google.golang.org/grpc v1.49.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/mod v0.13.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sync v0.4.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/tools v0.14.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect + google.golang.org/grpc v1.58.2 // indirect + google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/packages/zed-wasm/go.sum b/packages/zed-wasm/go.sum index 8e61eed971..9330966b06 100644 --- a/packages/zed-wasm/go.sum +++ b/packages/zed-wasm/go.sum @@ -1,5 +1,3 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= @@ -7,32 +5,25 @@ github.com/agnivade/levenshtein v1.1.1 h1:QY8M92nrzkmr798gCo3kmMyqXFzdQVpxLlGPRB github.com/agnivade/levenshtein v1.1.1/go.mod h1:veldBMzWxcCG2ZvUTKD2kJNRdCk5hVbJomOvKkmgYbo= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/apache/arrow/go/v11 v11.0.0-20221214174703-0dfec8e98f4f h1:QUx+UUDqXqbmYrTUhaLVv6UZFQ13DT3uyK8JvweSvO4= -github.com/apache/arrow/go/v11 v11.0.0-20221214174703-0dfec8e98f4f/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= -github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= -github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/apache/arrow/go/v14 v14.0.0 h1:NXfgmvrHAWSzPO1YNjDhO9VwYrUQI/kRvzy5dJuCIaY= +github.com/apache/arrow/go/v14 v14.0.0/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= +github.com/apache/thrift v0.17.0 h1:cMd2aj52n+8VoAtvSvLn4kDC3aZ6IAkBuqWQ2IDu7wo= +github.com/apache/thrift v0.17.0/go.mod h1:OLxhMRJxomX+1I/KUw03qoV3mMz16BwaKI+d4fPBx7Q= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.36.17 h1:8zTvseyGhgs3uQAzkgnFy7dvTo+ZnZLYmrhnopFxYME= github.com/aws/aws-sdk-go v1.36.17/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f h1:y06x6vGnFYfXUoVMbrcP1Uzpj4JG01eB5vRps9G8agM= github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f/go.mod h1:2stgcRjl6QmW+gU2h5E7BQXg4HU0gzxKWDuT5HviN9s= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/brimdata/zed v1.6.0 h1:/XB4xZZWAb9qbax0Fx0YuMAYfc7r/5EVvDfNgY8yOHo= -github.com/brimdata/zed v1.6.0/go.mod h1:WVzlx/gazFjLgHTCPsxYlU8eGxryCpii6nSVT/cJkJY= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/brimdata/zed v1.12.0 h1:JzDjjcdEI6GICUG07d2B3tQnRWcfUav9bglXzKnVXpY= +github.com/brimdata/zed v1.12.0/go.mod h1:bD4XpGLwJ2ZHUDBVtPuY+KCF5ogemaOXW2NTrVnRuaw= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -41,42 +32,24 @@ github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsY github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48 h1:fRzb/w+pyskVMQ+UbP35JkH8yB7MYb4q/qhBarqZE6g= github.com/dgryski/trifles v0.0.0-20200323201526-dd97f9abfb48/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fraugster/parquet-go v0.10.1-0.20220222153523-e6b70a8a7212 h1:u7X3aZRlWSm18x0EysX9szRULhH7QYQv7UkxW1yHbik= -github.com/fraugster/parquet-go v0.10.1-0.20220222153523-e6b70a8a7212/go.mod h1:dGzUxdNqXsAijatByVgbAWVPlFirnhknQbdazcUIjY0= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= -github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= -github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/flatbuffers v2.0.8+incompatible h1:ivUb1cGomAB101ZM1T0nOiWz9pSrTMoa9+EiY7igmkM= -github.com/google/flatbuffers v2.0.8+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb v1.7.6/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -84,59 +57,40 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGw github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= -github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/segmentio/ksuid v1.0.2 h1:9yBfKyw4ECGTdALaF09Snw3sLJmYIX6AbPJrAy6MrDc= github.com/segmentio/ksuid v1.0.2/go.mod h1:BXuJDr2byAiHuQaQtSKoXh1J0YmUDurywOXgB2w+OSU= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/teamortix/golang-wasm/wasm v0.0.0-20230308073412-915550b3b9ac h1:2K3l7bvK0s5x+Qkv2paUqlZm+FI5RzoJ/oQLUwWGYvk= github.com/teamortix/golang-wasm/wasm v0.0.0-20230308073412-915550b3b9ac/go.mod h1:nskvTyoGIaAsC+664SkRitVI1ft6dm1xerCr50YZsnY= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= @@ -149,88 +103,47 @@ go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo= -golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I= -golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc= -golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE= -golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= -golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= -gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= -gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987 h1:PDIOdWxZ8eRizhKa1AAvY53xsvLB1cWorMjslvY3VA8= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.49.0 h1:WTLtQzmQori5FUH25Pq4WT22oCsv8USpQ+F6rqtsmxw= -google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= +gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= +google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= +google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -238,5 +151,3 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/packages/zed-wasm/index.html b/packages/zed-wasm/index.html deleted file mode 100644 index c5866f1637..0000000000 --- a/packages/zed-wasm/index.html +++ /dev/null @@ -1,70 +0,0 @@ - - - - - Mocha Tests - - - - -
-

Manual Testing

-
-

Input File

- -
-
-

Program

- -
-
-

Output Format

- -
-
- -
-
-

View results in the devtools console.

-
-

Unit Tests

-
- - - - - - - - - - - - - diff --git a/packages/zed-wasm/lib/bridge.d.ts b/packages/zed-wasm/lib/bridge.d.ts deleted file mode 100644 index b889976ff9..0000000000 --- a/packages/zed-wasm/lib/bridge.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -declare module 'golang-wasm/src/bridge' { - export default function (arg: string): number; -} diff --git a/packages/zed-wasm/lib/bridge.js b/packages/zed-wasm/lib/bridge.js new file mode 100644 index 0000000000..52b6a38e75 --- /dev/null +++ b/packages/zed-wasm/lib/bridge.js @@ -0,0 +1,14 @@ +function wrapper(goFunc) { + return (...args) => { + const result = goFunc.apply(undefined, args); + if (result.error instanceof Error) { + throw result.error; + } + return result.result; + }; +} + +globalThis.__go_wasm__ = { + __wrapper__: wrapper, + __ready__: false, +}; diff --git a/packages/zed-wasm/lib/global-shim.js b/packages/zed-wasm/lib/global-shim.js deleted file mode 100644 index b7dd018eda..0000000000 --- a/packages/zed-wasm/lib/global-shim.js +++ /dev/null @@ -1,2 +0,0 @@ -const shim = globalThis; -export { shim as 'global' }; diff --git a/packages/zed-wasm/lib/node-globals.js b/packages/zed-wasm/lib/node-globals.js new file mode 100644 index 0000000000..43ec7632e4 --- /dev/null +++ b/packages/zed-wasm/lib/node-globals.js @@ -0,0 +1,9 @@ +import crypto from 'node:crypto'; +import getRandomValues from 'polyfill-crypto.getrandomvalues'; + +globalThis.crypto = { ...crypto, getRandomValues }; +globalThis.requestAnimationFrame = setTimeout; +globalThis.require = require; +globalThis.fs = require('fs'); +globalThis.TextEncoder = require('util').TextEncoder; +globalThis.TextDecoder = require('util').TextDecoder; diff --git a/packages/zed-wasm/lib/wasm_exec.d.ts b/packages/zed-wasm/lib/wasm_exec.d.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/zed-wasm/lib/wasm_exec.js b/packages/zed-wasm/lib/wasm_exec.js index 2128077b9d..bc6f210242 100644 --- a/packages/zed-wasm/lib/wasm_exec.js +++ b/packages/zed-wasm/lib/wasm_exec.js @@ -2,660 +2,560 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -'use strict'; +"use strict"; (() => { - const enosys = () => { - const err = new Error('not implemented'); - err.code = 'ENOSYS'; - return err; - }; - - if (!globalThis.fs) { - let outputBuf = ''; - globalThis.fs = { - constants: { - O_WRONLY: -1, - O_RDWR: -1, - O_CREAT: -1, - O_TRUNC: -1, - O_APPEND: -1, - O_EXCL: -1, - }, // unused - writeSync(fd, buf) { - outputBuf += decoder.decode(buf); - const nl = outputBuf.lastIndexOf('\n'); - if (nl != -1) { - console.log(outputBuf.substr(0, nl)); - outputBuf = outputBuf.substr(nl + 1); - } - return buf.length; - }, - write(fd, buf, offset, length, position, callback) { - if (offset !== 0 || length !== buf.length || position !== null) { - callback(enosys()); - return; - } - const n = this.writeSync(fd, buf); - callback(null, n); - }, - chmod(path, mode, callback) { - callback(enosys()); - }, - chown(path, uid, gid, callback) { - callback(enosys()); - }, - close(fd, callback) { - callback(enosys()); - }, - fchmod(fd, mode, callback) { - callback(enosys()); - }, - fchown(fd, uid, gid, callback) { - callback(enosys()); - }, - fstat(fd, callback) { - callback(enosys()); - }, - fsync(fd, callback) { - callback(null); - }, - ftruncate(fd, length, callback) { - callback(enosys()); - }, - lchown(path, uid, gid, callback) { - callback(enosys()); - }, - link(path, link, callback) { - callback(enosys()); - }, - lstat(path, callback) { - callback(enosys()); - }, - mkdir(path, perm, callback) { - callback(enosys()); - }, - open(path, flags, mode, callback) { - callback(enosys()); - }, - read(fd, buffer, offset, length, position, callback) { - callback(enosys()); - }, - readdir(path, callback) { - callback(enosys()); - }, - readlink(path, callback) { - callback(enosys()); - }, - rename(from, to, callback) { - callback(enosys()); - }, - rmdir(path, callback) { - callback(enosys()); - }, - stat(path, callback) { - callback(enosys()); - }, - symlink(path, link, callback) { - callback(enosys()); - }, - truncate(path, length, callback) { - callback(enosys()); - }, - unlink(path, callback) { - callback(enosys()); - }, - utimes(path, atime, mtime, callback) { - callback(enosys()); - }, - }; - } - - if (!globalThis.process) { - globalThis.process = { - getuid() { - return -1; - }, - getgid() { - return -1; - }, - geteuid() { - return -1; - }, - getegid() { - return -1; - }, - getgroups() { - throw enosys(); - }, - pid: -1, - ppid: -1, - umask() { - throw enosys(); - }, - cwd() { - throw enosys(); - }, - chdir() { - throw enosys(); - }, - }; - } - - if (!globalThis.crypto) { - throw new Error( - 'globalThis.crypto is not available, polyfill required (crypto.getRandomValues only)' - ); - } - - if (!globalThis.performance) { - throw new Error( - 'globalThis.performance is not available, polyfill required (performance.now only)' - ); - } - - if (!globalThis.TextEncoder) { - throw new Error( - 'globalThis.TextEncoder is not available, polyfill required' - ); - } - - if (!globalThis.TextDecoder) { - throw new Error( - 'globalThis.TextDecoder is not available, polyfill required' - ); - } - - const encoder = new TextEncoder('utf-8'); - const decoder = new TextDecoder('utf-8'); - - globalThis.Go = class { - constructor() { - this.argv = ['js']; - this.env = {}; - this.exit = (code) => { - if (code !== 0) { - console.warn('exit code:', code); - } - }; - this._exitPromise = new Promise((resolve) => { - this._resolveExitPromise = resolve; - }); - this._pendingEvent = null; - this._scheduledTimeouts = new Map(); - this._nextCallbackTimeoutID = 1; - - const setInt64 = (addr, v) => { - this.mem.setUint32(addr + 0, v, true); - this.mem.setUint32(addr + 4, Math.floor(v / 4294967296), true); - }; - - const getInt64 = (addr) => { - const low = this.mem.getUint32(addr + 0, true); - const high = this.mem.getInt32(addr + 4, true); - return low + high * 4294967296; - }; - - const loadValue = (addr) => { - const f = this.mem.getFloat64(addr, true); - if (f === 0) { - return undefined; - } - if (!isNaN(f)) { - return f; - } - - const id = this.mem.getUint32(addr, true); - return this._values[id]; - }; - - const storeValue = (addr, v) => { - const nanHead = 0x7ff80000; - - if (typeof v === 'number' && v !== 0) { - if (isNaN(v)) { - this.mem.setUint32(addr + 4, nanHead, true); - this.mem.setUint32(addr, 0, true); - return; - } - this.mem.setFloat64(addr, v, true); - return; - } - - if (v === undefined) { - this.mem.setFloat64(addr, 0, true); - return; - } - - let id = this._ids.get(v); - if (id === undefined) { - id = this._idPool.pop(); - if (id === undefined) { - id = this._values.length; - } - this._values[id] = v; - this._goRefCounts[id] = 0; - this._ids.set(v, id); - } - this._goRefCounts[id]++; - let typeFlag = 0; - switch (typeof v) { - case 'object': - if (v !== null) { - typeFlag = 1; - } - break; - case 'string': - typeFlag = 2; - break; - case 'symbol': - typeFlag = 3; - break; - case 'function': - typeFlag = 4; - break; - } - this.mem.setUint32(addr + 4, nanHead | typeFlag, true); - this.mem.setUint32(addr, id, true); - }; - - const loadSlice = (addr) => { - const array = getInt64(addr + 0); - const len = getInt64(addr + 8); - return new Uint8Array(this._inst.exports.mem.buffer, array, len); - }; - - const loadSliceOfValues = (addr) => { - const array = getInt64(addr + 0); - const len = getInt64(addr + 8); - const a = new Array(len); - for (let i = 0; i < len; i++) { - a[i] = loadValue(array + i * 8); - } - return a; - }; - - const loadString = (addr) => { - const saddr = getInt64(addr + 0); - const len = getInt64(addr + 8); - return decoder.decode( - new DataView(this._inst.exports.mem.buffer, saddr, len) - ); - }; - - const timeOrigin = Date.now() - performance.now(); - this.importObject = { - go: { - // Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters) - // may synchronously trigger a Go event handler. This makes Go code get executed in the middle of the imported - // function. A goroutine can switch to a new stack if the current stack is too small (see morestack function). - // This changes the SP, thus we have to update the SP used by the imported function. - - // func wasmExit(code int32) - 'runtime.wasmExit': (sp) => { - sp >>>= 0; - const code = this.mem.getInt32(sp + 8, true); - this.exited = true; - delete this._inst; - delete this._values; - delete this._goRefCounts; - delete this._ids; - delete this._idPool; - this.exit(code); - }, - - // func wasmWrite(fd uintptr, p unsafe.Pointer, n int32) - 'runtime.wasmWrite': (sp) => { - sp >>>= 0; - const fd = getInt64(sp + 8); - const p = getInt64(sp + 16); - const n = this.mem.getInt32(sp + 24, true); - fs.writeSync( - fd, - new Uint8Array(this._inst.exports.mem.buffer, p, n) - ); - }, - - // func resetMemoryDataView() - 'runtime.resetMemoryDataView': (sp) => { - sp >>>= 0; - this.mem = new DataView(this._inst.exports.mem.buffer); - }, - - // func nanotime1() int64 - 'runtime.nanotime1': (sp) => { - sp >>>= 0; - setInt64(sp + 8, (timeOrigin + performance.now()) * 1000000); - }, - - // func walltime() (sec int64, nsec int32) - 'runtime.walltime': (sp) => { - sp >>>= 0; - const msec = new Date().getTime(); - setInt64(sp + 8, msec / 1000); - this.mem.setInt32(sp + 16, (msec % 1000) * 1000000, true); - }, - - // func scheduleTimeoutEvent(delay int64) int32 - 'runtime.scheduleTimeoutEvent': (sp) => { - sp >>>= 0; - const id = this._nextCallbackTimeoutID; - this._nextCallbackTimeoutID++; - this._scheduledTimeouts.set( - id, - setTimeout( - () => { - this._resume(); - while (this._scheduledTimeouts.has(id)) { - // for some reason Go failed to register the timeout event, log and try again - // (temporary workaround for https://github.com/golang/go/issues/28975) - console.warn('scheduleTimeoutEvent: missed timeout event'); - this._resume(); - } - }, - getInt64(sp + 8) + 1 // setTimeout has been seen to fire up to 1 millisecond early - ) - ); - this.mem.setInt32(sp + 16, id, true); - }, - - // func clearTimeoutEvent(id int32) - 'runtime.clearTimeoutEvent': (sp) => { - sp >>>= 0; - const id = this.mem.getInt32(sp + 8, true); - clearTimeout(this._scheduledTimeouts.get(id)); - this._scheduledTimeouts.delete(id); - }, - - // func getRandomData(r []byte) - 'runtime.getRandomData': (sp) => { - sp >>>= 0; - crypto.getRandomValues(loadSlice(sp + 8)); - }, - - // func finalizeRef(v ref) - 'syscall/js.finalizeRef': (sp) => { - sp >>>= 0; - const id = this.mem.getUint32(sp + 8, true); - this._goRefCounts[id]--; - if (this._goRefCounts[id] === 0) { - const v = this._values[id]; - this._values[id] = null; - this._ids.delete(v); - this._idPool.push(id); - } - }, - - // func stringVal(value string) ref - 'syscall/js.stringVal': (sp) => { - sp >>>= 0; - storeValue(sp + 24, loadString(sp + 8)); - }, - - // func valueGet(v ref, p string) ref - 'syscall/js.valueGet': (sp) => { - sp >>>= 0; - const result = Reflect.get(loadValue(sp + 8), loadString(sp + 16)); - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 32, result); - }, - - // func valueSet(v ref, p string, x ref) - 'syscall/js.valueSet': (sp) => { - sp >>>= 0; - Reflect.set( - loadValue(sp + 8), - loadString(sp + 16), - loadValue(sp + 32) - ); - }, - - // func valueDelete(v ref, p string) - 'syscall/js.valueDelete': (sp) => { - sp >>>= 0; - Reflect.deleteProperty(loadValue(sp + 8), loadString(sp + 16)); - }, - - // func valueIndex(v ref, i int) ref - 'syscall/js.valueIndex': (sp) => { - sp >>>= 0; - storeValue( - sp + 24, - Reflect.get(loadValue(sp + 8), getInt64(sp + 16)) - ); - }, - - // valueSetIndex(v ref, i int, x ref) - 'syscall/js.valueSetIndex': (sp) => { - sp >>>= 0; - Reflect.set( - loadValue(sp + 8), - getInt64(sp + 16), - loadValue(sp + 24) - ); - }, - - // func valueCall(v ref, m string, args []ref) (ref, bool) - 'syscall/js.valueCall': (sp) => { - sp >>>= 0; - try { - const v = loadValue(sp + 8); - const m = Reflect.get(v, loadString(sp + 16)); - const args = loadSliceOfValues(sp + 32); - const result = Reflect.apply(m, v, args); - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 56, result); - this.mem.setUint8(sp + 64, 1); - } catch (err) { - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 56, err); - this.mem.setUint8(sp + 64, 0); - } - }, - - // func valueInvoke(v ref, args []ref) (ref, bool) - 'syscall/js.valueInvoke': (sp) => { - sp >>>= 0; - try { - const v = loadValue(sp + 8); - const args = loadSliceOfValues(sp + 16); - const result = Reflect.apply(v, undefined, args); - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 40, result); - this.mem.setUint8(sp + 48, 1); - } catch (err) { - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 40, err); - this.mem.setUint8(sp + 48, 0); - } - }, - - // func valueNew(v ref, args []ref) (ref, bool) - 'syscall/js.valueNew': (sp) => { - sp >>>= 0; - try { - const v = loadValue(sp + 8); - const args = loadSliceOfValues(sp + 16); - const result = Reflect.construct(v, args); - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 40, result); - this.mem.setUint8(sp + 48, 1); - } catch (err) { - sp = this._inst.exports.getsp() >>> 0; // see comment above - storeValue(sp + 40, err); - this.mem.setUint8(sp + 48, 0); - } - }, - - // func valueLength(v ref) int - 'syscall/js.valueLength': (sp) => { - sp >>>= 0; - setInt64(sp + 16, parseInt(loadValue(sp + 8).length)); - }, - - // valuePrepareString(v ref) (ref, int) - 'syscall/js.valuePrepareString': (sp) => { - sp >>>= 0; - const str = encoder.encode(String(loadValue(sp + 8))); - storeValue(sp + 16, str); - setInt64(sp + 24, str.length); - }, - - // valueLoadString(v ref, b []byte) - 'syscall/js.valueLoadString': (sp) => { - sp >>>= 0; - const str = loadValue(sp + 8); - loadSlice(sp + 16).set(str); - }, - - // func valueInstanceOf(v ref, t ref) bool - 'syscall/js.valueInstanceOf': (sp) => { - sp >>>= 0; - this.mem.setUint8( - sp + 24, - loadValue(sp + 8) instanceof loadValue(sp + 16) ? 1 : 0 - ); - }, - - // func copyBytesToGo(dst []byte, src ref) (int, bool) - 'syscall/js.copyBytesToGo': (sp) => { - sp >>>= 0; - const dst = loadSlice(sp + 8); - const src = loadValue(sp + 32); - if ( - !(src instanceof Uint8Array || src instanceof Uint8ClampedArray) - ) { - this.mem.setUint8(sp + 48, 0); - return; - } - const toCopy = src.subarray(0, dst.length); - dst.set(toCopy); - setInt64(sp + 40, toCopy.length); - this.mem.setUint8(sp + 48, 1); - }, - - // func copyBytesToJS(dst ref, src []byte) (int, bool) - 'syscall/js.copyBytesToJS': (sp) => { - sp >>>= 0; - const dst = loadValue(sp + 8); - const src = loadSlice(sp + 16); - if ( - !(dst instanceof Uint8Array || dst instanceof Uint8ClampedArray) - ) { - this.mem.setUint8(sp + 48, 0); - return; - } - const toCopy = src.subarray(0, dst.length); - dst.set(toCopy); - setInt64(sp + 40, toCopy.length); - this.mem.setUint8(sp + 48, 1); - }, - - debug: (value) => { - console.log(value); - }, - }, - }; - } - - async run(instance) { - if (!(instance instanceof WebAssembly.Instance)) { - throw new Error('Go.run: WebAssembly.Instance expected'); - } - this._inst = instance; - this.mem = new DataView(this._inst.exports.mem.buffer); - this._values = [ - // JS values that Go currently has references to, indexed by reference id - NaN, - 0, - null, - true, - false, - globalThis, - this, - ]; - this._goRefCounts = new Array(this._values.length).fill(Infinity); // number of references that Go has to a JS value, indexed by reference id - this._ids = new Map([ - // mapping from JS values to reference ids - [0, 1], - [null, 2], - [true, 3], - [false, 4], - [globalThis, 5], - [this, 6], - ]); - this._idPool = []; // unused ids that have been garbage collected - this.exited = false; // whether the Go program has exited - - // Pass command line arguments and environment variables to WebAssembly by writing them to the linear memory. - let offset = 4096; - - const strPtr = (str) => { - const ptr = offset; - const bytes = encoder.encode(str + '\0'); - new Uint8Array(this.mem.buffer, offset, bytes.length).set(bytes); - offset += bytes.length; - if (offset % 8 !== 0) { - offset += 8 - (offset % 8); - } - return ptr; - }; - - const argc = this.argv.length; - - const argvPtrs = []; - this.argv.forEach((arg) => { - argvPtrs.push(strPtr(arg)); - }); - argvPtrs.push(0); - - const keys = Object.keys(this.env).sort(); - keys.forEach((key) => { - argvPtrs.push(strPtr(`${key}=${this.env[key]}`)); - }); - argvPtrs.push(0); - - const argv = offset; - argvPtrs.forEach((ptr) => { - this.mem.setUint32(offset, ptr, true); - this.mem.setUint32(offset + 4, 0, true); - offset += 8; - }); - - // The linker guarantees global data starts from at least wasmMinDataAddr. - // Keep in sync with cmd/link/internal/ld/data.go:wasmMinDataAddr. - const wasmMinDataAddr = 4096 + 8192; - if (offset >= wasmMinDataAddr) { - throw new Error( - 'total length of command line and environment variables exceeds limit' - ); - } - - this._inst.exports.run(argc, argv); - if (this.exited) { - this._resolveExitPromise(); - } - await this._exitPromise; - } - - _resume() { - if (this.exited) { - throw new Error('Go program has already exited'); - } - this._inst.exports.resume(); - if (this.exited) { - this._resolveExitPromise(); - } - } - - _makeFuncWrapper(id) { - const go = this; - return function () { - const event = { id: id, this: this, args: arguments }; - go._pendingEvent = event; - go._resume(); - return event.result; - }; - } - }; + const enosys = () => { + const err = new Error("not implemented"); + err.code = "ENOSYS"; + return err; + }; + + if (!globalThis.fs) { + let outputBuf = ""; + globalThis.fs = { + constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1 }, // unused + writeSync(fd, buf) { + outputBuf += decoder.decode(buf); + const nl = outputBuf.lastIndexOf("\n"); + if (nl != -1) { + console.log(outputBuf.substring(0, nl)); + outputBuf = outputBuf.substring(nl + 1); + } + return buf.length; + }, + write(fd, buf, offset, length, position, callback) { + if (offset !== 0 || length !== buf.length || position !== null) { + callback(enosys()); + return; + } + const n = this.writeSync(fd, buf); + callback(null, n); + }, + chmod(path, mode, callback) { callback(enosys()); }, + chown(path, uid, gid, callback) { callback(enosys()); }, + close(fd, callback) { callback(enosys()); }, + fchmod(fd, mode, callback) { callback(enosys()); }, + fchown(fd, uid, gid, callback) { callback(enosys()); }, + fstat(fd, callback) { callback(enosys()); }, + fsync(fd, callback) { callback(null); }, + ftruncate(fd, length, callback) { callback(enosys()); }, + lchown(path, uid, gid, callback) { callback(enosys()); }, + link(path, link, callback) { callback(enosys()); }, + lstat(path, callback) { callback(enosys()); }, + mkdir(path, perm, callback) { callback(enosys()); }, + open(path, flags, mode, callback) { callback(enosys()); }, + read(fd, buffer, offset, length, position, callback) { callback(enosys()); }, + readdir(path, callback) { callback(enosys()); }, + readlink(path, callback) { callback(enosys()); }, + rename(from, to, callback) { callback(enosys()); }, + rmdir(path, callback) { callback(enosys()); }, + stat(path, callback) { callback(enosys()); }, + symlink(path, link, callback) { callback(enosys()); }, + truncate(path, length, callback) { callback(enosys()); }, + unlink(path, callback) { callback(enosys()); }, + utimes(path, atime, mtime, callback) { callback(enosys()); }, + }; + } + + if (!globalThis.process) { + globalThis.process = { + getuid() { return -1; }, + getgid() { return -1; }, + geteuid() { return -1; }, + getegid() { return -1; }, + getgroups() { throw enosys(); }, + pid: -1, + ppid: -1, + umask() { throw enosys(); }, + cwd() { throw enosys(); }, + chdir() { throw enosys(); }, + } + } + + if (!globalThis.crypto) { + throw new Error("globalThis.crypto is not available, polyfill required (crypto.getRandomValues only)"); + } + + if (!globalThis.performance) { + throw new Error("globalThis.performance is not available, polyfill required (performance.now only)"); + } + + if (!globalThis.TextEncoder) { + throw new Error("globalThis.TextEncoder is not available, polyfill required"); + } + + if (!globalThis.TextDecoder) { + throw new Error("globalThis.TextDecoder is not available, polyfill required"); + } + + const encoder = new TextEncoder("utf-8"); + const decoder = new TextDecoder("utf-8"); + + globalThis.Go = class { + constructor() { + this.argv = ["js"]; + this.env = {}; + this.exit = (code) => { + if (code !== 0) { + console.warn("exit code:", code); + } + }; + this._exitPromise = new Promise((resolve) => { + this._resolveExitPromise = resolve; + }); + this._pendingEvent = null; + this._scheduledTimeouts = new Map(); + this._nextCallbackTimeoutID = 1; + + const setInt64 = (addr, v) => { + this.mem.setUint32(addr + 0, v, true); + this.mem.setUint32(addr + 4, Math.floor(v / 4294967296), true); + } + + const setInt32 = (addr, v) => { + this.mem.setUint32(addr + 0, v, true); + } + + const getInt64 = (addr) => { + const low = this.mem.getUint32(addr + 0, true); + const high = this.mem.getInt32(addr + 4, true); + return low + high * 4294967296; + } + + const loadValue = (addr) => { + const f = this.mem.getFloat64(addr, true); + if (f === 0) { + return undefined; + } + if (!isNaN(f)) { + return f; + } + + const id = this.mem.getUint32(addr, true); + return this._values[id]; + } + + const storeValue = (addr, v) => { + const nanHead = 0x7FF80000; + + if (typeof v === "number" && v !== 0) { + if (isNaN(v)) { + this.mem.setUint32(addr + 4, nanHead, true); + this.mem.setUint32(addr, 0, true); + return; + } + this.mem.setFloat64(addr, v, true); + return; + } + + if (v === undefined) { + this.mem.setFloat64(addr, 0, true); + return; + } + + let id = this._ids.get(v); + if (id === undefined) { + id = this._idPool.pop(); + if (id === undefined) { + id = this._values.length; + } + this._values[id] = v; + this._goRefCounts[id] = 0; + this._ids.set(v, id); + } + this._goRefCounts[id]++; + let typeFlag = 0; + switch (typeof v) { + case "object": + if (v !== null) { + typeFlag = 1; + } + break; + case "string": + typeFlag = 2; + break; + case "symbol": + typeFlag = 3; + break; + case "function": + typeFlag = 4; + break; + } + this.mem.setUint32(addr + 4, nanHead | typeFlag, true); + this.mem.setUint32(addr, id, true); + } + + const loadSlice = (addr) => { + const array = getInt64(addr + 0); + const len = getInt64(addr + 8); + return new Uint8Array(this._inst.exports.mem.buffer, array, len); + } + + const loadSliceOfValues = (addr) => { + const array = getInt64(addr + 0); + const len = getInt64(addr + 8); + const a = new Array(len); + for (let i = 0; i < len; i++) { + a[i] = loadValue(array + i * 8); + } + return a; + } + + const loadString = (addr) => { + const saddr = getInt64(addr + 0); + const len = getInt64(addr + 8); + return decoder.decode(new DataView(this._inst.exports.mem.buffer, saddr, len)); + } + + const timeOrigin = Date.now() - performance.now(); + this.importObject = { + _gotest: { + add: (a, b) => a + b, + }, + gojs: { + // Go's SP does not change as long as no Go code is running. Some operations (e.g. calls, getters and setters) + // may synchronously trigger a Go event handler. This makes Go code get executed in the middle of the imported + // function. A goroutine can switch to a new stack if the current stack is too small (see morestack function). + // This changes the SP, thus we have to update the SP used by the imported function. + + // func wasmExit(code int32) + "runtime.wasmExit": (sp) => { + sp >>>= 0; + const code = this.mem.getInt32(sp + 8, true); + this.exited = true; + delete this._inst; + delete this._values; + delete this._goRefCounts; + delete this._ids; + delete this._idPool; + this.exit(code); + }, + + // func wasmWrite(fd uintptr, p unsafe.Pointer, n int32) + "runtime.wasmWrite": (sp) => { + sp >>>= 0; + const fd = getInt64(sp + 8); + const p = getInt64(sp + 16); + const n = this.mem.getInt32(sp + 24, true); + fs.writeSync(fd, new Uint8Array(this._inst.exports.mem.buffer, p, n)); + }, + + // func resetMemoryDataView() + "runtime.resetMemoryDataView": (sp) => { + sp >>>= 0; + this.mem = new DataView(this._inst.exports.mem.buffer); + }, + + // func nanotime1() int64 + "runtime.nanotime1": (sp) => { + sp >>>= 0; + setInt64(sp + 8, (timeOrigin + performance.now()) * 1000000); + }, + + // func walltime() (sec int64, nsec int32) + "runtime.walltime": (sp) => { + sp >>>= 0; + const msec = (new Date).getTime(); + setInt64(sp + 8, msec / 1000); + this.mem.setInt32(sp + 16, (msec % 1000) * 1000000, true); + }, + + // func scheduleTimeoutEvent(delay int64) int32 + "runtime.scheduleTimeoutEvent": (sp) => { + sp >>>= 0; + const id = this._nextCallbackTimeoutID; + this._nextCallbackTimeoutID++; + this._scheduledTimeouts.set(id, setTimeout( + () => { + this._resume(); + while (this._scheduledTimeouts.has(id)) { + // for some reason Go failed to register the timeout event, log and try again + // (temporary workaround for https://github.com/golang/go/issues/28975) + console.warn("scheduleTimeoutEvent: missed timeout event"); + this._resume(); + } + }, + getInt64(sp + 8), + )); + this.mem.setInt32(sp + 16, id, true); + }, + + // func clearTimeoutEvent(id int32) + "runtime.clearTimeoutEvent": (sp) => { + sp >>>= 0; + const id = this.mem.getInt32(sp + 8, true); + clearTimeout(this._scheduledTimeouts.get(id)); + this._scheduledTimeouts.delete(id); + }, + + // func getRandomData(r []byte) + "runtime.getRandomData": (sp) => { + sp >>>= 0; + crypto.getRandomValues(loadSlice(sp + 8)); + }, + + // func finalizeRef(v ref) + "syscall/js.finalizeRef": (sp) => { + sp >>>= 0; + const id = this.mem.getUint32(sp + 8, true); + this._goRefCounts[id]--; + if (this._goRefCounts[id] === 0) { + const v = this._values[id]; + this._values[id] = null; + this._ids.delete(v); + this._idPool.push(id); + } + }, + + // func stringVal(value string) ref + "syscall/js.stringVal": (sp) => { + sp >>>= 0; + storeValue(sp + 24, loadString(sp + 8)); + }, + + // func valueGet(v ref, p string) ref + "syscall/js.valueGet": (sp) => { + sp >>>= 0; + const result = Reflect.get(loadValue(sp + 8), loadString(sp + 16)); + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 32, result); + }, + + // func valueSet(v ref, p string, x ref) + "syscall/js.valueSet": (sp) => { + sp >>>= 0; + Reflect.set(loadValue(sp + 8), loadString(sp + 16), loadValue(sp + 32)); + }, + + // func valueDelete(v ref, p string) + "syscall/js.valueDelete": (sp) => { + sp >>>= 0; + Reflect.deleteProperty(loadValue(sp + 8), loadString(sp + 16)); + }, + + // func valueIndex(v ref, i int) ref + "syscall/js.valueIndex": (sp) => { + sp >>>= 0; + storeValue(sp + 24, Reflect.get(loadValue(sp + 8), getInt64(sp + 16))); + }, + + // valueSetIndex(v ref, i int, x ref) + "syscall/js.valueSetIndex": (sp) => { + sp >>>= 0; + Reflect.set(loadValue(sp + 8), getInt64(sp + 16), loadValue(sp + 24)); + }, + + // func valueCall(v ref, m string, args []ref) (ref, bool) + "syscall/js.valueCall": (sp) => { + sp >>>= 0; + try { + const v = loadValue(sp + 8); + const m = Reflect.get(v, loadString(sp + 16)); + const args = loadSliceOfValues(sp + 32); + const result = Reflect.apply(m, v, args); + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 56, result); + this.mem.setUint8(sp + 64, 1); + } catch (err) { + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 56, err); + this.mem.setUint8(sp + 64, 0); + } + }, + + // func valueInvoke(v ref, args []ref) (ref, bool) + "syscall/js.valueInvoke": (sp) => { + sp >>>= 0; + try { + const v = loadValue(sp + 8); + const args = loadSliceOfValues(sp + 16); + const result = Reflect.apply(v, undefined, args); + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 40, result); + this.mem.setUint8(sp + 48, 1); + } catch (err) { + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 40, err); + this.mem.setUint8(sp + 48, 0); + } + }, + + // func valueNew(v ref, args []ref) (ref, bool) + "syscall/js.valueNew": (sp) => { + sp >>>= 0; + try { + const v = loadValue(sp + 8); + const args = loadSliceOfValues(sp + 16); + const result = Reflect.construct(v, args); + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 40, result); + this.mem.setUint8(sp + 48, 1); + } catch (err) { + sp = this._inst.exports.getsp() >>> 0; // see comment above + storeValue(sp + 40, err); + this.mem.setUint8(sp + 48, 0); + } + }, + + // func valueLength(v ref) int + "syscall/js.valueLength": (sp) => { + sp >>>= 0; + setInt64(sp + 16, parseInt(loadValue(sp + 8).length)); + }, + + // valuePrepareString(v ref) (ref, int) + "syscall/js.valuePrepareString": (sp) => { + sp >>>= 0; + const str = encoder.encode(String(loadValue(sp + 8))); + storeValue(sp + 16, str); + setInt64(sp + 24, str.length); + }, + + // valueLoadString(v ref, b []byte) + "syscall/js.valueLoadString": (sp) => { + sp >>>= 0; + const str = loadValue(sp + 8); + loadSlice(sp + 16).set(str); + }, + + // func valueInstanceOf(v ref, t ref) bool + "syscall/js.valueInstanceOf": (sp) => { + sp >>>= 0; + this.mem.setUint8(sp + 24, (loadValue(sp + 8) instanceof loadValue(sp + 16)) ? 1 : 0); + }, + + // func copyBytesToGo(dst []byte, src ref) (int, bool) + "syscall/js.copyBytesToGo": (sp) => { + sp >>>= 0; + const dst = loadSlice(sp + 8); + const src = loadValue(sp + 32); + if (!(src instanceof Uint8Array || src instanceof Uint8ClampedArray)) { + this.mem.setUint8(sp + 48, 0); + return; + } + const toCopy = src.subarray(0, dst.length); + dst.set(toCopy); + setInt64(sp + 40, toCopy.length); + this.mem.setUint8(sp + 48, 1); + }, + + // func copyBytesToJS(dst ref, src []byte) (int, bool) + "syscall/js.copyBytesToJS": (sp) => { + sp >>>= 0; + const dst = loadValue(sp + 8); + const src = loadSlice(sp + 16); + if (!(dst instanceof Uint8Array || dst instanceof Uint8ClampedArray)) { + this.mem.setUint8(sp + 48, 0); + return; + } + const toCopy = src.subarray(0, dst.length); + dst.set(toCopy); + setInt64(sp + 40, toCopy.length); + this.mem.setUint8(sp + 48, 1); + }, + + "debug": (value) => { + console.log(value); + }, + } + }; + } + + async run(instance) { + if (!(instance instanceof WebAssembly.Instance)) { + throw new Error("Go.run: WebAssembly.Instance expected"); + } + this._inst = instance; + this.mem = new DataView(this._inst.exports.mem.buffer); + this._values = [ // JS values that Go currently has references to, indexed by reference id + NaN, + 0, + null, + true, + false, + globalThis, + this, + ]; + this._goRefCounts = new Array(this._values.length).fill(Infinity); // number of references that Go has to a JS value, indexed by reference id + this._ids = new Map([ // mapping from JS values to reference ids + [0, 1], + [null, 2], + [true, 3], + [false, 4], + [globalThis, 5], + [this, 6], + ]); + this._idPool = []; // unused ids that have been garbage collected + this.exited = false; // whether the Go program has exited + + // Pass command line arguments and environment variables to WebAssembly by writing them to the linear memory. + let offset = 4096; + + const strPtr = (str) => { + const ptr = offset; + const bytes = encoder.encode(str + "\0"); + new Uint8Array(this.mem.buffer, offset, bytes.length).set(bytes); + offset += bytes.length; + if (offset % 8 !== 0) { + offset += 8 - (offset % 8); + } + return ptr; + }; + + const argc = this.argv.length; + + const argvPtrs = []; + this.argv.forEach((arg) => { + argvPtrs.push(strPtr(arg)); + }); + argvPtrs.push(0); + + const keys = Object.keys(this.env).sort(); + keys.forEach((key) => { + argvPtrs.push(strPtr(`${key}=${this.env[key]}`)); + }); + argvPtrs.push(0); + + const argv = offset; + argvPtrs.forEach((ptr) => { + this.mem.setUint32(offset, ptr, true); + this.mem.setUint32(offset + 4, 0, true); + offset += 8; + }); + + // The linker guarantees global data starts from at least wasmMinDataAddr. + // Keep in sync with cmd/link/internal/ld/data.go:wasmMinDataAddr. + const wasmMinDataAddr = 4096 + 8192; + if (offset >= wasmMinDataAddr) { + throw new Error("total length of command line and environment variables exceeds limit"); + } + + this._inst.exports.run(argc, argv); + if (this.exited) { + this._resolveExitPromise(); + } + await this._exitPromise; + } + + _resume() { + if (this.exited) { + throw new Error("Go program has already exited"); + } + this._inst.exports.resume(); + if (this.exited) { + this._resolveExitPromise(); + } + } + + _makeFuncWrapper(id) { + const go = this; + return function () { + const event = { id: id, this: this, args: arguments }; + go._pendingEvent = event; + go._resume(); + return event.result; + }; + } + } })(); diff --git a/packages/zed-wasm/lib/wasm_exec_node.js b/packages/zed-wasm/lib/wasm_exec_node.js deleted file mode 100644 index bf0b5cab3c..0000000000 --- a/packages/zed-wasm/lib/wasm_exec_node.js +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2021 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -'use strict'; - -if (process.argv.length < 3) { - console.error('usage: go_js_wasm_exec [wasm binary] [arguments]'); - process.exit(1); -} - -globalThis.require = require; -globalThis.fs = require('fs'); -globalThis.TextEncoder = require('util').TextEncoder; -globalThis.TextDecoder = require('util').TextDecoder; - -globalThis.performance = { - now() { - const [sec, nsec] = process.hrtime(); - return sec * 1000 + nsec / 1000000; - }, -}; - -const crypto = require('crypto'); -globalThis.crypto = { - getRandomValues(b) { - crypto.randomFillSync(b); - }, -}; - -require('./wasm_exec'); - -const go = new Go(); -go.argv = process.argv.slice(2); -go.env = Object.assign({ TMPDIR: require('os').tmpdir() }, process.env); -go.exit = process.exit; -WebAssembly.instantiate(fs.readFileSync(process.argv[2]), go.importObject) - .then((result) => { - process.on('exit', (code) => { - // Node.js exits if no event handler is pending - if (code === 0 && !go.exited) { - // deadlock, make Go print error and stack traces - go._pendingEvent = { id: 0 }; - go._resume(); - } - }); - return go.run(result.instance); - }) - .catch((err) => { - console.error(err); - process.exit(1); - }); diff --git a/packages/zed-wasm/main.go b/packages/zed-wasm/main.go index e0c5997a75..96e05739ef 100644 --- a/packages/zed-wasm/main.go +++ b/packages/zed-wasm/main.go @@ -11,6 +11,7 @@ import ( "github.com/brimdata/zed" "github.com/brimdata/zed/compiler" + "github.com/brimdata/zed/compiler/parser" "github.com/brimdata/zed/pkg/storage" "github.com/brimdata/zed/runtime" "github.com/brimdata/zed/zbuf" @@ -21,6 +22,7 @@ import ( func main() { wasm.Expose("zq", zq) + wasm.Expose("parse", parse) wasm.Ready() <-make(chan struct{}) } @@ -59,7 +61,7 @@ func zq(opts opts) wasm.Promise { return "", errInvalidInput } zctx := zed.NewContext() - zr, err := anyio.NewReaderWithOpts(zctx, r, anyio.ReaderOpts{ + zr, err := anyio.NewReaderWithOpts(zctx, r, nil, anyio.ReaderOpts{ Format: opts.InputFormat, }) if err != nil { @@ -89,6 +91,24 @@ func zq(opts opts) wasm.Promise { }) } +func parse(program string) (interface{}, error) { + ast, err := parser.ParseZed(nil, program) + result := ParseResult{AST: ast} + if err != nil { + var ok bool + result.Error, ok = err.(*parser.Error) + if !ok { + return nil, err + } + } + return result, nil +} + +type ParseResult struct { + AST interface{} `wasm:"ast"` + Error *parser.Error `wasm:"error"` +} + func readableStream(readable js.Value) io.Reader { pr, pw := io.Pipe() go func() { diff --git a/packages/zed-wasm/package.json b/packages/zed-wasm/package.json index dc7e6c405d..3856f5240b 100644 --- a/packages/zed-wasm/package.json +++ b/packages/zed-wasm/package.json @@ -3,6 +3,25 @@ "version": "0.0.17", "homepage": "https://github.com/brimdata/zed", "type": "module", + "files": [ + "dist/browser.js", + "dist/index.js", + "dist/main.wasm", + "src/**/*.js" + ], + "main": "./dist/index.cjs", + "jsdelivr": "./dist/browser.js", + "scripts": { + "test": "run-s 'test:*'", + "test:node": "node --test test/node", + "test:browser": "playwright test", + "test:cjs": "jest test/cjs", + "build": "run-s 'build:*'", + "build:browser": "esbuild src/browser.js --outfile=dist/browser.js --format=esm --bundle", + "build:cjs": "esbuild src/index.js --outfile=dist/index.cjs --format=cjs --bundle --platform=node --packages=external", + "build:go": "GOARCH=wasm GOOS=js go build -tags=noasm -o dist/main.wasm main.go", + "start-test-server": "serve -p 2000" + }, "keywords": [ "zed", "data", @@ -11,10 +30,17 @@ "wasm" ], "devDependencies": { + "@playwright/test": "^1.40.1", "@types/golang-wasm": "^1.15.0", - "golang-wasm": "github:teamortix/golang-wasm#master" + "@types/node": "^20.11.0", + "esbuild": "^0.17.12", + "golang-wasm": "github:teamortix/golang-wasm#master", + "jest": "^29.4.1", + "npm-run-all": "^4.1.5", + "serve": "^14.2.1" }, "dependencies": { - "@brimdata/zed-js": "workspace:*" + "@brimdata/zed-js": "workspace:*", + "polyfill-crypto.getrandomvalues": "^1.0.0" } } diff --git a/packages/zed-wasm/playwright.config.ts b/packages/zed-wasm/playwright.config.ts new file mode 100644 index 0000000000..24a3883dc0 --- /dev/null +++ b/packages/zed-wasm/playwright.config.ts @@ -0,0 +1,81 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// require('dotenv').config(); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: './test/browser', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('/')`. */ + baseURL: 'http://127.0.0.1:2000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + + webServer: { + command: 'yarn run start-test-server', + url: 'http://127.0.0.1:2000', + reuseExistingServer: !process.env.CI, + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + // { + // name: 'firefox', + // use: { ...devices['Desktop Firefox'] }, + // }, + // + // { + // name: 'webkit', + // use: { ...devices['Desktop Safari'] }, + // }, + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ], + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://127.0.0.1:3000', + // reuseExistingServer: !process.env.CI, + // }, +}); diff --git a/packages/zed-wasm/project.json b/packages/zed-wasm/project.json index bb5d196f9e..48e71b0365 100644 --- a/packages/zed-wasm/project.json +++ b/packages/zed-wasm/project.json @@ -2,54 +2,5 @@ "name": "zed-wasm", "$schema": "../../node_modules/nx/schemas/project-schema.json", "sourceRoot": "packages/zed-wasm/src", - "projectType": "library", - "targets": { - "build": { -"dependsOn": ["build-go"], - "executor": "@nrwl/esbuild:esbuild", - "options": { - "main": "packages/zed-wasm/src/index.ts", - "tsConfig": "packages/zed-wasm/tsconfig.lib.json", - "outputPath": "dist/packages/zed-wasm", - "assets": ["packages/zed-wasm/*.md"], - "format": ["esm"], - "deleteOutputPath": false, - "esbuildOptions": { - "inject": ["packages/zed-wasm/lib/global-shim.js"] - } - } - }, - "build-go": { - "executor": "nx:run-commands", - "outputs": ["{workspaceRoot}/dist/packages/zed-wasm/main.wasm"], - "options": { - "command": "go build -tags=noasm -o ../../dist/packages/zed-wasm/main.wasm main.go", - "cwd": "packages/zed-wasm" - } - }, - "publish": { - "executor": "nx:run-commands", - "options": { - "command": "node tools/scripts/publish.mjs zed-wasm {args.ver} {args.tag}" - }, - "dependsOn": ["build"] - }, - "lint": { - "executor": "@nrwl/linter:eslint", - "outputs": ["{options.outputFile}"], - "options": { - "lintFilePatterns": ["packages/zed-wasm/**/*.ts"] - } - }, - "test-browser": { - "executor": "nx:run-commands", - "options": { - "commands": [ - "serve", - "open-cli http://localhost:3000/packages/zed-wasm/" - ] - } - } - }, - "tags": [] + "projectType": "library" } diff --git a/packages/zed-wasm/src/browser.js b/packages/zed-wasm/src/browser.js new file mode 100644 index 0000000000..ae748f4f65 --- /dev/null +++ b/packages/zed-wasm/src/browser.js @@ -0,0 +1,37 @@ +import '../lib/wasm_exec.js'; +import '../lib/bridge.js'; +import { createInterface } from './interface.js'; + +export async function initZedWasm(fetchRequest) { + const go = new Go(); + const { instance } = await WebAssembly.instantiateStreaming( + fetchRequest, + go.importObject + ); + go.run(instance); + return createInterface(__go_wasm__, { getInput }); +} + +function getInput(input) { + if (typeof input === 'string') return input; + if (input instanceof File) return input.stream(); + if (input instanceof Blob) return input.stream(); + if (input instanceof ReadableStream) return input; + if (input instanceof Response) return input.body; + if (Array.isArray(input)) return arrayStream(input); + if (input === undefined) return undefined; + if (input === null) return undefined; + throw new Error(`Unsupported input type provided to zq ${input}`); +} + +function arrayStream(input) { + const encoder = new TextEncoder(); + return new ReadableStream({ + start(ctl) { + for (const item of input) { + ctl.enqueue(encoder.encode(JSON.stringify(item) + '\n')); + } + ctl.close(); + }, + }); +} diff --git a/packages/zed-wasm/src/golang-wasm.d.ts b/packages/zed-wasm/src/golang-wasm.d.ts deleted file mode 100644 index 6fe715bb19..0000000000 --- a/packages/zed-wasm/src/golang-wasm.d.ts +++ /dev/null @@ -1,3 +0,0 @@ -declare module 'golang-wasm/src/bridge' { - export default function (arg: Promise): any; -} diff --git a/packages/zed-wasm/src/index.js b/packages/zed-wasm/src/index.js new file mode 100644 index 0000000000..51149aa9df --- /dev/null +++ b/packages/zed-wasm/src/index.js @@ -0,0 +1,22 @@ +import '../lib/node-globals.js'; +import '../lib/wasm_exec.js'; +import '../lib/bridge.js'; +import { readFile } from 'node:fs/promises'; +import { createInterface } from './interface.js'; + +export async function initZedWasm(wasmFilePath) { + const go = new Go(); + const { instance } = await WebAssembly.instantiate( + await readFile(wasmFilePath), + go.importObject + ); + go.run(instance); + return createInterface(__go_wasm__, { getInput }); +} + +function getInput(input) { + if (typeof input === 'string') return input; + if (input === undefined) return undefined; + if (input === null) return undefined; + throw new Error(`Unsupported input type provided to zq ${input}`); +} diff --git a/packages/zed-wasm/src/index.ts b/packages/zed-wasm/src/index.ts deleted file mode 100644 index ca4cbcb5ac..0000000000 --- a/packages/zed-wasm/src/index.ts +++ /dev/null @@ -1,52 +0,0 @@ -import '../lib/wasm_exec'; -import bridge from 'golang-wasm/src/bridge'; -import { LoadFormat, decode, ndjson } from '@brimdata/zed-js'; - -const url = new URL('.', import.meta.url); -const path = url.href + 'main.wasm'; -const wasm = await fetch(path); -const go = bridge(wasm.arrayBuffer()); - -export async function zq(opts: { - program?: string; - input?: string | File | Blob | ReadableStream | any[]; - inputFormat?: LoadFormat; - outputFormat?: 'js' | 'zed'; -}) { - const result = await go.zq({ - input: getInput(opts.input), - inputFormat: opts.inputFormat, - program: opts.program, - outputFormat: 'zjson', - }); - - const zed = decode(ndjson.parseLines(result)); - if (opts.outputFormat === 'zed') return zed; - return zed.map((val) => val.toJS()); -} - -function getInput( - input: string | File | Blob | ReadableStream | Response | undefined | any[] -) { - if (typeof input === 'string') return input; - if (input instanceof File) return input.stream(); - if (input instanceof Blob) return input.stream(); - if (input instanceof ReadableStream) return input; - if (input instanceof Response) return input.body; - if (Array.isArray(input)) return arrayStream(input); - if (input === undefined) return undefined; - if (input === null) return undefined; - throw new Error(`Unsupported input type provided to zq ${input}`); -} - -function arrayStream(input: any[]) { - const encoder = new TextEncoder(); - return new ReadableStream({ - start(ctl) { - for (const item of input) { - ctl.enqueue(encoder.encode(JSON.stringify(item) + '\n')); - } - ctl.close(); - }, - }); -} diff --git a/packages/zed-wasm/src/interface.js b/packages/zed-wasm/src/interface.js new file mode 100644 index 0000000000..c67ca1920d --- /dev/null +++ b/packages/zed-wasm/src/interface.js @@ -0,0 +1,24 @@ +import { decode, ndjson } from '@brimdata/zed-js'; + +export function createInterface(go, deps) { + return { + // program?: string, + // input?: string | File | Blob | ReadableStream | any[], + // inputFormat?: LoadFormat, + // outputFormat?: 'js' | 'zed', + zq: async (opts) => { + const result = await go.zq({ + input: deps.getInput(opts.input), + inputFormat: opts.inputFormat, + program: opts.program, + outputFormat: 'zjson', + }); + + const zed = decode(ndjson.parseLines(result)); + if (opts.outputFormat === 'zed') return zed; + return zed.map((val) => val.toJS()); + }, + + parse: (string) => go.parse(string), + }; +} diff --git a/packages/zed-wasm/test/browser/parse.spec.ts b/packages/zed-wasm/test/browser/parse.spec.ts new file mode 100644 index 0000000000..266ae08a96 --- /dev/null +++ b/packages/zed-wasm/test/browser/parse.spec.ts @@ -0,0 +1,12 @@ +import { test, expect } from '@playwright/test'; + +test.beforeEach(async ({ page }) => { + await page.goto('/test/pages/parse.html'); + await page.getByText('Wasm Ready').waitFor({ state: 'attached' }); +}); + +test('parse a simple zed string', async ({ page }) => { + await page.fill('input', 'zed := "world"'); + await page.click('button'); + await expect(page.locator('pre')).toContainText('"kind": "OpAssignment"'); +}); diff --git a/packages/zed-wasm/test/browser/zq.test.js b/packages/zed-wasm/test/browser/zq.test.js new file mode 100644 index 0000000000..ef1e3c58ae --- /dev/null +++ b/packages/zed-wasm/test/browser/zq.test.js @@ -0,0 +1,13 @@ +import { test, expect } from '@playwright/test'; + +test.beforeEach(async ({ page }) => { + await page.goto('/test/pages/zq.html'); + await page.getByText('Wasm Ready').waitFor({ state: 'attached' }); +}); + +test('run zq', async ({ page }) => { + await page.fill('[name=input]', '1 2 3'); + await page.fill('[name=script]', 'this * 10'); + await page.click('button'); + await expect(page.locator('pre')).toContainText('[10,20,30]'); +}); diff --git a/packages/zed-wasm/test/cjs/parse.test.js b/packages/zed-wasm/test/cjs/parse.test.js new file mode 100644 index 0000000000..904716fe7d --- /dev/null +++ b/packages/zed-wasm/test/cjs/parse.test.js @@ -0,0 +1,44 @@ +const { initZedWasm } = require('@brimdata/zed-wasm'); + +let wasm; +beforeAll(async () => { + wasm = await initZedWasm( + require.resolve('@brimdata/zed-wasm/dist/main.wasm') + ); +}); + +test('parse works in cjs env', async () => { + const resp = await wasm.parse('n := "james"'); + expect(resp).toMatchInlineSnapshot(` + { + "ast": [ + { + "assignments": [ + { + "kind": "Assignment", + "lhs": { + "kind": "ID", + "name": "n", + }, + "rhs": { + "kind": "Primitive", + "text": "james", + "type": "string", + }, + }, + ], + "kind": "OpAssignment", + }, + ], + "error": undefined, + } + `); +}); + +test('zq works in cjs env', async () => { + const resp = await wasm.zq({ + input: '1 2 3', + program: 'this * 10', + }); + expect(resp).toEqual([10, 20, 30]); +}); diff --git a/packages/zed-wasm/test/node/parser.test.js b/packages/zed-wasm/test/node/parser.test.js new file mode 100644 index 0000000000..ffc8684760 --- /dev/null +++ b/packages/zed-wasm/test/node/parser.test.js @@ -0,0 +1,37 @@ +import test from 'node:test'; +import assert from 'node:assert'; +import { initZedWasm } from '@brimdata/zed-wasm'; + +let parse; +test.before(async () => { + const wasm = await initZedWasm( + require.resolve('@brimdata/zed-wasm/dist/main.wasm') + ); + parse = wasm.parse; +}); + +test('parser', async () => { + const resp = await parse('me := "james"'); + assert.deepEqual(resp, { + error: undefined, + ast: [ + { + assignments: [ + { + kind: 'Assignment', + lhs: { + kind: 'ID', + name: 'me', + }, + rhs: { + kind: 'Primitive', + text: 'james', + type: 'string', + }, + }, + ], + kind: 'OpAssignment', + }, + ], + }); +}); diff --git a/packages/zed-wasm/test/node/zq.test.js b/packages/zed-wasm/test/node/zq.test.js new file mode 100644 index 0000000000..d29252fc03 --- /dev/null +++ b/packages/zed-wasm/test/node/zq.test.js @@ -0,0 +1,17 @@ +import test from 'node:test'; +import assert from 'node:assert'; +import { initZedWasm } from '@brimdata/zed-wasm'; + +let zq; +test.before(async () => { + const wasm = await initZedWasm( + require.resolve('@brimdata/zed-wasm/dist/main.wasm') + ); + zq = wasm.zq; +}); + +test('input is string', async (t) => { + const input = '1 2 3'; + const resp = await zq({ input, program: 'this + 1' }); + assert.deepEqual(resp, [2, 3, 4]); +}); diff --git a/packages/zed-wasm/test/pages/parse.html b/packages/zed-wasm/test/pages/parse.html new file mode 100644 index 0000000000..00b2d54c41 --- /dev/null +++ b/packages/zed-wasm/test/pages/parse.html @@ -0,0 +1,55 @@ + + + Parse Test + + + + + + +

Parse Test

+

Wasm Not Ready

+
+ + + +
+ 
+      Output
+
+ + diff --git a/packages/zed-wasm/test/pages/zq.html b/packages/zed-wasm/test/pages/zq.html new file mode 100644 index 0000000000..f6584adb32 --- /dev/null +++ b/packages/zed-wasm/test/pages/zq.html @@ -0,0 +1,60 @@ + + + Parse Test + + + + + + +

Parse Test

+

Wasm Not Ready

+
+ + + + + +
+ 
+      Output
+
+ + diff --git a/packages/zed-wasm/test/zq.test.js b/packages/zed-wasm/test/zq.test.js deleted file mode 100644 index a606c5471f..0000000000 --- a/packages/zed-wasm/test/zq.test.js +++ /dev/null @@ -1,51 +0,0 @@ -const assert = chai.assert; - -describe('zq', () => { - it('input is string', async () => { - const input = '1 2 3'; - const resp = await zq({ input, program: 'this + 1' }); - assert.deepEqual(resp, [2, 3, 4]); - }); - - it('input is file', async () => { - const input = new File(['1 2 3'], 'file.json'); - const resp = await zq({ input, program: 'this + 1' }); - - assert.deepEqual(resp, [2, 3, 4]); - }); - - it('input is blob', async () => { - const input = new Blob(['1 2 3']); - const resp = await zq({ input, program: 'this + 1' }); - assert.deepEqual(resp, [2, 3, 4]); - }); - - it('input is readable stream', async () => { - const input = new Blob(['1 2 3']).stream(); - const resp = await zq({ input, program: 'this + 1' }); - assert.deepEqual(resp, [2, 3, 4]); - }); - - it('input is a fetch', async () => { - const input = await fetch('./package.json'); - const resp = await zq({ input, program: 'yield name' }); - assert.deepEqual(resp, ['@brimdata/zed-wasm']); - }); - - it('input is an array of JS objects', async () => { - const input = [1, 2, 3]; - const zed = await zq({ input, program: 'this + 1', outputFormat: 'zed' }); - const resp = zed.map((val) => val.toJS()); - assert.deepEqual(resp, [2, 3, 4]); - }); - - it('works on 32kb+ file', async () => { - const count = 16000; - let str = ''; - for (let i = 0; i < count; i++) str += `1 `; - const input = new File([str], '32kb.txt'); - const resp = await zq({ input, program: 'count()' }); - - assert.deepEqual(resp, [count]); - }); -}); diff --git a/packages/zed-wasm/tsconfig.json b/packages/zed-wasm/tsconfig.json deleted file mode 100644 index 952b834959..0000000000 --- a/packages/zed-wasm/tsconfig.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "extends": "../../tsconfig.base.json", - "compilerOptions": { - "target": "es2017", - "module": "es2022", - "forceConsistentCasingInFileNames": true, - "strict": true, - "noImplicitOverride": true, - "noPropertyAccessFromIndexSignature": true, - "noImplicitReturns": true, - "noFallthroughCasesInSwitch": true, - }, - "files": [], - "include": [], - "references": [ - { - "path": "./tsconfig.lib.json" - }, - { - "path": "./tsconfig.spec.json" - } - ] -} diff --git a/packages/zed-wasm/tsconfig.lib.json b/packages/zed-wasm/tsconfig.lib.json deleted file mode 100644 index 58b793553b..0000000000 --- a/packages/zed-wasm/tsconfig.lib.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "extends": "./tsconfig.json", - "compilerOptions": { - "outDir": "../../dist/out-tsc", - "declaration": true, - "lib": ["es2020", "DOM"], - }, - "include": ["src/**/*.ts"], - "exclude": ["jest.config.ts", "src/**/*.spec.ts", "src/**/*.test.ts"] -} diff --git a/packages/zui-player/helpers/test-app.ts b/packages/zui-player/helpers/test-app.ts index 514621a2d7..79dea22085 100644 --- a/packages/zui-player/helpers/test-app.ts +++ b/packages/zui-player/helpers/test-app.ts @@ -99,9 +99,23 @@ export default class TestApp { } } + getLocationKey() { + return this.page + .locator('[data-location-key]') + .getAttribute('data-location-key'); + } + + waitForNextLocationKey(prevKey) { + return this.page + .locator(`[data-location-key="${prevKey}"]`) + .waitFor({ state: 'detached' }); + } + async query(zed: string): Promise { + const prevKey = await this.getLocationKey(); await this.setEditor(zed); await this.mainWin.getByRole('button', { name: 'Run Query' }).click(); + await this.waitForNextLocationKey(prevKey); await this.mainWin.getByRole('status', { name: 'fetching' }).isHidden(); } diff --git a/packages/zui-player/tests/queries.spec.ts b/packages/zui-player/tests/queries.spec.ts index cac158329a..75159eaa01 100644 --- a/packages/zui-player/tests/queries.spec.ts +++ b/packages/zui-player/tests/queries.spec.ts @@ -29,6 +29,7 @@ test.describe('Query tests', () => { "from 'sample.zeektsv' | 3 now", "from 'sample.zeektsv' | 2 now", "from 'sample.zeektsv' | 1 now", + "from 'sample.zeektsv' now", ]; expect(entries).toEqual(expected); }); diff --git a/tsconfig.base.json b/tsconfig.base.json index 29e3275df4..408a5545ed 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -18,7 +18,6 @@ "@brimdata/sample-data": ["packages/sample-data/src/index.ts"], "@brimdata/zed-js": ["packages/zed-js/src/index.ts"], "@brimdata/zed-node": ["packages/zed-node/src/index.ts"], - "@brimdata/zed-wasm": ["packages/zed-wasm/src/index.ts"], "zui-test-data": ["packages/zui-test-data/src/index.ts"] } }, diff --git a/yarn.lock b/yarn.lock index 327e063681..35bc5581d3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1935,8 +1935,15 @@ __metadata: resolution: "@brimdata/zed-wasm@workspace:packages/zed-wasm" dependencies: "@brimdata/zed-js": "workspace:*" + "@playwright/test": ^1.40.1 "@types/golang-wasm": ^1.15.0 + "@types/node": ^20.11.0 + esbuild: ^0.17.12 golang-wasm: "github:teamortix/golang-wasm#master" + jest: ^29.4.1 + npm-run-all: ^4.1.5 + polyfill-crypto.getrandomvalues: ^1.0.0 + serve: ^14.2.1 languageName: unknown linkType: soft @@ -3828,6 +3835,17 @@ __metadata: languageName: node linkType: hard +"@playwright/test@npm:^1.40.1": + version: 1.40.1 + resolution: "@playwright/test@npm:1.40.1" + dependencies: + playwright: 1.40.1 + bin: + playwright: cli.js + checksum: ae094e6cb809365c0707ee2b184e42d2a2542569ada020d2d44ca5866066941262bd9a67af185f86c2fb0133c9b712ea8cb73e2959a289e4261c5fd17077283c + languageName: node + linkType: hard + "@react-dnd/asap@npm:^4.0.0": version: 4.0.0 resolution: "@react-dnd/asap@npm:4.0.0" @@ -5118,6 +5136,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:^20.11.0": + version: 20.11.0 + resolution: "@types/node@npm:20.11.0" + dependencies: + undici-types: ~5.26.4 + checksum: 1bd6890db7e0404d11c33d28f46f19f73256f0ba35d19f0ef2a0faba09f366f188915fb9338eebebcc472075c1c4941e17c7002786aa69afa44980737846b200 + languageName: node + linkType: hard + "@types/normalize-package-data@npm:^2.4.0, @types/normalize-package-data@npm:^2.4.1": version: 2.4.1 resolution: "@types/normalize-package-data@npm:2.4.1" @@ -13777,6 +13804,13 @@ __metadata: languageName: node linkType: hard +"mersenne-twister@npm:^1.0.1": + version: 1.1.0 + resolution: "mersenne-twister@npm:1.1.0" + checksum: 7de1940ded117f2aad9320ae4d21d647b0ecf0667abbadcfe6a2835c669feb674ef46cb7a72da7af69a56d8b19e50e95e2fb7ef6d780efab7a6acd4d87f4cb2d + languageName: node + linkType: hard + "micromark@npm:~2.11.0": version: 2.11.4 resolution: "micromark@npm:2.11.4" @@ -15305,6 +15339,15 @@ __metadata: languageName: node linkType: hard +"playwright-core@npm:1.40.1": + version: 1.40.1 + resolution: "playwright-core@npm:1.40.1" + bin: + playwright-core: cli.js + checksum: 84d92fb9b86e3c225b16b6886bf858eb5059b4e60fa1205ff23336e56a06dcb2eac62650992dede72f406c8e70a7b6a5303e511f9b4bc0b85022ede356a01ee0 + languageName: node + linkType: hard + "playwright-core@npm:1.41.1": version: 1.41.1 resolution: "playwright-core@npm:1.41.1" @@ -15314,6 +15357,21 @@ __metadata: languageName: node linkType: hard +"playwright@npm:1.40.1": + version: 1.40.1 + resolution: "playwright@npm:1.40.1" + dependencies: + fsevents: 2.3.2 + playwright-core: 1.40.1 + dependenciesMeta: + fsevents: + optional: true + bin: + playwright: cli.js + checksum: 9e36791c1b4a649c104aa365fdd9d049924eeb518c5967c0e921aa38b9b00994aa6ee54784d6c2af194b3b494b6f69772673081ef53c6c4a4b2065af9955c4ba + languageName: node + linkType: hard + "playwright@npm:1.41.1": version: 1.41.1 resolution: "playwright@npm:1.41.1" @@ -15348,6 +15406,15 @@ __metadata: languageName: node linkType: hard +"polyfill-crypto.getrandomvalues@npm:^1.0.0": + version: 1.0.0 + resolution: "polyfill-crypto.getrandomvalues@npm:1.0.0" + dependencies: + mersenne-twister: ^1.0.1 + checksum: 73f0880b022af0c5930ef2d34a83a25e03196081082d8ceb204a3e5609a1a3d0dd1656a183073b17bc069a2c5749e44689cebdc4254dae6ed7e4e8b83f451964 + languageName: node + linkType: hard + "postcss-nested@npm:^4.2.1 || ^5.0.0": version: 5.0.6 resolution: "postcss-nested@npm:5.0.6" @@ -16803,6 +16870,27 @@ __metadata: languageName: node linkType: hard +"serve@npm:^14.2.1": + version: 14.2.1 + resolution: "serve@npm:14.2.1" + dependencies: + "@zeit/schemas": 2.29.0 + ajv: 8.11.0 + arg: 5.0.2 + boxen: 7.0.0 + chalk: 5.0.1 + chalk-template: 0.4.0 + clipboardy: 3.0.0 + compression: 1.7.4 + is-port-reachable: 4.0.0 + serve-handler: 6.1.5 + update-check: 1.5.4 + bin: + serve: build/main.js + checksum: c39a517b5d795a0a5c2f9fb9ff088b7e4962c579e34ace5b85dd62f93e0eacbc8a90359792c153c444a83258ffda392113dff7bfd10d41ced574a2d1886c2994 + languageName: node + linkType: hard + "set-blocking@npm:^2.0.0, set-blocking@npm:~2.0.0": version: 2.0.0 resolution: "set-blocking@npm:2.0.0"