Remove JS Zed Query Parser (#2972)

Removed zed parser.
Added domain models for better handler code.
Fixed e2e test flakiness.
Query is parsed async.
Error messages are better.

.github/actions/setup-zui/action.yml

@@ -1,7 +1,7 @@
 name: Setup Zui
 description: Shared steps for setting up Zui in workflows
 runs:
-  using: "composite"
+  using: 'composite'
   steps:
     - name: Install Go
       uses: actions/setup-go@v2
@@ -21,6 +21,10 @@ runs:
       run: yarn --inline-builds
       shell: bash
 
+    - name: Install Playwright Deps
+      run: yarn workspace @brimdata/zed-wasm run playwright install
+      shell: bash
+
     - name: Yarn Build
       run: yarn run build
       shell: bash

.node-version

@@ -1 +1 @@
-16.10.0
+18.18.2

apps/zui/.swcrc

@@ -14,5 +14,4 @@
     "type": "commonjs",
     "ignoreDynamic": true
   }
-
 }

apps/zui/CODE.md

@@ -81,13 +81,29 @@ _Main Process Initializers_
 
 Code that needs to be run one time before the app starts up can be put in an initializer. An initializer is a file that lives in the folder `src/electron/initializers/`. It must export a function named _initialize(main)_ that takes the Main Object as its only argument. See the FAQ for an example of creating a new initializer.
 
+_Query Session_
+
+This is a type of page that a tab can hold it the app. The page contains an editor pane, a results pane, and an inspector with various tabs related to querying data. These are session history, global history, data details, columns, and more.
+
 _Query_
 
-A query in the app is like a container object. It holds the name and id of the query. It does not contain the zed code. Those are stored in a QueryVersion. Each Query has many QueryVersions, showing the history of that query.
+A query in the app is like a container object. It holds the name and id of the query. It does not contain the zed code. Those are stored in a EditorSnapshot. Each Query has many EditorSnapshots, showing the history of that query.
+
+_Editor Snapshot_ - formerly QueryVersion
+
+This is an object that represents the state of a query editor at a given point in time. It contains fields like: pins, value, createdAt, lastRanAt, and queryId. The editor snapshot object will belong to either a named query, or a session query.
+
+_Query Text_
+
+This is text of the final Zed Query we will send to the backend.
 
 _Session Query_
 
-A session query is like an unnamed Query. Each session (tab) has exactly one SessionQuery associated with it. The SessionQuery has many QueryVersions associated with it.
+A session query is like an unnamed Query. Each session (tab) has exactly one SessionQuery associated with it. The SessionQuery has many EditorSnapshots associated with it.
+
+_Acitve Query_
+
+This refers to whatever query is currently being presented in the query session page. This can either be a query or a session query.
 
 _Store_
 

apps/zui/src/app/core/models/active-query.ts

@@ -1,4 +1,3 @@
-import {ZedAst} from "src/app/core/models/zed-ast"
 import {QueryModel} from "src/js/models/query-model"
 import {QueryVersion} from "src/js/state/QueryVersions/types"
 
@@ -63,8 +62,4 @@ export class ActiveQuery {
   toZed() {
     return QueryModel.versionToZed(this.version)
   }
-
-  toAst() {
-    return new ZedAst(QueryModel.versionToZed(this.version))
-  }
 }

apps/zui/src/app/core/models/zed-ast.ts

@@ -1,19 +1,8 @@
-import {parse as parseAst} from "zed/compiler/parser/parser"
-import {fieldExprToName} from "src/js/models/ast"
 import {toFieldPath} from "src/js/zed-script/toZedScript"
+import {fieldExprToName} from "./zed-expr"
 
 export class ZedAst {
-  public tree: any
-  public error: Error | null
-
-  constructor(public script: string) {
-    try {
-      this.tree = parseAst(script)
-    } catch (e) {
-      this.tree = null
-      this.error = e
-    }
-  }
+  constructor(public tree: any, public error: Error | null) {}
 
   get poolName() {
     const from = this.from
@@ -34,6 +23,12 @@ export class ZedAst {
     return trunks.filter((t) => t.source.kind === "Pool").map((t) => t.source)
   }
 
+  get groupByKeys() {
+    const g = this.ops.find((op) => op.kind === "Summarize")
+    const keys = g ? g.keys : []
+    return keys.map((k) => fieldExprToName(k.lhs || k.rhs))
+  }
+
   private _ops: any[]
   get ops() {
     if (this._ops) return this._ops
@@ -77,3 +72,11 @@ export class ZedAst {
 
 export const OP_EXPR_PROC = "OpExpr"
 export const PARALLEL_PROC = "Parallel"
+// are all these needed?
+export const HEAD_PROC = "Head"
+export const TAIL_PROC = "Tail"
+export const SORT_PROC = "Sort"
+export const FILTER_PROC = "Filter"
+export const PRIMITIVE_PROC = "Primitive"
+export const REGEXP_SEARCH_PROC = "RegexpSearch"
+export const ANALYTIC_PROCS = ["Summarize"]

apps/zui/src/app/core/models/zed-expr.ts

@@ -0,0 +1,32 @@
+import {toFieldPath} from "src/js/zed-script/toZedScript"
+
+export function fieldExprToName(expr) {
+  let s = _fieldExprToName(expr)
+  // const r = toFieldPath(s)
+  return s
+}
+
+function _fieldExprToName(expr): string | string[] {
+  switch (expr.kind) {
+    case "BinaryExpr":
+      if (expr.op == "." || expr.op == "[") {
+        return []
+          .concat(_fieldExprToName(expr.lhs), _fieldExprToName(expr.rhs))
+          .filter((n) => n !== "this")
+      }
+      return "<not-a-field>"
+    case "ID":
+      return expr.name
+    case "This":
+      return "this"
+    case "Primitive":
+      return expr.text
+    case "Call":
+      var args = expr.args
+        .map((e) => toFieldPath(_fieldExprToName(e)))
+        .join(",")
+      return `${expr.name}(${args})`
+    default:
+      return "<not-a-field>"
+  }
+}

apps/zui/src/app/core/models/zed-script.ts

@@ -1,13 +1,6 @@
-import {ZedAst} from "./zed-ast"
-
 export class ZedScript {
   constructor(public script: string) {}
 
-  private _ast: ZedAst
-  get ast() {
-    return this._ast || (this._ast = new ZedAst(this.script))
-  }
-
   isEmpty() {
     const lines = this.script.split("\n")
     const comment = /^\s*\/\/.*$/

apps/zui/src/app/core/state/create-wait-for-selector.ts

@@ -0,0 +1,32 @@
+export function createWaitForSelector(store) {
+  return function waitForSelector(
+    select,
+    options: {signal?: AbortSignal} = {}
+  ) {
+    let resolve
+    let targetValue
+
+    const unsubscribe = store.subscribe(() => {
+      const state = store.getState()
+      if (select(state) === targetValue) {
+        unsubscribe()
+        resolve()
+      }
+    })
+
+    let promise = new Promise((res, reject) => {
+      resolve = res
+      options.signal?.addEventListener("abort", () => {
+        unsubscribe()
+        reject(new DOMException("AbortError"))
+      })
+    })
+
+    return {
+      toReturn(value) {
+        targetValue = value
+        return promise
+      },
+    }
+  }
+}

apps/zui/src/app/features/right-pane/history/history-item.tsx

@@ -1,7 +1,6 @@
 import {formatDistanceToNowStrict} from "date-fns"
 import React, {useMemo} from "react"
 import {useSelector} from "react-redux"
-import {useZuiApi} from "src/app/core/context"
 import Current from "src/js/state/Current"
 import Queries from "src/js/state/Queries"
 import QueryVersions from "src/js/state/QueryVersions"
@@ -10,6 +9,7 @@ import {useEntryMenu} from "./use-entry-menu"
 import {State} from "src/js/state/types"
 import {ActiveQuery} from "src/app/core/models/active-query"
 import {NodeRendererProps} from "react-arborist"
+import {Snapshots} from "src/domain/handlers"
 
 const Wrap = styled.div`
   height: 100%;
@@ -101,7 +101,6 @@ function getTimestamp(active: ActiveQuery) {
 }
 
 export function HistoryItem({node}: NodeRendererProps<Props>) {
-  const api = useZuiApi()
   const {index, queryId, version} = node.data
   const onContextMenu = useEntryMenu(index)
   const sessionId = useSelector(Current.getSessionId)
@@ -118,7 +117,11 @@ export function HistoryItem({node}: NodeRendererProps<Props>) {
   const active = new ActiveQuery(session, query, versionObj)
   const onClick = () => {
     if (active.isDeleted()) return
-    api.queries.open(active.id(), {version: active.versionId(), history: false})
+    Snapshots.show({
+      sessionId,
+      namedQueryId: queryId,
+      snapshotId: version,
+    })
   }
   const type = getType(active)
   const value = getValue(active)

apps/zui/src/app/features/right-pane/history/section.tsx

@@ -7,7 +7,6 @@ import {isEmpty} from "lodash"
 import {EmptyText} from "../common"
 import {FillFlexParent} from "src/components/fill-flex-parent"
 import {Tree} from "react-arborist"
-import {useZuiApi} from "src/app/core/context"
 import {TREE_ITEM_HEIGHT} from "../../sidebar/item"
 
 const BG = styled.div`
@@ -18,7 +17,6 @@ const BG = styled.div`
 `
 
 export function HistorySection() {
-  const api = useZuiApi()
   const sessionHistory = useSelector(Current.getSessionHistory) || []
   const history = useMemo(
     () =>
@@ -46,12 +44,6 @@ export function HistorySection() {
               indent={8}
               disableDrag
               disableDrop
-              onActivate={(node) => {
-                api.queries.open(node.data.queryId, {
-                  version: node.data.version,
-                  history: false,
-                })
-              }}
             >
               {HistoryItem}
             </Tree>