[MODFQMMGR-174] Entity type creator (#235)

* Entity type creator: Initial + FQM connection

* Postgres connection

* Add entity type creation logic

* cleanerr connection dialogs

* Basic entity type editing

* Rich SQL support

* styling fix

* Sort ET keys

* Fields + sources

* value getters and friends

* remove ominous button

* format correctly

* Persist translations

* Improve valuegetter/etc formatting

* Support nested data types

* Verifying saving/retrieval

* Add query tool

* Minor fixes

* Database/jsonb inspector

* "docs"

* Update README.md

* better counting of aborted json scans

* add entity type  checklist to the pr template

PULL_REQUEST_TEMPLATE.md

@@ -9,3 +9,13 @@ _How does this change fulfill the purpose?_
 
 ## Learning
 _Describe the research stage. Add links to blog posts, patterns, libraries or addons used to solve this problem._
+
+## Pre-Merge Checklist
+
+If you are adding entity type(s), have you:
+- [ ] Added the JSON5 definition to the `src/main/resources/entity-types` directory?
+- [ ] Ensured that GETing the entity type at `/entity-types/{id}` works as expected?
+- [ ] Added translations for all fields, per the [translation guidelines](/translations/README.md)? (Check this by ensuring `GET /entity-types/{id}` does not have `mod-fqm-manager.entityType.` in the response)
+- [ ] Added views to liquibase, as applicable?
+- [ ] Added required interfaces to the module descriptor?
+- [ ] Checked that querying fields works correctly and all SQL is valid?

entity-type-creator/.gitignore

@@ -0,0 +1,36 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+.yarn/install-state.gz
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

entity-type-creator/README.md

@@ -0,0 +1,23 @@
+# Entity type creator
+
+This is a tool to assist in the creation of entity types for the `mod-fqm-manager`.
+
+## Getting Started
+
+First, install [Bun](https://bun.sh/). You may be able to use `npm`, `yarn`, or `pnpm` instead, however, nothing is guaranteed.
+
+Then, install the modules:
+
+```bash
+bun install
+```
+
+Then, run the development server:
+
+```bash
+bun run dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+Although not strictly necessary to run the application, it is **highly** recommended to connect the tool to a running instance of the `mod-fqm-manager` and database. Without these, many features may be disabled.

entity-type-creator/components/CheckValidity.tsx

@@ -0,0 +1,105 @@
+import { Done, Error, Pending, Schedule } from '@mui/icons-material';
+import { Alert, Button, Typography } from '@mui/material';
+import { useCallback, useState } from 'react';
+import { Socket } from 'socket.io-client';
+import { EntityType } from '../types';
+
+enum State {
+  NOT_STARTED,
+  STARTED,
+  PERSISTED,
+  DONE,
+  ERROR_PERSIST,
+  ERROR_QUERY,
+}
+
+export default function CheckValidity({
+  socket,
+  entityType,
+}: Readonly<{
+  socket: Socket;
+  entityType: EntityType | null;
+}>) {
+  const [state, setState] = useState<{ state: State; result?: string }>({ state: State.NOT_STARTED });
+
+  const run = useCallback(
+    (entityType: EntityType) => {
+      setState({ state: State.STARTED });
+
+      socket.emit('check-entity-type-validity', entityType);
+      socket.on(
+        'check-entity-type-validity-result',
+        (result: {
+          queried: boolean;
+          persisted: boolean;
+          queryError?: string;
+          persistError?: string;
+          queryResults?: string;
+        }) => {
+          if (result.queryError || result.persistError || result.queryResults) {
+            socket.off('check-entity-type-validity-result');
+          }
+
+          if (result.queryError) {
+            setState({ state: State.ERROR_QUERY, result: result.queryError });
+          } else if (result.persistError) {
+            setState({ state: State.ERROR_PERSIST, result: result.persistError });
+          } else if (result.queryResults) {
+            setState({ state: State.DONE, result: result.queryResults });
+          } else {
+            setState({ state: State.PERSISTED });
+          }
+        }
+      );
+    },
+    [state, socket]
+  );
+
+  if (entityType === null) {
+    return <p>Select an entity type first</p>;
+  }
+
+  return (
+    <>
+      <Typography>
+        Checks that <code>mod-fqm-manager</code> can successfully parse and handle the JSON representation of the entity
+        type.
+      </Typography>
+
+      <Button variant="outlined" onClick={() => run(entityType)}>
+        Run
+      </Button>
+
+      <Typography sx={{ display: 'flex', alignItems: 'center', gap: '0.5em', m: 2 }}>
+        {state.state === State.NOT_STARTED ? (
+          <Pending color="disabled" />
+        ) : state.state === State.STARTED ? (
+          <Schedule color="warning" />
+        ) : state.state === State.ERROR_PERSIST ? (
+          <Error color="error" />
+        ) : (
+          <Done color="success" />
+        )}
+        Persist to database
+      </Typography>
+      <Typography sx={{ display: 'flex', alignItems: 'center', gap: '0.5em', m: 2 }}>
+        {state.state === State.NOT_STARTED || state.state === State.STARTED ? (
+          <Pending color="disabled" />
+        ) : state.state === State.PERSISTED ? (
+          <Schedule color="warning" />
+        ) : state.state === State.ERROR_PERSIST || state.state === State.ERROR_QUERY ? (
+          <Error color="error" />
+        ) : (
+          <Done color="success" />
+        )}{' '}
+        Query <code>/entity-types/{entityType.id}</code>
+      </Typography>
+
+      {!!state.result && <pre>{state.result}</pre>}
+
+      <Alert severity="info">
+        Translations may not show correctly until the application is restarted, due to caching.
+      </Alert>
+    </>
+  );
+}

entity-type-creator/components/DBColumnInspector.tsx

@@ -0,0 +1,86 @@
+import { Typography } from '@mui/material';
+import { pascalCase } from 'change-case';
+import { compile } from 'json-schema-to-typescript';
+import { format } from 'prettier';
+import prettierPluginESTree from 'prettier/plugins/estree.mjs';
+import prettierPluginTS from 'prettier/plugins/typescript.mjs';
+import { useCallback, useState } from 'react';
+import { Socket } from 'socket.io-client';
+
+export default function DBColumnInspector({
+  socket,
+  db,
+  table,
+  column,
+  dataType,
+}: Readonly<{ socket: Socket; db: string; table: string; column: string; dataType: string }>) {
+  const [analysis, setAnalysis] = useState<{
+    scanned: number;
+    total: number;
+    finished: boolean;
+    result?: unknown;
+  } | null>(null);
+
+  const analyze = useCallback(() => {
+    setAnalysis({ scanned: 0, total: 0, finished: false });
+    socket.emit('analyze-jsonb', { db, table, column });
+
+    socket.on(
+      `analyze-jsonb-result-${db}-${table}-${column}`,
+      async (result: { scanned: number; total: number; finished: boolean; result?: unknown }) => {
+        if (result.finished) {
+          socket.off(`analyze-jsonb-result-${db}-${table}-${column}`);
+
+          setAnalysis({
+            ...result,
+            result: await format(
+              await compile(result.result as any, `${pascalCase(db)}${pascalCase(table)}${pascalCase(column)}Schema`, {
+                additionalProperties: false,
+                bannerComment: '',
+                format: false,
+                ignoreMinAndMaxItems: true,
+              }),
+              { parser: 'typescript', plugins: [prettierPluginTS, prettierPluginESTree] }
+            ),
+          });
+        } else {
+          setAnalysis(result);
+        }
+      }
+    );
+  }, [db, table, column, socket]);
+
+  return (
+    <li>
+      {column}: {dataType}{' '}
+      {dataType === 'jsonb' &&
+        (analysis ? (
+          analysis.finished ? (
+            <>
+              <button onClick={analyze}>re-analyze</button>
+              <Typography>
+                Scanned {analysis.scanned} of {analysis.total} records
+                <pre>{analysis.result as string}</pre>
+              </Typography>
+            </>
+          ) : (
+            <>
+              <button disabled>
+                analyzing ({analysis.scanned}/{analysis.total})
+              </button>
+              <button
+                onClick={() => {
+                  socket.emit(`abort-analyze-jsonb-${db}-${table}-${column}`);
+                  setAnalysis(null);
+                }}
+              >
+                abort
+              </button>
+            </>
+          )
+        ) : (
+          <button onClick={analyze}>analyze</button>
+        ))}
+    </li>
+  );
+}

entity-type-creator/components/DBInspector.tsx

@@ -0,0 +1,116 @@
+import { Schema } from '@/types';
+import { Box } from '@mui/system';
+import { useMemo } from 'react';
+import { Socket } from 'socket.io-client';
+import DBColumnInspector from './DBColumnInspector';
+
+export default function DBInspector({
+  socket,
+  schema,
+}: Readonly<{
+  socket: Socket;
+  schema: Schema;
+}>) {
+  const hierarchicalSchema = useMemo(() => {
+    const result: Record<
+      string,
+      {
+        routines: string[];
+        tables: Record<string, Record<string, string>>;
+        views: Record<string, Record<string, string>>;
+      }
+    > = {};
+    for (const db of Object.keys(schema.routines)) {
+      result[db] = { routines: schema.routines[db], tables: {}, views: {} };
+    }
+    for (const dbTableColumn of Object.keys(schema.typeMapping)) {
+      const [db, table, column] = dbTableColumn.split('.');
+      result[db] = result[db] ?? { routines: [], tables: {}, views: {} };
+
+      if (schema.isView[`${db}.${table}`]) {
+        result[db].views[table] = result[db].views[table] ?? {};
+        result[db].views[table][column] = schema.typeMapping[dbTableColumn];
+      } else {
+        result[db].tables[table] = result[db].tables[table] ?? {};
+        result[db].tables[table][column] = schema.typeMapping[dbTableColumn];
+      }
+    }
+    return result;
+  }, [schema]);
+
+  return (
+    <Box sx={{ fontFamily: 'monospace' }}>
+      {Object.keys(hierarchicalSchema)
+        .toSorted((a, b) => a.localeCompare(b))
+        .map((db) => (
+          <details key={db}>
+            <summary>
+              {db.replace('TENANT_', '')} (
+              {Object.keys(hierarchicalSchema[db].tables).length + Object.keys(hierarchicalSchema[db].views).length})
+            </summary>
+
+            <details style={{ marginLeft: '1em' }}>
+              <summary>🧮 Routines ({hierarchicalSchema[db].routines.length})</summary>
+
+              <ul>
+                {hierarchicalSchema[db].routines
+                  .toSorted((a, b) => a.localeCompare(b))
+                  .map((routine) => (
+                    <li key={routine}>{routine}</li>
+                  ))}
+              </ul>
+            </details>
+
+            {Object.keys(hierarchicalSchema[db].tables)
+              .toSorted((a, b) => a.localeCompare(b))
+              .map((table) => (
+                <details key={table} style={{ marginLeft: '1em' }}>
+                  <summary>
+                    💾 {table} ({Object.keys(hierarchicalSchema[db].tables[table]).length})
+                  </summary>
+
+                  <ul>
+                    {Object.keys(hierarchicalSchema[db].tables[table])
+                      .toSorted((a, b) => a.localeCompare(b))
+                      .map((column) => (
+                        <DBColumnInspector
+                          key={column}
+                          socket={socket}
+                          db={db}
+                          table={table}
+                          column={column}
+                          dataType={hierarchicalSchema[db].tables[table][column]}
+                        />
+                      ))}
+                  </ul>
+                </details>
+              ))}
+            {Object.keys(hierarchicalSchema[db].views)
+              .toSorted((a, b) => a.localeCompare(b))
+              .map((view) => (
+                <details key={view} style={{ marginLeft: '1em' }}>
+                  <summary>
+                    ✨ {view} ({Object.keys(hierarchicalSchema[db].views[view]).length})
+                  </summary>
+
+                  <ul>
+                    {Object.keys(hierarchicalSchema[db].views[view])
+                      .toSorted((a, b) => a.localeCompare(b))
+                      .map((column) => (
+                        <DBColumnInspector
+                          key={column}
+                          socket={socket}
+                          db={db}
+                          table={view}
+                          column={column}
+                          dataType={hierarchicalSchema[db].views[view][column]}
+                        />
+                      ))}
+                  </ul>
+                </details>
+              ))}
+          </details>
+        ))}
+    </Box>
+  );
+}