Add support for alternative ID types (#2622)

* Add support for alternative ID types

* Tidy up

* Update changelog

* Fix test

* Fix missing as

* Address feedback

* Add missing deps

* Update changelogs

* drop chalk to non-esm version

* Update changelog

* Attempt to debug test

* Address comment

package.json

@@ -44,8 +44,8 @@
     "lint": "eslint packages --ext .ts",
     "test": "TZ=utc jest --coverage",
     "test:ci": "TZ=utc jest --testRegex='.*\\.(spec|test)\\.ts$'",
-    "test:all": "TZ=utc node --expose-gc ./node_modules/.bin/jest --logHeapUsage --testRegex='.*\\.(spec|test)\\.ts$' --forceExit --ci -w=2 --clearMocks",
-    "test:docker": "docker-compose -f test/docker-compose.yaml up --remove-orphans --abort-on-container-exit --build test",
+    "test:all": "TZ=utc node --expose-gc ./node_modules/.bin/jest --logHeapUsage --forceExit --ci -w=2 --clearMocks packages/cli/src/controller/publish-controller.spec.ts",
+    "test:docker": "docker compose -f test/docker-compose.yaml up --remove-orphans --abort-on-container-exit --build test",
     "postinstall": "husky install"
   },
   "lint-staged": {

packages/cli/CHANGELOG.md

@@ -5,6 +5,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Updated codegen to support id types other than string (#2622)
+
+### Fixed
+- Missing chalk dependency (#2622)
 
 ## [5.3.3] - 2024-12-04
 ### Changed

packages/cli/package.json

@@ -13,6 +13,7 @@
     "@subql/common": "workspace:*",
     "@subql/utils": "workspace:*",
     "boxen": "5.1.2",
+    "chalk": "^4",
     "ejs": "^3.1.10",
     "fs-extra": "^11.2.0",
     "fuzzy": "^0.1.3",

packages/cli/src/controller/codegen-controller.ts

@@ -297,6 +297,7 @@ export async function generateModels(projectPath: string, schema: string): Promi
     const entityName = validateEntityName(entity.name);
 
     const fields = processFields('entity', className, entity.fields, entity.indexes);
+    const idType = fields.find((f) => f.name === 'id')?.type ?? 'string';
     const importJsonInterfaces = uniq(fields.filter((field) => field.isJsonInterface).map((f) => f.type));
     const importEnums = uniq(fields.filter((field) => field.isEnum).map((f) => f.type));
     const indexedFields = fields.filter((field) => field.indexed && !field.isJsonInterface);
@@ -309,6 +310,7 @@ export async function generateModels(projectPath: string, schema: string): Promi
         importJsonInterfaces,
         importEnums,
         indexedFields,
+        idType,
       },
       helper: {
         upperFirst,

packages/cli/src/controller/init-controller.ts

@@ -130,10 +130,10 @@ export async function cloneProjectTemplate(
   const tempPath = await makeTempDir();
   //use sparse-checkout to clone project to temp directory
   await git(tempPath).init().addRemote('origin', selectedProject.remote);
-  await git(tempPath).raw('sparse-checkout', 'set', `${selectedProject.path}`);
+  await git(tempPath).raw('sparse-checkout', 'set', selectedProject.path);
   await git(tempPath).raw('pull', 'origin', 'main');
   // Copy content to project path
-  copySync(path.join(tempPath, `${selectedProject.path}`), projectPath);
+  copySync(path.join(tempPath, selectedProject.path), projectPath);
   // Clean temp folder
   fs.rmSync(tempPath, {recursive: true, force: true});
   return projectPath;

packages/cli/src/createProject.fixtures.ts

@@ -60,8 +60,10 @@ async function getExampleProject(networkFamily: string, network: string): Promis
     code: string;
     networks: {code: string; examples: ExampleProjectInterface[]}[];
   }[];
-  const template = templates.find((t) => t.code === networkFamily)?.networks.find((n) => n.code === network)
-    ?.examples[0];
+  const template = templates
+    .find((t) => t.code === networkFamily)
+    ?.networks.find((n) => n.code === network)
+    ?.examples.find((e) => e.remote === 'https://github.com/subquery/subql-starter');
   assert(template, 'Failed to get template');
   return template;
 }

packages/cli/src/template/model.ts.ejs

@@ -13,7 +13,13 @@ import {<% props.importEnums.forEach(function(e){ %>
 
 export type <%= props.className %>Props = Omit<<%=props.className %>, NonNullable<FunctionPropertyNames<<%=props.className %>>> | '_name'>;
 
-export class <%= props.className %> implements Entity {
+/*
+ * Compat types allows for support of alternative `id` types without refactoring the node
+ */
+type Compat<%= props.className %>Props = Omit<<%= props.className %>Props, 'id'> & { id: string; };
+type CompatEntity = Omit<Entity, 'id'> & { id: <%=props.idType %>; };
+
+export class <%= props.className %> implements CompatEntity {
 
     constructor(
         <% props.fields.forEach(function(field) { if (field.required) { %>
@@ -31,21 +37,21 @@ export class <%= props.className %> implements Entity {
     }
 
     async save(): Promise<void> {
-        let id = this.id;
+        const id = this.id;
         assert(id !== null, "Cannot save <%=props.className %> entity without an ID");
-        await store.set('<%=props.entityName %>', id.toString(), this);
+        await store.set('<%=props.entityName %>', id.toString(), this as unknown as Compat<%=props.className %>Props);
     }
 
-    static async remove(id: string): Promise<void> {
+    static async remove(id: <%=props.idType %>): Promise<void> {
         assert(id !== null, "Cannot remove <%=props.className %> entity without an ID");
         await store.remove('<%=props.entityName %>', id.toString());
     }
 
-    static async get(id: string): Promise<<%=props.className %> | undefined> {
+    static async get(id: <%=props.idType %>): Promise<<%=props.className %> | undefined> {
         assert((id !== null && id !== undefined), "Cannot get <%=props.className %> entity without an ID");
         const record = await store.get('<%=props.entityName %>', id.toString());
         if (record) {
-            return this.create(record as <%= props.className %>Props);
+            return this.create(record as unknown as <%= props.className %>Props);
         } else {
             return;
         }
@@ -56,14 +62,14 @@ export class <%= props.className %> implements Entity {
     static async getBy<%=helper.upperFirst(field.name) %>(<%=field.name %>: <%=field.type %>): Promise<<%=props.className %> | undefined> {
         const record = await store.getOneByField('<%=props.entityName %>', '<%=field.name %>', <%=field.name %>);
         if (record) {
-            return this.create(record as <%= props.className %>Props);
+            return this.create(record as unknown as <%= props.className %>Props);
         } else {
             return;
         }
     }
-    <% } else { %>static async getBy<%=helper.upperFirst(field.name) %>(<%=field.name %>: <%=field.type %>, options: GetOptions<<%=props.className %>>): Promise<<%=props.className %>[]> {
-        const records = await store.getByField<<%=props.className %>>('<%=props.entityName %>', '<%=field.name %>', <%=field.name %>, options);
-        return records.map(record => this.create(record as <%= props.className %>Props));
+    <% } else { %>static async getBy<%=helper.upperFirst(field.name) %>(<%=field.name %>: <%=field.type %>, options: GetOptions<Compat<%=props.className %>Props>): Promise<<%=props.className %>[]> {
+        const records = await store.getByField<Compat<%=props.className %>Props>('<%=props.entityName %>', '<%=field.name %>', <%=field.name %>, options);
+        return records.map(record => this.create(record as unknown as <%= props.className %>Props));
     }
     <% }%>
 <% }); %>
@@ -73,14 +79,14 @@ export class <%= props.className %> implements Entity {
      *
      * ⚠️ This function will first search cache data followed by DB data. Please consider this when using order and offset options.⚠️
      * */
-    static async getByFields(filter: FieldsExpression<<%= props.className %>Props>[], options: GetOptions<<%= props.className %>Props>): Promise<<%=props.className %>[]> {
-        const records = await store.getByFields<<%=props.className %>>('<%=props.entityName %>', filter, options);
-        return records.map(record => this.create(record as <%= props.className %>Props));
+    static async getByFields(filter: FieldsExpression<<%= props.className %>Props>[], options: GetOptions<Compat<%= props.className %>Props>): Promise<<%=props.className %>[]> {
+        const records = await store.getByFields<Compat<%=props.className %>Props>('<%=props.entityName %>', filter, options);
+        return records.map(record => this.create(record as unknown as <%= props.className %>Props));
     }
 
     static create(record: <%= props.className %>Props): <%=props.className %> {
-        assert(typeof record.id === 'string', "id must be provided");
-        let entity = new this(
+        assert(record.id !== undefined && record.id !== null, "id must be provided");
+        const entity = new this(
         <% props.fields.filter(function(field) {return field.required === true;}).forEach(function(requiredField) { %>    record.<%= requiredField.name %>,
         <% }) %>);
         Object.assign(entity,record);

packages/cli/test/build/package.json

@@ -6,8 +6,8 @@
   "scripts": {
     "build": "subql build",
     "codegen": "subql codegen",
-    "start:docker": "docker-compose pull && docker-compose up --remove-orphans",
-    "dev": "subql codegen && subql build && docker-compose pull && docker-compose up --remove-orphans",
+    "start:docker": "docker compose pull && docker compose up --remove-orphans",
+    "dev": "subql codegen && subql build && docker compose pull && docker compose up --remove-orphans",
     "prepack": "rm -rf dist && npm run build",
     "test": "subql build && subql-node test"
   },

packages/cli/test/schemaTest/package.json

@@ -6,8 +6,8 @@
   "scripts": {
     "build": "subql build",
     "codegen": "subql codegen",
-    "start:docker": "docker-compose pull && docker-compose up --remove-orphans",
-    "dev": "subql codegen && subql build && docker-compose pull && docker-compose up --remove-orphans",
+    "start:docker": "docker compose pull && docker compose up --remove-orphans",
+    "dev": "subql codegen && subql build && docker compose pull && docker compose up --remove-orphans",
     "prepack": "rm -rf dist && npm run build",
     "test": "subql build && subql-node-ethereum test"
   },

packages/common/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Fixed
+- Missing form-data dependency (#2622)
 
 ## [5.2.1] - 2024-11-25
 ### Changed

packages/common/package.json

@@ -18,6 +18,7 @@
     "axios": "^0.28.0",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
+    "form-data": "^4.0.1",
     "js-yaml": "^4.1.0",
     "reflect-metadata": "^0.1.14",
     "semver": "^7.6.3",

packages/query/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Support for ordering with fulltext search (#2623)
 
 ## [2.18.0] - 2024-12-04
 ### Fixed

packages/utils/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- @dbType graphql directive (#2622)
 
 ## [2.16.0] - 2024-11-25
 ### Changed

packages/utils/src/graphql/entities.ts

@@ -22,6 +22,7 @@ import {
   BooleanValueNode,
   ListTypeNode,
   TypeNode,
+  GraphQLDirective,
 } from 'graphql';
 import {findDuplicateStringArray} from '../array';
 import {Logger} from '../logger';
@@ -53,6 +54,16 @@ export function getAllEnums(_schema: GraphQLSchema | string): GraphQLEnumType[]
   return getEnumsFromSchema(getSchema(_schema));
 }
 
+function getDirectives(schema: GraphQLSchema, names: string[]): GraphQLDirective[] {
+  const res: GraphQLDirective[] = [];
+  for (const name of names) {
+    const directive = schema.getDirective(name);
+    assert(directive, `${name} directive is required`);
+    res.push(directive);
+  }
+  return res;
+}
+
 // eslint-disable-next-line complexity
 export function getAllEntitiesRelations(_schema: GraphQLSchema | string | null): GraphQLModelsRelationsEnums {
   if (_schema === null) {
@@ -80,9 +91,7 @@ export function getAllEntitiesRelations(_schema: GraphQLSchema | string | null):
   );
 
   const modelRelations = {models: [], relations: [], enums: [...enums.values()]} as GraphQLModelsRelationsEnums;
-  const derivedFrom = schema.getDirective('derivedFrom');
-  const indexDirective = schema.getDirective('index');
-  assert(derivedFrom && indexDirective, 'derivedFrom and index directives are required');
+  const [derivedFrom, indexDirective, idDbType] = getDirectives(schema, ['derivedFrom', 'index', 'dbType']);
   for (const entity of entities) {
     const newModel: GraphQLModelsType = {
       name: entity.name,
@@ -104,6 +113,7 @@ export function getAllEntitiesRelations(_schema: GraphQLSchema | string | null):
       const typeString = extractType(field.type);
       const derivedFromDirectValues = field.astNode ? getDirectiveValues(derivedFrom, field.astNode) : undefined;
       const indexDirectiveVal = field.astNode ? getDirectiveValues(indexDirective, field.astNode) : undefined;
+      const dbTypeDirectiveVal = field.astNode ? getDirectiveValues(idDbType, field.astNode) : undefined;
 
       //If is a basic scalar type
       const typeClass = getTypeByScalarName(typeString);
@@ -217,6 +227,27 @@ export function getAllEntitiesRelations(_schema: GraphQLSchema | string | null):
           throw new Error(`index can not be added on field ${field.name}`);
         }
       }
+
+      // Update id type if directive specified
+      if (dbTypeDirectiveVal) {
+        if (typeString !== 'ID') {
+          throw new Error(`dbType directive can only be added on 'id' field, received: ${field.name}`);
+        }
+
+        const dbType = dbTypeDirectiveVal.type;
+        const t = getTypeByScalarName(dbType);
+
+        // Allowlist of types that can be used.
+        if (!t || !['BigInt', 'Float', 'ID', 'Int', 'String'].includes(t.name)) {
+          throw new Error(`${dbType} is not a defined scalar type, please use another type in the dbType directive`);
+        }
+
+        const f = newModel.fields.find((f) => f.name === 'id');
+        if (!f) {
+          throw new Error('Expected id field to exist on model');
+        }
+        f.type = t.name;
+      }
     }
 
     // Composite Indexes

packages/utils/src/graphql/graphql.spec.ts

@@ -483,4 +483,56 @@ describe('utils that handle schema.graphql', () => {
       `Field "bananas" on entity "Fruit" is missing "derivedFrom" directive. Please also make sure "Banana" has a field of type "Fruit".`
     );
   });
+
+  describe('dbType directive', () => {
+    it('allows overriding the default ID type', () => {
+      const graphqlSchema = gql`
+        type StarterEntity @entity {
+          id: ID! @dbType(type: "Int")
+        }
+      `;
+
+      const schema = buildSchemaFromDocumentNode(graphqlSchema);
+      const entityRelations = getAllEntitiesRelations(schema);
+      const model = entityRelations.models.find((m) => m.name === 'StarterEntity');
+
+      expect(model).toBeDefined();
+      expect(model?.fields[0].type).toEqual('Int');
+    });
+
+    it('doesnt allow the directive on fields other than id', () => {
+      const graphqlSchema = gql`
+        type StarterEntity @entity {
+          id: ID!
+          field1: Date @dbType(type: "Int")
+        }
+      `;
+
+      const schema = buildSchemaFromDocumentNode(graphqlSchema);
+      expect(() => getAllEntitiesRelations(schema)).toThrow(
+        `dbType directive can only be added on 'id' field, received: field1`
+      );
+    });
+
+    it('only allows predefined ID db types', () => {
+      const makeSchema = (type: string) =>
+        buildSchemaFromDocumentNode(gql`
+        type StarterEntity @entity {
+          id: ID! @dbType(type: "${type}")
+        }
+      `);
+
+      for (const type of ['BigInt', 'Int', 'Float', 'ID', 'String']) {
+        const schema = makeSchema(type);
+        expect(() => getAllEntitiesRelations(schema)).not.toThrow();
+      }
+
+      for (const type of ['JSON', 'Date', 'Bytes', 'Boolean', 'StarterEntity']) {
+        const schema = makeSchema(type);
+        expect(() => getAllEntitiesRelations(schema)).toThrow(
+          `${type} is not a defined scalar type, please use another type in the dbType directive`
+        );
+      }
+    });
+  });
 });

packages/utils/src/graphql/schema/directives.ts

@@ -10,4 +10,5 @@ export const directives = gql`
   directive @index(unique: Boolean) on FIELD_DEFINITION
   directive @compositeIndexes(fields: [[String]]!) on OBJECT
   directive @fullText(fields: [String!], language: String) on OBJECT
+  directive @dbType(type: String!) on FIELD_DEFINITION
 `;

test/docker-compose.yaml

@@ -1,4 +1,4 @@
-version: '3'
+version: "3"
 
 services:
   postgres:
@@ -31,4 +31,3 @@ services:
     command:
       - yarn
       - test:all
-

yarn.lock

@@ -6610,6 +6610,7 @@ __metadata:
     "@types/update-notifier": ^6
     "@types/websocket": ^1
     boxen: 5.1.2
+    chalk: ^4
     ejs: ^3.1.10
     eslint: ^8.8.0
     eslint-config-oclif: ^4.0.0
@@ -6773,6 +6774,7 @@ __metadata:
     axios: ^0.28.0
     class-transformer: ^0.5.1
     class-validator: ^0.14.1
+    form-data: ^4.0.1
     js-yaml: ^4.1.0
     reflect-metadata: ^0.1.14
     semver: ^7.6.3
@@ -12838,6 +12840,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"form-data@npm:^4.0.1":
+  version: 4.0.1
+  resolution: "form-data@npm:4.0.1"
+  dependencies:
+    asynckit: ^0.4.0
+    combined-stream: ^1.0.8
+    mime-types: ^2.1.12
+  checksum: ccee458cd5baf234d6b57f349fe9cc5f9a2ea8fd1af5ecda501a18fd1572a6dd3bf08a49f00568afd995b6a65af34cb8dec083cf9d582c4e621836499498dd84
+  languageName: node
+  linkType: hard
+
 "forwarded@npm:0.2.0":
   version: 0.2.0
   resolution: "forwarded@npm:0.2.0"