Merge pull request #522 from IKatsuba/fix-long-runs-end

fix(api): fix performance issue for /runs/end

apps/api/project.json

@@ -55,6 +55,16 @@
         ]
       }
     },
+    "create-migrations": {
+      "executor": "nx:run-commands",
+      "options": {
+        "commands": [
+          {
+            "command": "npm run migrations:create"
+          }
+        ]
+      }
+    },
     "lint": {
       "executor": "@nx/eslint:lint",
       "outputs": ["{options.outputFile}"]

libs/api/db/entities/src/lib/task/task.entity.ts

@@ -1,5 +1,6 @@
 import {
   Entity,
+  Index,
   LoadStrategy,
   ManyToOne,
   PrimaryKey,
@@ -29,6 +30,7 @@ export class TaskEntity implements Task {
   @Property({ nullable: true })
   configuration: string;
 
+  @Index()
   @Property()
   hash: string;
 

libs/api/db/entities/src/lib/task/task.service.ts

@@ -27,24 +27,10 @@ export class TaskService {
     });
   }
 
-  async findTaskWithoutCache(workspaceId: string, hash: string) {
-    return this.taskRepository.findOne(
-      {
-        execution: {
-          runGroup: {
-            workspace: {
-              id: workspaceId,
-            },
-          },
-        },
-        hash,
-        cacheStatus: 'cache-miss',
-      },
-      {
-        orderBy: {
-          startTime: 'desc',
-        },
-      }
-    );
+  async findTaskWithoutCache(hash: string) {
+    return this.taskRepository.findOne({
+      hash,
+      cacheStatus: 'cache-miss',
+    });
   }
 }

libs/api/http/runs/src/lib/runs.controller.ts

@@ -78,30 +78,30 @@ export class RunsController {
 
       const workspace = await this.workspaceService.getWorkspace(workspaceId);
 
-      for (const task of data.tasks) {
-        const prevTask =
-          task.cacheStatus === 'cache-miss'
-            ? task
-            : (await this.taskService.findTaskWithoutCache(
-                workspaceId,
-                task.hash
-              )) ?? task;
+      Promise.all(
+        data.tasks.map(async (task) => {
+          const prevTask =
+            task.cacheStatus === 'cache-miss'
+              ? task
+              : (await this.taskService.findTaskWithoutCache(task.hash)) ??
+                task;
 
-        const executionTime =
-          Date.parse(task.endTime) - Date.parse(task.startTime);
+          const executionTime =
+            Date.parse(task.endTime) - Date.parse(task.startTime);
 
-        const prevExecutionTime =
-          Date.parse(prevTask.endTime) - Date.parse(prevTask.startTime);
+          const prevExecutionTime =
+            Date.parse(prevTask.endTime) - Date.parse(prevTask.startTime);
 
-        const diff = prevExecutionTime - executionTime;
+          const diff = prevExecutionTime - executionTime;
 
-        await this.stats?.trackTaskExecutionTime?.(
-          workspace,
-          task,
-          executionTime,
-          diff
-        );
-      }
+          await this.stats?.trackTaskExecutionTime?.(
+            workspace,
+            task,
+            executionTime,
+            diff
+          );
+        })
+      ).catch(console.error);
 
       const runGroup =
         (await this.runGroupService.findOne(data.run.runGroup)) ??

migrations/.snapshot-postgres.json

@@ -327,6 +327,13 @@
       "name": "task_entity",
       "schema": "public",
       "indexes": [
+        {
+          "columnNames": ["hash"],
+          "composite": false,
+          "keyName": "task_entity_hash_index",
+          "primary": false,
+          "unique": false
+        },
         {
           "keyName": "task_entity_pkey",
           "columnNames": ["id"],

migrations/Migration20240507084336.ts

@@ -0,0 +1,13 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20240507084336 extends Migration {
+  async up(): Promise<void> {
+    this.addSql(
+      'create index "task_entity_hash_index" on "task_entity" ("hash");'
+    );
+  }
+
+  async down(): Promise<void> {
+    this.addSql('drop index "task_entity_hash_index";');
+  }
+}

package.json

@@ -3,6 +3,7 @@
   "version": "0.0.0-development",
   "license": "MIT",
   "scripts": {
+    "migrations:create": "ts-node -r tsconfig-paths/register -P scripts/tsconfig.json scripts/create-migrations.ts",
     "migrations:run": "ts-node -r tsconfig-paths/register -P scripts/tsconfig.json scripts/migrations.ts",
     "start:prod": "npm run migrations:run && node dist/apps/api/main.js"
   },

scripts/create-migrations.ts

@@ -0,0 +1,28 @@
+import { MikroORM } from '@mikro-orm/core';
+import { NestFactory } from '@nestjs/core';
+import { AppModule } from '../apps/api/src/app/app.module';
+import { Logger } from 'nestjs-pino';
+
+(async () => {
+  const app = await NestFactory.create(AppModule, {
+    autoFlushLogs: true,
+    bufferLogs: true,
+  });
+
+  const logger = app.get(Logger);
+
+  app.useLogger(logger);
+
+  logger.log('Starting migrations...');
+
+  const orm = app.get(MikroORM);
+
+  const migrator = orm.getMigrator();
+  await migrator.createMigration();
+
+  logger.log('Migrations created.');
+
+  await orm.close(true);
+})().catch((e) => {
+  console.error(e);
+});