diff --git a/apps/backend/src/app/modules/maps/map-list.service.spec.ts b/apps/backend/src/app/modules/maps/map-list.service.spec.ts index b7cbc9a26..cafad7e21 100644 --- a/apps/backend/src/app/modules/maps/map-list.service.spec.ts +++ b/apps/backend/src/app/modules/maps/map-list.service.spec.ts @@ -1,16 +1,24 @@ import { FlatMapList } from '@momentum/constants'; import { MapListService } from './map-list.service'; import { Test, TestingModule } from '@nestjs/testing'; -import { PRISMA_MOCK_PROVIDER } from '../../../../test/prisma-mock.const'; +import { + PRISMA_MOCK_PROVIDER, + PrismaMock +} from '../../../../test/prisma-mock.const'; import { mockDeep } from 'jest-mock-extended'; import { FileStoreService } from '../filestore/file-store.service'; +import { EXTENDED_PRISMA_SERVICE } from '../database/db.constants'; +import { promisify } from 'node:util'; +import * as zlib from 'node:zlib'; describe('MapListService', () => { describe('onModuleInit', () => { - let service: MapListService; + let service: MapListService, db: PrismaMock; const fileStoreMock = { listFileKeys: jest.fn(() => Promise.resolve([])), - deleteFiles: jest.fn() + storeFile: jest.fn(), + deleteFiles: jest.fn(), + deleteFile: jest.fn() }; beforeEach(async () => { @@ -25,57 +33,98 @@ describe('MapListService', () => { .compile(); service = module.get(MapListService); + db = module.get(EXTENDED_PRISMA_SERVICE); }); - it('should set version values based on files in storage', async () => { - fileStoreMock.listFileKeys.mockResolvedValueOnce([ - 'maplist/approved/1.dat' - ]); - fileStoreMock.listFileKeys.mockResolvedValueOnce([ - 'maplist/submissions/15012024.dat' - ]); + describe('onModuleInit', () => { + it('should set version values based on files in storage', async () => { + fileStoreMock.listFileKeys.mockResolvedValueOnce([ + 'maplist/approved/1.dat' + ]); + fileStoreMock.listFileKeys.mockResolvedValueOnce([ + 'maplist/submissions/15012024.dat' + ]); - await service.onModuleInit(); + await service.onModuleInit(); - expect(service['version']).toMatchObject({ - [FlatMapList.APPROVED]: 1, - [FlatMapList.SUBMISSION]: 15012024 + expect(service['version']).toMatchObject({ + [FlatMapList.APPROVED]: 1, + [FlatMapList.SUBMISSION]: 15012024 + }); + + expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled(); }); - expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled(); - }); + it('should set version to 0 when no versions exist in storage', async () => { + await service.onModuleInit(); - it('should set version to 0 when no versions exist in storage', async () => { - await service.onModuleInit(); + expect(service['version']).toMatchObject({ + [FlatMapList.APPROVED]: 0, + [FlatMapList.SUBMISSION]: 0 + }); - expect(service['version']).toMatchObject({ - [FlatMapList.APPROVED]: 0, - [FlatMapList.SUBMISSION]: 0 + expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled(); }); - expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled(); - }); + it('should pick most recent when multiple versions exist in storage, and wipe old versions', async () => { + fileStoreMock.listFileKeys.mockResolvedValueOnce([ + 'maplist/approved/4.dat', + 'maplist/approved/5.dat', + 'maplist/approved/3.dat', + 'maplist/approved/1.dat' + ]); - it('should pick most recent when multiple versions exist in storage, and wipe old versions', async () => { - fileStoreMock.listFileKeys.mockResolvedValueOnce([ - 'maplist/approved/4.dat', - 'maplist/approved/5.dat', - 'maplist/approved/3.dat', - 'maplist/approved/1.dat' - ]); + await service.onModuleInit(); - await service.onModuleInit(); + expect(service['version']).toMatchObject({ + [FlatMapList.APPROVED]: 5, + [FlatMapList.SUBMISSION]: 0 + }); - expect(service['version']).toMatchObject({ - [FlatMapList.APPROVED]: 5, - [FlatMapList.SUBMISSION]: 0 + expect(fileStoreMock.deleteFiles).toHaveBeenCalledWith([ + 'maplist/approved/4.dat', + 'maplist/approved/3.dat', + 'maplist/approved/1.dat' + ]); }); + }); + + describe('updateMapList', () => { + // prettier-ignore + const storedMap = { + id: 1, + name: 'The Map', + status: 0, + images: [ 'f2fecc26-34a0-448b-a3c7-007f43b9ec7e', 'a797e52e-3efc-4174-9f66-36e2c57ff55c', 'dee8bbd5-cec2-4341-9ddf-bdadd8337cdd' ], + info: { description: 'A map that makes me think I am becoming a better person', youtubeID: null, creationDate: '2024-09-27T10:18:42.318Z', mapID: 1 }, + leaderboards: [ { mapID: 12345, gamemode: 8, trackType: 0, trackNum: 1, style: 0, tier: 3, linear: false, type: 1, tags: [] } ], + credits: [ { type: 1, description: 'who am i', user: { id: 674, alias: 'John God', avatar: '0227a240393e6d62f539ee7b306dd048b0830eeb', steamID: '43576820710' } } ], + createdAt: '2024-09-27T22:31:12.846Z', + currentVersion: { id: 'fc89afc9-7ad2-4590-853c-a9ff4f41ddd5', versionNum: 3, bspHash: 'ddd39cbfc070e98e1e68131bab0f40df1d06645f', zoneHash: '608437d3bb461dd6e4abfff881f6b16827629d0b', hasVmf: false, submitterID: null, createdAt: '2024-09-27T18:12:52.465Z' } + }; + + it('should generate a Momentum Static Map List file and send to filestore', async () => { + db.mMap.findMany.mockResolvedValueOnce([storedMap as any]); - expect(fileStoreMock.deleteFiles).toHaveBeenCalledWith([ - 'maplist/approved/4.dat', - 'maplist/approved/3.dat', - 'maplist/approved/1.dat' - ]); + await service.updateMapList(FlatMapList.APPROVED); + + const buffer: Buffer = fileStoreMock.storeFile.mock.calls[0][0]; + expect(buffer.subarray(0, 4)).toMatchObject( + Buffer.from('MSML', 'utf8') + ); + expect(buffer.readUInt32LE(8)).toBe(1); + + const decompressed = await promisify(zlib.inflate)(buffer.subarray(12)); + expect(buffer.readUInt32LE(4)).toBe(decompressed.length); + + const parsed = JSON.parse(decompressed.toString('utf8')); + + // Can't do full toMatchObject, we've run through class-transformer. + expect(parsed[0]).toMatchObject({ + id: storedMap.id, + name: storedMap.name + }); + }); }); }); }); diff --git a/apps/backend/src/app/modules/maps/map-list.service.ts b/apps/backend/src/app/modules/maps/map-list.service.ts index 7dcc5668f..20660f77c 100644 --- a/apps/backend/src/app/modules/maps/map-list.service.ts +++ b/apps/backend/src/app/modules/maps/map-list.service.ts @@ -1,4 +1,4 @@ -import { Inject, Injectable, OnModuleInit } from '@nestjs/common'; +import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { FileStoreService } from '../filestore/file-store.service'; import { EXTENDED_PRISMA_SERVICE } from '../database/db.constants'; import { ExtendedPrismaService } from '../database/prisma.extension'; @@ -27,6 +27,8 @@ export class MapListService implements OnModuleInit { [FlatMapList.SUBMISSION]: 0 }; + private readonly logger = new Logger('Map List Service'); + async onModuleInit(): Promise { for (const type of [FlatMapList.APPROVED, FlatMapList.SUBMISSION]) { const keys = await this.fileStoreService.listFileKeys(mapListDir(type)); @@ -83,27 +85,67 @@ export class MapListService implements OnModuleInit { } }, createdAt: true, - currentVersion: { omit: { zones: true, changelog: true } }, + currentVersion: { omit: { zones: true, changelog: true, mapID: true } }, ...(type === FlatMapList.SUBMISSION - ? { submission: true, versions: { omit: { zones: true } } } + ? { + submission: true, + versions: { omit: { zones: true, mapID: true } } + } : {}) } }); + const t1 = Date.now(); + // Convert to DTO then serialize back to JSON so any class-transformer // transformations are applied. const mapListJson = JSON.stringify( maps.map((map) => instanceToPlain(plainToInstance(MapDto, map))) ); - const compressed = await promisify(zlib.deflate)(mapListJson); + + // Momentum Static Map List + // + // -- Header [12 bytes] -- + // Ident [4 bytes - "MSML" 4D 53 4D 4C] + // Length of uncompressed data [4 bytes - uint32 LE] + // Total number of maps [4 bytes - uint32 LE] + // + // -- Contents [Variable] -- + // Deflate compressed map list + + // We could get this way more memory-efficent by using streams, I just + // can't be fucked with streaming to the S3 API (see https://stackoverflow.com/a/73332454). + // This takes me ~100ms for ~3000 maps, could be brought down quite a bit + // with streams. + // + // Hilariously, class-transformer serialization takes about 10 TIMES the + // the time to JSON.stringify, compress, and concat the buffers. + // So this isn't worth optimising whilst we're still using that piece of + // crap library. + const uncompressed = Buffer.from(mapListJson); + const header = Buffer.alloc(12); + + header.write('MSML', 0, 'utf8'); + header.writeUInt32LE(uncompressed.length, 4); + header.writeUInt32LE(maps.length, 8); + + const compressed = await promisify(zlib.deflate)(uncompressed, { + level: 5 + }); + + const outBuf = Buffer.concat([header, compressed]); const oldVersion = this.version[type]; const newVersion = this.updateMapListVersion(type); const oldKey = mapListPath(type, oldVersion); const newKey = mapListPath(type, newVersion); + this.logger.log( + `Updating ${type} map list from v${oldVersion} to v${newVersion}, ${maps.length} maps, encoding took ${Date.now() - t1}ms` + ); + await this.fileStoreService.deleteFile(oldKey); - await this.fileStoreService.storeFile(compressed, newKey); + await this.fileStoreService.storeFile(outBuf, newKey); } private updateMapListVersion(type: FlatMapList): number { diff --git a/scripts/src/seed.script.ts b/scripts/src/seed.script.ts index 6a2520061..b81c1a50c 100644 --- a/scripts/src/seed.script.ts +++ b/scripts/src/seed.script.ts @@ -1032,9 +1032,12 @@ prismaWrapper(async (prisma: PrismaClient) => { } }, createdAt: true, - currentVersion: { omit: { zones: true, changelog: true } }, + currentVersion: { omit: { zones: true, changelog: true, mapID: true } }, ...(type === FlatMapList.SUBMISSION - ? { submission: true, versions: { omit: { zones: true } } } + ? { + submission: true, + versions: { omit: { zones: true, mapID: true } } + } : {}) } }); @@ -1077,12 +1080,29 @@ prismaWrapper(async (prisma: PrismaClient) => { writeFileSync(`./map-list-${type}.json`, mapListJson); } - const compressed = await promisify(zlib.deflate)(mapListJson); + // This is copied directly from map-list-service.ts, see there + const t1 = Date.now(); + + const uncompressed = Buffer.from(mapListJson); + const header = Buffer.alloc(12); + + header.write('MSML', 0, 'utf8'); + header.writeUInt32LE(uncompressed.length, 4); + header.writeUInt32LE(maps.length, 8); + + const compressed = await promisify(zlib.deflate)(uncompressed, { + level: 5 + }); + + const outBuf = Buffer.concat([header, compressed]); + + console.log(`Generated map list, encoding took ${Date.now() - t1}ms`); + await s3.send( new PutObjectCommand({ Bucket: s3BucketName, Key: mapListPath(type, 1), - Body: compressed + Body: outBuf }) ); }