Skip to content

Commit

Permalink
refactor(back): add header to static map list files
Browse files Browse the repository at this point in the history
  • Loading branch information
@tsa96
tsa96 committed Oct 18, 2024
1 parent 9cfb436 commit 434f8de
Show file tree
Hide file tree
Showing 3 changed files with 159 additions and 48 deletions.
127 changes: 88 additions & 39 deletions apps/backend/src/app/modules/maps/map-list.service.spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,24 @@
import { FlatMapList } from '@momentum/constants';
import { MapListService } from './map-list.service';
import { Test, TestingModule } from '@nestjs/testing';
import { PRISMA_MOCK_PROVIDER } from '../../../../test/prisma-mock.const';
import {
PRISMA_MOCK_PROVIDER,
PrismaMock
} from '../../../../test/prisma-mock.const';
import { mockDeep } from 'jest-mock-extended';
import { FileStoreService } from '../filestore/file-store.service';
import { EXTENDED_PRISMA_SERVICE } from '../database/db.constants';
import { promisify } from 'node:util';
import * as zlib from 'node:zlib';

describe('MapListService', () => {
describe('onModuleInit', () => {
let service: MapListService;
let service: MapListService, db: PrismaMock;
const fileStoreMock = {
listFileKeys: jest.fn(() => Promise.resolve([])),
deleteFiles: jest.fn()
storeFile: jest.fn(),
deleteFiles: jest.fn(),
deleteFile: jest.fn()
};

beforeEach(async () => {
Expand All @@ -25,57 +33,98 @@ describe('MapListService', () => {
.compile();

service = module.get(MapListService);
db = module.get(EXTENDED_PRISMA_SERVICE);
});

it('should set version values based on files in storage', async () => {
fileStoreMock.listFileKeys.mockResolvedValueOnce([
'maplist/approved/1.dat'
]);
fileStoreMock.listFileKeys.mockResolvedValueOnce([
'maplist/submissions/15012024.dat'
]);
describe('onModuleInit', () => {
it('should set version values based on files in storage', async () => {
fileStoreMock.listFileKeys.mockResolvedValueOnce([
'maplist/approved/1.dat'
]);
fileStoreMock.listFileKeys.mockResolvedValueOnce([
'maplist/submissions/15012024.dat'
]);

await service.onModuleInit();
await service.onModuleInit();

expect(service['version']).toMatchObject({
[FlatMapList.APPROVED]: 1,
[FlatMapList.SUBMISSION]: 15012024
expect(service['version']).toMatchObject({
[FlatMapList.APPROVED]: 1,
[FlatMapList.SUBMISSION]: 15012024
});

expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled();
});

expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled();
});
it('should set version to 0 when no versions exist in storage', async () => {
await service.onModuleInit();

it('should set version to 0 when no versions exist in storage', async () => {
await service.onModuleInit();
expect(service['version']).toMatchObject({
[FlatMapList.APPROVED]: 0,
[FlatMapList.SUBMISSION]: 0
});

expect(service['version']).toMatchObject({
[FlatMapList.APPROVED]: 0,
[FlatMapList.SUBMISSION]: 0
expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled();
});

expect(fileStoreMock.deleteFiles).not.toHaveBeenCalled();
});
it('should pick most recent when multiple versions exist in storage, and wipe old versions', async () => {
fileStoreMock.listFileKeys.mockResolvedValueOnce([
'maplist/approved/4.dat',
'maplist/approved/5.dat',
'maplist/approved/3.dat',
'maplist/approved/1.dat'
]);

it('should pick most recent when multiple versions exist in storage, and wipe old versions', async () => {
fileStoreMock.listFileKeys.mockResolvedValueOnce([
'maplist/approved/4.dat',
'maplist/approved/5.dat',
'maplist/approved/3.dat',
'maplist/approved/1.dat'
]);
await service.onModuleInit();

await service.onModuleInit();
expect(service['version']).toMatchObject({
[FlatMapList.APPROVED]: 5,
[FlatMapList.SUBMISSION]: 0
});

expect(service['version']).toMatchObject({
[FlatMapList.APPROVED]: 5,
[FlatMapList.SUBMISSION]: 0
expect(fileStoreMock.deleteFiles).toHaveBeenCalledWith([
'maplist/approved/4.dat',
'maplist/approved/3.dat',
'maplist/approved/1.dat'
]);
});
});

describe('updateMapList', () => {
// prettier-ignore
const storedMap = {
id: 1,
name: 'The Map',
status: 0,
images: [ 'f2fecc26-34a0-448b-a3c7-007f43b9ec7e', 'a797e52e-3efc-4174-9f66-36e2c57ff55c', 'dee8bbd5-cec2-4341-9ddf-bdadd8337cdd' ],
info: { description: 'A map that makes me think I am becoming a better person', youtubeID: null, creationDate: '2024-09-27T10:18:42.318Z', mapID: 1 },
leaderboards: [ { mapID: 12345, gamemode: 8, trackType: 0, trackNum: 1, style: 0, tier: 3, linear: false, type: 1, tags: [] } ],
credits: [ { type: 1, description: 'who am i', user: { id: 674, alias: 'John God', avatar: '0227a240393e6d62f539ee7b306dd048b0830eeb', steamID: '43576820710' } } ],
createdAt: '2024-09-27T22:31:12.846Z',
currentVersion: { id: 'fc89afc9-7ad2-4590-853c-a9ff4f41ddd5', versionNum: 3, bspHash: 'ddd39cbfc070e98e1e68131bab0f40df1d06645f', zoneHash: '608437d3bb461dd6e4abfff881f6b16827629d0b', hasVmf: false, submitterID: null, createdAt: '2024-09-27T18:12:52.465Z' }
};

it('should generate a Momentum Static Map List file and send to filestore', async () => {
db.mMap.findMany.mockResolvedValueOnce([storedMap as any]);

expect(fileStoreMock.deleteFiles).toHaveBeenCalledWith([
'maplist/approved/4.dat',
'maplist/approved/3.dat',
'maplist/approved/1.dat'
]);
await service.updateMapList(FlatMapList.APPROVED);

const buffer: Buffer = fileStoreMock.storeFile.mock.calls[0][0];
expect(buffer.subarray(0, 4)).toMatchObject(
Buffer.from('MSML', 'utf8')
);
expect(buffer.readUInt32LE(8)).toBe(1);

const decompressed = await promisify(zlib.inflate)(buffer.subarray(12));
expect(buffer.readUInt32LE(4)).toBe(decompressed.length);

const parsed = JSON.parse(decompressed.toString('utf8'));

// Can't do full toMatchObject, we've run through class-transformer.
expect(parsed[0]).toMatchObject({
id: storedMap.id,
name: storedMap.name
});
});
});
});
});
52 changes: 47 additions & 5 deletions apps/backend/src/app/modules/maps/map-list.service.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common';
import { FileStoreService } from '../filestore/file-store.service';
import { EXTENDED_PRISMA_SERVICE } from '../database/db.constants';
import { ExtendedPrismaService } from '../database/prisma.extension';
Expand Down Expand Up @@ -27,6 +27,8 @@ export class MapListService implements OnModuleInit {
[FlatMapList.SUBMISSION]: 0
};

private readonly logger = new Logger('Map List Service');

async onModuleInit(): Promise<void> {
for (const type of [FlatMapList.APPROVED, FlatMapList.SUBMISSION]) {
const keys = await this.fileStoreService.listFileKeys(mapListDir(type));
Expand Down Expand Up @@ -83,27 +85,67 @@ export class MapListService implements OnModuleInit {
}
},
createdAt: true,
currentVersion: { omit: { zones: true, changelog: true } },
currentVersion: { omit: { zones: true, changelog: true, mapID: true } },
...(type === FlatMapList.SUBMISSION
? { submission: true, versions: { omit: { zones: true } } }
? {
submission: true,
versions: { omit: { zones: true, mapID: true } }
}
: {})
}
});

const t1 = Date.now();

// Convert to DTO then serialize back to JSON so any class-transformer
// transformations are applied.
const mapListJson = JSON.stringify(
maps.map((map) => instanceToPlain(plainToInstance(MapDto, map)))
);
const compressed = await promisify(zlib.deflate)(mapListJson);

// Momentum Static Map List
//
// -- Header [12 bytes] --
// Ident [4 bytes - "MSML" 4D 53 4D 4C]
// Length of uncompressed data [4 bytes - uint32 LE]
// Total number of maps [4 bytes - uint32 LE]
//
// -- Contents [Variable] --
// Deflate compressed map list

// We could get this way more memory-efficent by using streams, I just
// can't be fucked with streaming to the S3 API (see https://stackoverflow.com/a/73332454).
// This takes me ~100ms for ~3000 maps, could be brought down quite a bit
// with streams.
//
// Hilariously, class-transformer serialization takes about 10 TIMES the
// the time to JSON.stringify, compress, and concat the buffers.
// So this isn't worth optimising whilst we're still using that piece of
// crap library.
const uncompressed = Buffer.from(mapListJson);
const header = Buffer.alloc(12);

header.write('MSML', 0, 'utf8');
header.writeUInt32LE(uncompressed.length, 4);
header.writeUInt32LE(maps.length, 8);

const compressed = await promisify(zlib.deflate)(uncompressed, {
level: 5
});

const outBuf = Buffer.concat([header, compressed]);

const oldVersion = this.version[type];
const newVersion = this.updateMapListVersion(type);
const oldKey = mapListPath(type, oldVersion);
const newKey = mapListPath(type, newVersion);

this.logger.log(
`Updating ${type} map list from v${oldVersion} to v${newVersion}, ${maps.length} maps, encoding took ${Date.now() - t1}ms`
);

await this.fileStoreService.deleteFile(oldKey);
await this.fileStoreService.storeFile(compressed, newKey);
await this.fileStoreService.storeFile(outBuf, newKey);
}

private updateMapListVersion(type: FlatMapList): number {
Expand Down
28 changes: 24 additions & 4 deletions scripts/src/seed.script.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1032,9 +1032,12 @@ prismaWrapper(async (prisma: PrismaClient) => {
}
},
createdAt: true,
currentVersion: { omit: { zones: true, changelog: true } },
currentVersion: { omit: { zones: true, changelog: true, mapID: true } },
...(type === FlatMapList.SUBMISSION
? { submission: true, versions: { omit: { zones: true } } }
? {
submission: true,
versions: { omit: { zones: true, mapID: true } }
}
: {})
}
});
Expand Down Expand Up @@ -1077,12 +1080,29 @@ prismaWrapper(async (prisma: PrismaClient) => {
writeFileSync(`./map-list-${type}.json`, mapListJson);
}

const compressed = await promisify(zlib.deflate)(mapListJson);
// This is copied directly from map-list-service.ts, see there
const t1 = Date.now();

const uncompressed = Buffer.from(mapListJson);
const header = Buffer.alloc(12);

header.write('MSML', 0, 'utf8');
header.writeUInt32LE(uncompressed.length, 4);
header.writeUInt32LE(maps.length, 8);

const compressed = await promisify(zlib.deflate)(uncompressed, {
level: 5
});

const outBuf = Buffer.concat([header, compressed]);

console.log(`Generated map list, encoding took ${Date.now() - t1}ms`);

await s3.send(
new PutObjectCommand({
Bucket: s3BucketName,
Key: mapListPath(type, 1),
Body: compressed
Body: outBuf
})
);
}
Expand Down

0 comments on commit 434f8de

Please sign in to comment.