Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Chore/add indexes to improve impact computing #1218

Merged
merged 3 commits into from
Dec 19, 2024
Merged

Chore/add indexes to improve impact computing #1218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
76b5b03
fix dependencies
alexeh Dec 19, 2024
b96752a
add indexes
alexeh Dec 19, 2024
f9fd3db
tmp: comment out import steps
alexeh Dec 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 70 additions & 70 deletions api/src/modules/import-data/sourcing-data/sourcing-data-import.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ export class SourcingDataImportService {

async importSourcingData(filePath: string, taskId: string): Promise<any> {
this.logger.log(`Starting import process`);
await this.fileService.isFilePresentInFs(filePath);
//await this.fileService.isFilePresentInFs(filePath);
try {
const parsedXLSXDataset: SourcingRecordsSheets =
await this.fileService.transformToJson(filePath, SHEETS_MAP);
Expand All @@ -77,80 +77,80 @@ export class SourcingDataImportService {
await this.excelValidator.validate(
parsedXLSXDataset as unknown as SourcingDataSheet,
);
if (validationErrors.length) {
throw new ExcelValidationError('Validation Errors', validationErrors);
}

//TODO: Implement transactional import. Move geocoding to first step

await this.dbCleaner.cleanDataBeforeImport();

const materials: Material[] =
await this.materialService.findAllUnpaginated();
if (!materials.length) {
throw new ServiceUnavailableException(
'No Materials found present in the DB. Please check the LandGriffon installation manual',
);
}
// if (validationErrors.length) {
// throw new ExcelValidationError('Validation Errors', validationErrors);
// }
//
// //TODO: Implement transactional import. Move geocoding to first step
//
// await this.dbCleaner.cleanDataBeforeImport();
//
// const materials: Material[] =
// await this.materialService.findAllUnpaginated();
// if (!materials.length) {
// throw new ServiceUnavailableException(
// 'No Materials found present in the DB. Please check the LandGriffon installation manual',
// );
// }
this.logger.log('Activating Indicators...');
const activeIndicators: Indicator[] =
await this.indicatorService.activateIndicators(
dtoMatchedData.indicators,
);
this.logger.log('Activating Materials...');
const activeMaterials: Material[] =
await this.materialService.activateMaterials(dtoMatchedData.materials);

await this.tasksService.updateImportTask({
taskId,
newLogs: [
`Activated indicators: ${activeIndicators
.map((i: Indicator) => i.name)
.join(', ')}`,
`Activated materials: ${activeMaterials
.map((i: Material) => i.hsCodeId)
.join(', ')}`,
],
});

const businessUnits: BusinessUnit[] =
await this.businessUnitService.createTree(dtoMatchedData.businessUnits);

const suppliers: Supplier[] = await this.supplierService.createTree(
dtoMatchedData.suppliers,
);

const { geoCodedSourcingData, errors } =
await this.geoCodingService.geoCodeLocations(
dtoMatchedData.sourcingData,
);
if (errors.length) {
throw new GeoCodingError(
'Import failed. There are GeoCoding errors present in the file',
errors,
);
}
const warnings: string[] = [];
geoCodedSourcingData.forEach((elem: SourcingData) => {
if (elem.locationWarning) warnings.push(elem.locationWarning);
});
warnings.length > 0 &&
(await this.tasksService.updateImportTask({
taskId,
newLogs: warnings,
}));

const sourcingDataWithOrganizationalEntities: SourcingLocation[] =
await this.relateSourcingDataWithOrganizationalEntities(
suppliers,
businessUnits,
materials,
geoCodedSourcingData,
);

await this.sourcingLocationService.save(
sourcingDataWithOrganizationalEntities,
);
// this.logger.log('Activating Materials...');
// const activeMaterials: Material[] =
// await this.materialService.activateMaterials(dtoMatchedData.materials);
//
// await this.tasksService.updateImportTask({
// taskId,
// newLogs: [
// `Activated indicators: ${activeIndicators
// .map((i: Indicator) => i.name)
// .join(', ')}`,
// `Activated materials: ${activeMaterials
// .map((i: Material) => i.hsCodeId)
// .join(', ')}`,
// ],
// });
//
// const businessUnits: BusinessUnit[] =
// await this.businessUnitService.createTree(dtoMatchedData.businessUnits);
//
// const suppliers: Supplier[] = await this.supplierService.createTree(
// dtoMatchedData.suppliers,
// );
//
// const { geoCodedSourcingData, errors } =
// await this.geoCodingService.geoCodeLocations(
// dtoMatchedData.sourcingData,
// );
// if (errors.length) {
// throw new GeoCodingError(
// 'Import failed. There are GeoCoding errors present in the file',
// errors,
// );
// }
// const warnings: string[] = [];
// geoCodedSourcingData.forEach((elem: SourcingData) => {
// if (elem.locationWarning) warnings.push(elem.locationWarning);
// });
// warnings.length > 0 &&
// (await this.tasksService.updateImportTask({
// taskId,
// newLogs: warnings,
// }));
//
// const sourcingDataWithOrganizationalEntities: SourcingLocation[] =
// await this.relateSourcingDataWithOrganizationalEntities(
// suppliers,
// businessUnits,
// materials,
// geoCodedSourcingData,
// );
//
// await this.sourcingLocationService.save(
// sourcingDataWithOrganizationalEntities,
// );

this.logger.log('Generating Indicator Records...');

Expand Down
4 changes: 4 additions & 0 deletions api/src/modules/indicator-coefficients/indicator-coefficient.entity.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {
BaseEntity,
Column,
Entity,
Index,
ManyToOne,
PrimaryGeneratedColumn,
} from 'typeorm';
Expand Down Expand Up @@ -31,17 +32,20 @@ export class IndicatorCoefficient extends BaseEntity {
@Column({ type: 'int' })
year!: number;

@Index()
@ManyToOne(() => AdminRegion, (ar: AdminRegion) => ar.indicatorCoefficients, {
nullable: true,
})
adminRegion: AdminRegion;

@Index()
@ManyToOne(
() => Indicator,
(indicator: Indicator) => indicator.indicatorCoefficients,
)
indicator!: Indicator;

@Index()
@ManyToOne(() => Material, (mat: Material) => mat.indicatorCoefficients)
material!: Material;
}
2 changes: 1 addition & 1 deletion api/src/modules/indicator-records/services/impact-calculation.dependencies.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ export const INDICATOR_NAME_CODE_TO_QUERY_MAP: {
},
[INDICATOR_NAME_CODES.WW]: {
[INDICATOR_NAME_CODES.WW]: () =>
`${get_indicator_coefficient_impact}('${INDICATOR_NAME_CODES.WW}', $3, $2) as "${INDICATOR_NAME_CODES.WU}"`,
`${get_indicator_coefficient_impact}('${INDICATOR_NAME_CODES.WW}', $3, $2) as "${INDICATOR_NAME_CODES.WW}"`,
},
[INDICATOR_NAME_CODES.WC]: {
[INDICATOR_NAME_CODES.WC]: () =>
Expand Down
3 changes: 3 additions & 0 deletions api/src/modules/sourcing-locations/sourcing-location.entity.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import {
Check,
Column,
Entity,
In,
Index,
JoinColumn,
ManyToOne,
Expand Down Expand Up @@ -136,6 +137,7 @@ export class SourcingLocation extends TimestampedBaseEntity {
material: Material;

@ApiPropertyOptional()
@Index('sourcing_location_material_id_index')
@Column()
materialId: string;

Expand All @@ -147,6 +149,7 @@ export class SourcingLocation extends TimestampedBaseEntity {
adminRegion: AdminRegion;

@Column({ nullable: true })
@Index('sourcing_location_admin_region_id_index')
@ApiPropertyOptional()
adminRegionId: string;

Expand Down
Loading