Skip to content

Commit

Permalink
MODFQMMGR-225: Enable cross-tenant querying in ECS environments (#342)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bvsharp
bvsharp authored Aug 2, 2024
1 parent 42cb138 commit df3b89f
Show file tree
Hide file tree
Showing 34 changed files with 910 additions and 309 deletions.
2 changes: 1 addition & 1 deletion src/main/java/org/folio/fqm/aspect/EntityTypePermissionsAspect.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ Object validatePermissionsWithContentsRequest(ProceedingJoinPoint joinPoint) thr
}

private EntityType getEntityTypeFromId(UUID entityTypeId) {
return entityTypeRepository.getEntityTypeDefinition(entityTypeId)
return entityTypeRepository.getEntityTypeDefinition(entityTypeId, null)
.orElseThrow(() -> new EntityTypeNotFoundException(entityTypeId));
}

Expand Down
13 changes: 7 additions & 6 deletions src/main/java/org/folio/fqm/repository/EntityTypeRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,18 +48,19 @@ public class EntityTypeRepository {
private final DSLContext jooqContext;
private final ObjectMapper objectMapper;

public Optional<EntityType> getEntityTypeDefinition(UUID entityTypeId) {
return getEntityTypeDefinitions(Collections.singleton(entityTypeId)).findFirst();
public Optional<EntityType> getEntityTypeDefinition(UUID entityTypeId, String tenantId) {
return getEntityTypeDefinitions(Collections.singleton(entityTypeId), tenantId).findFirst();
}

public Stream<EntityType> getEntityTypeDefinitions(Collection<UUID> entityTypeIds) {
public Stream<EntityType> getEntityTypeDefinitions(Collection<UUID> entityTypeIds, String tenantId) {
String tableName = tenantId == null ? TABLE_NAME : tenantId + "_mod_fqm_manager." + TABLE_NAME;
log.info("Getting definitions name for entity type ID: {}", entityTypeIds);

Field<String> definitionField = field(DEFINITION_FIELD_NAME, String.class);
Condition entityTypeIdCondition = isEmpty(entityTypeIds) ? trueCondition() : field("id").in(entityTypeIds);
return readerJooqContext
.select(definitionField)
.from(table(TABLE_NAME))
.from(table(tableName))
.where(entityTypeIdCondition)
.fetch(definitionField)
.stream()
Expand Down Expand Up @@ -99,8 +100,8 @@ public void replaceEntityTypeDefinitions(List<EntityType> entityTypes) {

private List<EntityTypeColumn> fetchColumnNamesForCustomFields(UUID entityTypeId, EntityType entityType) {
log.info("Getting columns for entity type ID: {}", entityTypeId);
EntityType entityTypeDefinition = entityTypeId.toString().equals(entityType.getId()) ? entityType : getEntityTypeDefinition(entityTypeId)
.orElseThrow(() -> new EntityTypeNotFoundException(entityTypeId));
EntityType entityTypeDefinition = entityTypeId.toString().equals(entityType.getId()) ? entityType :
getEntityTypeDefinition(entityTypeId, null).orElseThrow(() -> new EntityTypeNotFoundException(entityTypeId));
String sourceViewName = entityTypeDefinition.getSourceView();
String sourceViewExtractor = entityTypeDefinition.getSourceViewExtractor();

Expand Down
107 changes: 60 additions & 47 deletions src/main/java/org/folio/fqm/repository/IdStreamer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,18 +3,19 @@
import lombok.RequiredArgsConstructor;
import lombok.extern.log4j.Log4j2;
import org.folio.fqm.model.IdsWithCancelCallback;
import org.folio.fqm.service.CrossTenantQueryService;
import org.folio.fqm.service.EntityTypeFlatteningService;
import org.folio.fqm.service.FqlToSqlConverterService;
import org.folio.fqm.utils.IdColumnUtils;
import org.folio.fqm.utils.EntityTypeUtils;
import org.folio.fqm.utils.StreamHelper;
import org.folio.fql.model.Fql;
import org.folio.querytool.domain.dto.EntityType;
import org.folio.querytool.domain.dto.EntityTypeDefaultSort;
import org.folio.spring.FolioExecutionContext;
import org.jooq.Condition;
import org.jooq.Cursor;
import org.jooq.DSLContext;
import org.jooq.ResultQuery;
import org.jooq.SortField;
import org.jooq.Select;
import org.jooq.impl.DSL;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
Expand All @@ -30,7 +31,7 @@
import java.util.stream.Stream;

import static org.apache.commons.lang3.ObjectUtils.isEmpty;
import static org.folio.fqm.utils.IdColumnUtils.RESULT_ID_FIELD;
import static org.folio.fqm.utils.EntityTypeUtils.RESULT_ID_FIELD;
import static org.jooq.impl.DSL.field;
import static org.jooq.impl.DSL.table;

Expand All @@ -42,6 +43,8 @@ public class IdStreamer {
@Qualifier("readerJooqContext")
private final DSLContext jooqContext;
private final EntityTypeFlatteningService entityTypeFlatteningService;
private final CrossTenantQueryService crossTenantQueryService;
private final FolioExecutionContext executionContext;

/**
* Executes the given Fql Query and stream the result Ids back.
Expand All @@ -51,8 +54,8 @@ public int streamIdsInBatch(EntityType entityType,
Fql fql,
int batchSize,
Consumer<IdsWithCancelCallback> idsConsumer) {
Condition sqlWhereClause = FqlToSqlConverterService.getSqlCondition(fql.fqlCondition(), entityType);
return this.streamIdsInBatch(entityType, sortResults, sqlWhereClause, batchSize, idsConsumer);
List<String> tenantsToQuery = crossTenantQueryService.getTenantsToQuery(UUID.fromString(entityType.getId()));
return this.streamIdsInBatch(entityType, sortResults, fql, batchSize, idsConsumer, tenantsToQuery);
}

public List<List<String>> getSortedIds(String derivedTableName,
Expand All @@ -76,40 +79,40 @@ public List<List<String>> getSortedIds(String derivedTableName,

private int streamIdsInBatch(EntityType entityType,
boolean sortResults,
Condition sqlWhereClause,
int batchSize,
Consumer<IdsWithCancelCallback> idsConsumer) {
String finalJoinClause = entityTypeFlatteningService.getJoinClause(entityType);
Field<String[]> idValueGetter = IdColumnUtils.getResultIdValueGetter(entityType);
ResultQuery<Record1<String[]>> query = null;
if (!isEmpty(entityType.getGroupByFields())) {
Field<?>[] groupByFields = entityType
.getColumns()
.stream()
.filter(col -> entityType.getGroupByFields().contains(col.getName()))
.map(col -> col.getFilterValueGetter() == null ? col.getValueGetter() : col.getFilterValueGetter())
.map(DSL::field)
.toArray(Field[]::new);
query = jooqContext.dsl()
.select(field(idValueGetter))
.from(finalJoinClause)
.where(sqlWhereClause)
.groupBy(groupByFields)
.orderBy(getSortFields(entityType, sortResults))
.fetchSize(batchSize);
} else {
query = jooqContext.dsl()
.select(field(idValueGetter))
.from(finalJoinClause)
.where(sqlWhereClause)
.orderBy(getSortFields(entityType, sortResults))
.fetchSize(batchSize);
Fql fql, int batchSize,
Consumer<IdsWithCancelCallback> idsConsumer, List<String> tenantsToQuery) {
UUID entityTypeId = UUID.fromString(entityType.getId());
log.debug("List of tenants to query: {}", tenantsToQuery);
Field<String[]> idValueGetter = EntityTypeUtils.getResultIdValueGetter(entityType);
Select<Record1<String[]>> fullQuery = null;
for (String tenantId : tenantsToQuery) {
EntityType entityTypeDefinition = tenantId != null && tenantId.equals(executionContext.getTenantId()) ?
entityType : entityTypeFlatteningService.getFlattenedEntityType(entityTypeId, tenantId);
var currentIdValueGetter = EntityTypeUtils.getResultIdValueGetter(entityTypeDefinition);
String innerJoinClause = entityTypeFlatteningService.getJoinClause(entityTypeDefinition, tenantId);
Condition whereClause = FqlToSqlConverterService.getSqlCondition(fql.fqlCondition(), entityTypeDefinition);
ResultQuery<Record1<String[]>> innerQuery = buildQuery(
entityTypeDefinition,
currentIdValueGetter,
innerJoinClause,
whereClause,
sortResults,
batchSize
);
if (fullQuery == null) {
fullQuery = (Select<Record1<String[]>>) innerQuery;
} else {
fullQuery = fullQuery.unionAll((Select<Record1<String[]>>) innerQuery);
}
}
log.debug("Full query: {}", fullQuery);

try (
Cursor<Record1<String[]>> idsCursor = query.fetchLazy();
Cursor<Record1<String[]>> idsCursor = fullQuery.fetchLazy();
Stream<String[]> idStream = idsCursor
.stream()
// This is something we may want to revisit. This implementation is a bit hackish for cross-tenant queries,
// though it does work
.map(row -> row.getValue(idValueGetter));
Stream<List<String[]>> idsStream = StreamHelper.chunk(idStream, batchSize)
) {
Expand All @@ -123,19 +126,29 @@ private int streamIdsInBatch(EntityType entityType,
}
}

private List<SortField<Object>> getSortFields(EntityType entityType, boolean sortResults) {
if (sortResults && !isEmpty(entityType.getDefaultSort())) {
return entityType
.getDefaultSort()
private ResultQuery<Record1<String[]>> buildQuery(EntityType entityType, Field<String[]> idValueGetter, String finalJoinClause, Condition sqlWhereClause, boolean sortResults, int batchSize) {
if (!isEmpty(entityType.getGroupByFields())) {
Field<?>[] groupByFields = entityType
.getColumns()
.stream()
.map(IdStreamer::toSortField)
.toList();
.filter(col -> entityType.getGroupByFields().contains(col.getName()))
.map(col -> col.getFilterValueGetter() == null ? col.getValueGetter() : col.getFilterValueGetter())
.map(DSL::field)
.toArray(Field[]::new);
return jooqContext.dsl()
.select(field(idValueGetter))
.from(finalJoinClause)
.where(sqlWhereClause)
.groupBy(groupByFields)
.orderBy(EntityTypeUtils.getSortFields(entityType, sortResults))
.fetchSize(batchSize);
} else {
return jooqContext.dsl()
.select(field(idValueGetter))
.from(finalJoinClause)
.where(sqlWhereClause)
.orderBy(EntityTypeUtils.getSortFields(entityType, sortResults))
.fetchSize(batchSize);
}
return List.of();
}

private static SortField<Object> toSortField(EntityTypeDefaultSort entityTypeDefaultSort) {
Field<Object> field = field(entityTypeDefaultSort.getColumnName());
return entityTypeDefaultSort.getDirection() == EntityTypeDefaultSort.DirectionEnum.DESC ? field.desc() : field.asc();
}
}
2 changes: 1 addition & 1 deletion src/main/java/org/folio/fqm/repository/QueryResultsRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import java.util.List;
import java.util.UUID;

import static org.folio.fqm.utils.IdColumnUtils.RESULT_ID_FIELD;
import static org.folio.fqm.utils.EntityTypeUtils.RESULT_ID_FIELD;
import static org.jooq.impl.DSL.table;
import static org.jooq.impl.DSL.field;

Expand Down
Loading

0 comments on commit df3b89f

Please sign in to comment.