Skip to content

Commit

Permalink
fix(schema-compiler): fix FILTER_PARAMS to populate set and notSet fi…
Browse files Browse the repository at this point in the history 
…lters in cube's SQL query (#8937)

* fix(schema-compiler): fix FILTER_PARAMS to populate set and notSet filters in cube's SQL query

* Fix {DATE_OPERATORS}Where methods to return '1 = 1' when the required parameters are unavailable

* fix(ksql-driver): Kafka, list of brokers (#9009)

* docs: A few notes

* chore(cubesql): Do not call async Node functions while planning (#8793)

* Add postgress integration tests

---------

Co-authored-by: Konstantin Burkalev <[email protected]>
Co-authored-by: Dmitriy Rusov <[email protected]>
Co-authored-by: Igor Lukanin <[email protected]>
Co-authored-by: Alex Qyoun-ae <[email protected]>
  • Loading branch information
@sumeetgajjar @KSDaemon
@RusovDmitriy @igorlukanin @MazterQyou
5 people authored Dec 5, 2024
1 parent 8f45afa commit 6d82f18
Show file tree
Hide file tree
Showing 4 changed files with 291 additions and 16 deletions.
23 changes: 23 additions & 0 deletions packages/cubejs-schema-compiler/src/adapter/BaseFilter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ const dateTimeLocalURegex = /^\d\d\d\d-\d\d-\d\dT\d\d:\d\d:\d\d\.\d\d\d\d\d\d$/;
const dateRegex = /^\d\d\d\d-\d\d-\d\d$/;

export class BaseFilter extends BaseDimension {
public static readonly ALWAYS_TRUE: string = '1 = 1';

public readonly measure: any;

public readonly operator: any;
Expand Down Expand Up @@ -323,36 +325,57 @@ export class BaseFilter extends BaseDimension {

public inDateRangeWhere(column) {
const [from, to] = this.allocateTimestampParams();
if (!from || !to) {
return BaseFilter.ALWAYS_TRUE;
}
return this.query.timeRangeFilter(column, from, to);
}

public notInDateRangeWhere(column) {
const [from, to] = this.allocateTimestampParams();
if (!from || !to) {
return BaseFilter.ALWAYS_TRUE;
}
return this.query.timeNotInRangeFilter(column, from, to);
}

public onTheDateWhere(column) {
const [from, to] = this.allocateTimestampParams();
if (!from || !to) {
return BaseFilter.ALWAYS_TRUE;
}
return this.query.timeRangeFilter(column, from, to);
}

public beforeDateWhere(column) {
const [before] = this.allocateTimestampParams();
if (!before) {
return BaseFilter.ALWAYS_TRUE;
}
return this.query.beforeDateFilter(column, before);
}

public beforeOrOnDateWhere(column) {
const [before] = this.allocateTimestampParams();
if (!before) {
return BaseFilter.ALWAYS_TRUE;
}
return this.query.beforeOrOnDateFilter(column, before);
}

public afterDateWhere(column) {
const [after] = this.allocateTimestampParams();
if (!after) {
return BaseFilter.ALWAYS_TRUE;
}
return this.query.afterDateFilter(column, after);
}

public afterOrOnDateWhere(column) {
const [after] = this.allocateTimestampParams();
if (!after) {
return BaseFilter.ALWAYS_TRUE;
}
return this.query.afterOrOnDateFilter(column, after);
}

Expand Down
29 changes: 14 additions & 15 deletions packages/cubejs-schema-compiler/src/adapter/BaseQuery.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3805,7 +3805,7 @@ export class BaseQuery {

static renderFilterParams(filter, filterParamArgs, allocateParam, newGroupFilter, aliases) {
if (!filter) {
return '1 = 1';
return BaseFilter.ALWAYS_TRUE;
}

if (filter.operator === 'and' || filter.operator === 'or') {
Expand All @@ -3830,21 +3830,20 @@ export class BaseQuery {
if (!filterParamArg) {
throw new Error(`FILTER_PARAMS arg not found for ${filter.measure || filter.dimension}`);
}
if (
filterParams && filterParams.length
) {
if (typeof filterParamArg.__column() === 'function') {
// eslint-disable-next-line prefer-spread
return filterParamArg.__column().apply(
null,
filterParams.map(allocateParam),
);
} else {
return filter.conditionSql(filterParamArg.__column());
}
} else {
return '1 = 1';

if (typeof filterParamArg.__column() !== 'function') {
return filter.conditionSql(filterParamArg.__column());
}

if (!filterParams || !filterParams.length) {
return BaseFilter.ALWAYS_TRUE;
}

// eslint-disable-next-line prefer-spread
return filterParamArg.__column().apply(
null,
filterParams.map(allocateParam),
);
}

filterGroupFunction() {
Expand Down
177 changes: 176 additions & 1 deletion packages/cubejs-schema-compiler/test/integration/postgres/sql-generation.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ describe('SQL Generation', () => {
cube('visitor_checkins_sources', {
sql: \`
select id, source from visitor_checkins WHERE \${FILTER_PARAMS.visitor_checkins_sources.source.filter('source')}
select id, visitor_id, source from visitor_checkins WHERE \${FILTER_PARAMS.visitor_checkins_sources.source.filter('source')}
\`,
rewriteQueries: true,
Expand All @@ -419,12 +419,22 @@ describe('SQL Generation', () => {
}
},
measures: {
count: {
type: 'count'
}
},
dimensions: {
id: {
type: 'number',
sql: 'id',
primaryKey: true
},
visitor_id: {
type: 'number',
sql: 'visitor_id'
},
source: {
type: 'string',
sql: 'source'
Expand Down Expand Up @@ -1897,6 +1907,171 @@ describe('SQL Generation', () => {
])
);

it(
'equals NULL filter',
() => runQueryTest({
measures: [
'visitor_checkins_sources.count'
],
dimensions: [
'visitor_checkins_sources.visitor_id'
],
timeDimensions: [],
timezone: 'America/Los_Angeles',
filters: [{
dimension: 'visitor_checkins_sources.source',
operator: 'equals',
values: [null]
}],
order: [{
id: 'visitor_checkins_sources.visitor_id'
}]
}, [
{
visitor_checkins_sources__visitor_id: 1,
visitor_checkins_sources__count: '2'
},
{
visitor_checkins_sources__visitor_id: 2,
visitor_checkins_sources__count: '2'
},
{
visitor_checkins_sources__visitor_id: 3,
visitor_checkins_sources__count: '1'
}
])
);

it(
'notSet(IS NULL) filter',
() => runQueryTest({
measures: [
'visitor_checkins_sources.count'
],
dimensions: [
'visitor_checkins_sources.visitor_id'
],
timeDimensions: [],
timezone: 'America/Los_Angeles',
filters: [{
dimension: 'visitor_checkins_sources.source',
operator: 'notSet',
}],
order: [{
id: 'visitor_checkins_sources.visitor_id'
}]
}, [
{
visitor_checkins_sources__visitor_id: 1,
visitor_checkins_sources__count: '2'
},
{
visitor_checkins_sources__visitor_id: 2,
visitor_checkins_sources__count: '2'
},
{
visitor_checkins_sources__visitor_id: 3,
visitor_checkins_sources__count: '1'
}
])
);

it(
'notEquals NULL filter',
() => runQueryTest({
measures: [
'visitor_checkins_sources.count'
],
dimensions: [
'visitor_checkins_sources.visitor_id'
],
timeDimensions: [],
timezone: 'America/Los_Angeles',
filters: [{
dimension: 'visitor_checkins_sources.source',
operator: 'notEquals',
values: [null]
}],
order: [{
id: 'visitor_checkins_sources.visitor_id'
}]
}, [
{
visitor_checkins_sources__visitor_id: 1,
visitor_checkins_sources__count: '1'
}
])
);

it(
'set(IS NOT NULL) filter',
() => runQueryTest({
measures: [
'visitor_checkins_sources.count'
],
dimensions: [
'visitor_checkins_sources.visitor_id'
],
timeDimensions: [],
timezone: 'America/Los_Angeles',
filters: [{
dimension: 'visitor_checkins_sources.source',
operator: 'set',
}],
order: [{
id: 'visitor_checkins_sources.visitor_id'
}]
}, [
{
visitor_checkins_sources__visitor_id: 1,
visitor_checkins_sources__count: '1'
}
])
);

it(
'source is notSet(IS NULL) "or" source is google filter',
() => runQueryTest({
measures: [
'visitor_checkins_sources.count'
],
dimensions: [
'visitor_checkins_sources.visitor_id'
],
timeDimensions: [],
timezone: 'America/Los_Angeles',
filters: [{
or: [
{
dimension: 'visitor_checkins_sources.source',
operator: 'notSet',
},
{
dimension: 'visitor_checkins_sources.source',
operator: 'equals',
values: ['google']
}
]
}],
order: [{
id: 'visitor_checkins_sources.visitor_id'
}]
}, [
{
visitor_checkins_sources__visitor_id: 1,
visitor_checkins_sources__count: '3'
},
{
visitor_checkins_sources__visitor_id: 2,
visitor_checkins_sources__count: '2'
},
{
visitor_checkins_sources__visitor_id: 3,
visitor_checkins_sources__count: '1'
}
])
);

it('year granularity', () => runQueryTest({
measures: [
'visitors.visitor_count'
Expand Down
78 changes: 78 additions & 0 deletions packages/cubejs-schema-compiler/test/unit/base-query.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1784,6 +1784,84 @@ describe('SQL Generation', () => {
expect(cubeSQL).toMatch(/\(\s*\(.*type\s*=\s*\$\d\$.*OR.*type\s*=\s*\$\d\$.*\)\s*AND\s*\(.*type\s*=\s*\$\d\$.*OR.*type\s*=\s*\$\d\$.*\)\s*\)/);
});

it('equals NULL filter', async () => {
await compilers.compiler.compile();
const query = new BaseQuery(compilers, {
measures: ['Order.count'],
filters: [
{
and: [
{
member: 'Order.type',
operator: 'equals',
values: [null],
},
]
}
],
});
const cubeSQL = query.cubeSql('Order');
expect(cubeSQL).toContain('where (((type IS NULL)))');
});

it('notSet(IS NULL) filter', async () => {
await compilers.compiler.compile();
const query = new BaseQuery(compilers, {
measures: ['Order.count'],
filters: [
{
and: [
{
member: 'Order.type',
operator: 'notSet',
},
]
}
],
});
const cubeSQL = query.cubeSql('Order');
expect(cubeSQL).toContain('where (((type IS NULL)))');
});

it('notEquals NULL filter', async () => {
await compilers.compiler.compile();
const query = new BaseQuery(compilers, {
measures: ['Order.count'],
filters: [
{
and: [
{
member: 'Order.type',
operator: 'notEquals',
values: [null],
},
]
}
],
});
const cubeSQL = query.cubeSql('Order');
expect(cubeSQL).toContain('where (((type IS NOT NULL)))');
});

it('set(IS NOT NULL) filter', async () => {
await compilers.compiler.compile();
const query = new BaseQuery(compilers, {
measures: ['Order.count'],
filters: [
{
and: [
{
member: 'Order.type',
operator: 'set',
},
]
}
],
});
const cubeSQL = query.cubeSql('Order');
expect(cubeSQL).toContain('where (((type IS NOT NULL)))');
});

it('propagate filter params from view into cube\'s query', async () => {
await compilers.compiler.compile();
const query = new BaseQuery(compilers, {
Expand Down

0 comments on commit 6d82f18

Please sign in to comment.