cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change form of overviews queries so they can be optimized

Browse Source 
The PostgreSQL planner wasn't applying the spatial filtering of
tile bounds to the queries efficiently.
This commit is contained in:
Javier Goizueta 2016-03-02 19:25:08 +01:00
parent 2a789b5a5b
commit 4e40a61795
2 changed files with 49 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
lib/cartodb/utils/overviews_query_rewriter.js
View File

@ -74,14 +74,9 @@ function overviews_view_name(table) {
}
// replace a table name in a query by anoter name
function replace_table_in_query(sql, old_table_name, new_table_name) {
function replace_table_in_query(sql, old_table_name, replacement) {
var old_table = TableNameParser.parse(old_table_name);
var new_table = TableNameParser.parse(new_table_name);
var old_table_ident = TableNameParser.table_identifier(old_table);
var new_table_ident = TableNameParser.table_identifier(new_table);
// text that will be substituted by the table name pattern
var replacement = new_table_ident;
// regular expression prefix (beginning) to match a table name
function pattern_prefix(schema, identifier) {
@ -127,12 +122,13 @@ function replace_table_in_query(sql, old_table_name, new_table_name) {
function overviews_query(query, overviews, zoom_level_expression) {
var replaced_query = query;
var sql = "WITH\n _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
var replacement;
for ( var table in overviews ) {
if (overviews.hasOwnProperty(table)) {
var table_overviews = overviews[table];
var table_view = overviews_view_name(table);
replaced_query = replace_table_in_query(replaced_query, table, table_view);
sql += ",\n " + table_view + " AS (\n" + overviews_view_for_table(table, table_overviews) + "\n )";
replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n ) AS " + table_view;
replaced_query = replace_table_in_query(replaced_query, table, replacement);
}
}
if ( replaced_query !== query ) {
@ -173,7 +169,7 @@ OverviewsQueryRewriter.prototype.query = function(query, data) {
OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
return !!sql.match(
/^\s*SELECT\s+[\*\.a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
/^\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
);
};

118
test/unit/cartodb/overviews_query_rewriter.js
View File

@ -57,15 +57,15 @@ describe('Overviews query rewriter', function() {
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -83,13 +83,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -110,8 +109,8 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
@ -121,8 +120,7 @@ describe('Overviews query rewriter', function() {
SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -142,8 +140,8 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
@ -151,8 +149,7 @@ describe('Overviews query rewriter', function() {
SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -170,13 +167,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -194,13 +190,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
@ -219,13 +214,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
\"_vovw_table 1\" AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM public.\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM public.\"table 1\", _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM \"_vovw_table 1\"\
) AS \"_vovw_table 1\"\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -243,13 +237,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM \"user-1\".table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM \"user-1\".table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -269,13 +262,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
\"_vovw_table 1\" AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT * FROM (\
SELECT * FROM \"user-1\".\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM \"user-1\".\"table 1\", _vovw_scale WHERE _vovw_z > 2\
)\
SELECT * FROM \"_vovw_table 1\"\
) AS \"_vovw_table 1\"\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -294,44 +286,19 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT column1, column2, column3 FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT column1, column2, column3 FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with selected columns and all columns', function(done){
var sql = "SELECT table1.*, column1, column2, column3 FROM table1";
var data = {
overviews: {
table1: {
2: { table: 'table1_ov2' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with a semicolon', function(done){
var sql = "SELECT table1.*, column1, column2, column3 FROM table1;";
var sql = "SELECT column1, column2, column3 FROM table1;";
var data = {
overviews: {
table1: {
@ -340,22 +307,22 @@ describe('Overviews query rewriter', function() {
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT column1, column2, column3 FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1;\
) AS _vovw_table1;\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with extra whitespace', function(done){
var sql = " SELECT table1.* , column1,column2, column3 FROM table1 ";
var sql = " SELECT column1,column2, column3 FROM table1 ";
var data = {
overviews: {
table1: {
@ -366,13 +333,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
SELECT column1,column2, column3 FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT _vovw_table1.* , column1,column2, column3 FROM _vovw_table1\
) AS _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
@ -422,6 +388,10 @@ describe('Overviews query rewriter', function() {
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
sql = "SELECT table1.*, column1, column2, column3 FROM table1";
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
done();
});