cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #390 from CartoDB/overviews-optimization

Browse Source 
Optimize overviews queries
This commit is contained in:
Javier Goizueta 2016-03-03 18:59:24 +01:00
commit 47dfdf964e
2 changed files with 49 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
lib/cartodb/utils/overviews_query_rewriter.js
View File

@ -74,14 +74,9 @@ function overviews_view_name(table) {
} }
// replace a table name in a query by anoter name // replace a table name in a query by anoter name
function replace_table_in_query(sql, old_table_name, new_table_name) { function replace_table_in_query(sql, old_table_name, replacement) {
var old_table = TableNameParser.parse(old_table_name); var old_table = TableNameParser.parse(old_table_name);
var new_table = TableNameParser.parse(new_table_name);
var old_table_ident = TableNameParser.table_identifier(old_table); var old_table_ident = TableNameParser.table_identifier(old_table);
var new_table_ident = TableNameParser.table_identifier(new_table);
// text that will be substituted by the table name pattern
var replacement = new_table_ident;
// regular expression prefix (beginning) to match a table name // regular expression prefix (beginning) to match a table name
function pattern_prefix(schema, identifier) { function pattern_prefix(schema, identifier) {
@ -127,12 +122,13 @@ function replace_table_in_query(sql, old_table_name, new_table_name) {
function overviews_query(query, overviews, zoom_level_expression) { function overviews_query(query, overviews, zoom_level_expression) {
var replaced_query = query; var replaced_query = query;
var sql = "WITH\n _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )"; var sql = "WITH\n _vovw_scale AS ( SELECT " + zoom_level_expression + " AS _vovw_z )";
var replacement;
for ( var table in overviews ) { for ( var table in overviews ) {
if (overviews.hasOwnProperty(table)) { if (overviews.hasOwnProperty(table)) {
var table_overviews = overviews[table]; var table_overviews = overviews[table];
var table_view = overviews_view_name(table); var table_view = overviews_view_name(table);
replaced_query = replace_table_in_query(replaced_query, table, table_view); replacement = "(\n" + overviews_view_for_table(table, table_overviews) + "\n ) AS " + table_view;
sql += ",\n " + table_view + " AS (\n" + overviews_view_for_table(table, table_overviews) + "\n )"; replaced_query = replace_table_in_query(replaced_query, table, replacement);
} }
} }
if ( replaced_query !== query ) { if ( replaced_query !== query ) {
@ -173,7 +169,7 @@ OverviewsQueryRewriter.prototype.query = function(query, data) {
OverviewsQueryRewriter.prototype.is_supported_query = function(sql) { OverviewsQueryRewriter.prototype.is_supported_query = function(sql) {
return !!sql.match( return !!sql.match(
/^\s*SELECT\s+[\*\.a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i /^\s*SELECT\s+[\*a-z0-9_,\s]+?\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*$/i
); );
}; };

118
test/unit/cartodb/overviews_query_rewriter.js
View File

@ -57,15 +57,15 @@ describe('Overviews query rewriter', function() {
} }
}; };
var overviews_sql = overviewsQueryRewriter.query(sql, data); var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\ var expected_sql = "\
WITH\ WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
_vovw_table1 AS (\ SELECT * FROM (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\ SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\ UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0\
)\ ) AS _vovw_table1\
SELECT * FROM _vovw_table1\
"; ";
assertSameSql(overviews_sql, expected_sql); assertSameSql(overviews_sql, expected_sql);
done(); done();
@ -83,13 +83,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data); var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\ var expected_sql = "\
WITH\ WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
_vovw_table1 AS (\ SELECT * FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\ UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\ ) AS _vovw_table1\
SELECT * FROM _vovw_table1\
"; ";
assertSameSql(overviews_sql, expected_sql); assertSameSql(overviews_sql, expected_sql);
done(); done();
@ -110,8 +109,8 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data); var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\ var expected_sql = "\
WITH\ WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
_vovw_table1 AS (\ SELECT * FROM (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\ SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\ UNION ALL\
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\ SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
@ -121,8 +120,7 @@ describe('Overviews query rewriter', function() {
SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\ SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3\
UNION ALL\ UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3\
)\ ) AS _vovw_table1\
SELECT * FROM _vovw_table1\
"; ";
assertSameSql(overviews_sql, expected_sql); assertSameSql(overviews_sql, expected_sql);
done(); done();
@ -142,8 +140,8 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data); var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\ var expected_sql = "\
WITH\ WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
_vovw_table1 AS (\ SELECT * FROM (\
SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\ SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0\
UNION ALL\ UNION ALL\
SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\ SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1\
@ -151,8 +149,7 @@ describe('Overviews query rewriter', function() {
SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6\ SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6\
UNION ALL\ UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6\
)\ ) AS _vovw_table1\
SELECT * FROM _vovw_table1\
"; ";
assertSameSql(overviews_sql, expected_sql); assertSameSql(overviews_sql, expected_sql);
done(); done();
@ -170,13 +167,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data); var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\ var expected_sql = "\
WITH\ WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
_vovw_table1 AS (\ SELECT * FROM (\
SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\ UNION ALL\
SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\ SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
)\ ) AS _vovw_table1\
SELECT * FROM _vovw_table1\
"; ";
assertSameSql(overviews_sql, expected_sql); assertSameSql(overviews_sql, expected_sql);
done(); done();
@ -194,13 +190,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data); var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\ var expected_sql = "\
WITH\ WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
_vovw_table1 AS (\ SELECT * FROM (\
SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\ UNION ALL\
SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\ SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2\
)\ ) AS _vovw_table1\
SELECT * FROM _vovw_table1\
"; ";
assertSameSql(overviews_sql, expected_sql); assertSameSql(overviews_sql, expected_sql);
@ -219,13 +214,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data); var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\ var expected_sql = "\
WITH\ WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
\"_vovw_table 1\" AS (\ SELECT * FROM (\
SELECT * FROM public.\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\ SELECT * FROM public.\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\ UNION ALL\
SELECT * FROM public.\"table 1\", _vovw_scale WHERE _vovw_z > 2\ SELECT * FROM public.\"table 1\", _vovw_scale WHERE _vovw_z > 2\
)\ ) AS \"_vovw_table 1\"\
SELECT * FROM \"_vovw_table 1\"\
"; ";
assertSameSql(overviews_sql, expected_sql); assertSameSql(overviews_sql, expected_sql);
done(); done();
@ -243,13 +237,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data); var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\ var expected_sql = "\
WITH\ WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
_vovw_table1 AS (\ SELECT * FROM (\
SELECT * FROM \"user-1\".table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ SELECT * FROM \"user-1\".table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\ UNION ALL\
SELECT * FROM \"user-1\".table1, _vovw_scale WHERE _vovw_z > 2\ SELECT * FROM \"user-1\".table1, _vovw_scale WHERE _vovw_z > 2\
)\ ) AS _vovw_table1\
SELECT * FROM _vovw_table1\
"; ";
assertSameSql(overviews_sql, expected_sql); assertSameSql(overviews_sql, expected_sql);
done(); done();
@ -269,13 +262,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data); var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\ var expected_sql = "\
WITH\ WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
\"_vovw_table 1\" AS (\ SELECT * FROM (\
SELECT * FROM \"user-1\".\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\ SELECT * FROM \"user-1\".\"table 1_ov2\", _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\ UNION ALL\
SELECT * FROM \"user-1\".\"table 1\", _vovw_scale WHERE _vovw_z > 2\ SELECT * FROM \"user-1\".\"table 1\", _vovw_scale WHERE _vovw_z > 2\
)\ ) AS \"_vovw_table 1\"\
SELECT * FROM \"_vovw_table 1\"\
"; ";
assertSameSql(overviews_sql, expected_sql); assertSameSql(overviews_sql, expected_sql);
done(); done();
@ -294,44 +286,19 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data); var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\ var expected_sql = "\
WITH\ WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
_vovw_table1 AS (\ SELECT column1, column2, column3 FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\ UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\ ) AS _vovw_table1\
SELECT column1, column2, column3 FROM _vovw_table1\
";
assertSameSql(overviews_sql, expected_sql);
done();
});
it('generates query using overviews for queries with selected columns and all columns', function(done){
var sql = "SELECT table1.*, column1, column2, column3 FROM table1";
var data = {
overviews: {
table1: {
2: { table: 'table1_ov2' }
}
}
};
var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\
WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\
_vovw_table1 AS (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\
SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1\
"; ";
assertSameSql(overviews_sql, expected_sql); assertSameSql(overviews_sql, expected_sql);
done(); done();
}); });
it('generates query using overviews for queries with a semicolon', function(done){ it('generates query using overviews for queries with a semicolon', function(done){
var sql = "SELECT table1.*, column1, column2, column3 FROM table1;"; var sql = "SELECT column1, column2, column3 FROM table1;";
var data = { var data = {
overviews: { overviews: {
table1: { table1: {
@ -340,22 +307,22 @@ describe('Overviews query rewriter', function() {
} }
}; };
var overviews_sql = overviewsQueryRewriter.query(sql, data); var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\ var expected_sql = "\
WITH\ WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
_vovw_table1 AS (\ SELECT column1, column2, column3 FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\ UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\ ) AS _vovw_table1;\
SELECT _vovw_table1.*, column1, column2, column3 FROM _vovw_table1;\
"; ";
assertSameSql(overviews_sql, expected_sql); assertSameSql(overviews_sql, expected_sql);
done(); done();
}); });
it('generates query using overviews for queries with extra whitespace', function(done){ it('generates query using overviews for queries with extra whitespace', function(done){
var sql = " SELECT table1.* , column1,column2, column3 FROM table1 "; var sql = " SELECT column1,column2, column3 FROM table1 ";
var data = { var data = {
overviews: { overviews: {
table1: { table1: {
@ -366,13 +333,12 @@ describe('Overviews query rewriter', function() {
var overviews_sql = overviewsQueryRewriter.query(sql, data); var overviews_sql = overviewsQueryRewriter.query(sql, data);
var expected_sql = "\ var expected_sql = "\
WITH\ WITH\
_vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z ),\ _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )\
_vovw_table1 AS (\ SELECT column1,column2, column3 FROM (\
SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\ SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2\
UNION ALL\ UNION ALL\
SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\ SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2\
)\ ) AS _vovw_table1\
SELECT _vovw_table1.* , column1,column2, column3 FROM _vovw_table1\
"; ";
assertSameSql(overviews_sql, expected_sql); assertSameSql(overviews_sql, expected_sql);
done(); done();
@ -422,6 +388,10 @@ describe('Overviews query rewriter', function() {
overviews_sql = overviewsQueryRewriter.query(sql, data); overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql); assert.equal(overviews_sql, sql);
sql = "SELECT table1.*, column1, column2, column3 FROM table1";
overviews_sql = overviewsQueryRewriter.query(sql, data);
assert.equal(overviews_sql, sql);
done(); done();
}); });