cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #204 from CartoDB/CDB-3686

Browse Source 
Configurable QueryTablesAPI to call directly postgresql
This commit is contained in:
Raul Ochoa 2014-08-14 13:44:07 +02:00
commit c52c245b9d
8 changed files with 147 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
NEWS.md
View File

@ -1,6 +1,10 @@
1.15.1 -- 2014-mm-dd
--------------------
New features:
- Configurable QueryTablesAPI to call directly postgresql using cartodb-psql
or to keep using a request to the SQL API
1.15.0 -- 2014-08-13
--------------------
Enhancements:
@ -18,8 +22,10 @@ Enhancements:
- SQL API requests moved to its own entity
New features:
- Affected tables and last updated time for a query are performed in a single request to the SQL API
- Allow specifying the tile format, upgrades windshaft and grainstore dependencies for this matter
- Affected tables and last updated time for a query are performed in a single
request to the SQL API
- Allow specifying the tile format, upgrades windshaft and grainstore
dependencies for this matter
1.13.1 -- 2014-08-04

103
lib/cartodb/api/query_tables_api.js
View File

@ -1,4 +1,5 @@
var sqlApi = require('../sql/sql_api');
var sqlApi = require('../sql/sql_api'),
PSQL = require('cartodb-psql');
function QueryTablesApi() {
}
@ -33,7 +34,7 @@ QueryTablesApi.prototype.getLastUpdatedTime = function (username, api_key, table
});
};
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, api_key, sql, callback) {
QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, options, sql, callback) {
// Replace mapnik tokens
sql = sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
@ -45,20 +46,32 @@ QueryTablesApi.prototype.getAffectedTablesInQuery = function (username, api_key,
sql = 'SELECT CDB_QueryTables($windshaft$' + sql + '$windshaft$)';
// call sql api
sqlApi.query(username, api_key, sql, function(err, rows){
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not fetch source tables: ' + msg));
return;
}
var qtables = rows[0].cdb_querytables;
var tableNames = qtables.split(/^\{(.*)\}$/)[1];
tableNames = tableNames ? tableNames.split(',') : [];
callback(null, tableNames);
});
if (shouldQueryPostgresDirectly()) {
var psql = new PSQL(options);
psql.query(sql, function(err, resultSet) {
var rows = resultSet.rows || [];
handleAffectedTablesInQueryRows(err, rows, callback);
});
} else {
sqlApi.query(username, options.api_key, sql, function(err, rows) {
handleAffectedTablesInQueryRows(err, rows, callback);
});
}
};
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, api_key, sql, callback) {
function handleAffectedTablesInQueryRows(err, rows, callback) {
if (err){
var msg = err.message ? err.message : err;
callback(new Error('could not fetch source tables: ' + msg));
return;
}
var qtables = rows[0].cdb_querytables;
var tableNames = qtables.split(/^\{(.*)\}$/)[1];
tableNames = tableNames ? tableNames.split(',') : [];
callback(null, tableNames);
}
QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (username, options, sql, callback) {
sql = sql
.replace(affectedTableRegexCache.bbox, 'ST_MakeEnvelope(0,0,0,0)')
.replace(affectedTableRegexCache.pixel_width, '1')
@ -66,30 +79,48 @@ QueryTablesApi.prototype.getAffectedTablesAndLastUpdatedTime = function (usernam
;
var query = [
'SELECT',
'CDB_QueryTables($windshaft$' + sql + '$windshaft$) as tablenames,',
'EXTRACT(EPOCH FROM max(updated_at)) as max',
'WITH querytables AS (SELECT * FROM CDB_QueryTables($windshaft$' + sql + '$windshaft$) as tablenames)',
'SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max',
'FROM CDB_TableMetadata m',
'WHERE m.tabname = any (CDB_QueryTables($windshaft$' + sql + '$windshaft$)::regclass[])'
'WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])'
].join(' ');
sqlApi.query(username, api_key, query, function(err, rows){
if (err || rows.length === 0) {
var msg = err.message ? err.message : err;
callback(new Error('could not fetch affected tables and last updated time: ' + msg));
return;
}
var result = rows[0];
var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
tableNames = tableNames ? tableNames.split(',') : [];
var lastUpdatedTime = result.max || 0;
callback(null, {
affectedTables: tableNames,
lastUpdatedTime: lastUpdatedTime * 1000
if (shouldQueryPostgresDirectly()) {
var psql = new PSQL(options);
psql.query(query, function(err, resultSet) {
var rows = resultSet.rows || [];
handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback);
});
});
} else {
sqlApi.query(username, options.api_key, query, function(err, rows) {
handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback);
});
}
};
function handleAffectedTablesAndLastUpdatedTimeRows(err, rows, callback) {
if (err || rows.length === 0) {
var msg = err.message ? err.message : err;
callback(new Error('could not fetch affected tables and last updated time: ' + msg));
return;
}
var result = rows[0];
var tableNames = result.tablenames.split(/^\{(.*)\}$/)[1];
tableNames = tableNames ? tableNames.split(',') : [];
var lastUpdatedTime = result.max || 0;
callback(null, {
affectedTables: tableNames,
lastUpdatedTime: lastUpdatedTime * 1000
});
}
function shouldQueryPostgresDirectly() {
return global.environment
&& global.environment.enabledFeatures
&& global.environment.enabledFeatures.cdbQueryTablesFromPostgres;
}

18
lib/cartodb/server_options.js
View File

@ -192,7 +192,14 @@ module.exports = function(){
if ( req.profiler ) req.profiler.done('getSignerMapKey');
key = data;
}
queryTablesApi.getAffectedTablesInQuery(user, key, sql, this); // in addCacheChannel
queryTablesApi.getAffectedTablesInQuery(user, {
user: req.params.dbuser,
pass: req.params.dbpass,
host: req.params.dbhost,
port: req.params.dbport,
dbname: req.params.dbname,
api_key: key
}, sql, this); // in addCacheChannel
},
function finish(err, data) {
next(err,data);
@ -317,7 +324,14 @@ module.exports = function(){
Step(
function getAffectedTablesAndLastUpdatedTime() {
queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, key, sql, this);
queryTablesApi.getAffectedTablesAndLastUpdatedTime(usr, {
user: req.params.dbuser,
pass: req.params.dbpass,
host: req.params.dbhost,
port: req.params.dbport,
dbname: req.params.dbname,
api_key: key
}, sql, this);
},
function handleAffectedTablesAndLastUpdatedTime(err, result) {
if (req.profiler) req.profiler.done('queryTablesAndLastUpdated');

23
npm-shrinkwrap.json generated
View File

@ -937,6 +937,29 @@
"version": "0.9.0",
"from": "git://github.com/CartoDB/node-cartodb-redis.git#0.9.0"
},
"cartodb-psql": {
"version": "0.3.1",
"from": "git://github.com/CartoDB/node-cartodb-psql.git#0.3.1",
"dependencies": {
"pg": {
"version": "2.6.2",
"dependencies": {
"generic-pool": {
"version": "2.0.3"
},
"buffer-writer": {
"version": "1.0.0"
}
}
},
"step": {
"version": "0.0.5"
},
"underscore": {
"version": "1.6.0"
}
}
},
"redis-mpool": {
"version": "0.1.0",
"from": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",

1
package.json
View File

@ -28,6 +28,7 @@
"step": "~0.0.5",
"request": "2.9.202",
"cartodb-redis": "git://github.com/CartoDB/node-cartodb-redis.git#0.9.0",
"cartodb-psql": "git://github.com/CartoDB/node-cartodb-psql.git#0.3.1",
"redis-mpool": "https://github.com/CartoDB/node-redis-mpool/tarball/0.1.0",
"mapnik": "http://github.com/Vizzuality/node-mapnik/tarball/0.7.26-cdb1",
"lzma": "~1.2.3",

36
test/acceptance/multilayer.js
View File

@ -23,7 +23,11 @@ serverOptions = ServerOptions();
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
suite('multilayer', function() {
[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
var redis_client = redis.createClient(global.environment.redis.port);
var sqlapi_server;
@ -112,13 +116,12 @@ suite('multilayer', function() {
var jsonquery = cc.substring(dbname.length+1);
var sentquery = JSON.parse(jsonquery);
var expectedQuery = [layergroup.layers[0].options.sql, ';', layergroup.layers[1].options.sql].join('');
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
assert.equal(sentquery.q, 'WITH querytables AS (SELECT * FROM CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$) as tablenames, EXTRACT(EPOCH FROM max(updated_at)) as max'
+ '$windshaft$) as tablenames)'
+ ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
+ ' FROM CDB_TableMetadata m'
+ ' WHERE m.tabname = any (CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$)::regclass[])');
+ ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
assert.imageEqualsFile(res.body, 'test/fixtures/test_table_0_0_0_multilayer1.png', IMAGE_EQUALS_HIGHER_TOLERANCE_PER_MIL,
function(err, similarity) {
@ -395,13 +398,12 @@ suite('multilayer', function() {
.replace(/!bbox!/g, 'ST_MakeEnvelope(0,0,0,0)')
.replace(/!pixel_width!/g, '1')
.replace(/!pixel_height!/g, '1');
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
assert.equal(sentquery.q, 'WITH querytables AS (SELECT * FROM CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$) as tablenames, EXTRACT(EPOCH FROM max(updated_at)) as max'
+ '$windshaft$) as tablenames)'
+ ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
+ ' FROM CDB_TableMetadata m'
+ ' WHERE m.tabname = any (CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$)::regclass[])');
+ ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err, similarity) {
@ -433,13 +435,12 @@ suite('multilayer', function() {
.replace('!bbox!', 'ST_MakeEnvelope(0,0,0,0)')
.replace('!pixel_width!', '1')
.replace('!pixel_height!', '1');
assert.equal(sentquery.q, 'SELECT CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$) as tablenames, EXTRACT(EPOCH FROM max(updated_at)) as max'
+ ' FROM CDB_TableMetadata m'
+ ' WHERE m.tabname = any (CDB_QueryTables($windshaft$'
assert.equal(sentquery.q, 'WITH querytables AS (SELECT * FROM CDB_QueryTables($windshaft$'
+ expectedQuery
+ '$windshaft$)::regclass[])');
+ '$windshaft$) as tablenames)'
+ ' SELECT (SELECT tablenames FROM querytables), EXTRACT(EPOCH FROM max(updated_at)) as max'
+ ' FROM CDB_TableMetadata m'
+ ' WHERE m.tabname = any ((SELECT tablenames from querytables)::regclass[])');
assert.imageEqualsFile(res.body, 'test/fixtures/test_multilayer_bbox.png', IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err, similarity) {
@ -1345,3 +1346,4 @@ suite('multilayer', function() {
});
});

8
test/acceptance/server.js
View File

@ -19,7 +19,12 @@ var serverOptions = require(__dirname + '/../../lib/cartodb/server_options')();
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
suite('server', function() {
[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
suite('multilayer:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
var redis_client = redis.createClient(global.environment.redis.port);
var sqlapi_server;
@ -1389,3 +1394,4 @@ suite('server', function() {
});
});

7
test/acceptance/templates.js
View File

@ -25,7 +25,11 @@ var serverOptions = ServerOptions();
var server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
suite('template_api', function() {
[true, false].forEach(function(cdbQueryTablesFromPostgresEnabledValue) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: cdbQueryTablesFromPostgresEnabledValue};
suite('template_api:postgres=' + cdbQueryTablesFromPostgresEnabledValue, function() {
var redis_client = redis.createClient(global.environment.redis.port);
var sqlapi_server;
@ -1948,3 +1952,4 @@ suite('template_api', function() {
});
});