cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'release/staging'

Browse Source
This commit is contained in:
Luis Bosque 2012-11-15 13:49:32 +01:00
commit 731cbd66f8
13 changed files with 1369 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
NEWS.md
View File

@ -1,3 +1,31 @@
1.3.1 (DD/MM/YY)
-----
* Support 'format' and 'filename' params in POST
* Fix oAuth bug introduced by 'skipfields' param in 1.3.0 (#69)
1.3.0 (DD/MM/YY)
-----
* Support for specifying a filename for exports (#64)
* Support for specifying a list of fields to skip from output (#63)
* Add 'cache_policy' parameter (#62)
1.2.1 (DD/MM/YY)
-----
* Added timeout default to 600 miliseconds in cluster.js
1.2.0 (DD/MM/YY)
-----
* New output formats:
* ESRI Shapefile (format=shp)
* SVG (format=svg)
* KML (format=kml)
* Advertise header presence in CSV Content-Type
* Fix CSV output with no rows (#60)
* Use "attachment" Content-Disposition for all output formats (#61)
* Only use last format parameter when multiple are requested
* Return a 400 response on unsupported format request
* Added X-Prototype-Version, X-CSRF-Token to Access-Control-Allow-Headers
1.1.0 (30/10/12)
-----
* Fixed problem in cluster2 with pidfile name

441
app/controllers/app.js
View File

@ -14,6 +14,9 @@
// eg. vizzuality.cartodb.com/api/v1/?sql=SELECT * from my_table
//
//
var path = require('path');
var express = require('express')
, app = express.createServer(
express.logger({
@ -23,6 +26,10 @@ var express = require('express')
, Step = require('step')
, csv = require('csv')
, crypto = require('crypto')
, fs = require('fs')
, zlib = require('zlib')
, util = require('util')
, spawn = require('child_process').spawn
, Meta = require(global.settings.app_root + '/app/models/metadata')
, oAuth = require(global.settings.app_root + '/app/models/oauth')
, PSQL = require(global.settings.app_root + '/app/models/psql')
@ -38,26 +45,64 @@ app.all('/api/v1/sql', function(req, res) { handleQuery(req, res) } );
app.all('/api/v1/sql.:f', function(req, res) { handleQuery(req, res) } );
app.get('/api/v1/cachestatus', function(req, res) { handleCacheStatus(req, res) } );
// Return true of the given query may write to the database
//
// NOTE: this is a fuzzy check, the return could be true even
// if the query doesn't really write anything.
// But you can be pretty sure of a false return.
//
function queryMayWrite(sql) {
var mayWrite = false;
var pattern = RegExp("(alter|insert|update|delete|create|drop)", "i");
if ( pattern.test(sql) ) {
mayWrite = true;
}
return mayWrite;
}
// Return database username from user_id
// NOTE: a "null" user_id is a request to use the public user
function userid_to_dbuser(user_id) {
if ( _.isString(user_id) )
return _.template(global.settings.db_user, {user_id: user_id});
return "publicuser" // FIXME: make configurable
};
function sanitize_filename(filename) {
filename = path.basename(filename, path.extname(filename));
filename = filename.replace(/[;()\[\]<>'"\s]/g, '_');
//console.log("Sanitized: " + filename);
return filename;
}
// request handlers
function handleQuery(req, res) {
var supportedFormats = ['json', 'geojson', 'csv', 'svg', 'shp', 'kml'];
var svg_width = 1024.0;
var svg_height = 768.0;
// extract input
var body = (req.body) ? req.body : {};
var sql = req.query.q || body.q; // HTTP GET and POST store in different vars
var api_key = req.query.api_key || body.api_key;
var database = req.query.database; // TODO: Depricate
var database = req.query.database; // TODO: Deprecate
var limit = parseInt(req.query.rows_per_page);
var offset = parseInt(req.query.page);
var format = req.query.format;
var dp = req.query.dp; // decimal point digits (defaults to 6)
var gn = "the_geom"; // TODO: read from configuration file
var svg_width = 1024.0;
var svg_height = 768.0;
var requestedFormat = req.query.format || body.format;
var format = _.isArray(requestedFormat) ? _.last(requestedFormat) : requestedFormat;
var requestedFilename = req.query.filename || body.filename
var filename = requestedFilename;
var requestedSkipfields = req.query.skipfields || body.skipfields;
var skipfields = requestedSkipfields ? requestedSkipfields.split(',') : [];
var dp = req.query.dp || body.dp; // decimal point digits (defaults to 6)
var gn = "the_geom"; // TODO: read from configuration file
var user_id;
// sanitize and apply defaults to input
dp = (dp === "" || _.isUndefined(dp)) ? '6' : dp;
format = (format === "" || _.isUndefined(format)) ? null : format.toLowerCase();
format = (format === "" || _.isUndefined(format)) ? 'json' : format.toLowerCase();
filename = (filename === "" || _.isUndefined(filename)) ? 'cartodb-query' : sanitize_filename(filename);
sql = (sql === "" || _.isUndefined(sql)) ? null : sql;
database = (database === "" || _.isUndefined(database)) ? null : database;
limit = (_.isNumber(limit)) ? limit : null;
@ -67,6 +112,10 @@ function handleQuery(req, res) {
var start = new Date().getTime();
try {
if ( -1 === supportedFormats.indexOf(format) )
throw new Error("Invalid format: " + format);
if (!_.isString(sql)) throw new Error("You must indicate a sql query");
// initialise MD5 key of sql for cache lookups
@ -75,6 +124,8 @@ function handleQuery(req, res) {
// placeholder for connection
var pg;
var authenticated;
// 1. Get database from redis via the username stored in the host header subdomain
// 2. Run the request through OAuth to get R/W user id if signed
// 3. Get the list of tables affected by the query
@ -108,17 +159,20 @@ function handleQuery(req, res) {
oAuth.verifyRequest(req, this);
}
},
function queryExplain(err, user_id){
function queryExplain(err, data){
if (err) throw err;
user_id = data;
// store postgres connection
pg = new PSQL(user_id, database, limit, offset);
authenticated = ! _.isNull(user_id);
// get all the tables from Cache or SQL
if (!_.isNull(tableCache[sql_md5]) && !_.isUndefined(tableCache[sql_md5])){
tableCache[sql_md5].hits++;
return true;
} else{
pg.query("SELECT CDB_QueryTables($quotesql$" + sql + "$quotesql$)", this);
} else {
pg.query("SELECT CDB_QueryTables($quotesql$" + sql + "$quotesql$)", this);
}
},
function queryResult(err, result){
@ -127,15 +181,18 @@ function handleQuery(req, res) {
// store explain result in local Cache
if (_.isUndefined(tableCache[sql_md5])){
tableCache[sql_md5] = result;
tableCache[sql_md5].may_write = queryMayWrite(sql);
tableCache[sql_md5].hits = 1; //initialise hit counter
}
// TODO: refactor formats to external object
if (format === 'geojson'){
sql = ['SELECT *, ST_AsGeoJSON(the_geom,',dp,') as the_geom FROM (', sql, ') as foo'].join("");
} else if (format === 'svg'){
} else if (format === 'shp') {
return null;
} else if (format === 'svg') {
var svg_ratio = svg_width/svg_height;
sql = 'WITH source AS ( ' + sql + '), extent AS ( '
sql = 'WITH source AS ( ' + sql + '), extent AS ( '
+ ' SELECT ST_Extent(' + gn + ') AS e FROM source '
+ '), extent_info AS ( SELECT e, '
+ 'st_xmin(e) as ex0, st_ymax(e) as ey0, '
@ -161,23 +218,40 @@ function handleQuery(req, res) {
function setHeaders(err, result){
if (err) throw err;
// configure headers for geojson/CSV
res.header("Content-Disposition", getContentDisposition(format));
// configure headers for given format
var use_inline = !requestedFormat && !requestedFilename;
res.header("Content-Disposition", getContentDisposition(format, filename, use_inline));
res.header("Content-Type", getContentType(format));
// allow cross site post
setCrossDomain(res);
// set cache headers
res.header('Last-Modified', new Date().toUTCString());
res.header('Cache-Control', 'no-cache,max-age=3600,must-revalidate, public');
res.header('X-Cache-Channel', generateCacheKey(database, tableCache[sql_md5]));
res.header('X-Cache-Channel', generateCacheKey(database, tableCache[sql_md5], authenticated));
var cache_policy = req.query.cache_policy;
if ( cache_policy == 'persist' ) {
res.header('Cache-Control', 'public,max-age=31536000'); // 1 year
} else {
// TODO: set ttl=0 when tableCache[sql_md5].may_write is true ?
var ttl = 3600;
res.header('Last-Modified', new Date().toUTCString());
res.header('Cache-Control', 'no-cache,max-age='+ttl+',must-revalidate,public');
}
return result;
},
function packageResults(err, result){
if (err) throw err;
if ( skipfields.length ){
for ( var i=0; i<result.rows.length; ++i ) {
for ( var j=0; j<skipfields.length; ++j ) {
delete result.rows[i][skipfields[j]];
}
}
}
// TODO: refactor formats to external object
if (format === 'geojson'){
toGeoJSON(result, res, this);
@ -185,22 +259,25 @@ function handleQuery(req, res) {
toSVG(result.rows, gn, this);
} else if (format === 'csv'){
toCSV(result, res, this);
} else {
// TODO: error out if 'format' resolves to an unsupported format !
} else if ( format === 'shp'){
toSHP(database, user_id, gn, sql, skipfields, filename, res, this);
} else if ( format === 'kml'){
toKML(database, user_id, gn, sql, skipfields, res, this);
} else if ( format === 'json'){
var end = new Date().getTime();
var json_result = {'time' : (end - start)/1000};
json_result.total_rows = result.rowCount;
json_result.rows = result.rows;
return json_result;
}
else throw new Error("Unexpected format in packageResults: " + format);
},
function sendResults(err, out){
if (err) throw err;
// return to browser
res.send(out);
if ( out ) res.send(out);
},
function errorHandle(err, result){
handleException(err, res);
@ -281,7 +358,7 @@ function toSVG(rows, gn, callback){
polys.push('<path d="' + g + '" />');
}
if ( ! bbox ) {
if ( ! bbox ) {
// Parse layer extent: "BOX(x y, X Y)"
// NOTE: the name of the extent field is
// determined by the same code adding the
@ -291,9 +368,9 @@ function toSVG(rows, gn, callback){
bbox = bbox.match(/BOX\(([^ ]*) ([^ ,]*),([^ ]*) ([^)]*)\)/);
bbox = {
xmin: parseFloat(bbox[1]),
ymin: parseFloat(bbox[2]),
ymin: parseFloat(bbox[2]),
xmax: parseFloat(bbox[3]),
ymax: parseFloat(bbox[4])
ymax: parseFloat(bbox[4])
};
}
});
@ -320,7 +397,7 @@ function toSVG(rows, gn, callback){
bbox.width = bbox.xmax - bbox.xmin;
bbox.height = bbox.ymax - bbox.ymin;
root_tag += 'viewBox="' + bbox.xmin + ' ' + (-bbox.ymax) + ' '
+ bbox.width + ' ' + bbox.height + '" ';
+ bbox.width + ' ' + bbox.height + '" ';
}
root_tag += 'style="fill-opacity:' + fill_opacity
+ '; stroke:' + stroke_color
@ -343,7 +420,7 @@ function toSVG(rows, gn, callback){
function toCSV(data, res, callback){
try{
// pull out keys for column headers
var columns = _.keys(data.rows[0]);
var columns = data.rows.length ? _.keys(data.rows[0]) : [];
// stream the csv out over http
csv()
@ -355,7 +432,289 @@ function toCSV(data, res, callback){
}
}
function getContentDisposition(format){
// Internal function usable by all OGR-driven outputs
function toOGR(dbname, user_id, gcol, sql, skipfields, res, out_format, out_filename, callback) {
var ogr2ogr = 'ogr2ogr'; // FIXME: make configurable
var dbhost = global.settings.db_host;
var dbport = global.settings.db_port;
var dbuser = userid_to_dbuser(user_id);
var dbpass = ''; // turn into a parameter..
var tmpdir = '/tmp'; // FIXME: make configurable
var columns = [];
Step (
function fetchColumns() {
var colsql = 'SELECT * FROM (' + sql + ') as _cartodbsqlapi LIMIT 1';
var pg = new PSQL(user_id, dbname, 1, 0);
pg.query(colsql, this);
},
function spawnDumper(err, result) {
if (err) throw err;
if ( ! result.rows.length )
throw new Error("Query returns no rows");
// Skip system columns
for (var k in result.rows[0]) {
if ( skipfields.indexOf(k) != -1 ) continue;
if ( k == "the_geom_webmercator" ) continue;
columns.push('"' + k + '"');
}
//console.log(columns.join(','));
var next = this;
sql = 'SELECT ' + columns.join(',')
+ ' FROM (' + sql + ') as _cartodbsqlapi';
var child = spawn(ogr2ogr, [
'-f', out_format,
out_filename,
"PG:host=" + dbhost
+ " user=" + dbuser
+ " dbname=" + dbname
+ " password=" + dbpass
+ " tables=fake" // trick to skip query to geometry_columns
+ "",
'-sql', sql
]);
/*
console.log(['ogr2ogr',
'-f', out_format,
out_filename,
"'PG:host=" + dbhost
+ " user=" + dbuser
+ " dbname=" + dbname
+ " password=" + dbpass
+ " tables=fake" // trick to skip query to geometry_columns
+ "'",
'-sql "', sql, '"'].join(' '));
*/
var stdout = '';
child.stdout.on('data', function(data) {
stdout += data;
//console.log('stdout: ' + data);
});
var stderr = '';
child.stderr.on('data', function(data) {
stderr += data;
console.log('ogr2ogr stderr: ' + data);
});
child.on('exit', function(code) {
if ( code ) {
next(new Error("ogr2ogr returned an error (error code " + code + ")\n" + stderr));
} else {
next(null);
}
});
},
function finish(err) {
callback(err);
}
);
}
function toSHP(dbname, user_id, gcol, sql, skipfields, filename, res, callback) {
var zip = 'zip'; // FIXME: make configurable
var tmpdir = '/tmp'; // FIXME: make configurable
var outdirpath = tmpdir + '/sqlapi-shapefile-' + generateMD5(sql);
var shapefile = outdirpath + '/' + filename + '.shp';
// TODO: following tests:
// - fetch with no auth [done]
// - fetch with auth [done]
// - fetch same query concurrently
// - fetch query with no "the_geom" column
// TODO: Check if the file already exists
// (should mean another export of the same query is in progress)
Step (
function createOutDir() {
fs.mkdir(outdirpath, 0777, this);
},
function spawnDumper(err) {
if ( err ) {
if ( err.code == 'EEXIST' ) {
// TODO: this could mean another request for the same
// resource is in progress, in which case we might want
// to queue the response to after it's completed...
console.log("Reusing existing SHP output directory for query: " + sql);
} else {
throw err;
}
}
toOGR(dbname, user_id, gcol, sql, skipfields, res, 'ESRI Shapefile', shapefile, this);
},
function zipAndSendDump(err) {
if ( err ) throw err;
var next = this;
var dir = outdirpath;
var zipfile = dir + '.zip';
var child = spawn(zip, ['-qrj', '-', dir ]);
child.stdout.on('data', function(data) {
res.write(data);
});
var stderr = '';
child.stderr.on('data', function(data) {
stderr += data;
console.log('zip stderr: ' + data);
});
child.on('exit', function(code) {
if (code) {
res.statusCode = 500;
//res.send(stderr);
}
//console.log("Zip complete, zip return code was " + code);
next(null);
});
},
function cleanupDir(topError) {
var next = this;
//console.log("Cleaning up " + outdirpath);
// Unlink the dir content
var unlinkall = function(dir, files, finish) {
var f = files.shift();
if ( ! f ) { finish(null); return; }
var fn = dir + '/' + f;
fs.unlink(fn, function(err) {
if ( err ) {
console.log("Unlinking " + fn + ": " + err);
finish(err);
}
else unlinkall(dir, files, finish)
});
}
fs.readdir(outdirpath, function(err, files) {
if ( err ) {
if ( err.code != 'ENOENT' ) {
next(new Error([topError, err].join('\n')));
} else {
next(topError);
}
} else {
unlinkall(outdirpath, files, function(err) {
fs.rmdir(outdirpath, function(err) {
if ( err ) console.log("Removing dir " + path + ": " + err);
next(topError);
});
});
}
});
},
function finish(err) {
if ( err ) callback(err);
else {
res.end();
callback(null);
}
}
);
}
function toKML(dbname, user_id, gcol, sql, skipfields, res, callback) {
var zip = 'zip'; // FIXME: make configurable
var tmpdir = '/tmp'; // FIXME: make configurable
var outdirpath = tmpdir + '/sqlapi-kmloutput-' + generateMD5(sql);
var dumpfile = outdirpath + '/cartodb-query.kml';
// TODO: following tests:
// - fetch with no auth
// - fetch with auth
// - fetch same query concurrently
// - fetch query with no "the_geom" column
Step (
function createOutDir() {
fs.mkdir(outdirpath, 0777, this);
},
function spawnDumper(err) {
if ( err ) {
if ( err.code == 'EEXIST' ) {
// TODO: this could mean another request for the same
// resource is in progress, in which case we might want
// to queue the response to after it's completed...
console.log("Reusing existing KML output directory for query: " + sql);
} else {
throw err;
}
}
toOGR(dbname, user_id, gcol, sql, skipfields, res, 'KML', dumpfile, this);
},
function sendResults(err) {
if ( ! err ) {
var stream = fs.createReadStream(dumpfile);
util.pump(stream, res);
}
// cleanup output dir (should be safe to unlink)
var topError = err;
var next = this;
//console.log("Cleaning up " + outdirpath);
// Unlink the dir content
var unlinkall = function(dir, files, finish) {
var f = files.shift();
if ( ! f ) { finish(null); return; }
var fn = dir + '/' + f;
fs.unlink(fn, function(err) {
if ( err ) {
console.log("Unlinking " + fn + ": " + err);
finish(err);
}
else unlinkall(dir, files, finish)
});
}
fs.readdir(outdirpath, function(err, files) {
if ( err ) {
if ( err.code != 'ENOENT' ) {
next(new Error([topError, err].join('\n')));
} else {
next(topError);
}
} else {
unlinkall(outdirpath, files, function(err) {
fs.rmdir(outdirpath, function(err) {
if ( err ) console.log("Removing dir " + path + ": " + err);
next(topError);
});
});
}
});
},
function finish(err) {
if ( err ) callback(err);
else {
res.end();
callback(null);
}
}
);
}
function getContentDisposition(format, filename, inline) {
var ext = 'json';
if (format === 'geojson'){
ext = 'geojson';
@ -366,28 +725,44 @@ function getContentDisposition(format){
else if (format === 'svg'){
ext = 'svg';
}
else if (format === 'shp'){
ext = 'zip';
}
else if (format === 'kml'){
ext = 'kml';
}
var time = new Date().toUTCString();
return 'inline; filename=cartodb-query.' + ext + '; modification-date="' + time + '";';
return ( inline ? 'inline' : 'attachment' ) +'; filename=' + filename + '.' + ext + '; modification-date="' + time + '";';
}
function getContentType(format){
var type = "application/json; charset=utf-8";
if (format === 'csv'){
type = "text/csv; charset=utf-8";
type = "text/csv; charset=utf-8; header=present";
}
else if (format === 'svg'){
type = "image/svg+xml; charset=utf-8";
}
else if (format === 'shp'){
type = "application/zip; charset=utf-8";
}
else if (format === 'kml'){
type = "application/kml; charset=utf-8";
}
return type;
}
function setCrossDomain(res){
res.header("Access-Control-Allow-Origin", "*");
res.header("Access-Control-Allow-Headers", "X-Requested-With");
res.header("Access-Control-Allow-Headers", "X-Requested-With, X-Prototype-Version, X-CSRF-Token");
}
function generateCacheKey(database,tables){
return database + ":" + tables.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1];
function generateCacheKey(database,tables,is_authenticated){
if ( is_authenticated && tables.may_write ) {
return "NONE";
} else {
return database + ":" + tables.rows[0].cdb_querytables.split(/^\{(.*)\}$/)[1];
}
}
function generateMD5(data){

6
app/models/apikey_auth.js
View File

@ -88,7 +88,7 @@ module.exports = (function() {
* Get privacy for cartodb table
*
* @param req - standard req object. Importantly contains table and host information
* @param callback - user_id if ok, null if auth fails
* @param callback - err, user_id (null if no auth)
*/
me.verifyRequest = function(req, callback) {
var that = this;
@ -108,8 +108,8 @@ module.exports = (function() {
}
},
function (err, user_id){
if (err) throw err;
callback(false, user_id);
if (err) callback(err);
else callback(false, user_id);
}
);
};

2
app/models/metadata.js
View File

@ -72,4 +72,4 @@ module.exports = function() {
};
return me;
}();
}();

3
cluster.js
View File

@ -32,7 +32,8 @@ var cluster = new Cluster({
port: global.settings.node_port,
host: global.settings.node_host,
monHost: global.settings.node_host,
monPort: global.settings.node_port+1
monPort: global.settings.node_port+1,
timeout: 600000
});
cluster.listen(function(cb) {

16
doc/API.md
View File

@ -12,7 +12,14 @@ Supported query string parameters:
'format': Specifies which format to use for the response.
Supported formats: JSON (the default), GeoJSON,
CSV, SVG
CSV, SVG, SHP
'filename': Sets the filename to use for the query result
file attachment
'skipfields':
Comma separate list of fields that are not wanted
in output. Only useful with "SELECT *" queries.
'dp': Number of digits after the decimal point.
Only affects format=GeoJSON.
@ -20,6 +27,13 @@ Supported query string parameters:
'api_key': Needed to authenticate in order to modify the database.
'cache_policy':
Set to "persist" to have the server send an Cache-Control
header requesting caching devices to keep the response
cached as much as possible. This is best used with a
timestamp value in cache_buster for manual control of
updates.
Response formats
----------------

15
npm-shrinkwrap.json generated
View File

@ -1,9 +1,9 @@
{
"name": "cartodb_api",
"version": "1.1.0",
"version": "1.3.0",
"dependencies": {
"cluster2": {
"version": "0.3.5-cdb01",
"version": "0.3.5-cdb02",
"from": "git://github.com/CartoDB/cluster2.git#cdb_production",
"dependencies": {
"ejs": {
@ -218,6 +218,17 @@
"csv": {
"version": "0.0.13"
},
"zipfile": {
"version": "0.3.2"
},
"libxmljs": {
"version": "0.6.1",
"dependencies": {
"bindings": {
"version": "1.0.0"
}
}
},
"mocha": {
"version": "1.2.1",
"dependencies": {

6
package.json
View File

@ -2,7 +2,7 @@
"private": true,
"name": "cartodb_api",
"description": "high speed SQL api for cartodb",
"version": "1.1.0",
"version": "1.3.1",
"author": {
"name": "Simon Tokumine, Sandro Santilli, Vizzuality",
"url": "http://vizzuality.com",
@ -23,7 +23,9 @@
"csv":"0.0.13"
},
"devDependencies": {
"mocha": "1.2.1"
"mocha": "1.2.1",
"zipfile": "~0.3.2",
"libxmljs": "~0.6.1"
},
"scripts": {
"test": "test/run_tests.sh"

933
test/acceptance/app.test.js
View File

File diff suppressed because it is too large Load Diff

2
test/prepare_db.sh
View File

@ -34,6 +34,8 @@ echo "preparing postgres..."
dropdb ${TEST_DB} # 2> /dev/null # error expected if doesn't exist, but not otherwise
createdb -Ttemplate_postgis -EUTF8 ${TEST_DB} || die "Could not create test database"
psql -f test.sql ${TEST_DB}
psql -f support/CDB_QueryStatements.sql ${TEST_DB}
psql -f support/CDB_QueryTables.sql ${TEST_DB}
echo "preparing redis..."
echo "HSET rails:users:vizzuality id 1" | redis-cli -p ${REDIS_PORT} -n 5

13
test/support/CDB_QueryStatements.sql Normal file
View File

@ -0,0 +1,13 @@
-- Return an array of statements found in the given query text
--
-- Curtesy of Hubert Lubaczewski (depesz)
--
CREATE OR REPLACE FUNCTION CDB_QueryStatements(query text)
RETURNS SETOF TEXT AS $$
SELECT stmt FROM (
SELECT btrim(q[1], E' \n\t\r;') as stmt FROM (
SELECT regexp_matches( $1, $REG$((?:[^'"$;]+|"[^"]*"|'(?:[^']*|'')*'|(\$[^$]*\$).*?\2)+)$REG$, 'g' ) as q
) i
) j
WHERE stmt <> '';
$$ language sql;

54
test/support/CDB_QueryTables.sql Normal file
View File

@ -0,0 +1,54 @@
-- Return an array of table names scanned by a given query
--
-- Requires PostgreSQL 9.x+
--
CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
RETURNS name[]
AS $$
DECLARE
exp XML;
tables NAME[];
rec RECORD;
rec2 RECORD;
BEGIN
tables := '{}';
FOR rec IN SELECT CDB_QueryStatements(query) q LOOP
BEGIN
EXECUTE 'EXPLAIN (FORMAT XML) ' || rec.q INTO STRICT exp;
EXCEPTION WHEN others THEN
RAISE WARNING 'Cannot explain query: % (%)', rec.q, SQLERRM;
CONTINUE;
END;
-- Now need to extract all values of <Relation-Name>
--RAISE DEBUG 'Explain: %', exp;
FOR rec2 IN WITH
inp AS ( SELECT xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]) as x )
SELECT unnest(x)::name as p from inp
LOOP
--RAISE DEBUG 'tab: %', rec2.p;
tables := array_append(tables, rec2.p);
END LOOP;
-- RAISE DEBUG 'Tables: %', tables;
END LOOP;
-- RAISE DEBUG 'Tables: %', tables;
-- Remove duplicates and sort by name
IF array_upper(tables, 1) > 0 THEN
WITH dist as ( SELECT DISTINCT unnest(tables)::text as p ORDER BY p )
SELECT array_agg(p) from dist into tables;
END IF;
--RAISE DEBUG 'Tables: %', tables;
return tables;
END
$$ LANGUAGE 'plpgsql' VOLATILE STRICT;

26
test/test.sql
View File

@ -20,31 +20,6 @@ SET search_path = public, pg_catalog;
SET default_tablespace = '';
SET default_with_oids = false;
-- Return an array of table names used by a given query
CREATE OR REPLACE FUNCTION CDB_QueryTables(query text)
RETURNS name[]
AS $$
DECLARE
exp XML;
tables NAME[];
BEGIN
EXECUTE 'EXPLAIN (FORMAT XML) ' || query INTO STRICT exp;
-- Now need to extract all values of <Relation-Name>
--RAISE DEBUG 'Explain: %', exp;
tables := xpath('//x:Relation-Name/text()', exp, ARRAY[ARRAY['x', 'http://www.postgresql.org/2009/explain']]);
--RAISE DEBUG 'Tables: %', tables;
return tables;
END
$$ LANGUAGE 'plpgsql' VOLATILE STRICT;
-- first table
DROP TABLE IF EXISTS untitle_table_4;
CREATE TABLE untitle_table_4 (
@ -133,6 +108,7 @@ CREATE INDEX test_table_the_geom_webmercator_idx_p ON private_table USING gist (
CREATE USER publicuser WITH PASSWORD '';
CREATE USER test_cartodb_user_1 WITH PASSWORD '';
GRANT ALL ON TABLE untitle_table_4 TO test_cartodb_user_1;
GRANT SELECT ON TABLE untitle_table_4 TO publicuser;
GRANT ALL ON TABLE private_table TO test_cartodb_user_1;
GRANT ALL ON SEQUENCE test_table_cartodb_id_seq_p TO test_cartodb_user_1