cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into upgrade-cartodb-psql

Browse Source
This commit is contained in:
Raul Ochoa 2017-08-11 11:57:16 +02:00
commit 32154b67c6
13 changed files with 357 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.jshintrc
View File

@ -40,7 +40,7 @@
"debug" : false, // true: Allow debugger statements e.g. browser breakpoints.
// "eqnull" : false, // true: Tolerate use of `== null`
// "es5" : false, // true: Allow ES5 syntax (ex: getters and setters)
// "esnext" : false, // true: Allow ES.next (ES6) syntax (ex: `const`)
"esnext" : true, // true: Allow ES.next (ES6) syntax (ex: `const`)
// "moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features)
// // (ex: `for each`, multiple try/catch, function expression…)
// "evil" : false, // true: Tolerate use of `eval` and `new Function()`

1
.travis.yml
View File

@ -30,5 +30,4 @@ env:
language: node_js
node_js:
- "0.10"
- "6"

9
NEWS.md
View File

@ -1,12 +1,19 @@
#Changelog
## 1.46.2
## 1.47.1
Released 2017-mm-dd
Announcements:
* Upgrade cartodb-psql to 0.8.0.
* Content edits to doc/version.md
## 1.47.0
Released 2017-08-10
Announcements:
* Now export and query APIs respond with `429 You are over the limits` when a query or export command overcomes the pre-configured user's timeout.
## 1.46.1
Released 2017-07-01

7
app/controllers/query_controller.js
View File

@ -50,6 +50,7 @@ QueryController.prototype.handleQuery = function (req, res) {
var skipfields;
var dp = params.dp; // decimal point digits (defaults to 6)
var gn = "the_geom"; // TODO: read from configuration file
var userLimits;
if ( req.profiler ) {
req.profiler.start('sqlapi.query');
@ -122,12 +123,13 @@ QueryController.prototype.handleQuery = function (req, res) {
function getUserDBInfo() {
self.userDatabaseService.getConnectionParams(new AuthApi(req, params), cdbUsername, this);
},
function queryExplain(err, dbParams, authDbParams) {
function queryExplain(err, dbParams, authDbParams, userTimeoutLimits) {
assert.ifError(err);
var next = this;
dbopts = dbParams;
userLimits = userTimeoutLimits;
if ( req.profiler ) {
req.profiler.done('setDBAuth');
@ -217,7 +219,8 @@ QueryController.prototype.handleQuery = function (req, res) {
filename: filename,
bufferedRows: global.settings.bufferedRows,
callback: params.callback,
abortChecker: checkAborted
abortChecker: checkAborted,
timeout: userLimits.timeout
};
if ( req.profiler ) {

19
app/models/formats/ogr.js
View File

@ -65,6 +65,8 @@ OgrFormat.prototype.toOGR = function(options, out_format, out_filename, callback
var dbpass = dbopts.pass;
var dbname = dbopts.dbname;
var timeout = options.timeout;
var that = this;
var columns = [];
@ -167,9 +169,20 @@ OgrFormat.prototype.toOGR = function(options, out_format, out_filename, callback
ogrargs.push('-nln', out_layername);
// TODO: research if `exec` could fit better than `spawn`
var child = spawn(ogr2ogr, ogrargs);
var timedOut = false;
var ogrTimeout;
if (timeout > 0) {
ogrTimeout = setTimeout(function () {
timedOut = true;
child.kill();
}, timeout);
}
child.on('error', function (err) {
clearTimeout(ogrTimeout);
next(err);
});
@ -180,6 +193,12 @@ OgrFormat.prototype.toOGR = function(options, out_format, out_filename, callback
});
child.on('exit', function(code) {
clearTimeout(ogrTimeout);
if (timedOut) {
return next(new Error('You are over platform\'s limits. Please contact us to know more details'));
}
if (code !== 0) {
var errMessage = 'ogr2ogr command return code ' + code;
if (stderrData.length > 0) {

8
app/postgresql/error_handler.js
View File

@ -40,12 +40,14 @@ ErrorHandler.prototype.getStatus = function() {
statusError = 401;
}
if (message === conditionToMessage[pgErrorCodes.conditionToCode.query_canceled]) {
statusError = 429;
}
return statusError;
};
var conditionToMessage = {};
conditionToMessage[pgErrorCodes.conditionToCode.query_canceled] = [
"Your query was not able to finish.",
"Either you have too many queries running or the one you are trying to run is too expensive.",
"Try again."
'You are over platform\'s limits. Please contact us to know more details'
].join(' ');

27
app/services/user_database_service.js
View File

@ -54,7 +54,26 @@ UserDatabaseService.prototype.getConnectionParams = function (authApi, cdbUserna
apiKey: dbParams.apikey
}, next);
},
function setDBAuth(err, isAuthenticated) {
function getUserLimits (err, isAuthenticated) {
var next = this;
if (err) {
return next(err);
}
self.metadataBackend.getUserTimeoutRenderLimits(cdbUsername, function (err, timeoutRenderLimit) {
if (err) {
return next(err);
}
var userLimits = {
timeout: isAuthenticated ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic
};
next(null, isAuthenticated, userLimits);
});
},
function setDBAuth(err, isAuthenticated, userLimits) {
if (err) {
throw err;
}
@ -76,14 +95,14 @@ UserDatabaseService.prototype.getConnectionParams = function (authApi, cdbUserna
var authDbOpts = _.defaults({user: user, pass: pass}, dbopts);
return this(null, dbopts, authDbOpts);
return this(null, dbopts, authDbOpts, userLimits);
},
function errorHandle(err, dbopts, authDbOpts) {
function errorHandle(err, dbopts, authDbOpts, userLimits) {
if (err) {
return callback(err);
}
callback(null, dbopts, authDbOpts);
callback(null, dbopts, authDbOpts, userLimits);
}
);

8
npm-shrinkwrap.json generated
View File

@ -1,6 +1,6 @@
{
"name": "cartodb_sql_api",
"version": "1.46.2",
"version": "1.47.1",
"dependencies": {
"accepts": {
"version": "1.2.13",
@ -158,9 +158,9 @@
"resolved": "https://registry.npmjs.org/cartodb-query-tables/-/cartodb-query-tables-0.2.0.tgz"
},
"cartodb-redis": {
"version": "0.13.2",
"from": "cartodb-redis@0.13.2",
"resolved": "https://registry.npmjs.org/cartodb-redis/-/cartodb-redis-0.13.2.tgz"
"version": "0.14.0",
"from": "cartodb-redis@0.14.0",
"resolved": "https://registry.npmjs.org/cartodb-redis/-/cartodb-redis-0.14.0.tgz"
},
"caseless": {
"version": "0.11.0",

4
package.json
View File

@ -5,7 +5,7 @@
"keywords": [
"cartodb"
],
"version": "1.46.2",
"version": "1.47.1",
"repository": {
"type": "git",
"url": "git://github.com/CartoDB/CartoDB-SQL-API.git"
@ -21,7 +21,7 @@
"bunyan": "1.8.1",
"cartodb-psql": "0.8.0",
"cartodb-query-tables": "0.2.0",
"cartodb-redis": "0.13.2",
"cartodb-redis": "0.14.0",
"debug": "2.2.0",
"express": "~4.13.3",
"log4js": "cartodb/log4js-node#cdb",

13
test/acceptance/app.test.js
View File

@ -782,10 +782,19 @@ it('GET with callback must return 200 status error even if it is an error', func
method: 'GET'
},
{
status: 400
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(err, res) {
assert.ok(res.body.match(/was not able to finish.*try again/i));
var error = JSON.parse(res.body);
assert.deepEqual(error, {
error: [
'You are over platform\'s limits. Please contact us to know more details'
]
});
done();
});
});

190
test/acceptance/export/timeout.js Normal file
View File

@ -0,0 +1,190 @@
const TestClient = require('../../support/test-client');
require('../../support/assert');
var assert = require('assert');
var querystring = require('querystring');
describe('timeout', function () {
describe('export database', function () {
const databaseTimeoutQuery = `
select
ST_SetSRID(ST_Point(0, 0), 4326) as the_geom,
pg_sleep(0.2) as sleep,
1 as value
`;
const scenarios = [
{
desc: 'CSV',
format: 'csv',
contentType: 'application/x-www-form-urlencoded',
parser: querystring.stringify,
// only: true,
skip: true
},
{
query: databaseTimeoutQuery,
desc: 'Geopackage',
format: 'gpkg'
},
{
query: databaseTimeoutQuery,
desc: 'KML',
format: 'kml'
},
{
query: databaseTimeoutQuery,
desc: 'Shapefile',
format: 'shp'
},
{
query: databaseTimeoutQuery,
desc: 'Spatialite',
format: 'spatialite'
},
{
query: databaseTimeoutQuery,
desc: 'Array Buffer',
format: 'arraybuffer'
},
{
query: databaseTimeoutQuery,
desc: 'GeoJSON',
format: 'geojson'
},
{
query: databaseTimeoutQuery,
desc: 'JSON',
format: 'json'
},
{
query: databaseTimeoutQuery,
desc: 'SVG',
format: 'svg'
},
{
query: databaseTimeoutQuery,
desc: 'TopoJSON',
format: 'topojson'
}
];
beforeEach(function (done) {
this.testClient = new TestClient();
this.testClient.setUserDatabaseTimeoutLimit('localhost', 100, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit('localhost', 2000, done);
});
scenarios.forEach((scenario) => {
const test = scenario.only ? it.only : scenario.skip ? it.skip : it;
test(`${scenario.desc} export exceeding statement timeout responds 429 Over Limits`, function (done) {
const override = {
'Content-Type': scenario.contentType,
parser: scenario.parser,
anonymous: true,
format: scenario.format,
response: {
status: 429
}
};
this.testClient.getResult(scenario.query, override, (err, res) => {
assert.ifError(err);
assert.deepEqual(res, {
error: [
'You are over platform\'s limits. Please contact us to know more details'
]
});
done();
});
});
});
});
describe('export ogr command timeout', function () {
const ogrCommandTimeoutQuery = `
select
ST_SetSRID(ST_Point(0, 0), 4326) as the_geom,
pg_sleep(0.2) as sleep,
1 as value
`;
const scenarios = [
{
query: ogrCommandTimeoutQuery,
desc: 'CSV',
format: 'csv',
contentType: 'application/x-www-form-urlencoded',
parser: querystring.stringify,
// only: true,
// skip: true
},
{
query: ogrCommandTimeoutQuery,
filename: 'wadus_gpkg_filename',
desc: 'Geopackage',
format: 'gpkg'
},
{
query: ogrCommandTimeoutQuery,
desc: 'KML',
format: 'kml'
},
{
query: ogrCommandTimeoutQuery,
desc: 'Shapefile',
format: 'shp'
},
{
query: ogrCommandTimeoutQuery,
desc: 'Spatialite',
format: 'spatialite'
}
];
beforeEach(function (done) {
this.testClient = new TestClient();
this.testClient.setUserRenderTimeoutLimit('vizzuality', 100, done);
});
afterEach(function (done) {
this.testClient.setUserRenderTimeoutLimit('vizzuality', 0, done);
});
scenarios.forEach((scenario) => {
const test = scenario.only ? it.only : scenario.skip ? it.skip : it;
test(`${scenario.desc} export exceeding statement timeout responds 429 Over Limits`, function (done) {
const override = {
'Content-Type': scenario.contentType,
parser: scenario.parser,
anonymous: true,
format: scenario.format,
filename: scenario.filename,
response: {
status: 429
}
};
this.testClient.getResult(scenario.query, override, (err, res) => {
assert.ifError(err);
assert.deepEqual(res, {
error: [
'You are over platform\'s limits. Please contact us to know more details'
]
});
done();
});
});
});
});
});

4
test/support/redis_utils.js
View File

@ -33,3 +33,7 @@ var pool = new RedisPool(redisConfig);
module.exports.getPool = function getPool() {
return pool;
};
module.exports.configureUserMetadata = function configureUserMetadata(action, params, callback) {
metadataBackend.redisCmd(5, action, params, callback);
}

89
test/support/test-client.js
View File

@ -3,6 +3,12 @@
require('../helper');
var assert = require('assert');
var appServer = require('../../app/server');
var redisUtils = require('./redis_utils');
const step = require('step');
const PSQL = require('cartodb-psql');
const _ = require('underscore');
// TODO: remove after upgrading cartodb-psql to 0.9.0
const pg = require('pg');
function response(code) {
return {
@ -35,20 +41,26 @@ TestClient.prototype.getResult = function(query, override, callback) {
url: this.getUrl(override),
headers: {
host: this.getHost(override),
'Content-Type': 'application/json'
'Content-Type': this.getContentType(override)
},
method: 'POST',
data: JSON.stringify({
q: query
data: this.getParser(override)({
q: query,
format: this.getFormat(override),
filename: this.getFilename(override)
})
},
RESPONSE.OK,
this.getExpectedResponse(override),
function (err, res) {
if (err) {
return callback(err);
}
var result = JSON.parse(res.body);
if (res.statusCode > 299) {
return callback(null, result);
}
return callback(null, result.rows || []);
}
);
@ -58,6 +70,75 @@ TestClient.prototype.getHost = function(override) {
return override.host || this.config.host || 'vizzuality.cartodb.com';
};
TestClient.prototype.getContentType = function(override) {
return override['Content-Type'] || this.config['Content-Type'] || 'application/json';
};
TestClient.prototype.getParser = function (override) {
return override.parser || this.config.parser || JSON.stringify
}
TestClient.prototype.getUrl = function(override) {
if (override.anonymous) {
return '/api/v1/sql?';
}
return '/api/v2/sql?api_key=' + (override.apiKey || this.config.apiKey || '1234');
};
TestClient.prototype.getExpectedResponse = function (override) {
return override.response || this.config.response || RESPONSE.OK;
};
TestClient.prototype.getFormat = function (override) {
return override.format || this.config.format || undefined;
};
TestClient.prototype.getFilename = function (override) {
return override.filename || this.config.filename || undefined;
};
TestClient.prototype.setUserRenderTimeoutLimit = function (user, userTimeoutLimit, callback) {
const userTimeoutLimitsKey = `limits:timeout:${user}`;
const params = [
userTimeoutLimitsKey,
'render', userTimeoutLimit,
'render_public', userTimeoutLimit
];
redisUtils.configureUserMetadata('hmset', params, callback);
};
TestClient.prototype.setUserDatabaseTimeoutLimit = function (user, timeoutLimit, callback) {
const dbname = _.template(global.settings.db_base_name, { user_id: 1 });
const dbuser = _.template(global.settings.db_user, { user_id: 1 })
const pass = _.template(global.settings.db_user_pass, { user_id: 1 })
const publicuser = global.settings.db_pubuser;
// TODO: We do need to upgrade cartodb-psql to 0.9.0 to use psql.end() instead.
// we need to guarantee all new connections have the new settings
pg.end();
const psql = new PSQL({
user: 'postgres',
dbname: dbname,
host: global.settings.db_host,
port: global.settings.db_port
});
step(
function configureTimeouts () {
const timeoutSQLs = [
`ALTER ROLE "${publicuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`,
`ALTER ROLE "${dbuser}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`,
`ALTER DATABASE "${dbname}" SET STATEMENT_TIMEOUT TO ${timeoutLimit}`
];
const group = this.group();
timeoutSQLs.forEach(sql => psql.query(sql, group()));
},
callback
);
};