cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into limit-batch-queries

Browse Source
This commit is contained in:
Raul Ochoa 2016-10-06 12:46:34 +02:00
commit 7c7320061f
78 changed files with 2413 additions and 1614 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.travis.yml
View File

@ -1,4 +1,4 @@
before_script:
before_install:
- lsb_release -a
- sudo mv /etc/apt/sources.list.d/pgdg.list* /tmp
- sudo apt-get -qq purge postgis* postgresql*

2
Makefile
View File

@ -17,7 +17,7 @@ TEST_SUITE := $(shell find test/{acceptance,unit,integration} -name "*.js")
TEST_SUITE_UNIT := $(shell find test/unit -name "*.js")
TEST_SUITE_INTEGRATION := $(shell find test/integration -name "*.js")
TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance -name "*.js")
TEST_SUITE_BATCH := $(shell find test -name "*job*.js")
TEST_SUITE_BATCH := $(shell find test/*/batch -name "*.js")
test:
@echo "***tests***"

54
NEWS.md
View File

@ -1,10 +1,62 @@
1.34.3 - 2016-mm-dd
1.37.2 - 2016-mm-dd
-------------------
Announcements:
* limited batch queries to 12 hours
1.37.1 - 2016-10-05
-------------------
Bug fixes:
* Body parser accepting multipart requests.
1.37.0 - 2016-10-04
-------------------
Enhancements:
* Migrate to Express.js 4.x series.
1.36.2 - 2016-10-03
-------------------
Bug fixes:
- Batch Queries logs: use path instead of stream to be able to reopen FD.
1.36.1 - 2016-09-30
-------------------
Enhancements:
* Tag fallback jobs logs.
1.36.0 - 2016-09-30
-------------------
New features:
* Log queries from batch fallback jobs.
Enhancements:
* assert.response following callback(err, obj) pattern.
1.35.0 - 2016-09-15
-------------------
New features:
* Allow to use `--config /path/to/config.js` to specify configuration file.
- Environment will be loaded from config file if `environment` key is present, otherwise it keeps current behaviour.
Bug fixes:
* Allow to use absolute paths for log files.
Announcements:
* Removes support for optional rollbar logging.
1.34.2 - 2016-08-30
-------------------

96
app.js
View File

@ -9,65 +9,67 @@
* environments: [development, test, production]
*
*/
var _ = require('underscore');
var fs = require('fs');
var path = require('path');
var ENV = process.env.NODE_ENV || 'development';
var argv = require('yargs')
.usage('Usage: $0 <environment> [options]')
.help('h')
.example(
'$0 production -c /etc/sql-api/config.js',
'start server in production environment with /etc/sql-api/config.js as config file'
)
.alias('h', 'help')
.alias('c', 'config')
.nargs('c', 1)
.describe('c', 'Load configuration from path')
.argv;
if (process.argv[2]) {
ENV = process.argv[2];
var environmentArg = argv._[0] || process.env.NODE_ENV || 'development';
var configurationFile = path.resolve(argv.config || './config/environments/' + environmentArg + '.js');
if (!fs.existsSync(configurationFile)) {
console.error('Configuration file "%s" does not exist', configurationFile);
process.exit(1);
}
process.env.NODE_ENV = ENV;
global.settings = require(configurationFile);
var ENVIRONMENT = argv._[0] || process.env.NODE_ENV || global.settings.environment;
process.env.NODE_ENV = ENVIRONMENT;
var availableEnvironments = ['development', 'production', 'test', 'staging'];
// sanity check arguments
if (availableEnvironments.indexOf(ENV) === -1) {
console.error("\nnode app.js [environment]");
console.error("environments: " + availableEnvironments.join(', '));
if (availableEnvironments.indexOf(ENVIRONMENT) === -1) {
console.error("node app.js [environment]");
console.error("Available environments: " + availableEnvironments.join(', '));
process.exit(1);
}
// set Node.js app settings and boot
global.settings = require(__dirname + '/config/settings');
var env = require(__dirname + '/config/environments/' + ENV);
env.api_hostname = require('os').hostname().split('.')[0];
_.extend(global.settings, env);
global.settings.api_hostname = require('os').hostname().split('.')[0];
global.log4js = require('log4js');
var log4js_config = {
appenders: [],
replaceConsole:true
var log4jsConfig = {
appenders: [],
replaceConsole: true
};
if ( env.log_filename ) {
var logdir = path.dirname(env.log_filename);
// See cwd inlog4js.configure call below
logdir = path.resolve(__dirname, logdir);
if ( ! fs.existsSync(logdir) ) {
console.error("Log filename directory does not exist: " + logdir);
if ( global.settings.log_filename ) {
var logFilename = path.resolve(global.settings.log_filename);
var logDirectory = path.dirname(logFilename);
if (!fs.existsSync(logDirectory)) {
console.error("Log filename directory does not exist: " + logDirectory);
process.exit(1);
}
console.log("Logs will be written to " + env.log_filename);
log4js_config.appenders.push(
{ type: "file", filename: env.log_filename }
console.log("Logs will be written to " + logFilename);
log4jsConfig.appenders.push(
{ type: "file", absolute: true, filename: logFilename }
);
} else {
log4js_config.appenders.push(
log4jsConfig.appenders.push(
{ type: "console", layout: { type:'basic' } }
);
}
if ( global.settings.rollbar ) {
log4js_config.appenders.push({
type: __dirname + "/app/models/log4js_rollbar.js",
options: global.settings.rollbar
});
}
global.log4js.configure(log4js_config, { cwd: __dirname });
global.log4js.configure(log4jsConfig);
global.logger = global.log4js.getLogger();
@ -78,12 +80,14 @@ if ( ! global.settings.base_url ) {
var version = require("./package").version;
var app = require(global.settings.app_root + '/app/app')();
app.listen(global.settings.node_port, global.settings.node_host, function() {
console.log(
"CartoDB SQL API %s listening on %s:%s with base_url %s (%s)",
version, global.settings.node_host, global.settings.node_port, global.settings.base_url, ENV
);
var server = require('./app/server')();
var listener = server.listen(global.settings.node_port, global.settings.node_host);
listener.on('listening', function() {
console.info('Using configuration file "%s"', configurationFile);
console.log(
"CartoDB SQL API %s listening on %s:%s PID=%d (%s)",
version, global.settings.node_host, global.settings.node_port, process.pid, ENVIRONMENT
);
});
process.on('uncaughtException', function(err) {
@ -92,15 +96,19 @@ process.on('uncaughtException', function(err) {
process.on('SIGHUP', function() {
global.log4js.clearAndShutdownAppenders(function() {
global.log4js.configure(log4js_config);
global.log4js.configure(log4jsConfig);
global.logger = global.log4js.getLogger();
console.log('Log files reloaded');
});
if (server.batch && server.batch.logger) {
server.batch.logger.reopenFileStreams();
}
});
process.on('SIGTERM', function () {
app.batch.stop();
app.batch.drain(function (err) {
server.batch.stop();
server.batch.drain(function (err) {
if (err) {
console.log('Exit with error');
return process.exit(1);

4
app/controllers/health_check_controller.js
View File

@ -24,11 +24,11 @@ HealthCheckController.prototype.handleHealthCheck = function (req, res) {
if (err) {
response.err = err.message;
}
res.send(response, ok ? 200 : 503);
res.status(ok ? 200 : 503).send(response);
});
} else {
res.send({enabled: false, ok: true}, 200);
res.status(200).send({enabled: false, ok: true});
}
};

286
app/controllers/job_controller.js
View File

@ -1,14 +1,11 @@
'use strict';
var _ = require('underscore');
var step = require('step');
var assert = require('assert');
var util = require('util');
var AuthApi = require('../auth/auth_api');
var CdbRequest = require('../models/cartodb_request');
var userMiddleware = require('../middlewares/user');
var authenticatedMiddleware = require('../middlewares/authenticated-request');
var handleException = require('../utils/error_handler');
var cdbReq = new CdbRequest();
var ONE_KILOBYTE_IN_BYTES = 1024;
var MAX_LIMIT_QUERY_SIZE_IN_KB = 8;
@ -46,245 +43,70 @@ module.exports.MAX_LIMIT_QUERY_SIZE_IN_BYTES = MAX_LIMIT_QUERY_SIZE_IN_BYTES;
module.exports.getMaxSizeErrorMessage = getMaxSizeErrorMessage;
JobController.prototype.route = function (app) {
app.post(global.settings.base_url + '/sql/job', bodyPayloadSizeMiddleware, this.createJob.bind(this));
app.get(global.settings.base_url + '/sql/job/:job_id', this.getJob.bind(this));
app.delete(global.settings.base_url + '/sql/job/:job_id', this.cancelJob.bind(this));
app.post(
global.settings.base_url + '/sql/job',
bodyPayloadSizeMiddleware, userMiddleware, authenticatedMiddleware(this.userDatabaseService),
this.createJob.bind(this)
);
app.get(
global.settings.base_url + '/sql/job/:job_id',
userMiddleware, authenticatedMiddleware(this.userDatabaseService),
this.getJob.bind(this)
);
app.delete(
global.settings.base_url + '/sql/job/:job_id',
userMiddleware, authenticatedMiddleware(this.userDatabaseService),
this.cancelJob.bind(this)
);
};
JobController.prototype.cancelJob = function (req, res) {
var self = this;
var job_id = req.params.job_id;
var body = (req.body) ? req.body : {};
var params = _.extend({}, req.query, body); // clone so don't modify req.params or req.body so oauth is not broken
var cdbUsername = cdbReq.userByReq(req);
if ( req.profiler ) {
req.profiler.start('sqlapi.job');
req.profiler.done('init');
}
step(
function getUserDBInfo() {
var next = this;
var authApi = new AuthApi(req, params);
self.userDatabaseService.getConnectionParams(authApi, cdbUsername, next);
},
function cancelJob(err, userDatabase) {
assert.ifError(err);
if (!userDatabase.authenticated) {
throw new Error('permission denied');
}
var next = this;
if ( req.profiler ) {
req.profiler.done('setDBAuth');
}
self.jobService.cancel(job_id, function (err, job) {
if (err) {
return next(err);
}
next(null, {
job: job.serialize(),
host: userDatabase.host
});
});
},
function handleResponse(err, result) {
if ( err ) {
return handleException(err, res);
}
if (global.settings.api_hostname) {
res.header('X-Served-By-Host', global.settings.api_hostname);
}
if (result.host) {
res.header('X-Served-By-DB-Host', result.host);
}
if ( req.profiler ) {
req.profiler.done('cancelJob');
req.profiler.end();
req.profiler.sendStats();
res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
}
if ( err ) {
self.statsdClient.increment('sqlapi.job.error');
} else {
self.statsdClient.increment('sqlapi.job.success');
}
res.send(result.job);
}
);
this.jobService.cancel(req.params.job_id, jobResponse(req, res, this.statsdClient, 'cancel'));
};
JobController.prototype.getJob = function (req, res) {
var self = this;
var job_id = req.params.job_id;
var body = (req.body) ? req.body : {};
var params = _.extend({}, req.query, body); // clone so don't modify req.params or req.body so oauth is not broken
var cdbUsername = cdbReq.userByReq(req);
if ( req.profiler ) {
req.profiler.start('sqlapi.job');
req.profiler.done('init');
}
step(
function getUserDBInfo() {
var next = this;
var authApi = new AuthApi(req, params);
self.userDatabaseService.getConnectionParams(authApi, cdbUsername, next);
},
function getJob(err, userDatabase) {
assert.ifError(err);
if (!userDatabase.authenticated) {
throw new Error('permission denied');
}
var next = this;
if ( req.profiler ) {
req.profiler.done('setDBAuth');
}
self.jobService.get(job_id, function (err, job) {
if (err) {
return next(err);
}
next(null, {
job: job.serialize(),
host: userDatabase.host
});
});
},
function handleResponse(err, result) {
if ( err ) {
return handleException(err, res);
}
if (global.settings.api_hostname) {
res.header('X-Served-By-Host', global.settings.api_hostname);
}
if (result.host) {
res.header('X-Served-By-DB-Host', result.host);
}
if ( req.profiler ) {
req.profiler.done('getJob');
req.profiler.end();
req.profiler.sendStats();
res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
}
if ( err ) {
self.statsdClient.increment('sqlapi.job.error');
} else {
self.statsdClient.increment('sqlapi.job.success');
}
res.send(result.job);
}
);
this.jobService.get(req.params.job_id, jobResponse(req, res, this.statsdClient, 'retrieve'));
};
JobController.prototype.createJob = function (req, res) {
var self = this;
var body = (req.body) ? req.body : {};
var params = _.extend({}, req.query, body); // clone so don't modify req.params or req.body so oauth is not broken
var sql = (params.query === "" || _.isUndefined(params.query)) ? null : params.query;
var cdbUsername = cdbReq.userByReq(req);
if ( req.profiler ) {
req.profiler.start('sqlapi.job');
req.profiler.done('init');
}
var data = {
user: req.context.user,
query: sql,
host: req.context.userDatabase.host
};
step(
function getUserDBInfo() {
var next = this;
var authApi = new AuthApi(req, params);
self.userDatabaseService.getConnectionParams(authApi, cdbUsername, next);
},
function persistJob(err, userDatabase) {
assert.ifError(err);
if (!userDatabase.authenticated) {
throw new Error('permission denied');
}
var next = this;
if ( req.profiler ) {
req.profiler.done('setDBAuth');
}
var data = {
user: cdbUsername,
query: sql,
host: userDatabase.host
};
self.jobService.create(data, function (err, job) {
if (err) {
return next(err);
}
next(null, {
job: job.serialize(),
host: userDatabase.host
});
});
},
function handleResponse(err, result) {
if ( err ) {
return handleException(err, res);
}
if (global.settings.api_hostname) {
res.header('X-Served-By-Host', global.settings.api_hostname);
}
if (result.host) {
res.header('X-Served-By-DB-Host', result.host);
}
if ( req.profiler ) {
req.profiler.done('persistJob');
req.profiler.end();
req.profiler.sendStats();
res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
}
if ( err ) {
self.statsdClient.increment('sqlapi.job.error');
} else {
self.statsdClient.increment('sqlapi.job.success');
}
console.info(JSON.stringify({
type: 'sql_api_batch_job',
username: cdbUsername,
action: 'create',
job_id: result.job.job_id
}));
res.status(201).send(result.job);
}
);
this.jobService.create(data, jobResponse(req, res, this.statsdClient, 'create', 201));
};
function jobResponse(req, res, statsdClient, action, status) {
return function handler(err, job) {
status = status || 200;
if (err) {
statsdClient.increment('sqlapi.job.error');
return handleException(err, res);
}
res.header('X-Served-By-DB-Host', req.context.userDatabase.host);
req.profiler.done(action);
req.profiler.end();
req.profiler.sendStats();
res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
statsdClient.increment('sqlapi.job.success');
console.info(JSON.stringify({
type: 'sql_api_batch_job',
username: req.context.user,
action: action,
job_id: job.job_id
}));
res.status(status).send(job.serialize());
};
}

3
app/controllers/query_controller.js
View File

@ -228,9 +228,6 @@ QueryController.prototype.handleQuery = function (req, res) {
};
}
if (global.settings.api_hostname) {
res.header('X-Served-By-Host', global.settings.api_hostname);
}
if (dbopts.host) {
res.header('X-Served-By-DB-Host', dbopts.host);
}

35
app/middlewares/authenticated-request.js Normal file
View File

@ -0,0 +1,35 @@
'use strict';
var _ = require('underscore');
var AuthApi = require('../auth/auth_api');
var handleException = require('../utils/error_handler');
function authenticatedMiddleware(userDatabaseService) {
return function middleware(req, res, next) {
req.profiler.start('sqlapi.job');
req.profiler.done('init');
var body = (req.body) ? req.body : {};
// clone so don't modify req.params or req.body so oauth is not broken
var params = _.extend({}, req.query, body);
var authApi = new AuthApi(req, params);
userDatabaseService.getConnectionParams(authApi, req.context.user, function cancelJob(err, userDatabase) {
req.profiler.done('setDBAuth');
if (err) {
return handleException(err, res);
}
if (!userDatabase.authenticated) {
return handleException(new Error('permission denied'), res);
}
req.context.userDatabase = userDatabase;
return next(null);
});
};
}
module.exports = authenticatedMiddleware;

145
app/middlewares/body-parser.js Normal file
View File

@ -0,0 +1,145 @@
/*!
* Connect - bodyParser
* Copyright(c) 2010 Sencha Inc.
* Copyright(c) 2011 TJ Holowaychuk
* MIT Licensed
*/
/**
* Module dependencies.
*/
var qs = require('qs');
var multer = require('multer');
/**
* Extract the mime type from the given request's
* _Content-Type_ header.
*
* @param {IncomingMessage} req
* @return {String}
* @api private
*/
function mime(req) {
var str = req.headers['content-type'] || '';
return str.split(';')[0];
}
/**
* Parse request bodies.
*
* By default _application/json_, _application/x-www-form-urlencoded_,
* and _multipart/form-data_ are supported, however you may map `connect.bodyParser.parse[contentType]`
* to a function receiving `(req, options, callback)`.
*
* Examples:
*
* connect.createServer(
* connect.bodyParser()
* , function(req, res) {
* res.end('viewing user ' + req.body.user.name);
* }
* );
*
* $ curl -d 'user[name]=tj' http://localhost/
* $ curl -d '{"user":{"name":"tj"}}' -H "Content-Type: application/json" http://localhost/
*
* Multipart req.files:
*
* As a security measure files are stored in a separate object, stored
* as `req.files`. This prevents attacks that may potentially alter
* filenames, and depending on the application gain access to restricted files.
*
* Multipart configuration:
*
* The `options` passed are provided to each parser function.
* The _multipart/form-data_ parser merges these with formidable's
* IncomingForm object, allowing you to tweak the upload directory,
* size limits, etc. For example you may wish to retain the file extension
* and change the upload directory:
*
* server.use(bodyParser({ uploadDir: '/www/mysite.com/uploads' }));
*
* View [node-formidable](https://github.com/felixge/node-formidable) for more information.
*
* If you wish to use formidable directly within your app, and do not
* desire this behaviour for multipart requests simply remove the
* parser:
*
* delete connect.bodyParser.parse['multipart/form-data'];
*
* Or
*
* delete express.bodyParser.parse['multipart/form-data'];
*
* @param {Object} options
* @return {Function}
* @api public
*/
exports = module.exports = function bodyParser(options){
options = options || {};
return function bodyParser(req, res, next) {
if (req.body) {
return next();
}
req.body = {};
if ('GET' === req.method || 'HEAD' === req.method) {
return next();
}
var parser = exports.parse[mime(req)];
if (parser) {
parser(req, options, next);
} else {
next();
}
};
};
/**
* Parsers.
*/
exports.parse = {};
/**
* Parse application/x-www-form-urlencoded.
*/
exports.parse['application/x-www-form-urlencoded'] = function(req, options, fn){
var buf = '';
req.setEncoding('utf8');
req.on('data', function(chunk){ buf += chunk; });
req.on('end', function(){
try {
req.body = buf.length ? qs.parse(buf) : {};
fn();
} catch (err){
fn(err);
}
});
};
/**
* Parse application/json.
*/
exports.parse['application/json'] = function(req, options, fn){
var buf = '';
req.setEncoding('utf8');
req.on('data', function(chunk){ buf += chunk; });
req.on('end', function(){
try {
req.body = buf.length ? JSON.parse(buf) : {};
fn();
} catch (err){
fn(err);
}
});
};
var multipartMiddleware = multer({ limits: { fieldSize: Infinity } });
exports.parse['multipart/form-data'] = multipartMiddleware.none();

7
app/middlewares/user.js Normal file
View File

@ -0,0 +1,7 @@
var CdbRequest = require('../models/cartodb_request');
var cdbRequest = new CdbRequest();
module.exports = function userMiddleware(req, res, next) {
req.context.user = cdbRequest.userByReq(req);
next();
};

51
app/models/log4js_rollbar.js
View File

@ -1,51 +0,0 @@
var rollbar = require("rollbar");
/**
* Rollbar Appender. Sends logging events to Rollbar using node-rollbar
*
* @param config object with rollbar configuration data
* {
* token: 'your-secret-token',
* options: node-rollbar options
* }
*/
function rollbarAppender(config) {
var opt = config.options;
rollbar.init(opt.token, opt.options);
return function(loggingEvent) {
/*
For logger.trace('one','two','three'):
{ startTime: Wed Mar 12 2014 16:27:40 GMT+0100 (CET),
categoryName: '[default]',
data: [ 'one', 'two', 'three' ],
level: { level: 5000, levelStr: 'TRACE' },
logger: { category: '[default]', _events: { log: [Object] } } }
*/
// Levels:
// TRACE 5000
// DEBUG 10000
// INFO 20000
// WARN 30000
// ERROR 40000
// FATAL 50000
//
// We only log error and higher errors
//
if ( loggingEvent.level.level < 40000 ) {
return;
}
rollbar.reportMessage(loggingEvent.data);
};
}
function configure(config) {
return rollbarAppender(config);
}
exports.name = "rollbar";
exports.appender = rollbarAppender;
exports.configure = configure;

39
app/app.js → app/server.js
View File

@ -15,8 +15,9 @@
//
var express = require('express');
var bodyParser = require('./middlewares/body-parser');
var os = require('os');
var Profiler = require('step-profiler');
var Profiler = require('./stats/profiler-proxy');
var StatsD = require('node-statsd').StatsD;
var _ = require('underscore');
var LRU = require('lru-cache');
@ -50,7 +51,7 @@ require('./utils/date_to_json');
// jshint maxcomplexity:12
function App() {
var app = express.createServer();
var app = express();
var redisConfig = {
host: global.settings.redis_host,
@ -102,16 +103,6 @@ function App() {
}
};
app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level:'info'})));
} else {
// Express logger uses tokens as described here: http://www.senchalabs.org/connect/logger.html
express.logger.token('sql', function(req) {
return app.getSqlQueryFromRequestBody(req);
});
app.use(express.logger({
buffer: true,
format: global.settings.log_format ||
':remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type]'
}));
}
// Initialize statsD client if requested
@ -156,12 +147,20 @@ function App() {
app.use(cors());
// Use step-profiler
if ( global.settings.useProfiler ) {
app.use(function(req, res, next) {
req.profiler = new Profiler({statsd_client:statsd_client});
app.use(function bootstrap$prepareRequestResponse(req, res, next) {
req.context = req.context || {};
if (global.settings.api_hostname) {
res.header('X-Served-By-Host', global.settings.api_hostname);
}
var profile = global.settings.useProfiler;
req.profiler = new Profiler({
profile: profile,
statsd_client: statsd_client
});
next();
});
}
});
// Set connection timeout
if ( global.settings.hasOwnProperty('node_socket_timeout') ) {
@ -172,9 +171,11 @@ function App() {
});
}
app.use(express.bodyParser());
app.use(bodyParser());
app.enable('jsonp callback');
app.set("trust proxy", true);
app.disable('x-powered-by');
app.disable('etag');
// basic routing
@ -209,7 +210,7 @@ function App() {
var isBatchProcess = process.argv.indexOf('--no-batch') === -1;
if (global.settings.environment !== 'test' && isBatchProcess) {
app.batch = batchFactory(metadataBackend, redisConfig, statsd_client);
app.batch = batchFactory(metadataBackend, redisConfig, statsd_client, global.settings.batch_log_filename);
app.batch.start();
}

53
app/stats/profiler-proxy.js Normal file
View File

@ -0,0 +1,53 @@
var Profiler = require('step-profiler');
/**
* Proxy to encapsulate node-step-profiler module so there is no need to check if there is an instance
*/
function ProfilerProxy(opts) {
this.profile = !!opts.profile;
this.profiler = null;
if (!!opts.profile) {
this.profiler = new Profiler({statsd_client: opts.statsd_client});
}
}
ProfilerProxy.prototype.done = function(what) {
if (this.profile) {
this.profiler.done(what);
}
};
ProfilerProxy.prototype.end = function() {
if (this.profile) {
this.profiler.end();
}
};
ProfilerProxy.prototype.start = function(what) {
if (this.profile) {
this.profiler.start(what);
}
};
ProfilerProxy.prototype.add = function(what) {
if (this.profile) {
this.profiler.add(what || {});
}
};
ProfilerProxy.prototype.sendStats = function() {
if (this.profile) {
this.profiler.sendStats();
}
};
ProfilerProxy.prototype.toString = function() {
return this.profile ? this.profiler.toString() : "";
};
ProfilerProxy.prototype.toJSONString = function() {
return this.profile ? this.profiler.toJSONString() : "{}";
};
module.exports = ProfilerProxy;

18
app/utils/error_handler.js
View File

@ -25,14 +25,22 @@ module.exports = function handleException(err, res) {
// Force inline content disposition
res.header("Content-Disposition", 'inline');
if ( res.req && res.req.profiler ) {
res.req.profiler.done('finish');
res.header('X-SQLAPI-Profiler', res.req.profiler.toJSONString());
var req = res.req;
if (req && req.profiler ) {
req.profiler.done('finish');
res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
}
res.send(msg, getStatusError(pgErrorHandler, res.req));
res.header('Content-Type', 'application/json; charset=utf-8');
res.status(getStatusError(pgErrorHandler, req));
if (req.query && req.query.callback) {
res.jsonp(msg);
} else {
res.json(msg);
}
if ( res.req && res.req.profiler ) {
if (req && req.profiler) {
res.req.profiler.sendStats();
}
};

91
batch/README.md Normal file
View File

@ -0,0 +1,91 @@
# Batch Queries
This document describes the currently supported query types, and what they are missing in terms of features.
## Job types
### Simple
```json
{
"query": "update ..."
}
```
Does not support main fallback queries. Ideally it should support something like:
```json
{
"query": "update ...",
"onsuccess": "select 'general success fallback'",
"onerror": "select 'general error fallback'"
}
```
### Multiple
```json
{
"query": [
"update ...",
"select ... into ..."
]
}
```
Does not support main fallback queries. Ideally it should support something like:
```json
{
"query": [
"update ...",
"select ... into ..."
],
"onsuccess": "select 'general success fallback'",
"onerror": "select 'general error fallback'"
}
```
### Fallback
```json
{
"query": {
"query": [
{
"query": "select 1",
"onsuccess": "select 'success fallback query 1'",
"onerror": "select 'error fallback query 1'"
},
{
"query": "select 2",
"onerror": "select 'error fallback query 2'"
}
],
"onsuccess": "select 'general success fallback'",
"onerror": "select 'general error fallback'"
}
}
```
It's weird to have two nested `query` attributes. Also, it's not possible to mix _plain_ with _fallback_ ones.
Ideally it should support something like:
```json
{
"query": [
{
"query": "select 1",
"onsuccess": "select 'success fallback query 1'",
"onerror": "select 'error fallback query 1'"
},
"select 2"
],
"onsuccess": "select 'general success fallback'",
"onerror": "select 'general error fallback'"
}
}
```
Where you don't need a nested `query` attribute, it's just an array as in Multiple job type, and you can mix objects and
plain queries.

29
batch/batch-logger.js Normal file
View File

@ -0,0 +1,29 @@
'use strict';
var bunyan = require('bunyan');
function BatchLogger (path) {
var stream = {
level: 'info'
};
if (path) {
stream.path = path;
} else {
stream.stream = process.stdout;
}
this.path = path;
this.logger = bunyan.createLogger({
name: 'batch-queries',
streams: [stream]
});
}
module.exports = BatchLogger;
BatchLogger.prototype.log = function (job) {
return job.log(this.logger);
};
BatchLogger.prototype.reopenFileStreams = function () {
this.logger.reopenFileStreams();
};

5
batch/batch.js
View File

@ -7,12 +7,13 @@ var forever = require('./util/forever');
var queue = require('queue-async');
var jobStatus = require('./job_status');
function Batch(jobSubscriber, jobQueuePool, jobRunner, jobService) {
function Batch(jobSubscriber, jobQueuePool, jobRunner, jobService, logger) {
EventEmitter.call(this);
this.jobSubscriber = jobSubscriber;
this.jobQueuePool = jobQueuePool;
this.jobRunner = jobRunner;
this.jobService = jobService;
this.logger = logger;
}
util.inherits(Batch, EventEmitter);
@ -90,6 +91,8 @@ Batch.prototype._consumeJobs = function (host, queue, callback) {
debug('Job %s %s in %s', job_id, job.data.status, host);
}
self.logger.log(job);
self.emit('job:' + job.data.status, job_id);
callback();

7
batch/index.js
View File

@ -1,6 +1,5 @@
'use strict';
var RedisPool = require('redis-mpool');
var _ = require('underscore');
var JobRunner = require('./job_runner');
@ -14,9 +13,10 @@ var JobPublisher = require('./job_publisher');
var JobQueue = require('./job_queue');
var JobBackend = require('./job_backend');
var JobService = require('./job_service');
var BatchLogger = require('./batch-logger');
var Batch = require('./batch');
module.exports = function batchFactory (metadataBackend, redisConfig, statsdClient) {
module.exports = function batchFactory (metadataBackend, redisConfig, statsdClient, loggerPath) {
var redisPoolSubscriber = new RedisPool(_.extend(redisConfig, { name: 'batch-subscriber'}));
var redisPoolPublisher = new RedisPool(_.extend(redisConfig, { name: 'batch-publisher'}));
var queueSeeker = new QueueSeeker(metadataBackend);
@ -30,6 +30,7 @@ module.exports = function batchFactory (metadataBackend, redisConfig, statsdClie
var jobCanceller = new JobCanceller(userDatabaseMetadataService);
var jobService = new JobService(jobBackend, jobCanceller);
var jobRunner = new JobRunner(jobService, jobQueue, queryRunner, statsdClient);
var logger = new BatchLogger(loggerPath);
return new Batch(jobSubscriber, jobQueuePool, jobRunner, jobService);
return new Batch(jobSubscriber, jobQueuePool, jobRunner, jobService, logger);
};

4
batch/models/job_base.js
View File

@ -111,3 +111,7 @@ JobBase.prototype.serialize = function () {
return data;
};
JobBase.prototype.log = function(/*logger*/) {
return false;
};

71
batch/models/job_fallback.js
View File

@ -7,6 +7,9 @@ var QueryFallback = require('./query/query_fallback');
var MainFallback = require('./query/main_fallback');
var QueryFactory = require('./query/query_factory');
var JobUtils = require('./job_state_machine');
var jobUtils = new JobUtils();
function JobFallback(jobDefinition) {
JobBase.call(this, jobDefinition);
@ -206,3 +209,71 @@ JobFallback.prototype.getLastFinishedStatus = function () {
return this.isFinalStatus(status) ? status : lastFinished;
}.bind(this), jobStatus.DONE);
};
JobFallback.prototype.log = function(logger) {
if (!isFinished(this)) {
return false;
}
var queries = this.data.query.query;
for (var i = 0; i < queries.length; i++) {
var query = queries[i];
var logEntry = {
time: query.started_at,
endtime: query.ended_at,
username: this.data.user,
dbhost: this.data.host,
job: this.data.job_id,
elapsed: elapsedTime(query.started_at, query.ended_at)
};
var queryId = query.id;
var tag = 'query';
if (queryId) {
logEntry.query_id = queryId;
var node = parseQueryId(queryId);
if (node) {
logEntry.analysis = node.analysisId;
logEntry.node = node.nodeId;
logEntry.type = node.nodeType;
tag = 'analysis';
}
}
logger.info(logEntry, tag);
}
return true;
};
function isFinished (job) {
return jobUtils.isFinalStatus(job.data.status) &&
(!job.data.fallback_status || jobUtils.isFinalStatus(job.data.fallback_status));
}
function parseQueryId (queryId) {
var data = queryId.split(':');
if (data.length === 3) {
return {
analysisId: data[0],
nodeId: data[1],
nodeType: data[2]
};
}
return null;
}
function elapsedTime (started_at, ended_at) {
if (!started_at || !ended_at) {
return;
}
var start = new Date(started_at);
var end = new Date(ended_at);
return end.getTime() - start.getTime();
}

1
config/environments/development.js.example
View File

@ -31,6 +31,7 @@ module.exports.db_port = '5432';
module.exports.db_batch_port = '5432';
module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours
module.exports.batch_query_timeout || 12 * 3600 * 1000; // 12 hours in milliseconds
module.exports.batch_log_filename = 'logs/batch-queries.log';
// Max database connections in the pool
// Subsequent connections will wait for a free slot.
// NOTE: not used by OGR-mediated accesses

9
config/environments/production.js.example
View File

@ -32,6 +32,7 @@ module.exports.db_port = '6432';
module.exports.db_batch_port = '5432';
module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours
module.exports.batch_query_timeout || 12 * 3600 * 1000; // 12 hours in milliseconds
module.exports.batch_log_filename = 'logs/batch-queries.log';
// Max database connections in the pool
// Subsequent connections will wait for a free slot.i
// NOTE: not used by OGR-mediated accesses
@ -65,14 +66,6 @@ module.exports.tableCacheMaxAge = 1000*60*10;
module.exports.tmpDir = '/tmp';
// change ogr2ogr command or path
module.exports.ogr2ogrCommand = 'ogr2ogr';
// Optional rollbar support
module.exports.rollbar = {
token: 'secret',
// See http://github.com/rollbar/node_rollbar#configuration-reference
options: {
handler: 'inline'
}
}
// Optional statsd support
module.exports.statsd = {
host: 'localhost',

9
config/environments/staging.js.example
View File

@ -32,6 +32,7 @@ module.exports.db_port = '6432';
module.exports.db_batch_port = '5432';
module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours
module.exports.batch_query_timeout || 12 * 3600 * 1000; // 12 hours in milliseconds
module.exports.batch_log_filename = 'logs/batch-queries.log';
// Max database connections in the pool
// Subsequent connections will wait for a free slot.
// NOTE: not used by OGR-mediated accesses
@ -65,14 +66,6 @@ module.exports.tableCacheMaxAge = 1000*60*10;
module.exports.tmpDir = '/tmp';
// change ogr2ogr command or path
module.exports.ogr2ogrCommand = 'ogr2ogr';
// Optional rollbar support
module.exports.rollbar = {
token: 'secret',
// See http://github.com/rollbar/node_rollbar#configuration-reference
options: {
handler: 'inline'
}
}
// Optional statsd support
module.exports.statsd = {
host: 'localhost',

1
config/environments/test.js.example
View File

@ -29,6 +29,7 @@ module.exports.db_port = '5432';
module.exports.db_batch_port = '5432';
module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours
module.exports.batch_query_timeout || 12 * 3600 * 1000; // 12 hours in milliseconds
module.exports.batch_log_filename = 'logs/batch-queries.log';
// Max database connections in the pool
// Subsequent connections will wait for a free slot.
// NOTE: not used by OGR-mediated accesses

4
config/settings.js
View File

@ -1,4 +0,0 @@
var path = require('path');
module.exports.app_root = path.join(__dirname, '..');

5
configure vendored
View File

@ -20,9 +20,8 @@ while test -n "$1"; do
PGPORT=`echo "$1" | cut -d= -f2`
;;
*)
echo "Unknown option '$1'" >&2
usage >&2
exit 1
echo "Unused option '$1'" >&2
;;
esac
shift
done

5
doc/batch_queries.md
View File

@ -482,10 +482,11 @@ In some scenarios, you may need to fetch the output of a job. If that is the cas
For best practices, follow these recommended usage notes when using Batch Queries:
- Batch Queries are not intended for large query payloads (e.g: inserting thousands of rows), use the [Import API](https://carto.com/docs/carto-engine/import-api/) for this type of data management
- Batch Queries are recommended for INSERT, UPDATE, and CREATE queries that manipulate and create new data, such as creating expensive indexes, applying updates over large tables, and creating tables from complex queries. Batch queries have no effect for SELECT queries that retrieve data but do not store the results in a table. For example, running a batch query using `SELECT * from my_dataset` will not produce any results.
- Batch Queries are not intended for large query payloads (e.g: inserting thousands of rows), use the [Import API](https://carto.com/docs/carto-engine/import-api/) for this type of data management.
- There is a limit of 8kb per job. The following error message appears if your job exceeds this size:
`Your payload is too large. Max size allowed is 8192 (8kb)`
- Only the `query` element of the job scheme can be modified. All other elements of the job schema are defined by the Batch Query and are read-only

924
npm-shrinkwrap.json generated
View File

@ -1,7 +1,132 @@
{
"name": "cartodb_sql_api",
"version": "1.34.2",
"version": "1.37.2",
"dependencies": {
"bunyan": {
"version": "1.8.1",
"from": "bunyan@1.8.1",
"resolved": "https://registry.npmjs.org/bunyan/-/bunyan-1.8.1.tgz",
"dependencies": {
"dtrace-provider": {
"version": "0.6.0",
"from": "dtrace-provider@>=0.6.0 <0.7.0",
"resolved": "https://registry.npmjs.org/dtrace-provider/-/dtrace-provider-0.6.0.tgz",
"dependencies": {
"nan": {
"version": "2.4.0",
"from": "nan@>=2.0.8 <3.0.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.4.0.tgz"
}
}
},
"mv": {
"version": "2.1.1",
"from": "mv@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
"dependencies": {
"mkdirp": {
"version": "0.5.1",
"from": "mkdirp@>=0.5.1 <0.6.0",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
"dependencies": {
"minimist": {
"version": "0.0.8",
"from": "minimist@0.0.8",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz"
}
}
},
"ncp": {
"version": "2.0.0",
"from": "ncp@>=2.0.0 <2.1.0",
"resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz"
},
"rimraf": {
"version": "2.4.5",
"from": "rimraf@>=2.4.0 <2.5.0",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
"dependencies": {
"glob": {
"version": "6.0.4",
"from": "glob@>=6.0.1 <7.0.0",
"resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
"dependencies": {
"inflight": {
"version": "1.0.5",
"from": "inflight@>=1.0.4 <2.0.0",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.5.tgz",
"dependencies": {
"wrappy": {
"version": "1.0.2",
"from": "wrappy@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz"
}
}
},
"inherits": {
"version": "2.0.3",
"from": "inherits@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz"
},
"minimatch": {
"version": "3.0.3",
"from": "minimatch@>=2.0.0 <3.0.0||>=3.0.0 <4.0.0",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.3.tgz",
"dependencies": {
"brace-expansion": {
"version": "1.1.6",
"from": "brace-expansion@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.6.tgz",
"dependencies": {
"balanced-match": {
"version": "0.4.2",
"from": "balanced-match@>=0.4.1 <0.5.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-0.4.2.tgz"
},
"concat-map": {
"version": "0.0.1",
"from": "concat-map@0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz"
}
}
}
}
},
"once": {
"version": "1.4.0",
"from": "once@>=1.3.0 <2.0.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"dependencies": {
"wrappy": {
"version": "1.0.2",
"from": "wrappy@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz"
}
}
},
"path-is-absolute": {
"version": "1.0.1",
"from": "path-is-absolute@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz"
}
}
}
}
}
}
},
"safe-json-stringify": {
"version": "1.0.3",
"from": "safe-json-stringify@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/safe-json-stringify/-/safe-json-stringify-1.0.3.tgz"
},
"moment": {
"version": "2.15.1",
"from": "moment@>=2.10.6 <3.0.0",
"resolved": "https://registry.npmjs.org/moment/-/moment-2.15.1.tgz"
}
}
},
"cartodb-psql": {
"version": "0.6.1",
"from": "cartodb-psql@>=0.6.0 <0.7.0",
@ -56,36 +181,268 @@
}
},
"express": {
"version": "2.5.11",
"from": "express@>=2.5.11 <2.6.0",
"resolved": "https://registry.npmjs.org/express/-/express-2.5.11.tgz",
"version": "4.13.4",
"from": "express@>=4.13.3 <4.14.0",
"resolved": "https://registry.npmjs.org/express/-/express-4.13.4.tgz",
"dependencies": {
"connect": {
"version": "1.9.2",
"from": "connect@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/connect/-/connect-1.9.2.tgz",
"accepts": {
"version": "1.2.13",
"from": "accepts@>=1.2.12 <1.3.0",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.2.13.tgz",
"dependencies": {
"formidable": {
"version": "1.0.17",
"from": "formidable@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/formidable/-/formidable-1.0.17.tgz"
"mime-types": {
"version": "2.1.12",
"from": "mime-types@>=2.1.11 <2.2.0",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.12.tgz",
"dependencies": {
"mime-db": {
"version": "1.24.0",
"from": "mime-db@>=1.24.0 <1.25.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.24.0.tgz"
}
}
},
"negotiator": {
"version": "0.5.3",
"from": "negotiator@0.5.3",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.5.3.tgz"
}
}
},
"mime": {
"version": "1.2.4",
"from": "mime@1.2.4",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.2.4.tgz"
"array-flatten": {
"version": "1.1.1",
"from": "array-flatten@1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz"
},
"content-disposition": {
"version": "0.5.1",
"from": "content-disposition@0.5.1",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.1.tgz"
},
"content-type": {
"version": "1.0.2",
"from": "content-type@>=1.0.1 <1.1.0",
"resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.2.tgz"
},
"cookie": {
"version": "0.1.5",
"from": "cookie@0.1.5",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.1.5.tgz"
},
"cookie-signature": {
"version": "1.0.6",
"from": "cookie-signature@1.0.6",
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz"
},
"depd": {
"version": "1.1.0",
"from": "depd@>=1.1.0 <1.2.0",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.0.tgz"
},
"escape-html": {
"version": "1.0.3",
"from": "escape-html@>=1.0.3 <1.1.0",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz"
},
"etag": {
"version": "1.7.0",
"from": "etag@>=1.7.0 <1.8.0",
"resolved": "https://registry.npmjs.org/etag/-/etag-1.7.0.tgz"
},
"finalhandler": {
"version": "0.4.1",
"from": "finalhandler@0.4.1",
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-0.4.1.tgz",
"dependencies": {
"unpipe": {
"version": "1.0.0",
"from": "unpipe@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz"
}
}
},
"fresh": {
"version": "0.3.0",
"from": "fresh@0.3.0",
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.3.0.tgz"
},
"merge-descriptors": {
"version": "1.0.1",
"from": "merge-descriptors@1.0.1",
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz"
},
"methods": {
"version": "1.1.2",
"from": "methods@>=1.1.2 <1.2.0",
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz"
},
"on-finished": {
"version": "2.3.0",
"from": "on-finished@>=2.3.0 <2.4.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
"dependencies": {
"ee-first": {
"version": "1.1.1",
"from": "ee-first@1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
}
}
},
"parseurl": {
"version": "1.3.1",
"from": "parseurl@>=1.3.1 <1.4.0",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.1.tgz"
},
"path-to-regexp": {
"version": "0.1.7",
"from": "path-to-regexp@0.1.7",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz"
},
"proxy-addr": {
"version": "1.0.10",
"from": "proxy-addr@>=1.0.10 <1.1.0",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-1.0.10.tgz",
"dependencies": {
"forwarded": {
"version": "0.1.0",
"from": "forwarded@>=0.1.0 <0.2.0",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.0.tgz"
},
"ipaddr.js": {
"version": "1.0.5",
"from": "ipaddr.js@1.0.5",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.0.5.tgz"
}
}
},
"qs": {
"version": "0.4.2",
"from": "qs@>=0.4.0 <0.5.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-0.4.2.tgz"
"version": "4.0.0",
"from": "qs@4.0.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-4.0.0.tgz"
},
"mkdirp": {
"version": "0.3.0",
"from": "mkdirp@0.3.0",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.3.0.tgz"
"range-parser": {
"version": "1.0.3",
"from": "range-parser@>=1.0.3 <1.1.0",
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.0.3.tgz"
},
"send": {
"version": "0.13.1",
"from": "send@0.13.1",
"resolved": "https://registry.npmjs.org/send/-/send-0.13.1.tgz",
"dependencies": {
"destroy": {
"version": "1.0.4",
"from": "destroy@>=1.0.4 <1.1.0",
"resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz"
},
"http-errors": {
"version": "1.3.1",
"from": "http-errors@>=1.3.1 <1.4.0",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
"dependencies": {
"inherits": {
"version": "2.0.3",
"from": "inherits@>=2.0.1 <2.1.0",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz"
}
}
},
"mime": {
"version": "1.3.4",
"from": "mime@1.3.4",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz"
},
"ms": {
"version": "0.7.1",
"from": "ms@0.7.1",
"resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
},
"statuses": {
"version": "1.2.1",
"from": "statuses@>=1.2.1 <1.3.0",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
}
}
},
"serve-static": {
"version": "1.10.3",
"from": "serve-static@>=1.10.2 <1.11.0",
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.10.3.tgz",
"dependencies": {
"send": {
"version": "0.13.2",
"from": "send@0.13.2",
"resolved": "https://registry.npmjs.org/send/-/send-0.13.2.tgz",
"dependencies": {
"destroy": {
"version": "1.0.4",
"from": "destroy@>=1.0.4 <1.1.0",
"resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz"
},
"http-errors": {
"version": "1.3.1",
"from": "http-errors@>=1.3.1 <1.4.0",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.3.1.tgz",
"dependencies": {
"inherits": {
"version": "2.0.3",
"from": "inherits@>=2.0.1 <2.1.0",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz"
}
}
},
"mime": {
"version": "1.3.4",
"from": "mime@1.3.4",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.3.4.tgz"
},
"ms": {
"version": "0.7.1",
"from": "ms@0.7.1",
"resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
},
"statuses": {
"version": "1.2.1",
"from": "statuses@>=1.2.1 <1.3.0",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.2.1.tgz"
}
}
}
}
},
"type-is": {
"version": "1.6.13",
"from": "type-is@>=1.6.6 <1.7.0",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.13.tgz",
"dependencies": {
"media-typer": {
"version": "0.3.0",
"from": "media-typer@0.3.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
},
"mime-types": {
"version": "2.1.12",
"from": "mime-types@>=2.1.11 <2.2.0",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.12.tgz",
"dependencies": {
"mime-db": {
"version": "1.24.0",
"from": "mime-db@>=1.24.0 <1.25.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.24.0.tgz"
}
}
}
}
},
"utils-merge": {
"version": "1.0.0",
"from": "utils-merge@1.0.0",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.0.tgz"
},
"vary": {
"version": "1.0.1",
"from": "vary@>=1.0.1 <1.1.0",
"resolved": "https://registry.npmjs.org/vary/-/vary-1.0.1.tgz"
}
}
},
@ -120,9 +477,9 @@
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
},
"inherits": {
"version": "2.0.1",
"version": "2.0.3",
"from": "inherits@>=2.0.1 <2.1.0",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.1.tgz"
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz"
}
}
},
@ -143,6 +500,171 @@
"from": "lru-cache@>=2.5.0 <2.6.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.5.2.tgz"
},
"multer": {
"version": "1.2.0",
"from": "multer@>=1.2.0 <1.3.0",
"resolved": "https://registry.npmjs.org/multer/-/multer-1.2.0.tgz",
"dependencies": {
"append-field": {
"version": "0.1.0",
"from": "append-field@>=0.1.0 <0.2.0",
"resolved": "https://registry.npmjs.org/append-field/-/append-field-0.1.0.tgz"
},
"busboy": {
"version": "0.2.13",
"from": "busboy@>=0.2.11 <0.3.0",
"resolved": "https://registry.npmjs.org/busboy/-/busboy-0.2.13.tgz",
"dependencies": {
"dicer": {
"version": "0.2.5",
"from": "dicer@0.2.5",
"resolved": "https://registry.npmjs.org/dicer/-/dicer-0.2.5.tgz",
"dependencies": {
"streamsearch": {
"version": "0.1.2",
"from": "streamsearch@0.1.2",
"resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-0.1.2.tgz"
}
}
},
"readable-stream": {
"version": "1.1.14",
"from": "readable-stream@>=1.1.0 <1.2.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz",
"dependencies": {
"core-util-is": {
"version": "1.0.2",
"from": "core-util-is@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz"
},
"isarray": {
"version": "0.0.1",
"from": "isarray@0.0.1",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz"
},
"string_decoder": {
"version": "0.10.31",
"from": "string_decoder@>=0.10.0 <0.11.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
},
"inherits": {
"version": "2.0.3",
"from": "inherits@>=2.0.1 <2.1.0",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz"
}
}
}
}
},
"concat-stream": {
"version": "1.5.2",
"from": "concat-stream@>=1.5.0 <2.0.0",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.5.2.tgz",
"dependencies": {
"inherits": {
"version": "2.0.3",
"from": "inherits@>=2.0.1 <2.1.0",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz"
},
"typedarray": {
"version": "0.0.6",
"from": "typedarray@>=0.0.5 <0.1.0",
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz"
},
"readable-stream": {
"version": "2.0.6",
"from": "readable-stream@>=2.0.0 <2.1.0",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.0.6.tgz",
"dependencies": {
"core-util-is": {
"version": "1.0.2",
"from": "core-util-is@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz"
},
"isarray": {
"version": "1.0.0",
"from": "isarray@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz"
},
"process-nextick-args": {
"version": "1.0.7",
"from": "process-nextick-args@>=1.0.6 <1.1.0",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz"
},
"string_decoder": {
"version": "0.10.31",
"from": "string_decoder@>=0.10.0 <0.11.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz"
},
"util-deprecate": {
"version": "1.0.2",
"from": "util-deprecate@>=1.0.1 <1.1.0",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz"
}
}
}
}
},
"mkdirp": {
"version": "0.5.1",
"from": "mkdirp@>=0.5.1 <0.6.0",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
"dependencies": {
"minimist": {
"version": "0.0.8",
"from": "minimist@0.0.8",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz"
}
}
},
"object-assign": {
"version": "3.0.0",
"from": "object-assign@>=3.0.0 <4.0.0",
"resolved": "https://registry.npmjs.org/object-assign/-/object-assign-3.0.0.tgz"
},
"on-finished": {
"version": "2.3.0",
"from": "on-finished@>=2.3.0 <3.0.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
"dependencies": {
"ee-first": {
"version": "1.1.1",
"from": "ee-first@1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz"
}
}
},
"type-is": {
"version": "1.6.13",
"from": "type-is@>=1.6.4 <2.0.0",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.13.tgz",
"dependencies": {
"media-typer": {
"version": "0.3.0",
"from": "media-typer@0.3.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz"
},
"mime-types": {
"version": "2.1.12",
"from": "mime-types@>=2.1.2 <2.2.0",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.12.tgz",
"dependencies": {
"mime-db": {
"version": "1.24.0",
"from": "mime-db@>=1.24.0 <1.25.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.24.0.tgz"
}
}
}
}
},
"xtend": {
"version": "4.0.1",
"from": "xtend@>=4.0.0 <5.0.0",
"resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz"
}
}
},
"node-statsd": {
"version": "0.0.7",
"from": "node-statsd@>=0.0.7 <0.1.0",
@ -165,6 +687,11 @@
}
}
},
"qs": {
"version": "6.2.1",
"from": "qs@>=6.2.1 <6.3.0",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.2.1.tgz"
},
"queue-async": {
"version": "1.0.7",
"from": "queue-async@>=1.0.7 <1.1.0",
@ -204,23 +731,6 @@
}
}
},
"rollbar": {
"version": "0.3.13",
"from": "rollbar@>=0.3.2 <0.4.0",
"resolved": "https://registry.npmjs.org/rollbar/-/rollbar-0.3.13.tgz",
"dependencies": {
"lru-cache": {
"version": "2.2.4",
"from": "lru-cache@>=2.2.1 <2.3.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-2.2.4.tgz"
},
"json-stringify-safe": {
"version": "5.0.1",
"from": "json-stringify-safe@>=5.0.0 <5.1.0",
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz"
}
}
},
"step": {
"version": "0.0.6",
"from": "step@>=0.0.5 <0.1.0",
@ -254,6 +764,336 @@
"version": "1.6.0",
"from": "underscore@>=1.6.0 <1.7.0",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.6.0.tgz"
},
"yargs": {
"version": "5.0.0",
"from": "yargs@>=5.0.0 <5.1.0",
"resolved": "https://registry.npmjs.org/yargs/-/yargs-5.0.0.tgz",
"dependencies": {
"cliui": {
"version": "3.2.0",
"from": "cliui@>=3.2.0 <4.0.0",
"resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz",
"dependencies": {
"strip-ansi": {
"version": "3.0.1",
"from": "strip-ansi@>=3.0.1 <4.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"dependencies": {
"ansi-regex": {
"version": "2.0.0",
"from": "ansi-regex@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz"
}
}
},
"wrap-ansi": {
"version": "2.0.0",
"from": "wrap-ansi@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.0.0.tgz"
}
}
},
"decamelize": {
"version": "1.2.0",
"from": "decamelize@>=1.1.1 <2.0.0",
"resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz"
},
"get-caller-file": {
"version": "1.0.2",
"from": "get-caller-file@>=1.0.1 <2.0.0",
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.2.tgz"
},
"lodash.assign": {
"version": "4.2.0",
"from": "lodash.assign@>=4.2.0 <5.0.0",
"resolved": "https://registry.npmjs.org/lodash.assign/-/lodash.assign-4.2.0.tgz"
},
"os-locale": {
"version": "1.4.0",
"from": "os-locale@>=1.4.0 <2.0.0",
"resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz",
"dependencies": {
"lcid": {
"version": "1.0.0",
"from": "lcid@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/lcid/-/lcid-1.0.0.tgz",
"dependencies": {
"invert-kv": {
"version": "1.0.0",
"from": "invert-kv@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz"
}
}
}
}
},
"read-pkg-up": {
"version": "1.0.1",
"from": "read-pkg-up@>=1.0.1 <2.0.0",
"resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-1.0.1.tgz",
"dependencies": {
"find-up": {
"version": "1.1.2",
"from": "find-up@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/find-up/-/find-up-1.1.2.tgz",
"dependencies": {
"path-exists": {
"version": "2.1.0",
"from": "path-exists@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz"
},
"pinkie-promise": {
"version": "2.0.1",
"from": "pinkie-promise@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz",
"dependencies": {
"pinkie": {
"version": "2.0.4",
"from": "pinkie@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz"
}
}
}
}
},
"read-pkg": {
"version": "1.1.0",
"from": "read-pkg@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-1.1.0.tgz",
"dependencies": {
"load-json-file": {
"version": "1.1.0",
"from": "load-json-file@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz",
"dependencies": {
"graceful-fs": {
"version": "4.1.9",
"from": "graceful-fs@>=4.1.2 <5.0.0",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.9.tgz"
},
"parse-json": {
"version": "2.2.0",
"from": "parse-json@>=2.2.0 <3.0.0",
"resolved": "https://registry.npmjs.org/parse-json/-/parse-json-2.2.0.tgz",
"dependencies": {
"error-ex": {
"version": "1.3.0",
"from": "error-ex@>=1.2.0 <2.0.0",
"resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.0.tgz",
"dependencies": {
"is-arrayish": {
"version": "0.2.1",
"from": "is-arrayish@>=0.2.1 <0.3.0",
"resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz"
}
}
}
}
},
"pify": {
"version": "2.3.0",
"from": "pify@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz"
},
"pinkie-promise": {
"version": "2.0.1",
"from": "pinkie-promise@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz",
"dependencies": {
"pinkie": {
"version": "2.0.4",
"from": "pinkie@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz"
}
}
},
"strip-bom": {
"version": "2.0.0",
"from": "strip-bom@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz",
"dependencies": {
"is-utf8": {
"version": "0.2.1",
"from": "is-utf8@>=0.2.0 <0.3.0",
"resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz"
}
}
}
}
},
"normalize-package-data": {
"version": "2.3.5",
"from": "normalize-package-data@>=2.3.2 <3.0.0",
"resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.3.5.tgz",
"dependencies": {
"hosted-git-info": {
"version": "2.1.5",
"from": "hosted-git-info@>=2.1.4 <3.0.0",
"resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.1.5.tgz"
},
"is-builtin-module": {
"version": "1.0.0",
"from": "is-builtin-module@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz",
"dependencies": {
"builtin-modules": {
"version": "1.1.1",
"from": "builtin-modules@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz"
}
}
},
"semver": {
"version": "5.3.0",
"from": "semver@>=2.0.0 <3.0.0||>=3.0.0 <4.0.0||>=4.0.0 <5.0.0||>=5.0.0 <6.0.0",
"resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz"
},
"validate-npm-package-license": {
"version": "3.0.1",
"from": "validate-npm-package-license@>=3.0.1 <4.0.0",
"resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz",
"dependencies": {
"spdx-correct": {
"version": "1.0.2",
"from": "spdx-correct@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-1.0.2.tgz",
"dependencies": {
"spdx-license-ids": {
"version": "1.2.2",
"from": "spdx-license-ids@>=1.0.2 <2.0.0",
"resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-1.2.2.tgz"
}
}
},
"spdx-expression-parse": {
"version": "1.0.3",
"from": "spdx-expression-parse@>=1.0.0 <1.1.0",
"resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-1.0.3.tgz"
}
}
}
}
},
"path-type": {
"version": "1.1.0",
"from": "path-type@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/path-type/-/path-type-1.1.0.tgz",
"dependencies": {
"graceful-fs": {
"version": "4.1.9",
"from": "graceful-fs@>=4.1.2 <5.0.0",
"resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.1.9.tgz"
},
"pify": {
"version": "2.3.0",
"from": "pify@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz"
},
"pinkie-promise": {
"version": "2.0.1",
"from": "pinkie-promise@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/pinkie-promise/-/pinkie-promise-2.0.1.tgz",
"dependencies": {
"pinkie": {
"version": "2.0.4",
"from": "pinkie@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/pinkie/-/pinkie-2.0.4.tgz"
}
}
}
}
}
}
}
}
},
"require-directory": {
"version": "2.1.1",
"from": "require-directory@>=2.1.1 <3.0.0",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz"
},
"require-main-filename": {
"version": "1.0.1",
"from": "require-main-filename@>=1.0.1 <2.0.0",
"resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz"
},
"set-blocking": {
"version": "2.0.0",
"from": "set-blocking@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/set-blocking/-/set-blocking-2.0.0.tgz"
},
"string-width": {
"version": "1.0.2",
"from": "string-width@>=1.0.2 <2.0.0",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
"dependencies": {
"code-point-at": {
"version": "1.0.1",
"from": "code-point-at@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.0.1.tgz",
"dependencies": {
"number-is-nan": {
"version": "1.0.1",
"from": "number-is-nan@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz"
}
}
},
"is-fullwidth-code-point": {
"version": "1.0.0",
"from": "is-fullwidth-code-point@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
"dependencies": {
"number-is-nan": {
"version": "1.0.1",
"from": "number-is-nan@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz"
}
}
},
"strip-ansi": {
"version": "3.0.1",
"from": "strip-ansi@>=3.0.1 <4.0.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"dependencies": {
"ansi-regex": {
"version": "2.0.0",
"from": "ansi-regex@>=2.0.0 <3.0.0",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.0.0.tgz"
}
}
}
}
},
"which-module": {
"version": "1.0.0",
"from": "which-module@>=1.0.0 <2.0.0",
"resolved": "https://registry.npmjs.org/which-module/-/which-module-1.0.0.tgz"
},
"window-size": {
"version": "0.2.0",
"from": "window-size@>=0.2.0 <0.3.0",
"resolved": "https://registry.npmjs.org/window-size/-/window-size-0.2.0.tgz"
},
"y18n": {
"version": "3.2.1",
"from": "y18n@>=3.2.1 <4.0.0",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz"
},
"yargs-parser": {
"version": "3.2.0",
"from": "yargs-parser@>=3.2.0 <4.0.0",
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-3.2.0.tgz",
"dependencies": {
"camelcase": {
"version": "3.0.0",
"from": "camelcase@>=3.0.0 <4.0.0",
"resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz"
}
}
}
}
}
}
}

12
package.json
View File

@ -5,7 +5,7 @@
"keywords": [
"cartodb"
],
"version": "1.34.3",
"version": "1.37.2",
"repository": {
"type": "git",
"url": "git://github.com/CartoDB/CartoDB-SQL-API.git"
@ -17,23 +17,26 @@
"Sandro Santilli <strk@vizzuality.com>"
],
"dependencies": {
"bunyan": "1.8.1",
"cartodb-psql": "~0.6.0",
"cartodb-query-tables": "0.2.0",
"cartodb-redis": "0.13.1",
"debug": "2.2.0",
"express": "~2.5.11",
"express": "~4.13.3",
"log4js": "cartodb/log4js-node#cdb",
"lru-cache": "~2.5.0",
"multer": "~1.2.0",
"node-statsd": "~0.0.7",
"node-uuid": "^1.4.7",
"oauth-client": "0.3.0",
"qs": "~6.2.1",
"queue-async": "~1.0.7",
"redis-mpool": "0.4.0",
"rollbar": "~0.3.2",
"step": "~0.0.5",
"step-profiler": "~0.3.0",
"topojson": "0.0.8",
"underscore": "~1.6.0"
"underscore": "~1.6.0",
"yargs": "~5.0.0"
},
"devDependencies": {
"istanbul": "~0.4.2",
@ -43,7 +46,6 @@
"jshint": "~2.6.0",
"zipfile": "~0.5.0",
"libxmljs": "~0.8.1",
"qs": "6.2.0",
"sqlite3": "~3.0.8"
},
"scripts": {

6
test/acceptance/app.auth.test.js
View File

@ -1,6 +1,6 @@
require('../helper');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../app/server')();
var assert = require('../support/assert');
describe('app.auth', function() {
@ -40,7 +40,7 @@ describe('app.auth', function() {
scenarios.forEach(function(scenario) {
it(scenario.desc, function(done) {
assert.response(app, {
assert.response(server, {
// view prepare_db.sh to find public table name and structure
url: scenario.url,
headers: {
@ -49,7 +49,7 @@ describe('app.auth', function() {
method: 'GET'
},
{},
function(res) {
function(err, res) {
assert.equal(res.statusCode, scenario.statusCode, res.statusCode + ': ' + res.body);
done();
}

350
test/acceptance/app.test.js
View File

File diff suppressed because it is too large Load Diff

16
test/acceptance/backend_crash.js
View File

@ -27,17 +27,14 @@ it('does not hang server', function(done){
var db_port_backup = global.settings.db_port;
global.settings.db_host = 'localhost';
global.settings.db_port = sql_server_port;
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../app/server')();
step(
function sendQuery() {
var next = this;
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT+1',
method: 'GET',
headers: {host: 'vizzuality.localhost' }
},{}, function(res, err) {
next(err, res);
});
},{}, this);
},
function checkResponse(err, res) {
assert.ifError(err);
@ -49,14 +46,11 @@ it('does not hang server', function(done){
return null;
},
function sendAnotherQuery() {
var next = this;
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT+2',
method: 'GET',
headers: {host: 'vizzuality.localhost' }
},{}, function(res, err) {
next(err, res);
});
},{}, this);
},
function checkResponse(err, res) {
assert.ifError(err);

32
test/acceptance/batch.test.js → test/acceptance/batch/batch.test.js
View File

@ -1,29 +1,23 @@
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
require('../../helper');
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var _ = require('underscore');
var RedisPool = require('redis-mpool');
var queue = require('queue-async');
var batchFactory = require('../../batch');
var batchFactory = require('../../../batch/index');
var JobPublisher = require('../../batch/job_publisher');
var JobQueue = require('../../batch/job_queue');
var JobBackend = require('../../batch/job_backend');
var JobService = require('../../batch/job_service');
var UserDatabaseMetadataService = require('../../batch/user_database_metadata_service');
var JobCanceller = require('../../batch/job_canceller');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var JobPublisher = require('../../../batch/job_publisher');
var JobQueue = require('../../../batch/job_queue');
var JobBackend = require('../../../batch/job_backend');
var JobService = require('../../../batch/job_service');
var UserDatabaseMetadataService = require('../../../batch/user_database_metadata_service');
var JobCanceller = require('../../../batch/job_canceller');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
describe('batch module', function() {
var dbInstance = 'localhost';
var username = 'vizzuality';
var redisPoolPublisher = new RedisPool(_.extend(redisConfig, { name: 'batch-publisher'}));
var redisPoolPublisher = new RedisPool(_.extend(redisUtils.getConfig(), { name: 'batch-publisher'}));
var jobPublisher = new JobPublisher(redisPoolPublisher);
var jobQueue = new JobQueue(metadataBackend, jobPublisher);
var jobBackend = new JobBackend(metadataBackend, jobQueue);
@ -31,7 +25,7 @@ describe('batch module', function() {
var jobCanceller = new JobCanceller(userDatabaseMetadataService);
var jobService = new JobService(jobBackend, jobCanceller);
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();

35
test/acceptance/job.callback-template.test.js → test/acceptance/batch/job.callback-template.test.js
View File

@ -1,24 +1,17 @@
require('../helper');
require('../../helper');
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var server = require('../../../app/server')();
var querystring = require('qs');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var jobStatus = require('../../batch/job_status');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var batchFactory = require('../../../batch/index');
var jobStatus = require('../../../batch/job_status');
describe('Batch API callback templates', function () {
function createJob(jobDefinition, callback) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -28,7 +21,7 @@ describe('Batch API callback templates', function () {
data: querystring.stringify(jobDefinition)
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return callback(err);
}
@ -37,7 +30,7 @@ describe('Batch API callback templates', function () {
}
function getJobStatus(jobId, callback) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + jobId + '?api_key=1234&',
headers: {
host: 'vizzuality.cartodb.com'
@ -45,7 +38,7 @@ describe('Batch API callback templates', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return callback(err);
}
@ -54,7 +47,7 @@ describe('Batch API callback templates', function () {
}
function getQueryResult(query, callback) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql?' + querystring.stringify({q: query, api_key: 1234}),
headers: {
host: 'vizzuality.cartodb.com'
@ -62,7 +55,7 @@ describe('Batch API callback templates', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return callback(err);
}
@ -95,7 +88,7 @@ describe('Batch API callback templates', function () {
assert.equal(actual.onerror, expected.onerror);
}
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();

215
test/acceptance/job.fallback.test.js → test/acceptance/batch/job.fallback.test.js
View File

@ -1,19 +1,12 @@
require('../helper');
require('../../helper');
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var server = require('../../../app/server')();
var querystring = require('qs');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var jobStatus = require('../../batch/job_status');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var batchFactory = require('../../../batch/index');
var jobStatus = require('../../../batch/job_status');
describe('Batch API fallback job', function () {
@ -36,7 +29,7 @@ describe('Batch API fallback job', function () {
assert.equal(actual.onerror, expected.onerror);
}
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();
@ -52,7 +45,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -69,7 +62,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -89,7 +82,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -98,7 +91,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -120,7 +113,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -137,7 +130,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -156,7 +149,7 @@ describe('Batch API fallback job', function () {
}]
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -165,7 +158,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -187,7 +180,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -204,7 +197,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -224,7 +217,7 @@ describe('Batch API fallback job', function () {
}]
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -233,7 +226,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -255,7 +248,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -272,7 +265,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -293,7 +286,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -302,7 +295,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -325,7 +318,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -342,7 +335,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -361,7 +354,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -370,7 +363,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -392,7 +385,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -409,7 +402,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -429,7 +422,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -438,7 +431,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -461,7 +454,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -478,7 +471,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -498,7 +491,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -507,7 +500,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -529,7 +522,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -546,7 +539,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -565,7 +558,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -574,7 +567,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -597,7 +590,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -615,7 +608,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -636,7 +629,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -645,7 +638,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -667,7 +660,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -687,7 +680,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -712,7 +705,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -721,7 +714,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -743,7 +736,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -763,7 +756,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -789,7 +782,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -798,7 +791,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -821,7 +814,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -841,7 +834,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -867,7 +860,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -876,7 +869,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -898,7 +891,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -918,7 +911,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -943,7 +936,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -952,7 +945,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -974,7 +967,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -991,7 +984,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1012,7 +1005,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1021,7 +1014,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1044,7 +1037,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1061,7 +1054,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1081,7 +1074,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1090,7 +1083,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1112,7 +1105,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1132,7 +1125,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1158,7 +1151,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1167,7 +1160,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1189,7 +1182,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1209,7 +1202,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1235,7 +1228,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1244,7 +1237,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1266,7 +1259,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1287,7 +1280,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1313,7 +1306,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1322,7 +1315,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1345,7 +1338,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1366,7 +1359,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1393,7 +1386,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1402,7 +1395,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1425,7 +1418,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1443,7 +1436,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1464,7 +1457,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1473,7 +1466,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1503,7 +1496,7 @@ describe('Batch API fallback job', function () {
"onsuccess": "SELECT pg_sleep(0)"
};
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1512,7 +1505,7 @@ describe('Batch API fallback job', function () {
method: 'DELETE'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1531,7 +1524,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1549,7 +1542,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1570,7 +1563,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1579,7 +1572,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1612,7 +1605,7 @@ describe('Batch API fallback job', function () {
"onsuccess": "SELECT pg_sleep(0)"
};
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1621,7 +1614,7 @@ describe('Batch API fallback job', function () {
method: 'DELETE'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1640,7 +1633,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1661,7 +1654,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1688,7 +1681,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1697,7 +1690,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1719,7 +1712,7 @@ describe('Batch API fallback job', function () {
var fallbackJob = {};
it('should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1740,7 +1733,7 @@ describe('Batch API fallback job', function () {
})
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}
@ -1770,7 +1763,7 @@ describe('Batch API fallback job', function () {
};
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -1779,7 +1772,7 @@ describe('Batch API fallback job', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return done(err);
}

26
test/acceptance/job.query.limit.test.js → test/acceptance/batch/job.query.limit.test.js
View File

@ -12,11 +12,11 @@
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../helper');
var JobController = require('../../app/controllers/job_controller');
var redisUtils = require('../support/redis_utils');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../support/assert');
require('../../helper');
var JobController = require('../../../app/controllers/job_controller');
var redisUtils = require('../../support/redis_utils');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var querystring = require('qs');
function payload(query) {
@ -42,7 +42,7 @@ describe('job query limit', function() {
it('POST /api/v2/sql/job with a invalid query size should respond with 400 query too long', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -51,7 +51,7 @@ describe('job query limit', function() {
})
}, {
status: 400
}, function (res) {
}, function (err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [expectedErrorMessage(queryTooLong)] });
done();
@ -60,7 +60,7 @@ describe('job query limit', function() {
it('POST /api/v2/sql/job with a valid query size should respond with 201 created', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -69,7 +69,7 @@ describe('job query limit', function() {
})
}, {
status: 201
}, function (res) {
}, function (err, res) {
var job = JSON.parse(res.body);
assert.ok(job.job_id);
done();
@ -78,7 +78,7 @@ describe('job query limit', function() {
it('POST /api/v2/sql/job with a invalid query size should consider multiple queries', function (done){
var queries = [queryTooLong, 'select 1'];
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -87,7 +87,7 @@ describe('job query limit', function() {
})
}, {
status: 400
}, function (res) {
}, function (err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [expectedErrorMessage(queries)] });
done();
@ -104,7 +104,7 @@ describe('job query limit', function() {
onsuccess: "SELECT * FROM untitle_table_4 limit 3"
}]
};
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -113,7 +113,7 @@ describe('job query limit', function() {
})
}, {
status: 400
}, function (res) {
}, function (err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [expectedErrorMessage(fallbackQueries)] });
done();

52
test/acceptance/job.test.js → test/acceptance/batch/job.test.js
View File

@ -12,11 +12,11 @@
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../helper');
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var querystring = require('querystring');
describe('job module', function() {
@ -27,7 +27,7 @@ describe('job module', function() {
});
it('POST /api/v2/sql/job should respond with 200 and the created job', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -36,7 +36,7 @@ describe('job module', function() {
})
}, {
status: 201
}, function(res) {
}, function(err, res) {
job = JSON.parse(res.body);
assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
assert.ok(job.job_id);
@ -48,14 +48,14 @@ describe('job module', function() {
it('POST /api/v2/sql/job without query should respond with 400 and the corresponding message of error',
function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
data: querystring.stringify({})
}, {
status: 400
}, function(res) {
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] });
done();
@ -63,7 +63,7 @@ describe('job module', function() {
});
it('POST /api/v2/sql/job with bad query param should respond with 400 and message of error', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -72,7 +72,7 @@ describe('job module', function() {
})
}, {
status: 400
}, function(res) {
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [ 'You must indicate a valid SQL' ] });
done();
@ -80,7 +80,7 @@ describe('job module', function() {
});
it('POST /api/v2/sql/job with wrong api key should respond with 401 permission denied', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=wrong',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -89,7 +89,7 @@ describe('job module', function() {
})
}, {
status: 401
}, function(res) {
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [ 'permission denied' ] });
done();
@ -97,7 +97,7 @@ describe('job module', function() {
});
it('POST /api/v2/sql/job with wrong host header should respond with 404 not found', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=wrong',
headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -106,7 +106,7 @@ describe('job module', function() {
})
}, {
status: 404
}, function(res) {
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, {
error: [
@ -119,13 +119,13 @@ describe('job module', function() {
});
it('GET /api/v2/sql/job/:job_id should respond with 200 and the requested job', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var jobGot = JSON.parse(res.body);
assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
assert.equal(jobGot.query, "SELECT * FROM untitle_table_4");
@ -135,13 +135,13 @@ describe('job module', function() {
});
it('GET /api/v2/sql/job/:job_id with wrong api key should respond with 401 permission denied', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 401
}, function(res) {
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [ 'permission denied' ] });
done();
@ -149,13 +149,13 @@ describe('job module', function() {
});
it('GET /api/v2/sql/job/:jobId with wrong jobId header respond with 400 and an error', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/irrelevantJob?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 400
}, function(res) {
}, function(err, res) {
var error = JSON.parse(res.body);
console.log(error);
assert.deepEqual(error , {
@ -166,13 +166,13 @@ describe('job module', function() {
});
it('DELETE /api/v2/sql/job/:job_id should respond with 200 and the requested job', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var jobCancelled = JSON.parse(res.body);
assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
assert.equal(jobCancelled.job_id, job.job_id);
@ -184,13 +184,13 @@ describe('job module', function() {
});
it('DELETE /api/v2/sql/job/:job_id with wrong api key should respond with 401 permission denied', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + job.job_id + '?api_key=wrong',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 401
}, function(res) {
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error, { error: [ 'permission denied' ] });
done();
@ -198,13 +198,13 @@ describe('job module', function() {
});
it('DELETE /api/v2/sql/job/ with wrong host header respond with 404 not found', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + job.job_id + '?api_key=1234',
headers: { 'host': 'wrong-host.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 404
}, function(res) {
}, function(err, res) {
var error = JSON.parse(res.body);
assert.deepEqual(error , {
error: [

31
test/acceptance/job.timing.test.js → test/acceptance/batch/job.timing.test.js
View File

@ -1,24 +1,17 @@
require('../helper');
require('../../helper');
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var server = require('../../../app/server')();
var querystring = require('qs');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var jobStatus = require('../../batch/job_status');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var batchFactory = require('../../../batch');
var jobStatus = require('../../../batch/job_status');
describe('Batch API query timing', function () {
function createJob(jobDefinition, callback) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
@ -28,7 +21,7 @@ describe('Batch API query timing', function () {
data: querystring.stringify(jobDefinition)
}, {
status: 201
}, function (res, err) {
}, function (err, res) {
if (err) {
return callback(err);
}
@ -37,7 +30,7 @@ describe('Batch API query timing', function () {
}
function getJobStatus(jobId, callback) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + jobId + '?api_key=1234&',
headers: {
host: 'vizzuality.cartodb.com'
@ -45,7 +38,7 @@ describe('Batch API query timing', function () {
method: 'GET'
}, {
status: 200
}, function (res, err) {
}, function (err, res) {
if (err) {
return callback(err);
}
@ -72,7 +65,7 @@ describe('Batch API query timing', function () {
assert.equal(actual.onerror, expected.onerror);
}
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();

33
test/acceptance/job.use-case-1.test.js → test/acceptance/batch/job.use-case-1.test.js
View File

@ -12,24 +12,17 @@
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../helper');
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var querystring = require('querystring');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var batchFactory = require('../../../batch/index');
describe('Use case 1: cancel and modify a done job', function () {
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();
@ -44,7 +37,7 @@ describe('Use case 1: cancel and modify a done job', function () {
var doneJob = {};
it('Step 1, should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -53,7 +46,7 @@ describe('Use case 1: cancel and modify a done job', function () {
})
}, {
status: 201
}, function (res) {
}, function (err, res) {
doneJob = JSON.parse(res.body);
done();
});
@ -61,13 +54,13 @@ describe('Use case 1: cancel and modify a done job', function () {
it('Step 2, job should be done', function (done) {
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function (res) {
}, function (err, res) {
var job = JSON.parse(res.body);
if (job.status === "done") {
clearInterval(interval);
@ -83,13 +76,13 @@ describe('Use case 1: cancel and modify a done job', function () {
});
it('Step 3, cancel a done job should give an error', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 400
}, function(res) {
}, function(err, res) {
var errors = JSON.parse(res.body);
assert.equal(errors.error[0], "Cannot set status from done to cancelled");
done();

33
test/acceptance/job.use-case-10.test.js → test/acceptance/batch/job.use-case-10.test.js
View File

@ -12,24 +12,17 @@
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../helper');
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var querystring = require('querystring');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var batchFactory = require('../../../batch/index');
describe('Use case 10: cancel and modify a done multiquery job', function () {
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();
@ -44,7 +37,7 @@ describe('Use case 10: cancel and modify a done multiquery job', function () {
var doneJob = {};
it('Step 1, should create a multiquery job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -57,7 +50,7 @@ describe('Use case 10: cancel and modify a done multiquery job', function () {
})
}, {
status: 201
}, function (res) {
}, function (err, res) {
doneJob = JSON.parse(res.body);
done();
});
@ -65,13 +58,13 @@ describe('Use case 10: cancel and modify a done multiquery job', function () {
it('Step 2, multiquery job should be done', function (done) {
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function (res) {
}, function (err, res) {
var job = JSON.parse(res.body);
if (job.status === "done") {
clearInterval(interval);
@ -87,13 +80,13 @@ describe('Use case 10: cancel and modify a done multiquery job', function () {
});
it('Step 3, cancel a done multiquery job should give an error', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 400
}, function(res) {
}, function(err, res) {
var errors = JSON.parse(res.body);
assert.equal(errors.error[0], "Cannot set status from done to cancelled");
done();

41
test/acceptance/job.use-case-2.test.js → test/acceptance/batch/job.use-case-2.test.js
View File

@ -12,24 +12,17 @@
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../helper');
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var querystring = require('querystring');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var batchFactory = require('../../../batch/index');
describe('Use case 2: cancel a running job', function() {
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();
@ -45,7 +38,7 @@ describe('Use case 2: cancel a running job', function() {
var cancelledJob = {};
it('Step 1, should create a new job', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -54,7 +47,7 @@ describe('Use case 2: cancel a running job', function() {
})
}, {
status: 201
}, function(res) {
}, function(err, res) {
runningJob = JSON.parse(res.body);
done();
});
@ -62,13 +55,13 @@ describe('Use case 2: cancel a running job', function() {
it('Step 2, job should be running', function (done){
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var job = JSON.parse(res.body);
if (job.status === "running") {
clearInterval(interval);
@ -82,13 +75,13 @@ describe('Use case 2: cancel a running job', function() {
});
it('Step 3, cancel a job', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 200
}, function(res) {
}, function(err, res) {
cancelledJob = JSON.parse(res.body);
assert.equal(cancelledJob.status, "cancelled");
done();
@ -96,13 +89,13 @@ describe('Use case 2: cancel a running job', function() {
});
it('Step 4, job should be cancelled', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var job = JSON.parse(res.body);
if (job.status === "cancelled") {
done();
@ -113,13 +106,13 @@ describe('Use case 2: cancel a running job', function() {
});
it('Step 5, cancel a cancelled should give an error', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + cancelledJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 400
}, function(res) {
}, function(err, res) {
var errors = JSON.parse(res.body);
assert.equal(errors.error[0], "Cannot set status from cancelled to cancelled");
done();

41
test/acceptance/job.use-case-3.test.js → test/acceptance/batch/job.use-case-3.test.js
View File

@ -12,24 +12,17 @@
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../helper');
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var querystring = require('querystring');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var batchFactory = require('../../../batch/index');
describe('Use case 3: cancel a pending job', function() {
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();
@ -45,7 +38,7 @@ describe('Use case 3: cancel a pending job', function() {
var pendingJob = {};
it('Step 1, should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -54,14 +47,14 @@ describe('Use case 3: cancel a pending job', function() {
})
}, {
status: 201
}, function (res) {
}, function (err, res) {
runningJob = JSON.parse(res.body);
done();
});
});
it('Step 2, should create a another job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -70,7 +63,7 @@ describe('Use case 3: cancel a pending job', function() {
})
}, {
status: 201
}, function(res) {
}, function(err, res) {
pendingJob = JSON.parse(res.body);
done();
});
@ -78,13 +71,13 @@ describe('Use case 3: cancel a pending job', function() {
it('Step 3, job should be pending', function (done){
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + pendingJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var job = JSON.parse(res.body);
if (job.status === "pending") {
clearInterval(interval);
@ -98,13 +91,13 @@ describe('Use case 3: cancel a pending job', function() {
});
it('Step 4, cancel a pending job should be cancelled', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + pendingJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var jobGot = JSON.parse(res.body);
assert.equal(jobGot.job_id, pendingJob.job_id);
assert.equal(jobGot.status, "cancelled");
@ -113,13 +106,13 @@ describe('Use case 3: cancel a pending job', function() {
});
it('Step 5, running job should be cancelled', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var cancelledJob = JSON.parse(res.body);
assert.equal(cancelledJob.status, "cancelled");
done();

37
test/acceptance/job.use-case-4.test.js → test/acceptance/batch/job.use-case-4.test.js
View File

@ -12,24 +12,17 @@
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../helper');
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var querystring = require('querystring');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var batchFactory = require('../../../batch');
describe('Use case 4: modify a pending job', function() {
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();
@ -45,7 +38,7 @@ describe('Use case 4: modify a pending job', function() {
var pendingJob = {};
it('Step 1, should create a job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -54,14 +47,14 @@ describe('Use case 4: modify a pending job', function() {
})
}, {
status: 201
}, function(res) {
}, function(err, res) {
runningJob = JSON.parse(res.body);
done();
});
});
it('Step 2, should create another job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -70,7 +63,7 @@ describe('Use case 4: modify a pending job', function() {
})
}, {
status: 201
}, function(res) {
}, function(err, res) {
pendingJob = JSON.parse(res.body);
done();
});
@ -78,13 +71,13 @@ describe('Use case 4: modify a pending job', function() {
it('Step 3, job should be pending', function (done){
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + pendingJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var job = JSON.parse(res.body);
if (job.status === "pending") {
clearInterval(interval);
@ -98,13 +91,13 @@ describe('Use case 4: modify a pending job', function() {
});
it('Step 5, running job should be cancelled', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var cancelledJob = JSON.parse(res.body);
assert.equal(cancelledJob.status, "cancelled");
done();

33
test/acceptance/job.use-case-5.test.js → test/acceptance/batch/job.use-case-5.test.js
View File

@ -12,24 +12,17 @@
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../helper');
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var querystring = require('querystring');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var batchFactory = require('../../../batch');
describe('Use case 5: modify a running job', function() {
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();
@ -44,7 +37,7 @@ describe('Use case 5: modify a running job', function() {
var runningJob = {};
it('Step 1, should create job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -53,7 +46,7 @@ describe('Use case 5: modify a running job', function() {
})
}, {
status: 201
}, function (res) {
}, function (err, res) {
runningJob = JSON.parse(res.body);
done();
});
@ -61,13 +54,13 @@ describe('Use case 5: modify a running job', function() {
it('Step 2, job should be running', function (done){
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var job = JSON.parse(res.body);
if (job.status === "running") {
clearInterval(interval);
@ -81,13 +74,13 @@ describe('Use case 5: modify a running job', function() {
});
it('Step 4, running job should be cancelled', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var cancelledJob = JSON.parse(res.body);
assert.equal(cancelledJob.status, "cancelled");
done();

29
test/acceptance/job.use-case-6.test.js → test/acceptance/batch/job.use-case-6.test.js
View File

@ -12,24 +12,17 @@
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../helper');
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var querystring = require('querystring');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var batchFactory = require('../../../batch');
describe('Use case 6: modify a done job', function() {
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();
@ -44,7 +37,7 @@ describe('Use case 6: modify a done job', function() {
var doneJob = {};
it('Step 1, should create job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -53,7 +46,7 @@ describe('Use case 6: modify a done job', function() {
})
}, {
status: 201
}, function (res) {
}, function (err, res) {
doneJob = JSON.parse(res.body);
done();
});
@ -61,13 +54,13 @@ describe('Use case 6: modify a done job', function() {
it('Step 2, job should be done', function (done) {
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + doneJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var job = JSON.parse(res.body);
if (job.status === "done") {
clearInterval(interval);

33
test/acceptance/job.use-case-7.test.js → test/acceptance/batch/job.use-case-7.test.js
View File

@ -12,24 +12,17 @@
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../helper');
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var querystring = require('querystring');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var batchFactory = require('../../../batch');
describe('Use case 7: cancel a job with quotes', function() {
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();
@ -44,7 +37,7 @@ describe('Use case 7: cancel a job with quotes', function() {
var runningJob = {};
it('Step 1, should create job with quotes', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -53,7 +46,7 @@ describe('Use case 7: cancel a job with quotes', function() {
})
}, {
status: 201
}, function (res) {
}, function (err, res) {
runningJob = JSON.parse(res.body);
done();
});
@ -61,13 +54,13 @@ describe('Use case 7: cancel a job with quotes', function() {
it('Step 2, job should be running', function (done){
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var job = JSON.parse(res.body);
if (job.status === "running") {
clearInterval(interval);
@ -81,13 +74,13 @@ describe('Use case 7: cancel a job with quotes', function() {
});
it('Step 3, running job should be cancelled', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var cancelledJob = JSON.parse(res.body);
assert.equal(cancelledJob.status, "cancelled");
done();

41
test/acceptance/job.use-case-8.test.js → test/acceptance/batch/job.use-case-8.test.js
View File

@ -12,24 +12,17 @@
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../helper');
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var querystring = require('querystring');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var batchFactory = require('../../../batch');
describe('Use case 8: cancel a running multiquery job', function() {
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();
@ -45,7 +38,7 @@ describe('Use case 8: cancel a running multiquery job', function() {
var cancelledJob = {};
it('Step 1, should create a new multiquery job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -58,7 +51,7 @@ describe('Use case 8: cancel a running multiquery job', function() {
})
}, {
status: 201
}, function(res) {
}, function(err, res) {
runningJob = JSON.parse(res.body);
done();
});
@ -66,13 +59,13 @@ describe('Use case 8: cancel a running multiquery job', function() {
it('Step 2, multiquery job should be running', function (done){
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var job = JSON.parse(res.body);
if (job.status === "running") {
clearInterval(interval);
@ -86,13 +79,13 @@ describe('Use case 8: cancel a running multiquery job', function() {
});
it('Step 3, cancel a multiquery job', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 200
}, function(res) {
}, function(err, res) {
cancelledJob = JSON.parse(res.body);
assert.equal(cancelledJob.status, "cancelled");
done();
@ -100,13 +93,13 @@ describe('Use case 8: cancel a running multiquery job', function() {
});
it('Step 4, multiquery job should be cancelled', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var job = JSON.parse(res.body);
if (job.status === "cancelled") {
done();
@ -117,13 +110,13 @@ describe('Use case 8: cancel a running multiquery job', function() {
});
it('Step 5, cancel a cancelled multiquery job should give an error', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + cancelledJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 400
}, function(res) {
}, function(err, res) {
var errors = JSON.parse(res.body);
assert.equal(errors.error[0], "Cannot set status from cancelled to cancelled");
done();

37
test/acceptance/job.use-case-9.test.js → test/acceptance/batch/job.use-case-9.test.js
View File

@ -12,24 +12,17 @@
* HSET rails:users:vizzuality database_name cartodb_test_user_1_db
*
*/
require('../helper');
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var assert = require('../support/assert');
var redisUtils = require('../support/redis_utils');
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var querystring = require('querystring');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var batchFactory = require('../../batch');
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var batchFactory = require('../../../batch');
describe('Use case 9: modify a pending multiquery job', function() {
var batch = batchFactory(metadataBackend, redisConfig);
var batch = batchFactory(metadataBackend, redisUtils.getConfig());
before(function (done) {
batch.start();
@ -45,7 +38,7 @@ describe('Use case 9: modify a pending multiquery job', function() {
var pendingJob = {};
it('Step 1, should create a multiquery job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -57,14 +50,14 @@ describe('Use case 9: modify a pending multiquery job', function() {
})
}, {
status: 201
}, function(res) {
}, function(err, res) {
runningJob = JSON.parse(res.body);
done();
});
});
it('Step 2, should create another multiquery job', function (done) {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST',
@ -76,7 +69,7 @@ describe('Use case 9: modify a pending multiquery job', function() {
})
}, {
status: 201
}, function(res) {
}, function(err, res) {
pendingJob = JSON.parse(res.body);
done();
});
@ -84,13 +77,13 @@ describe('Use case 9: modify a pending multiquery job', function() {
it('Step 3, multiquery job should be pending', function (done){
var interval = setInterval(function () {
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + pendingJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'GET'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var job = JSON.parse(res.body);
if (job.status === "pending") {
clearInterval(interval);
@ -104,13 +97,13 @@ describe('Use case 9: modify a pending multiquery job', function() {
});
it('Step 5, running multiquery job should be cancelled', function (done){
assert.response(app, {
assert.response(server, {
url: '/api/v2/sql/job/' + runningJob.job_id + '?api_key=1234',
headers: { 'host': 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'DELETE'
}, {
status: 200
}, function(res) {
}, function(err, res) {
var cancelledJob = JSON.parse(res.body);
assert.equal(cancelledJob.status, "cancelled");
done();

13
test/acceptance/export/arraybuffer.js
View File

@ -2,24 +2,21 @@ require('../../helper');
require('../../support/assert');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../../app/server')();
var assert = require('assert');
var querystring = require('querystring');
// allow lots of emitters to be set to silence warning
app.setMaxListeners(0);
describe('export.arraybuffer', function() {
it('GET /api/v1/sql as arraybuffer ', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: 'SELECT cartodb_id,name,1::integer,187.9 FROM untitle_table_4',
format: 'arraybuffer'
}),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "application/octet-stream");
done();
@ -27,14 +24,14 @@ it('GET /api/v1/sql as arraybuffer ', function(done){
});
it('GET /api/v1/sql as arraybuffer does not support geometry types ', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: 'SELECT cartodb_id, the_geom FROM untitle_table_4',
format: 'arraybuffer'
}),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 400, res.body);
var result = JSON.parse(res.body);
assert.equal(result.error[0], "geometry types are not supported");

57
test/acceptance/export/csv.js
View File

@ -2,29 +2,26 @@ require('../../helper');
require('../../support/assert');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../../app/server')();
var assert = require('assert');
var querystring = require('querystring');
// allow lots of emitters to be set to silence warning
app.setMaxListeners(0);
describe('export.csv', function() {
it('CSV format', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: 'SELECT * FROM untitle_table_4 WHERE cartodb_id = 1',
format: 'csv'
}),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'CSV is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.csv/gi.test(cd));
var ct = res.header('Content-Type');
var ct = res.headers['content-type'];
assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct);
var rows = res.body.split(/\r\n/);
@ -39,7 +36,7 @@ it('CSV format', function(done){
});
it('CSV format, bigger than 81920 bytes', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql',
data: querystring.stringify({
q: 'SELECT 0 as fname FROM generate_series(0,81920)',
@ -47,7 +44,7 @@ it('CSV format, bigger than 81920 bytes', function(done){
}),
headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST'
},{ }, function(res){
},{ }, function(err, res){
assert.ok(res.body.length > 81920, 'CSV smaller than expected: ' + res.body.length);
done();
});
@ -55,33 +52,33 @@ it('CSV format, bigger than 81920 bytes', function(done){
it('CSV format from POST', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql',
data: querystring.stringify({q: "SELECT * FROM untitle_table_4 LIMIT 1", format: 'csv'}),
headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'CSV is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.csv/gi.test(cd));
var ct = res.header('Content-Type');
var ct = res.headers['content-type'];
assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct);
done();
});
});
it('CSV format, custom filename', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv&filename=mycsv.csv',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'CSV is not disposed as attachment: ' + cd);
assert.equal(true, /filename=mycsv.csv/gi.test(cd), cd);
var ct = res.header('Content-Type');
var ct = res.headers['content-type'];
assert.equal(true, /header=present/.test(ct), "CSV doesn't advertise header presence: " + ct);
var row0 = res.body.substring(0, res.body.search(/[\n\r]/)).split(',');
var checkFields = { name: true, cartodb_id: true, the_geom: true, the_geom_webmercator: true };
@ -98,12 +95,12 @@ it('CSV format, custom filename', function(done){
});
it('skipfields controls fields included in CSV output', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=csv' +
'&skipfields=unexistant,cartodb_id',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var row0 = res.body.substring(0, res.body.search(/[\n\r]/)).split(',');
var checkFields = { name: true, cartodb_id: false, the_geom: true, the_geom_webmercator: true };
@ -120,12 +117,12 @@ it('skipfields controls fields included in CSV output', function(done){
});
it('GET /api/v1/sql as csv', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20cartodb_id,ST_AsEWKT(the_geom)%20as%20geom%20FROM%20untitle_table_4%20LIMIT%201' +
'&format=csv',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var expected = 'cartodb_id,geom\r\n1,"SRID=4326;POINT(-3.699732 40.423012)"\r\n';
assert.equal(res.body, expected);
@ -135,11 +132,11 @@ it('GET /api/v1/sql as csv', function(done){
// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/60
it('GET /api/v1/sql as csv with no rows', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20true%20WHERE%20false&format=csv',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var obtained_lines = res.body.split('\r\n');
assert.ok(obtained_lines.length <= 2, // may or may not have an header
@ -150,11 +147,11 @@ it('GET /api/v1/sql as csv with no rows', function(done){
});
it('GET /api/v1/sql as csv, properly escaped', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20cartodb_id,%20address%20FROM%20untitle_table_4%20LIMIT%201&format=csv',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var expected = 'cartodb_id,address\r\n1,"Calle de Pérez Galdós 9, Madrid, Spain"\r\n';
assert.equal(res.body, expected);
@ -166,7 +163,7 @@ it('GET /api/v1/sql as csv, concurrently', function(done){
var concurrency = 4;
var waiting = concurrency;
function validate(res){
function validate(err, res){
var expected = 'cartodb_id,address\r\n1,"Calle de Pérez Galdós 9, Madrid, Spain"\r\n';
assert.equal(res.body, expected);
if ( ! --waiting ) {
@ -174,7 +171,7 @@ it('GET /api/v1/sql as csv, concurrently', function(done){
}
}
for (var i=0; i<concurrency; ++i) {
assert.response(app,
assert.response(server,
{
url: '/api/v1/sql?q=SELECT%20cartodb_id,%20address%20FROM%20untitle_table_4%20LIMIT%201&format=csv',
headers: {host: 'vizzuality.cartodb.com'},
@ -191,7 +188,7 @@ it('GET /api/v1/sql as csv, concurrently', function(done){
it('expects 1200 rows in public table', function(done){
var limit = 1200;
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: "SELECT * from populated_places_simple_reduced limit " + limit,
format: 'csv'
@ -202,7 +199,7 @@ it('GET /api/v1/sql as csv, concurrently', function(done){
{
status: 200
},
function(res) {
function(err, res) {
var headersPlusExtraLine = 2;
assert.equal(res.body.split('\n').length, limit + headersPlusExtraLine);
done();

60
test/acceptance/export/geojson.js
View File

@ -1,13 +1,9 @@
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var querystring = require('querystring');
// allow lots of emitters to be set to silence warning
// TODO: check if still needed ...
app.setMaxListeners(0);
// use dec_sep for internationalization
var checkDecimals = function(x, dec_sep){
var tmp='' + x;
@ -23,13 +19,13 @@ describe('export.geojson', function() {
// GEOJSON tests
it('GET /api/v1/sql with SQL parameter, ensuring content-disposition set to geojson', function(done) {
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'GEOJSON is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.geojson/gi.test(cd));
done();
@ -37,14 +33,14 @@ it('GET /api/v1/sql with SQL parameter, ensuring content-disposition set to geoj
});
it('POST /api/v1/sql with SQL parameter, ensuring content-disposition set to geojson', function(done) {
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql',
data: querystring.stringify({q: "SELECT * FROM untitle_table_4", format: 'geojson' }),
headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'GEOJSON is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.geojson/gi.test(cd));
done();
@ -52,37 +48,37 @@ it('POST /api/v1/sql with SQL parameter, ensuring content-disposition set to geo
});
it('uses the last format parameter when multiple are used', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?format=csv&q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /filename=cartodb-query.geojson/gi.test(cd));
done();
});
});
it('uses custom filename', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson&filename=x',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /filename=x.geojson/gi.test(cd), cd);
done();
});
});
it('does not include the_geom and the_geom_webmercator properties by default', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var parsed_body = JSON.parse(res.body);
var row0 = parsed_body.features[0].properties;
@ -99,11 +95,11 @@ it('does not include the_geom and the_geom_webmercator properties by default', f
});
it('skipfields controls fields included in GeoJSON output', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4&format=geojson&skipfields=unexistant,cartodb_id',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var parsed_body = JSON.parse(res.body);
var row0 = parsed_body.features[0].properties;
@ -121,14 +117,14 @@ it('skipfields controls fields included in GeoJSON output', function(done){
it('GET /api/v1/sql as geojson limiting decimal places', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: 'SELECT ST_MakePoint(0.123,2.3456) as the_geom',
format: 'geojson',
dp: '1'}),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var result = JSON.parse(res.body);
assert.equal(1, checkDecimals(result.features[0].geometry.coordinates[0], '.'));
@ -137,13 +133,13 @@ it('GET /api/v1/sql as geojson limiting decimal places', function(done){
});
it('GET /api/v1/sql as geojson with default dp as 6', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: 'SELECT ST_MakePoint(0.12345678,2.3456787654) as the_geom',
format: 'geojson'}),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var result = JSON.parse(res.body);
assert.equal(6, checkDecimals(result.features[0].geometry.coordinates[0], '.'));
@ -152,16 +148,16 @@ it('GET /api/v1/sql as geojson with default dp as 6', function(done){
});
it('null geometries in geojson output', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: "SELECT 1 as gid, 'U' as name, null::geometry as the_geom ",
format: 'geojson'
}),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'GEOJSON is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.geojson/gi.test(cd));
var gjson = JSON.parse(res.body);
@ -177,14 +173,14 @@ it('null geometries in geojson output', function(done){
});
it('stream response handle errors', function(done) {
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: "SELECTT 1 as gid, null::geometry as the_geom ",
format: 'geojson'
}),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 400, res.body);
var geoJson = JSON.parse(res.body);
assert.ok(geoJson.error);
@ -195,14 +191,14 @@ it('stream response handle errors', function(done) {
});
it('stream response with empty result set has valid output', function(done) {
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: "SELECT 1 as gid, null::geometry as the_geom limit 0",
format: 'geojson'
}),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var geoJson = JSON.parse(res.body);
var expectedGeoJson = {"type": "FeatureCollection", "features": []};

10
test/acceptance/export/geopackage.js
View File

@ -1,6 +1,6 @@
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var sqlite = require('sqlite3');
var fs = require('fs');
@ -11,11 +11,11 @@ describe('geopackage query', function(){
var base_url = '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=gpkg&filename=' + table_name;
it('returns a valid geopackage database', function(done){
assert.response(app, {
assert.response(server, {
url: base_url,
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res) {
},{ }, function(err, res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers["content-type"], "application/x-sqlite3; charset=utf-8");
assert.notEqual(res.headers["content-disposition"].indexOf(table_name + ".gpkg"), -1);
@ -30,12 +30,12 @@ describe('geopackage query', function(){
});
it('gets database and geopackage schema', function(done){
assert.response(app, {
assert.response(server, {
url: base_url,
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res) {
},{ }, function(err, res) {
var tmpfile = '/tmp/a_geopackage_file.gpkg';
try {
fs.writeFileSync(tmpfile, res.body, 'binary');

129
test/acceptance/export/kml.js
View File

@ -1,14 +1,9 @@
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var querystring = require('querystring');
var libxmljs = require('libxmljs');
var http = require('http');
var server_utils = require('../../support/server_utils');
// allow lots of emitters to be set to silence warning
app.setMaxListeners(0);
describe('export.kml', function() {
@ -109,13 +104,13 @@ var extractFolderName = function(kml) {
// KML tests
it('KML format, unauthenticated', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
var row0 = res.body;
@ -132,14 +127,14 @@ it('KML format, unauthenticated', function(done){
});
it('KML format, unauthenticated, POST', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql',
data: 'q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml',
headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
done();
@ -147,7 +142,7 @@ it('KML format, unauthenticated, POST', function(done){
});
it('KML format, bigger than 81920 bytes', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql',
data: querystring.stringify({
q: 'SELECT 0 as fname FROM generate_series(0,81920)',
@ -155,9 +150,9 @@ it('KML format, bigger than 81920 bytes', function(done){
}),
headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
assert.ok(res.body.length > 81920, 'KML smaller than expected: ' + res.body.length);
@ -166,13 +161,13 @@ it('KML format, bigger than 81920 bytes', function(done){
});
it('KML format, skipfields', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&skipfields=address,cartodb_id',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
var row0 = res.body;
@ -189,13 +184,13 @@ it('KML format, skipfields', function(done){
});
it('KML format, unauthenticated, custom filename', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&filename=kmltest',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
assert.equal(true, /filename=kmltest.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
var name = extractFolderName(res.body);
@ -205,13 +200,13 @@ it('KML format, unauthenticated, custom filename', function(done){
});
it('KML format, authenticated', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=kml&api_key=1234',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /filename=cartodb-query.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
done();
});
@ -228,57 +223,39 @@ it('KML format, unauthenticated, concurrent requests', function(done){
var concurrency = 4;
var waiting = concurrency;
function onResponse(res) {
//console.log("Response started");
res.body = '';
//res.setEncoding('binary');
res.on('data', function(chunk){ res.body += chunk; });
res.on('end', function(){
//console.log("Response ended");
assert.equal(res.statusCode, 200, res.body);
assert.ok(res.body);
var snippet = res.body.substr(0, 5);
assert.equal(snippet, "<?xml");
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
assert.equal(true, /filename=multi.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
if ( ! --waiting ) {
app.close();
done();
}
});
function validate(err, res) {
//console.log("Response ended");
assert.equal(res.statusCode, 200, res.body);
assert.ok(res.body);
var snippet = res.body.substr(0, 5);
assert.equal(snippet, "<?xml");
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'KML is not disposed as attachment: ' + cd);
assert.equal(true, /filename=multi.kml/gi.test(cd), 'Unexpected KML filename: ' + cd);
if ( ! --waiting ) {
done();
}
}
function onError(err) {
console.log("Response error" + err);
}
var request = {
method: 'GET',
headers: { host: 'vizzuality.cartodb.com' },
url: '/api/v1/sql?' + query
};
server_utils.startOnNextPort(app, function() {
var port = app.address().port;
//console.log("Listening on port " + port);
for (var i=0; i<concurrency; ++i) {
for (var i=0; i<concurrency; ++i) {
//console.log("Sending request");
http.request({
host: 'localhost',
port: port,
path: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
agent: false // or should this be true ?
})
.on('response', onResponse)
.on('error', onError)
.end();
}
});
assert.response(server, request, { status: 200 }, validate);
}
});
// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/60
it('GET /api/v1/sql as kml with no rows', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20true%20WHERE%20false&format=kml',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
// NOTE: GDAL-1.11+ added 'id="root_doc"' attribute to the output
var pat = new RegExp('^<\\?xml version="1.0" encoding="utf-8" \\?>' +
@ -294,14 +271,14 @@ it('GET /api/v1/sql as kml with no rows', function(done){
// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/90
it('GET /api/v1/sql as kml with ending semicolon', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: 'SELECT true WHERE false;',
format: 'kml'
}),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
// NOTE: GDAL-1.11+ added 'id="root_doc"' attribute to the output
var pat = new RegExp('^<\\?xml version="1.0" encoding="utf-8" \\?>' +
@ -317,14 +294,14 @@ it('GET /api/v1/sql as kml with ending semicolon', function(done){
// See https://github.com/CartoDB/cartodb/issues/276
it('check point coordinates, unauthenticated', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: 'SELECT * from untitle_table_4 WHERE cartodb_id = -1',
format: 'kml'
}),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var coords = extractCoordinates(res.body);
assert(coords, 'No coordinates in ' + res.body);
@ -335,7 +312,7 @@ it('check point coordinates, unauthenticated', function(done){
// See https://github.com/CartoDB/cartodb/issues/276
it('check point coordinates, authenticated', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: 'SELECT * from untitle_table_4 WHERE cartodb_id = -1',
api_key: 1234,
@ -343,7 +320,7 @@ it('check point coordinates, authenticated', function(done){
}),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var coords = extractCoordinates(res.body);
assert(coords, 'No coordinates in ' + res.body);
@ -357,7 +334,7 @@ it('check point coordinates, authenticated', function(done){
it('expects ' + limit + ' placemarks in public table', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql',
data: querystring.stringify({
q: "SELECT * from populated_places_simple_reduced limit " + limit,
@ -369,7 +346,7 @@ it('check point coordinates, authenticated', function(done){
{
status: 200
},
function(res) {
function(err, res) {
assert.equal(res.body.match(/<Placemark>/g).length, limit);
done();
}
@ -378,7 +355,7 @@ it('check point coordinates, authenticated', function(done){
it('expects ' + limit + ' placemarks in private table using the API KEY', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: "SELECT * from populated_places_simple_reduced limit " + limit,
api_key: 1234,
@ -390,7 +367,7 @@ it('check point coordinates, authenticated', function(done){
{
status: 200
},
function(res) {
function(err, res) {
assert.equal(res.body.match(/<Placemark>/g).length, limit);
done();
}
@ -399,7 +376,7 @@ it('check point coordinates, authenticated', function(done){
it('should work with queries returning no results', function(done) {
assert.response(
app,
server,
{
url: "/api/v1/sql?" + querystring.stringify({
q: "SELECT * FROM populated_places_simple_reduced LIMIT 0",
@ -414,7 +391,7 @@ it('check point coordinates, authenticated', function(done){
{
status: 200
},
function(res) {
function(err, res) {
assert.equal(res.body.match(/<Placemark>/g), null);
done();
}

121
test/acceptance/export/shapefile.js
View File

@ -1,6 +1,6 @@
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var querystring = require('querystring');
var shapefile = require('shapefile');
@ -8,28 +8,25 @@ var _ = require('underscore');
var zipfile = require('zipfile');
var fs = require('fs');
// allow lots of emitters to be set to silence warning
app.setMaxListeners(0);
describe('export.shapefile', function() {
// SHP tests
it('SHP format, unauthenticated', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp',
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
var tmpfile = '/tmp/myshape.zip';
var err = fs.writeFileSync(tmpfile, res.body, 'binary');
if (err) {
return done(err);
var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
if (writeErr) {
return done(writeErr);
}
var zf = new zipfile.ZipFile(tmpfile);
assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
@ -43,14 +40,14 @@ it('SHP format, unauthenticated', function(done){
});
it('SHP format, unauthenticated, POST', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql',
data: 'q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp',
headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
done();
@ -58,7 +55,7 @@ it('SHP format, unauthenticated, POST', function(done){
});
it('SHP format, big size, POST', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql',
data: querystring.stringify({
q: 'SELECT 0 as fname, st_makepoint(i,i) FROM generate_series(0,81920) i',
@ -66,9 +63,9 @@ it('SHP format, big size, POST', function(done){
}),
headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
assert.ok(res.body.length > 81920, 'SHP smaller than expected: ' + res.body.length);
@ -77,20 +74,20 @@ it('SHP format, big size, POST', function(done){
});
it('SHP format, unauthenticated, with custom filename', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=myshape',
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
assert.equal(true, /filename=myshape.zip/gi.test(cd));
var tmpfile = '/tmp/myshape.zip';
var err = fs.writeFileSync(tmpfile, res.body, 'binary');
if (err) {
return done(err);
var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
if (writeErr) {
return done(writeErr);
}
var zf = new zipfile.ZipFile(tmpfile);
assert.ok(_.contains(zf.names, 'myshape.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
@ -103,21 +100,21 @@ it('SHP format, unauthenticated, with custom filename', function(done){
});
it('SHP format, unauthenticated, with custom, dangerous filename', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=b;"%20()[]a',
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var fname = "b_______a";
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
assert.equal(true, /filename=b_______a.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
var tmpfile = '/tmp/myshape.zip';
var err = fs.writeFileSync(tmpfile, res.body, 'binary');
if (err) {
return done(err);
var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
if (writeErr) {
return done(writeErr);
}
var zf = new zipfile.ZipFile(tmpfile);
assert.ok(_.contains(zf.names, fname + '.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
@ -130,19 +127,19 @@ it('SHP format, unauthenticated, with custom, dangerous filename', function(done
});
it('SHP format, authenticated', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&api_key=1234',
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
var tmpfile = '/tmp/myshape.zip';
var err = fs.writeFileSync(tmpfile, res.body, 'binary');
if (err) {
return done(err);
var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
if (writeErr) {
return done(writeErr);
}
var zf = new zipfile.ZipFile(tmpfile);
assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
@ -163,17 +160,17 @@ it('SHP format, unauthenticated, with utf8 data', function(done){
format: 'shp',
filename: 'myshape'
});
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var tmpfile = '/tmp/myshape.zip';
var err = fs.writeFileSync(tmpfile, res.body, 'binary');
if (err) {
return done(err);
var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
if (writeErr) {
return done(writeErr);
}
var zf = new zipfile.ZipFile(tmpfile);
var buffer = zf.readFileSync('myshape.dbf');
@ -190,12 +187,12 @@ it('mixed type geometry', function(done){
q: "SELECT 'POINT(0 0)'::geometry as g UNION ALL SELECT 'LINESTRING(0 0, 1 0)'::geometry",
format: 'shp'
});
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
assert.deepEqual(res.headers['content-disposition'], 'inline');
assert.equal(res.statusCode, 400, res.statusCode + ': ' +res.body);
@ -215,12 +212,12 @@ it('errors are not confused with warnings', function(done){
].join(" UNION ALL "),
format: 'shp'
});
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
assert.deepEqual(res.headers['content-disposition'], 'inline');
assert.equal(res.statusCode, 400, res.statusCode + ': ' +res.body);
@ -238,17 +235,17 @@ it('skipfields controls fields included in SHP output', function(done){
skipfields: 'skipme',
filename: 'myshape'
});
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var tmpfile = '/tmp/myshape.zip';
var err = fs.writeFileSync(tmpfile, res.body, 'binary');
if (err) {
return done(err);
var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
if (writeErr) {
return done(writeErr);
}
var zf = new zipfile.ZipFile(tmpfile);
var buffer = zf.readFileSync('myshape.dbf');
@ -262,14 +259,14 @@ it('skipfields controls fields included in SHP output', function(done){
it('SHP format, concurrently', function(done){
var concurrency = 1;
var waiting = concurrency;
function validate(res){
var cd = res.header('Content-Disposition');
function validate(err, res){
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
var tmpfile = '/tmp/myshape.zip';
var err = fs.writeFileSync(tmpfile, res.body, 'binary');
if (err) {
return done(err);
var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
if (writeErr) {
return done(writeErr);
}
var zf = new zipfile.ZipFile(tmpfile);
assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
@ -284,7 +281,7 @@ it('SHP format, concurrently', function(done){
}
for (var i=0; i<concurrency; ++i) {
assert.response(
app,
server,
{
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp',
headers: {host: 'vizzuality.cartodb.com'},
@ -305,19 +302,19 @@ it('point with null first', function(done){
q: "SELECT null::geometry as g UNION ALL SELECT 'SRID=4326;POINT(0 0)'::geometry",
format: 'shp'
});
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
encoding: 'binary',
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
var tmpfile = '/tmp/myshape.zip';
var err = fs.writeFileSync(tmpfile, res.body, 'binary');
if (err) {
return done(err);
var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
if (writeErr) {
return done(writeErr);
}
var zf = new zipfile.ZipFile(tmpfile);
assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
@ -336,7 +333,7 @@ it('point with null first', function(done){
var filename = 'test_1200';
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: "SELECT * from populated_places_simple_reduced limit " + limit,
format: 'shp',
@ -349,7 +346,7 @@ it('point with null first', function(done){
{
status: 200
},
function(res, err) {
function(err, res) {
if (err) {
return done(err);
}

14
test/acceptance/export/spatialite.js
View File

@ -1,17 +1,17 @@
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var sqlite = require('sqlite3');
describe('spatialite query', function(){
it('returns a valid sqlite database', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=spatialite',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res) {
},{ }, function(err, res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers["content-type"], "application/x-sqlite3; charset=utf-8");
var db = new sqlite.Database(':memory:', res.body);
@ -25,11 +25,11 @@ describe('spatialite query', function(){
});
it('different file name', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=spatialite&filename=manolo',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res) {
},{ }, function(err, res) {
assert.equal(res.headers["content-type"], "application/x-sqlite3; charset=utf-8");
assert.notEqual(res.headers["content-disposition"].indexOf("manolo.sqlite"), -1);
done();
@ -37,11 +37,11 @@ describe('spatialite query', function(){
});
it('gets database schema', function(done){
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=spatialite',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res) {
},{ }, function(err, res) {
var db = new sqlite.Database(':memory:', res.body);
var schemaQuery = "SELECT name, sql FROM sqlite_master WHERE type='table' ORDER BY name";
var qr = db.get(schemaQuery, function(err){

65
test/acceptance/export/svg.js
View File

@ -1,12 +1,9 @@
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var querystring = require('querystring');
// allow lots of emitters to be set to silence warning
app.setMaxListeners(0);
describe('export.svg', function() {
it('GET /api/v1/sql with SVG format', function(done){
@ -14,15 +11,15 @@ it('GET /api/v1/sql with SVG format', function(done){
q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ",
format: "svg"
});
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8');
assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0" />') > 0, res.body );
// TODO: test viewBox
done();
@ -34,17 +31,17 @@ it('POST /api/v1/sql with SVG format', function(done){
q: "SELECT 1 as cartodb_id, ST_MakeLine(ST_MakePoint(10, 10), ST_MakePoint(1034, 778)) AS the_geom ",
format: "svg"
});
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql',
data: query,
headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
method: 'POST'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'SVG is not disposed as attachment: ' + cd);
assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8');
assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0" />') > 0, res.body );
// TODO: test viewBox
done();
@ -57,15 +54,15 @@ it('GET /api/v1/sql with SVG format and custom filename', function(done){
format: "svg",
filename: 'mysvg'
});
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.ok(/filename=mysvg.svg/gi.test(cd), cd);
assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8');
assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0" />') > 0, res.body );
// TODO: test viewBox
done();
@ -77,15 +74,15 @@ it('GET /api/v1/sql with SVG format and centered point', function(done){
q: "SELECT 1 as cartodb_id, ST_MakePoint(5000, -54) AS the_geom ",
format: "svg"
});
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8');
assert.ok( res.body.indexOf('cx="0" cy="0"') > 0, res.body );
// TODO: test viewBox
// TODO: test radius
@ -99,29 +96,29 @@ it('GET /api/v1/sql with SVG format and trimmed decimals', function(done){
format: "svg",
dp: 2
};
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify(queryobj),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8');
assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0 500.12 167.01" />') > 0, res.body );
// TODO: test viewBox
queryobj.dp = 3;
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify(queryobj),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{}, function(res) {
},{}, function(err, res) {
assert.equal(res.statusCode, 200, res.body);
var cd = res.header('Content-Disposition');
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'SVG is not disposed as attachment: ' + cd);
assert.ok(/filename=cartodb-query.svg/gi.test(cd), cd);
assert.equal(res.header('Content-Type'), 'image/svg+xml; charset=utf-8');
assert.equal(res.headers['content-type'], 'image/svg+xml; charset=utf-8');
assert.ok( res.body.indexOf('<path d="M 0 768 L 1024 0 500.123 167.012" />') > 0, res.body );
// TODO: test viewBox
done();
@ -137,11 +134,11 @@ it('SVG format with "the_geom" in skipfields', function(done){
format: "svg",
skipfields: "the_geom"
});
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
assert.deepEqual(res.headers['content-disposition'], 'inline');
@ -157,11 +154,11 @@ it('SVG format with missing "the_geom" field', function(done){
q: "SELECT 1 as cartodb_id, ST_MakePoint(5000, -54) AS something_else ",
format: "svg"
});
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?' + query,
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{ }, function(res){
},{ }, function(err, res){
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
assert.deepEqual(JSON.parse(res.body), {
error:['column "the_geom" does not exist']
@ -172,7 +169,7 @@ it('SVG format with missing "the_geom" field', function(done){
it('should close on error and error must be the only key in the body', function(done) {
assert.response(
app,
server,
{
url: "/api/v1/sql?" + querystring.stringify({
q: "SELECT the_geom, 100/(cartodb_id - 3) cdb_ratio FROM untitle_table_4",
@ -186,7 +183,7 @@ it('SVG format with missing "the_geom" field', function(done){
{
status: 400
},
function(res) {
function(err, res) {
var parsedBody = JSON.parse(res.body);
assert.deepEqual(Object.keys(parsedBody), ['error']);
assert.deepEqual(parsedBody.error, ["division by zero"]);

30
test/acceptance/export/topojson.js
View File

@ -1,14 +1,10 @@
require('../../helper');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../../app/server')();
var assert = require('../../support/assert');
var querystring = require('querystring');
var _ = require('underscore');
// allow lots of emitters to be set to silence warning
app.setMaxListeners(0);
describe('export.topojson', function() {
// TOPOJSON tests
@ -29,7 +25,7 @@ describe('export.topojson', function() {
}
it('GET two polygons sharing an edge as topojson', function(done){
assert.response(app,
assert.response(server,
getRequest(
"SELECT 1 as gid, 'U' as name, 'POLYGON((-5 0,5 0,0 5,-5 0))'::geometry as the_geom " +
" UNION ALL " +
@ -38,8 +34,8 @@ it('GET two polygons sharing an edge as topojson', function(done){
{
status: 200
},
function(res) {
var cd = res.header('Content-Disposition');
function(err, res) {
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'TOPOJSON is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.topojson/gi.test(cd));
var topojson = JSON.parse(res.body);
@ -135,7 +131,7 @@ it('GET two polygons sharing an edge as topojson', function(done){
});
it('null geometries', function(done){
assert.response(app, getRequest(
assert.response(server, getRequest(
"SELECT 1 as gid, 'U' as name, 'POLYGON((-5 0,5 0,0 5,-5 0))'::geometry as the_geom " +
" UNION ALL " +
"SELECT 2, 'D', null::geometry as the_geom "
@ -143,8 +139,8 @@ it('null geometries', function(done){
{
status: 200
},
function(res) {
var cd = res.header('Content-Disposition');
function(err, res) {
var cd = res.headers['content-disposition'];
assert.equal(true, /^attachment/.test(cd), 'TOPOJSON is not disposed as attachment: ' + cd);
assert.equal(true, /filename=cartodb-query.topojson/gi.test(cd));
var topojson = JSON.parse(res.body);
@ -193,7 +189,7 @@ it('null geometries', function(done){
});
it('skipped fields are not returned', function(done) {
assert.response(app,
assert.response(server,
getRequest(
"SELECT 1 as gid, 'U' as name, 'POLYGON((-5 0,5 0,0 5,-5 0))'::geometry as the_geom",
{
@ -203,7 +199,7 @@ it('null geometries', function(done){
{
status: 200
},
function(res) {
function(err, res) {
var parsedBody = JSON.parse(res.body);
assert.equal(parsedBody.objects[0].properties.gid, 1, 'gid was expected property');
assert.ok(!parsedBody.objects[0].properties.name);
@ -214,7 +210,7 @@ it('null geometries', function(done){
it('jsonp callback is invoked', function(done){
assert.response(
app,
server,
getRequest(
"SELECT 1 as gid, 'U' as name, 'POLYGON((-5 0,5 0,0 5,-5 0))'::geometry as the_geom",
{
@ -224,7 +220,7 @@ it('null geometries', function(done){
{
status: 200
},
function(res) {
function(err, res) {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var didRunJsonCallback = false;
// jshint ignore:start
@ -242,7 +238,7 @@ it('null geometries', function(done){
it('should close on error and error must be the only key in the body', function(done) {
assert.response(
app,
server,
{
url: "/api/v1/sql?" + querystring.stringify({
q: "SELECT the_geom, 100/(cartodb_id - 3) cdb_ratio FROM untitle_table_4",
@ -256,7 +252,7 @@ it('null geometries', function(done){
{
status: 400
},
function(res) {
function(err, res) {
var parsedBody = JSON.parse(res.body);
assert.deepEqual(Object.keys(parsedBody), ['error']);
assert.deepEqual(parsedBody.error, ["division by zero"]);

9
test/acceptance/frontend_abort.js
View File

@ -30,19 +30,16 @@ it('aborts request', function(done){
var db_port_backup = global.settings.db_port;
global.settings.db_host = 'localhost';
global.settings.db_port = sql_server_port;
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../app/server')();
var timeout;
step(
function sendQuery() {
var next = this;
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT+1',
method: 'GET',
timeout: 1,
headers: {host: 'vizzuality.localhost' }
},{}, function(res, err) {
next(err, res);
});
},{}, this);
},
function checkResponse(err/*, res*/) {
assert(err); // expect timeout

10
test/acceptance/health_check.js
View File

@ -2,7 +2,7 @@ require('../helper');
require('../support/assert');
var assert = require('assert');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../app/server')();
describe('health checks', function() {
@ -26,12 +26,12 @@ describe('health checks', function() {
it('returns 200 and ok=true with disabled configuration', function(done) {
global.settings.health.enabled = false;
assert.response(app,
assert.response(server,
healthCheckRequest,
{
status: 200
},
function(res, err) {
function(err, res) {
assert.ok(!err);
var parsed = JSON.parse(res.body);
@ -45,12 +45,12 @@ describe('health checks', function() {
});
it('returns 200 and ok=true with enabled configuration', function(done) {
assert.response(app,
assert.response(server,
healthCheckRequest,
{
status: 200
},
function(res, err) {
function(err, res) {
assert.ok(!err);
var parsed = JSON.parse(res.body);

14
test/acceptance/last-modified-header.js
View File

@ -1,6 +1,6 @@
require('../helper');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../app/server')();
var assert = require('../support/assert');
var qs = require('querystring');
@ -37,7 +37,7 @@ describe('last modified header', function() {
}).join(' UNION ALL '),
api_key: 1234
});
assert.response(app,
assert.response(server,
{
url: '/api/v1/sql?' + query,
headers: {
@ -48,7 +48,7 @@ describe('last modified header', function() {
{
statusCode: 200
},
function(res) {
function(err, res) {
assert.equal(res.headers['last-modified'], scenario.expectedLastModified);
done();
}
@ -66,7 +66,7 @@ describe('last modified header', function() {
Date.now = function() {
return fixedDateNow;
};
assert.response(app,
assert.response(server,
{
url: '/api/v1/sql?' + query,
headers: {
@ -77,7 +77,7 @@ describe('last modified header', function() {
{
statusCode: 200
},
function(res) {
function(err, res) {
Date.now = dateNowFn;
assert.equal(res.headers['last-modified'], new Date(fixedDateNow).toUTCString());
done();
@ -95,7 +95,7 @@ describe('last modified header', function() {
Date.now = function() {
return fixedDateNow;
};
assert.response(app,
assert.response(server,
{
url: '/api/v1/sql?' + query,
headers: {
@ -106,7 +106,7 @@ describe('last modified header', function() {
{
statusCode: 200
},
function(res) {
function(err, res) {
Date.now = dateNowFn;
assert.equal(res.headers['last-modified'], new Date(fixedDateNow).toUTCString());
done();

8
test/acceptance/logging.js
View File

@ -1,6 +1,6 @@
require('../helper');
var app = require(global.settings.app_root + '/app/app');
var appServer = require('../../app/server');
var assert = require('../support/assert');
var qs = require('querystring');
var log4js = require('log4js');
@ -32,7 +32,7 @@ describe('Logging SQL query on POST requests', function() {
}
]
});
server = app();
server = appServer();
});
after(function() {
@ -105,7 +105,7 @@ describe('Logging SQL query on POST requests', function() {
return result;
};
assert.response(server, scenario.request, RESPONSE_OK, function(res, err) {
assert.response(server, scenario.request, RESPONSE_OK, function(err) {
assert.ok(!err);
assert.equal(called, 1);
@ -137,7 +137,7 @@ describe('Logging SQL query on POST requests', function() {
}
},
RESPONSE_OK,
function(res, err) {
function(err) {
assert.ok(!err);
assert.equal(called, 1);

12
test/acceptance/query-tables-api-cache.js
View File

@ -2,14 +2,14 @@ require('../helper');
var qs = require('querystring');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../app/server')();
var assert = require('../support/assert');
describe('query-tables-api', function() {
function getCacheStatus(callback) {
assert.response(
app,
server,
{
method: 'GET',
url: '/api/v1/cachestatus'
@ -17,7 +17,7 @@ describe('query-tables-api', function() {
{
status: 200
},
function(res) {
function(err, res) {
callback(null, JSON.parse(res.body));
}
);
@ -38,7 +38,7 @@ describe('query-tables-api', function() {
};
it('should create a key in affected tables cache', function(done) {
assert.response(app, request, RESPONSE_OK, function(res, err) {
assert.response(server, request, RESPONSE_OK, function(err) {
assert.ok(!err, err);
getCacheStatus(function(err, cacheStatus) {
@ -52,7 +52,7 @@ describe('query-tables-api', function() {
});
it('should use cache to retrieve affected tables', function(done) {
assert.response(app, request, RESPONSE_OK, function(res, err) {
assert.response(server, request, RESPONSE_OK, function(err) {
assert.ok(!err, err);
getCacheStatus(function(err, cacheStatus) {
@ -76,7 +76,7 @@ describe('query-tables-api', function() {
},
method: 'GET'
};
assert.response(app, authenticatedRequest, RESPONSE_OK, function(res, err) {
assert.response(server, authenticatedRequest, RESPONSE_OK, function(err) {
assert.ok(!err, err);
getCacheStatus(function(err, cacheStatus) {

18
test/acceptance/regressions.js
View File

@ -1,6 +1,6 @@
require('../helper');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../app/server')();
var assert = require('../support/assert');
var qs = require('querystring');
@ -25,22 +25,22 @@ describe('regressions', function() {
statusCode: 200
};
assert.response(app, createRequest('CREATE TABLE "foo.bar" (a int);'), responseOk,
function(res, err) {
assert.response(server, createRequest('CREATE TABLE "foo.bar" (a int);'), responseOk,
function(err) {
if (err) {
return done(err);
}
assert.response(app, createRequest('INSERT INTO "foo.bar" (a) values (1), (2)'), responseOk,
function(res, err) {
assert.response(server, createRequest('INSERT INTO "foo.bar" (a) values (1), (2)'), responseOk,
function(err, res) {
if (err) {
return done(err);
}
var parsedBody = JSON.parse(res.body);
assert.equal(parsedBody.total_rows, 2);
assert.response(app, createRequest('SELECT * FROM "foo.bar"'), responseOk,
function(res, err) {
assert.response(server, createRequest('SELECT * FROM "foo.bar"'), responseOk,
function(err, res) {
if (err) {
return done(err);
}
@ -50,7 +50,9 @@ describe('regressions', function() {
var parsedBody = JSON.parse(res.body);
assert.equal(parsedBody.total_rows, 2);
assert.deepEqual(parsedBody.rows, [{ a: 1 }, { a: 2 }]);
done();
// delete table
assert.response(server, createRequest('DROP TABLE "foo.bar"'), responseOk, done);
}
);
}

10
test/acceptance/stream-responses.js
View File

@ -1,6 +1,6 @@
require('../helper');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../app/server')();
var assert = require('../support/assert');
var querystring = require('querystring');
@ -32,10 +32,10 @@ describe('stream-responses', function() {
it('should close on error and error message must be part of the response', function(done) {
assert.response(
app,
server,
createFailingQueryRequest(),
okResponse,
function(res) {
function(err, res) {
var parsedBody = JSON.parse(res.body);
assert.equal(parsedBody.rows.length, 2);
assert.deepEqual(parsedBody.fields, {
@ -54,10 +54,10 @@ describe('stream-responses', function() {
it('should close on error and error message must be part of the response', function(done) {
assert.response(
app,
server,
createFailingQueryRequest('geojson'),
okResponse,
function(res) {
function(err, res) {
var parsedBody = JSON.parse(res.body);
assert.equal(parsedBody.features.length, 2);
assert.deepEqual(parsedBody.error, ["division by zero"]);

21
test/acceptance/surrogate-key.js
View File

@ -1,14 +1,11 @@
require('../helper');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../app/server')();
var assert = require('../support/assert');
var querystring = require('querystring');
var QueryTables = require('cartodb-query-tables');
var _ = require('underscore');
// allow lots of emitters to be set to silence warning
app.setMaxListeners(0);
describe('Surrogate-Key header', function() {
function createGetRequest(sqlQuery) {
@ -43,7 +40,7 @@ describe('Surrogate-Key header', function() {
function tableNamesInSurrogateKeyHeader(expectedTableNames, done) {
return function(res) {
return function(err, res) {
surrogateKeyHasTables(res.headers['surrogate-key'], expectedTableNames);
done();
};
@ -53,7 +50,7 @@ describe('Surrogate-Key header', function() {
var sql = "SELECT a.name as an, b.name as bn FROM untitle_table_4 a " +
"left join private_table b ON (a.cartodb_id = b.cartodb_id)";
assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([
assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([
{dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'private_table'},
{dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'untitle_table_4'}
], done));
@ -62,7 +59,7 @@ describe('Surrogate-Key header', function() {
it('supports multistatements', function(done) {
var sql = "SELECT * FROM untitle_table_4; SELECT * FROM private_table";
assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([
assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([
{dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'private_table'},
{dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'untitle_table_4'}
], done));
@ -71,7 +68,7 @@ describe('Surrogate-Key header', function() {
it('supports explicit transactions', function(done) {
var sql = "BEGIN; SELECT * FROM untitle_table_4; COMMIT; BEGIN; SELECT * FROM private_table; COMMIT;";
assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([
assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([
{dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'private_table'},
{dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'untitle_table_4'}
], done));
@ -80,14 +77,14 @@ describe('Surrogate-Key header', function() {
it('survives partial transactions', function(done) {
var sql = "BEGIN; SELECT * FROM untitle_table_4";
assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([
assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInSurrogateKeyHeader([
{dbname: 'cartodb_test_user_1_db', schema_name: 'public', table_name: 'untitle_table_4'}
], done));
});
it('should not add header for functions', function(done) {
var sql = "SELECT format('%s', 'wadus')";
assert.response(app, createGetRequest(sql), RESPONSE_OK, function(res) {
assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) {
assert.ok(!res.headers.hasOwnProperty('surrogate-key'), res.headers['surrogate-key']);
done();
});
@ -95,7 +92,7 @@ describe('Surrogate-Key header', function() {
it('should not add header for CDB_QueryTables', function(done) {
var sql = "SELECT CDB_QueryTablesText('select * from untitle_table_4')";
assert.response(app, createGetRequest(sql), RESPONSE_OK, function(res) {
assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) {
assert.ok(!res.headers.hasOwnProperty('surrogate-key'), res.headers['surrogate-key']);
done();
});
@ -103,7 +100,7 @@ describe('Surrogate-Key header', function() {
it('should not add header for non table results', function(done) {
var sql = "SELECT 'wadus'::text";
assert.response(app, createGetRequest(sql), RESPONSE_OK, function(res) {
assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) {
assert.ok(!res.headers.hasOwnProperty('surrogate-key'), res.headers['surrogate-key']);
done();
});

9
test/acceptance/timeout.js
View File

@ -26,17 +26,14 @@ it('after configured milliseconds', function(done){
//console.log("settings:"); console.dir(global.settings);
var timeoutBackup = global.settings.node_socket_timeout;
global.settings.node_socket_timeout = testTimeout;
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../app/server')();
step(
function sendLongQuery() {
var next = this;
assert.response(app, {
assert.response(server, {
url: '/api/v1/sql?q=SELECT+count(*)+FROM+generate_series(1,100000)',
method: 'GET',
headers: {host: 'vizzuality.localhost' }
},{}, function(res, err) {
next(err, res);
});
},{}, this);
},
function checkResponse(err/*, res*/) {
assert.ok(err);

7
test/acceptance/transaction.js
View File

@ -10,12 +10,13 @@ describe('transaction', function() {
var server;
before(function(done) {
server = require(global.settings.app_root + '/app/app')();
server.listen(SERVER_PORT, '127.0.0.1', done);
server = require('../../app/server')();
this.listener = server.listen(SERVER_PORT, '127.0.0.1');
this.listener.on('listening', done);
});
after(function(done) {
server.close(done);
this.listener.close(done);
});
var sqlRequest = request.defaults({

21
test/acceptance/x-cache-channel.js
View File

@ -1,13 +1,10 @@
require('../helper');
var app = require(global.settings.app_root + '/app/app')();
var server = require('../../app/server')();
var assert = require('../support/assert');
var querystring = require('querystring');
var _ = require('underscore');
// allow lots of emitters to be set to silence warning
app.setMaxListeners(0);
describe('X-Cache-Channel header', function() {
function createGetRequest(sqlQuery) {
@ -42,7 +39,7 @@ describe('X-Cache-Channel header', function() {
}
function tableNamesInCacheChannelHeader(expectedTableNames, done) {
return function(res) {
return function(err, res) {
xCacheChannelHeaderHasTables(res.headers['x-cache-channel'], expectedTableNames);
done();
};
@ -52,7 +49,7 @@ describe('X-Cache-Channel header', function() {
var sql = "SELECT a.name as an, b.name as bn FROM untitle_table_4 a " +
"left join private_table b ON (a.cartodb_id = b.cartodb_id)";
assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([
assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([
'public.private_table',
'public.untitle_table_4'
], done));
@ -61,7 +58,7 @@ describe('X-Cache-Channel header', function() {
it('supports multistatements', function(done) {
var sql = "SELECT * FROM untitle_table_4; SELECT * FROM private_table";
assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([
assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([
'public.private_table',
'public.untitle_table_4'
], done));
@ -70,7 +67,7 @@ describe('X-Cache-Channel header', function() {
it('supports explicit transactions', function(done) {
var sql = "BEGIN; SELECT * FROM untitle_table_4; COMMIT; BEGIN; SELECT * FROM private_table; COMMIT;";
assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([
assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([
'public.private_table',
'public.untitle_table_4'
], done));
@ -79,14 +76,14 @@ describe('X-Cache-Channel header', function() {
it('survives partial transactions', function(done) {
var sql = "BEGIN; SELECT * FROM untitle_table_4";
assert.response(app, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([
assert.response(server, createGetRequest(sql), RESPONSE_OK, tableNamesInCacheChannelHeader([
'public.untitle_table_4'
], done));
});
it('should not add header for functions', function(done) {
var sql = "SELECT format('%s', 'wadus')";
assert.response(app, createGetRequest(sql), RESPONSE_OK, function(res) {
assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) {
assert.ok(!res.headers.hasOwnProperty('x-cache-channel'), res.headers['x-cache-channel']);
done();
});
@ -94,7 +91,7 @@ describe('X-Cache-Channel header', function() {
it('should not add header for CDB_QueryTables', function(done) {
var sql = "SELECT CDB_QueryTablesText('select * from untitle_table_4')";
assert.response(app, createGetRequest(sql), RESPONSE_OK, function(res) {
assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) {
assert.ok(!res.headers.hasOwnProperty('x-cache-channel'), res.headers['x-cache-channel']);
done();
});
@ -102,7 +99,7 @@ describe('X-Cache-Channel header', function() {
it('should not add header for non table results', function(done) {
var sql = "SELECT 'wadus'::text";
assert.response(app, createGetRequest(sql), RESPONSE_OK, function(res) {
assert.response(server, createGetRequest(sql), RESPONSE_OK, function(err, res) {
assert.ok(!res.headers.hasOwnProperty('x-cache-channel'), res.headers['x-cache-channel']);
done();
});

6
test/helper.js
View File

@ -1,5 +1 @@
var _ = require('underscore');
global.settings = require(__dirname + '/../config/settings');
var env = require(__dirname + '/../config/environments/test');
_.extend(global.settings, env);
global.settings = require('../config/environments/test');

14
test/integration/batch/batch.multiquery.test.js
View File

@ -5,15 +5,7 @@ var assert = require('../../support/assert');
var redisUtils = require('../../support/redis_utils');
var queue = require('queue-async');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var StatsD = require('node-statsd').StatsD;
var statsdClient = new StatsD(global.settings.statsd);
@ -29,7 +21,7 @@ var JobQueue = require(BATCH_SOURCE + 'job_queue');
var JobBackend = require(BATCH_SOURCE + 'job_backend');
var JobFactory = require(BATCH_SOURCE + 'models/job_factory');
var redisPoolPublisher = new RedisPool(_.extend(redisConfig, { name: 'batch-publisher'}));
var redisPoolPublisher = new RedisPool(_.extend(redisUtils.getConfig(), { name: 'batch-publisher'}));
var jobPublisher = new JobPublisher(redisPoolPublisher);
var jobQueue = new JobQueue(metadataBackend, jobPublisher);
var jobBackend = new JobBackend(metadataBackend, jobQueue);
@ -67,7 +59,7 @@ function assertJob(job, expectedStatus, done) {
}
describe('batch multiquery', function() {
var batch = batchFactory(metadataBackend, redisConfig, statsdClient);
var batch = batchFactory(metadataBackend, redisUtils.getConfig(), statsdClient);
before(function (done) {
batch.start();

12
test/integration/batch/job_backend.test.js
View File

@ -15,16 +15,8 @@ var JobPublisher = require(BATCH_SOURCE + 'job_publisher');
var JobFactory = require(BATCH_SOURCE + 'models/job_factory');
var jobStatus = require(BATCH_SOURCE + 'job_status');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var redisPoolPublisher = new RedisPool(_.extend(redisConfig, { name: 'batch-publisher'}));
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var redisPoolPublisher = new RedisPool(_.extend(redisUtils.getConfig(), { name: 'batch-publisher'}));
var jobPublisher = new JobPublisher(redisPoolPublisher);
var jobQueue = new JobQueue(metadataBackend, jobPublisher);

12
test/integration/batch/job_canceller.test.js
View File

@ -17,16 +17,8 @@ var UserDatabaseMetadataService = require(BATCH_SOURCE + 'user_database_metadata
var JobCanceller = require(BATCH_SOURCE + 'job_canceller');
var PSQL = require('cartodb-psql');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var redisPoolPublisher = new RedisPool(_.extend(redisConfig, { name: 'batch-publisher'}));
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var redisPoolPublisher = new RedisPool(_.extend(redisUtils.getConfig(), { name: 'batch-publisher'}));
var jobPublisher = new JobPublisher(redisPoolPublisher);
var jobQueue = new JobQueue(metadataBackend, jobPublisher);
var jobBackend = new JobBackend(metadataBackend, jobQueue);

13
test/integration/batch/job_publisher.test.js
View File

@ -8,19 +8,12 @@ var assert = require('../../support/assert');
var _ = require('underscore');
var RedisPool = require('redis-mpool');
var redisUtils = require('../../support/redis_utils');
var JobPublisher = require(BATCH_SOURCE + 'job_publisher');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var redisPoolPublisher = new RedisPool(_.extend(redisConfig, { name: 'batch-publisher'}));
var redisPoolSubscriber = new RedisPool(_.extend(redisConfig, { name: 'batch-subscriber'}));
var redisPoolPublisher = new RedisPool(_.extend(redisUtils.getConfig(), { name: 'batch-publisher'}));
var redisPoolSubscriber = new RedisPool(_.extend(redisUtils.getConfig(), { name: 'batch-subscriber'}));
var HOST = 'wadus';
var CHANNEL = 'batch:hosts';

11
test/integration/batch/job_runner.test.js
View File

@ -19,16 +19,9 @@ var JobService = require(BATCH_SOURCE + 'job_service');
var JobRunner = require(BATCH_SOURCE + 'job_runner');
var QueryRunner = require(BATCH_SOURCE + 'query_runner');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var redisPoolPublisher = new RedisPool(_.extend(redisConfig, { name: 'batch-publisher'}));
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var redisPoolPublisher = new RedisPool(_.extend(redisUtils.getConfig(), { name: 'batch-publisher'}));
var jobPublisher = new JobPublisher(redisPoolPublisher);
var jobQueue = new JobQueue(metadataBackend, jobPublisher);
var jobBackend = new JobBackend(metadataBackend, jobQueue);

12
test/integration/batch/job_service.test.js
View File

@ -18,16 +18,8 @@ var JobCanceller = require(BATCH_SOURCE + 'job_canceller');
var JobService = require(BATCH_SOURCE + 'job_service');
var PSQL = require('cartodb-psql');
var redisConfig = {
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
};
var metadataBackend = require('cartodb-redis')(redisConfig);
var redisPoolPublisher = new RedisPool(_.extend(redisConfig, { name: 'batch-publisher'}));
var metadataBackend = require('cartodb-redis')(redisUtils.getConfig());
var redisPoolPublisher = new RedisPool(_.extend(redisUtils.getConfig(), { name: 'batch-publisher'}));
var jobPublisher = new JobPublisher(redisPoolPublisher);
var jobQueue = new JobQueue(metadataBackend, jobPublisher);
var jobBackend = new JobBackend(metadataBackend, jobQueue);

195
test/support/assert.js
View File

@ -1,145 +1,74 @@
var http = require('http');
var assert = module.exports = exports = require('assert');
var request = require('request');
/**
* Assert response from `server` with
* the given `req` object and `res` assertions object.
*
* @param {Server} server
* @param {Object} req
* @param {Object|Function} res
* @param {String|Function} msg
*/
assert.response = function(server, req, res, msg){
var port = 5555;
function check(){
try {
server.__port = server.address().port;
server.__listening = true;
} catch (err) {
process.nextTick(check);
return;
assert.response = function(server, req, res, callback) {
if (!callback) {
callback = res;
res = {};
}
var port = 5555,
host = '127.0.0.1';
var listeningAttempts = 0;
var listener;
function listen() {
if (listeningAttempts > 25) {
return callback(new Error('Tried too many ports'));
}
if (server.__deferred) {
server.__deferred.forEach(function(args){
assert.response.apply(assert, args);
});
server.__deferred = null;
}
}
// Check that the server is ready or defer
if (!server.fd) {
server.__deferred = server.__deferred || [];
server.listen(server.__port = port++, '127.0.0.1', check);
} else if (!server.__port) {
server.__deferred = server.__deferred || [];
process.nextTick(check);
}
// The socket was created but is not yet listening, so keep deferring
if (!server.__listening) {
server.__deferred.push(arguments);
return;
}
// Callback as third or fourth arg
var callback = typeof res === 'function'
? res
: typeof msg === 'function'
? msg
: function(){};
// Default messate to test title
if (typeof msg === 'function') msg = null;
msg = msg || assert.testTitle;
msg += '. ';
// Pending responses
server.__pending = server.__pending || 0;
server.__pending++;
// Create client
if (!server.fd) {
server.listen(server.__port = port++, '127.0.0.1', issue);
} else {
issue();
}
function issue(){
// Issue request
var timer,
method = req.method || 'GET',
status = res.status || res.statusCode,
data = req.data || req.body,
requestTimeout = req.timeout || 0,
encoding = req.encoding || 'utf8';
var request = http.request({
host: '127.0.0.1',
port: server.__port,
path: req.url,
method: method,
headers: req.headers,
agent: false
listener = server.listen(port, host);
listener.on('error', function() {
port++;
listeningAttempts++;
listen();
});
listener.on('listening', onServerListening);
}
var check = function() {
if (--server.__pending === 0) {
server.close();
server.__listening = false;
}
listen();
// jshint maxcomplexity:10
function onServerListening() {
var status = res.status || res.statusCode;
var requestParams = {
url: 'http://' + host + ':' + port + req.url,
method: req.method || 'GET',
headers: req.headers || {},
timeout: req.timeout || 0,
encoding: req.encoding || 'utf8'
};
// Timeout
if (requestTimeout) {
timer = setTimeout(function(){
check();
delete req.timeout;
request.destroy(); // will trigger 'error' event
}, requestTimeout);
if (req.body || req.data) {
requestParams.body = req.body || req.data;
}
if (data) request.write(data);
request(requestParams, function assert$response$requestHandler(error, response, body) {
listener.close(function() {
if (error) {
return callback(error);
}
request.on('error', function(err){
check();
callback(null, err);
});
request.on('response', function(response){
response.body = '';
response.setEncoding(encoding);
response.on('data', function(chunk){ response.body += chunk; });
response.on('end', function(){
if (timer) clearTimeout(timer);
check();
response = response || {};
response.body = response.body || body;
// Assert response body
if (res.body !== undefined) {
var eql = res.body instanceof RegExp
? res.body.test(response.body)
: res.body === response.body;
if (res.body) {
var eql = res.body instanceof RegExp ? res.body.test(response.body) : res.body === response.body;
assert.ok(
eql,
msg + 'Invalid response body.\n'
+ ' Expected: ' + res.body + '\n'
+ ' Got: ' + response.body
colorize('[red]{Invalid response body.}\n' +
' Expected: [green]{' + res.body + '}\n' +
' Got: [red]{' + response.body + '}')
);
}
// Assert response status
if (typeof status === 'number') {
assert.equal(
response.statusCode,
status,
msg + colorize('Invalid response status code.\n'
+ ' Expected: [green]{' + status + '}\n'
+ ' Got: [red]{' + response.statusCode + '}\n'
+ ' Response body: ' + response.body)
assert.equal(response.statusCode, status,
colorize('[red]{Invalid response status code.}\n' +
' Expected: [green]{' + status + '}\n' +
' Got: [red]{' + response.statusCode + '}\n' +
' Body: ' + response.body)
);
}
@ -150,25 +79,21 @@ assert.response = function(server, req, res, msg){
var name = keys[i],
actual = response.headers[name.toLowerCase()],
expected = res.headers[name],
eql = expected instanceof RegExp
? expected.test(actual)
: expected == actual;
assert.ok(
eql,
msg + colorize('Invalid response header [bold]{' + name + '}.\n'
+ ' Expected: [green]{' + expected + '}\n'
+ ' Got: [red]{' + actual + '}\n'
+ ' Response body: ' + response.body)
headerEql = expected instanceof RegExp ? expected.test(actual) : expected === actual;
assert.ok(headerEql,
colorize('Invalid response header [bold]{' + name + '}.\n' +
' Expected: [green]{' + expected + '}\n' +
' Got: [red]{' + actual + '}')
);
}
}
callback(response);
// Callback
callback(null, response);
});
});
request.end();
}
}
};
/**

4
test/support/redis_utils.js
View File

@ -22,3 +22,7 @@ module.exports.clean = function clean(pattern, callback) {
metadataBackend.redisCmd(5, 'DEL', keys, callback);
});
};
module.exports.getConfig = function getConfig() {
return redisConfig;
};

17
test/support/server_utils.js
View File

@ -1,17 +0,0 @@
var utils = {};
utils.startOnNextPort = function(server, issue, start_port) {
var port = start_port || 5555;
server.on('error', function(e) {
console.log("Port " + port + " already in use, retrying");
utils.startOnNextPort(server, issue, port+1);
});
server.listen(port, '127.0.0.1', issue);
}
module.exports = utils;