cartodb4/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

first commit

Browse Source
This commit is contained in:
zhongjin 2023-05-19 00:42:48 +08:00
commit 53de9c6c51
243 changed files with 39485 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
.gitignore vendored Normal file
View File

@ -0,0 +1,16 @@
config/environments/*.js
logs/*.log
pids/*.pid
*.sock
test/tmp/*
node_modules*
.idea/*
.vscode/
tools/munin/cdbsqlapi.conf
test/redis.pid
test/test.log
test/acceptance/oauth/venv/*
coverage/
npm-debug.log
log/*.log
yarn.lock

3
.jshintignore Normal file
View File

@ -0,0 +1,3 @@
test/support/
test/websocket_test/
app/models/formats/pg/topojson.js

98
.jshintrc Normal file
View File

@ -0,0 +1,98 @@
{
// // JSHint Default Configuration File (as on JSHint website)
// // See http://jshint.com/docs/ for more details
//
// "maxerr" : 50, // {int} Maximum error before stopping
//
// // Enforcing
// "bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.)
// "camelcase" : false, // true: Identifiers must be in camelCase
"curly" : true, // true: Require {} for every new block or scope
"eqeqeq" : true, // true: Require triple equals (===) for comparison
"forin" : true, // true: Require filtering for..in loops with obj.hasOwnProperty()
"freeze" : true, // true: prohibits overwriting prototypes of native objects such as Array, Date etc.
"immed" : true, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());`
// "indent" : 4, // {int} Number of spaces to use for indentation
// "latedef" : false, // true: Require variables/functions to be defined before being used
"newcap" : true, // true: Require capitalization of all constructor functions e.g. `new F()`
"noarg" : true, // true: Prohibit use of `arguments.caller` and `arguments.callee`
// "noempty" : true, // true: Prohibit use of empty blocks
"nonbsp" : true, // true: Prohibit "non-breaking whitespace" characters.
"nonew" : true, // true: Prohibit use of constructors for side-effects (without assignment)
// "plusplus" : false, // true: Prohibit use of `++` & `--`
// "quotmark" : false, // Quotation mark consistency:
// // false : do nothing (default)
// // true : ensure whatever is used is consistent
// // "single" : require single quotes
// // "double" : require double quotes
"undef" : true, // true: Require all non-global variables to be declared (prevents global leaks)
"unused" : true, // true: Require all defined variables be used
// "strict" : true, // true: Requires all functions run in ES5 Strict Mode
// "maxparams" : false, // {int} Max number of formal params allowed per function
// "maxdepth" : false, // {int} Max depth of nested blocks (within functions)
// "maxstatements" : false, // {int} Max number statements per function
"maxcomplexity" : 6, // {int} Max cyclomatic complexity per function
"maxlen" : 120, // {int} Max number of characters per line
//
// // Relaxing
// "asi" : false, // true: Tolerate Automatic Semicolon Insertion (no semicolons)
// "boss" : false, // true: Tolerate assignments where comparisons would be expected
"debug" : false, // true: Allow debugger statements e.g. browser breakpoints.
// "eqnull" : false, // true: Tolerate use of `== null`
// "es5" : false, // true: Allow ES5 syntax (ex: getters and setters)
"esnext" : true, // true: Allow ES.next (ES6) syntax (ex: `const`)
// "moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features)
// // (ex: `for each`, multiple try/catch, function expression…)
// "evil" : false, // true: Tolerate use of `eval` and `new Function()`
// "expr" : false, // true: Tolerate `ExpressionStatement` as Programs
// "funcscope" : false, // true: Tolerate defining variables inside control statements
// "globalstrict" : false, // true: Allow global "use strict" (also enables 'strict')
// "iterator" : false, // true: Tolerate using the `__iterator__` property
// "lastsemic" : false, // true: Tolerate omitting a semicolon for the last statement of a 1-line block
// "laxbreak" : false, // true: Tolerate possibly unsafe line breakings
// "laxcomma" : false, // true: Tolerate comma-first style coding
// "loopfunc" : false, // true: Tolerate functions being defined in loops
// "multistr" : false, // true: Tolerate multi-line strings
// "noyield" : false, // true: Tolerate generator functions with no yield statement in them.
// "notypeof" : false, // true: Tolerate invalid typeof operator values
// "proto" : false, // true: Tolerate using the `__proto__` property
// "scripturl" : false, // true: Tolerate script-targeted URLs
// "shadow" : false, // true: Allows re-define variables later in code e.g. `var x=1; x=2;`
// "sub" : false, // true: Tolerate using `[]` notation when it can still be expressed in dot notation
// "supernew" : false, // true: Tolerate `new function () { ... };` and `new Object;`
// "validthis" : false, // true: Tolerate using this in a non-constructor function
//
// // Environments
// "browser" : true, // Web Browser (window, document, etc)
// "browserify" : false, // Browserify (node.js code in the browser)
// "couch" : false, // CouchDB
// "devel" : true, // Development/debugging (alert, confirm, etc)
// "dojo" : false, // Dojo Toolkit
// "jasmine" : false, // Jasmine
// "jquery" : false, // jQuery
"mocha" : true, // Mocha
// "mootools" : false, // MooTools
"node" : true, // Node.js
// "nonstandard" : false, // Widely adopted globals (escape, unescape, etc)
// "prototypejs" : false, // Prototype and Scriptaculous
// "qunit" : false, // QUnit
// "rhino" : false, // Rhino
// "shelljs" : false, // ShellJS
// "worker" : false, // Web Workers
// "wsh" : false, // Windows Scripting Host
// "yui" : false, // Yahoo User Interface
// Custom Globals
"globals" : { // additional predefined global variables
"suite": true,
"suiteSetup": true,
"test": true,
"suiteTeardown": true,
"beforeEach": true,
"afterEach": true,
"before": true,
"after": true,
"describe": true,
"it": true
}
}

79
.travis.yml Normal file
View File

@ -0,0 +1,79 @@
jobs:
include:
- sudo: required
services:
- docker
language: generic
before_install: docker pull carto/nodejs6-xenial-pg101
script: npm run docker-test -- nodejs6
- sudo: required
services:
- docker
language: generic
before_install: docker pull carto/nodejs10-xenial-pg101:postgis-2.4.4.5
script: npm run docker-test -- nodejs10
- dist: precise
addons:
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- g++-4.8
postgresql: 9.5
before_install:
# Add custom PPAs from cartodb
- sudo add-apt-repository -y ppa:cartodb/postgresql-9.5
- sudo add-apt-repository -y ppa:cartodb/gis
- sudo add-apt-repository -y ppa:cartodb/gis-testing
- sudo apt-get update
# Force instalation of libgeos-3.5.0 (presumably needed because of existing version of postgis)
- sudo apt-get -y install libgeos-3.5.0=3.5.0-1cdb2
# Install postgres db and build deps
- sudo /etc/init.d/postgresql stop # stop travis default instance
- sudo apt-get -y remove --purge postgresql-9.1
- sudo apt-get -y remove --purge postgresql-9.2
- sudo apt-get -y remove --purge postgresql-9.3
- sudo apt-get -y remove --purge postgresql-9.4
- sudo apt-get -y remove --purge postgresql-9.5
- sudo apt-get -y remove --purge postgresql-9.6
- sudo rm -rf /var/lib/postgresql/
- sudo rm -rf /var/log/postgresql/
- sudo rm -rf /etc/postgresql/
- sudo apt-get -y remove --purge postgis-2.2
- sudo apt-get -y autoremove
- sudo apt-get -y install postgresql-9.5-postgis-scripts=2.2.2.0-cdb2
- sudo apt-get -y install postgresql-9.5=9.5.2-3cdb3 --allow-downgrades
- sudo apt-get -y install postgresql-server-dev-9.5=9.5.2-3cdb3
- sudo apt-get -y install postgresql-plpython-9.5=9.5.2-3cdb3 --allow-downgrades
- sudo apt-get -y install postgresql-9.5-postgis-2.2=2.2.2.0-cdb2
- sudo apt-get install -q gdal-bin
- sudo apt-get install -q ogr2ogr2-static-bin
- sudo apt-get install -y libc6
- wget http://download.redis.io/releases/redis-4.0.8.tar.gz
- tar xvzf redis-4.0.8.tar.gz
- cd redis-4.0.8
- make
- sudo make install
- cd ..
- rm redis-4.0.8.tar.gz
# configure it to accept local connections from postgres
- echo -e "# TYPE DATABASE USER ADDRESS METHOD \nlocal all postgres trust\nlocal all all trust\nhost all all 127.0.0.1/32 trust" \
| sudo tee /etc/postgresql/9.5/main/pg_hba.conf
- sudo /etc/init.d/postgresql restart 9.5
- psql -c 'create database template_postgis;' -U postgres
- psql -c 'CREATE EXTENSION postgis;' -U postgres -d template_postgis
- ./configure
env:
- PGUSER=postgres CXX=g++-4.8
language: node_js
node_js:
- "6"

11
CONTRIBUTING.md Normal file
View File

@ -0,0 +1,11 @@
Contributing
---
The issue tracker is at [github.com/CartoDB/CartoDB-SQL-API](https://github.com/CartoDB/CartoDB-SQL-API).
We love pull requests from everyone, see [Contributing to Open Source on GitHub](https://guides.github.com/activities/contributing-to-open-source/#contributing).
## Submitting Contributions
* You will need to sign a Contributor License Agreement (CLA) before making a submission. [Learn more here](https://carto.com/contributions).

17
HOWTO_RELEASE Normal file
View File

@ -0,0 +1,17 @@
1. Test (make clean all check), fix if broken before proceeding
2. Ensure proper version in package.json and package-lock.json
3. Ensure NEWS section exists for the new version, review it, add release date
4. Commit package.json, npm-shrinwrap.json, NEWS
5. git tag -a Major.Minor.Patch # use NEWS section as content
6. Stub NEWS/package for next version
Versions:
Bugfix releases increment Patch component of version.
Feature releases increment Minor and set Patch to zero.
If backward compatibility is broken, increment Major and
set to zero Minor and Patch.
Branches named 'b<Major>.<Minor>' are kept for any critical
fix that might need to be shipped before next feature release
is ready.

27
LICENSE Normal file
View File

@ -0,0 +1,27 @@
Copyright (c) 2015, CartoDB
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

47
Makefile Normal file
View File

@ -0,0 +1,47 @@
SHELL=/bin/bash
all:
npm install
clean:
rm -rf node_modules/*
check:
npm test
jshint:
@echo "***jshint***"
@./node_modules/.bin/jshint app/ batch/ test/ app.js
TEST_SUITE := $(shell find test/{unit,integration,acceptance} -name "*.js")
TEST_SUITE_UNIT := $(shell find test/unit -name "*.js")
TEST_SUITE_INTEGRATION := $(shell find test/integration -name "*.js")
TEST_SUITE_ACCEPTANCE := $(shell find test/acceptance -name "*.js")
TEST_SUITE_BATCH := $(shell find test/*/batch -name "*.js")
test:
@echo "***tests***"
@$(SHELL) test/run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE)
test-unit:
@echo "***unit tests***"
@$(SHELL) test/run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_UNIT)
test-integration:
@echo "***integration tests***"
@$(SHELL) test/run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_INTEGRATION)
test-acceptance:
@echo "***acceptance tests***"
@$(SHELL) test/run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_ACCEPTANCE)
test-batch:
@echo "***batch queries tests***"
@$(SHELL) test/run_tests.sh ${RUNTESTFLAGS} $(TEST_SUITE_BATCH)
test-all: test jshint
coverage:
@RUNTESTFLAGS=--with-coverage make test
.PHONY: test coverage

1037
NEWS.md Normal file
View File

File diff suppressed because it is too large Load Diff

82
README.md Normal file
View File

@ -0,0 +1,82 @@
SQL API for carto.com
========================
[![Build Status](https://travis-ci.org/CartoDB/CartoDB-SQL-API.png?branch=master)](https://travis-ci.org/CartoDB/CartoDB-SQL-API)
Provides a node.js based API for running SQL queries against CartoDB.
* Users are authenticated over OAuth or via an API KEY.
* Authenticated requests to this API should always be made over SSL.
core requirements
-----------------
* Node >= 10.14.2 or 6.9.2
* npm >= 6.4.1 || 3.10.9 || 3.10.10
* Postgres `9.3+`.
* Postgis `2.2`.
* [CartoDB Postgres Extension](https://github.com/CartoDB/cartodb-postgresql/blob/0.19.2/README.md) `0.19+`.
* GDAL `1.11.0` (bin utils). See [installing GDAL](http://trac.osgeo.org/gdal/wiki/DownloadingGdalBinaries)
* zip commandline tool.
* Redis `3`, recommended reversion `3.0.2`.
Install dependencies
--------------------
- Node.js >= 10.14.2:
```
$ mv npm-shrinkwrap.json npm-shrinkwrap.json.backup
$ npm ci
$ mv npm-shrinkwrap.json.backup npm-shrinkwrap.json
```
- Node.js 6.9.2:
```sh
npm install
```
usage
-----
Create and edit config/environments/<environment>.js from .js.example files.
You may find the ./configure script useful to make an edited copy for you,
see ```./configure --help``` for a list of supported switches.
Make sure redis is running and knows about active cartodb user.
Make sure your PostgreSQL server is running, is accessible on
the host and port specified in the <environment> file, has
a 'publicuser' role (or whatever you set ``db_pubuser`` configuration
directive to) and trusts user authentication from localhost
connections.
```sh
node app.js <environment>
```
Supported <environment> values are development, test, production
See doc/API.md for API documentation.
For examples of use, see under test/.
tests
-----
Run with:
```sh
npm test
```
If any issue arise see test/README.md
Note that the environment should be set to ensure the default
PostgreSQL user is superuser (PGUSER=postgres make check).
Contributing
---
See [CONTRIBUTING.md](CONTRIBUTING.md).

241
app.js Executable file
View File

@ -0,0 +1,241 @@
#!/usr/bin/env node
'use strict';
/*
* SQL API loader
* ===============
*
* node app [environment]
*
* environments: [development, test, production]
*
*/
var fs = require('fs');
var path = require('path');
const fqdn = require('@carto/fqdn-sync');
var argv = require('yargs')
.usage('Usage: $0 <environment> [options]')
.help('h')
.example(
'$0 production -c /etc/sql-api/config.js',
'start server in production environment with /etc/sql-api/config.js as config file'
)
.alias('h', 'help')
.alias('c', 'config')
.nargs('c', 1)
.describe('c', 'Load configuration from path')
.argv;
var environmentArg = argv._[0] || process.env.NODE_ENV || 'development';
var configurationFile = path.resolve(argv.config || './config/environments/' + environmentArg + '.js');
if (!fs.existsSync(configurationFile)) {
console.error('Configuration file "%s" does not exist', configurationFile);
process.exit(1);
}
global.settings = require(configurationFile);
var ENVIRONMENT = argv._[0] || process.env.NODE_ENV || global.settings.environment;
process.env.NODE_ENV = ENVIRONMENT;
var availableEnvironments = ['development', 'production', 'test', 'staging'];
// sanity check arguments
if (availableEnvironments.indexOf(ENVIRONMENT) === -1) {
console.error("node app.js [environment]");
console.error("Available environments: " + availableEnvironments.join(', '));
process.exit(1);
}
global.settings.api_hostname = fqdn.hostname();
global.log4js = require('log4js');
var log4jsConfig = {
appenders: [],
replaceConsole: true
};
if ( global.settings.log_filename ) {
var logFilename = path.resolve(global.settings.log_filename);
var logDirectory = path.dirname(logFilename);
if (!fs.existsSync(logDirectory)) {
console.error("Log filename directory does not exist: " + logDirectory);
process.exit(1);
}
console.log("Logs will be written to " + logFilename);
log4jsConfig.appenders.push(
{ type: "file", absolute: true, filename: logFilename }
);
} else {
log4jsConfig.appenders.push(
{ type: "console", layout: { type:'basic' } }
);
}
global.log4js.configure(log4jsConfig);
global.logger = global.log4js.getLogger();
// kick off controller
if ( ! global.settings.base_url ) {
global.settings.base_url = '/api/*';
}
var version = require("./package").version;
var StatsClient = require('./app/stats/client');
if (global.settings.statsd) {
// Perform keyword substitution in statsd
if (global.settings.statsd.prefix) {
global.settings.statsd.prefix = global.settings.statsd.prefix.replace(/:host/, fqdn.reverse());
}
}
var statsClient = StatsClient.getInstance(global.settings.statsd);
var server = require('./app/server')(statsClient);
var listener = server.listen(global.settings.node_port, global.settings.node_host);
listener.on('listening', function() {
console.info("Using Node.js %s", process.version);
console.info('Using configuration file "%s"', configurationFile);
console.log(
"CartoDB SQL API %s listening on %s:%s PID=%d (%s)",
version, global.settings.node_host, global.settings.node_port, process.pid, ENVIRONMENT
);
});
process.on('uncaughtException', function(err) {
global.logger.error('Uncaught exception: ' + err.stack);
});
process.on('SIGHUP', function() {
global.log4js.clearAndShutdownAppenders(function() {
global.log4js.configure(log4jsConfig);
global.logger = global.log4js.getLogger();
console.log('Log files reloaded');
});
if (server.batch && server.batch.logger) {
server.batch.logger.reopenFileStreams();
}
if (server.dataIngestionLogger) {
server.dataIngestionLogger.reopenFileStreams();
}
});
process.on('SIGTERM', function () {
server.batch.stop();
server.batch.drain(function (err) {
if (err) {
console.log('Exit with error');
return process.exit(1);
}
console.log('Exit gracefully');
process.exit(0);
});
});
function isGteMinVersion(version, minVersion) {
var versionMatch = /[a-z]?([0-9]*)/.exec(version);
if (versionMatch) {
var majorVersion = parseInt(versionMatch[1], 10);
if (Number.isFinite(majorVersion)) {
return majorVersion >= minVersion;
}
}
return false;
}
setInterval(function memoryUsageMetrics () {
let memoryUsage = process.memoryUsage();
Object.keys(memoryUsage).forEach(property => {
statsClient.gauge(`sqlapi.memory.${property}`, memoryUsage[property]);
});
}, 5000);
function getCPUUsage (oldUsage) {
let usage;
if (oldUsage && oldUsage._start) {
usage = Object.assign({}, process.cpuUsage(oldUsage._start.cpuUsage));
usage.time = Date.now() - oldUsage._start.time;
} else {
usage = Object.assign({}, process.cpuUsage());
usage.time = process.uptime() * 1000; // s to ms
}
usage.percent = (usage.system + usage.user) / (usage.time * 10);
Object.defineProperty(usage, '_start', {
value: {
cpuUsage: process.cpuUsage(),
time: Date.now()
}
});
return usage;
}
let previousCPUUsage = getCPUUsage();
setInterval(function cpuUsageMetrics () {
const CPUUsage = getCPUUsage(previousCPUUsage);
Object.keys(CPUUsage).forEach(property => {
statsClient.gauge(`sqlapi.cpu.${property}`, CPUUsage[property]);
});
previousCPUUsage = CPUUsage;
}, 5000);
if (global.gc && isGteMinVersion(process.version, 6)) {
var gcInterval = Number.isFinite(global.settings.gc_interval) ?
global.settings.gc_interval :
10000;
if (gcInterval > 0) {
setInterval(function gcForcedCycle() {
global.gc();
}, gcInterval);
}
}
const gcStats = require('gc-stats')();
gcStats.on('stats', function ({ pauseMS, gctype }) {
statsClient.timing('sqlapi.gc', pauseMS);
statsClient.timing(`sqlapi.gctype.${getGCTypeValue(gctype)}`, pauseMS);
});
function getGCTypeValue (type) {
// 1: Scavenge (minor GC)
// 2: Mark/Sweep/Compact (major GC)
// 4: Incremental marking
// 8: Weak/Phantom callback processing
// 15: All
let value;
switch (type) {
case 1:
value = 'Scavenge';
break;
case 2:
value = 'MarkSweepCompact';
break;
case 4:
value = 'IncrementalMarking';
break;
case 8:
value = 'ProcessWeakCallbacks';
break;
case 15:
value = 'All';
break;
default:
value = 'Unkown';
break;
}
return value;
}

74
app/auth/apikey.js Normal file
View File

@ -0,0 +1,74 @@
'use strict';
/**
* this module allows to auth user using an pregenerated api key
*/
function ApikeyAuth(req, metadataBackend, username, apikeyToken) {
this.req = req;
this.metadataBackend = metadataBackend;
this.username = username;
this.apikeyToken = apikeyToken;
}
module.exports = ApikeyAuth;
function usernameMatches(basicAuthUsername, requestUsername) {
return !(basicAuthUsername && (basicAuthUsername !== requestUsername));
}
ApikeyAuth.prototype.verifyCredentials = function (callback) {
this.metadataBackend.getApikey(this.username, this.apikeyToken, (err, apikey) => {
if (err) {
err.http_status = 500;
err.message = 'Unexpected error';
return callback(err);
}
if (isApiKeyFound(apikey)) {
if (!usernameMatches(apikey.user, this.username)) {
const usernameError = new Error('Forbidden');
usernameError.type = 'auth';
usernameError.subtype = 'api-key-username-mismatch';
usernameError.http_status = 403;
return callback(usernameError);
}
if (!apikey.grantsSql) {
const forbiddenError = new Error('forbidden');
forbiddenError.http_status = 403;
return callback(forbiddenError);
}
return callback(null, getAuthorizationLevel(apikey));
} else {
const apiKeyNotFoundError = new Error('Unauthorized');
apiKeyNotFoundError.type = 'auth';
apiKeyNotFoundError.subtype = 'api-key-not-found';
apiKeyNotFoundError.http_status = 401;
return callback(apiKeyNotFoundError);
}
});
};
ApikeyAuth.prototype.hasCredentials = function () {
return !!this.apikeyToken;
};
ApikeyAuth.prototype.getCredentials = function () {
return this.apikeyToken;
};
function getAuthorizationLevel(apikey) {
return apikey.type;
}
function isApiKeyFound(apikey) {
return apikey.type !== null &&
apikey.user !== null &&
apikey.databasePassword !== null &&
apikey.databaseRole !== null;
}

48
app/auth/auth_api.js Normal file
View File

@ -0,0 +1,48 @@
'use strict';
var ApiKeyAuth = require('./apikey'),
OAuthAuth = require('./oauth');
function AuthApi(req, requestParams) {
this.req = req;
this.authBackend = getAuthBackend(req, requestParams);
this._hasCredentials = null;
}
AuthApi.prototype.getType = function () {
if (this.authBackend instanceof ApiKeyAuth) {
return 'apiKey';
} else if (this.authBackend instanceof OAuthAuth) {
return 'oAuth';
}
};
AuthApi.prototype.hasCredentials = function() {
if (this._hasCredentials === null) {
this._hasCredentials = this.authBackend.hasCredentials();
}
return this._hasCredentials;
};
AuthApi.prototype.getCredentials = function() {
return this.authBackend.getCredentials();
};
AuthApi.prototype.verifyCredentials = function(callback) {
if (this.hasCredentials()) {
this.authBackend.verifyCredentials(callback);
} else {
callback(null, false);
}
};
function getAuthBackend(req, requestParams) {
if (requestParams.api_key) {
return new ApiKeyAuth(req, requestParams.metadataBackend, requestParams.user, requestParams.api_key);
} else {
return new OAuthAuth(req, requestParams.metadataBackend);
}
}
module.exports = AuthApi;

192
app/auth/oauth.js Normal file
View File

@ -0,0 +1,192 @@
'use strict';
// too bound to the request object, but ok for now
var _ = require('underscore');
var OAuthUtil = require('oauth-client');
var step = require('step');
var CdbRequest = require('../models/cartodb_request');
var cdbReq = new CdbRequest();
var oAuth = (function(){
var me = {
oauth_database: 3,
oauth_user_key: "rails:oauth_access_tokens:<%= oauth_access_key %>",
is_oauth_request: true
};
// oauth token cases:
// * in GET request
// * in header
me.parseTokens = function(req){
var query_oauth = _.clone(req.method === "POST" ? req.body: req.query);
var header_oauth = {};
var oauth_variables = ['oauth_body_hash',
'oauth_consumer_key',
'oauth_token',
'oauth_signature_method',
'oauth_signature',
'oauth_timestamp',
'oauth_nonce',
'oauth_version'];
// pull only oauth tokens out of query
var non_oauth = _.difference(_.keys(query_oauth), oauth_variables);
_.each(non_oauth, function(key){ delete query_oauth[key]; });
// pull oauth tokens out of header
var header_string = req.headers.authorization;
if (!_.isUndefined(header_string)) {
_.each(oauth_variables, function(oauth_key){
var matched_string = header_string.match(new RegExp(oauth_key + '=\"([^\"]+)\"'));
if (!_.isNull(matched_string)) {
header_oauth[oauth_key] = decodeURIComponent(matched_string[1]);
}
});
}
//merge header and query oauth tokens. preference given to header oauth
return _.defaults(header_oauth, query_oauth);
};
// remove oauthy tokens from an object
me.splitParams = function(obj) {
var removed = null;
for (var prop in obj) {
if (/^oauth_\w+$/.test(prop)) {
if(!removed) {
removed = {};
}
removed[prop] = obj[prop];
delete obj[prop];
}
}
return removed;
};
me.getAllowedHosts= function() {
var oauthConfig = global.settings.oauth || {};
return oauthConfig.allowedHosts || ['carto.com', 'cartodb.com'];
};
// do new fancy get User ID
me.verifyRequest = function(req, metadataBackend, callback) {
var that = this;
//TODO: review this
var httpProto = req.protocol;
if(!httpProto || (httpProto !== 'http' && httpProto !== 'https')) {
var msg = "Unknown HTTP protocol " + httpProto + ".";
var unknownProtocolErr = new Error(msg);
unknownProtocolErr.http_status = 500;
return callback(unknownProtocolErr);
}
var username = cdbReq.userByReq(req);
var requestTokens;
var signature;
step(
function getTokensFromURL(){
return oAuth.parseTokens(req);
},
function getOAuthHash(err, _requestTokens) {
if (err) {
throw err;
}
// this is oauth request only if oauth headers are present
this.is_oauth_request = !_.isEmpty(_requestTokens);
if (this.is_oauth_request) {
requestTokens = _requestTokens;
that.getOAuthHash(metadataBackend, requestTokens.oauth_token, this);
} else {
return null;
}
},
function regenerateSignature(err, oAuthHash){
if (err) {
throw err;
}
if (!this.is_oauth_request) {
return null;
}
var consumer = OAuthUtil.createConsumer(oAuthHash.consumer_key, oAuthHash.consumer_secret);
var access_token = OAuthUtil.createToken(oAuthHash.access_token_token, oAuthHash.access_token_secret);
var signer = OAuthUtil.createHmac(consumer, access_token);
var method = req.method;
var hostsToValidate = {};
var requestHost = req.headers.host;
hostsToValidate[requestHost] = true;
that.getAllowedHosts().forEach(function(allowedHost) {
hostsToValidate[username + '.' + allowedHost] = true;
});
that.splitParams(req.query);
// remove oauth_signature from body
if(req.body) {
delete req.body.oauth_signature;
}
signature = requestTokens.oauth_signature;
// remove signature from requestTokens
delete requestTokens.oauth_signature;
var requestParams = _.extend({}, req.body, requestTokens, req.query);
var hosts = Object.keys(hostsToValidate);
var requestSignatures = hosts.map(function(host) {
var url = httpProto + '://' + host + req.path;
return signer.sign(method, url, requestParams);
});
return requestSignatures.reduce(function(validSignature, requestSignature) {
if (signature === requestSignature && !_.isUndefined(requestSignature)) {
validSignature = true;
}
return validSignature;
}, false);
},
function finishValidation(err, hasValidSignature) {
const authorizationLevel = hasValidSignature ? 'master' : null;
return callback(err, authorizationLevel);
}
);
};
me.getOAuthHash = function(metadataBackend, oAuthAccessKey, callback){
metadataBackend.getOAuthHash(oAuthAccessKey, callback);
};
return me;
})();
function OAuthAuth(req, metadataBackend) {
this.req = req;
this.metadataBackend = metadataBackend;
this.isOAuthRequest = null;
}
OAuthAuth.prototype.verifyCredentials = function(callback) {
if (this.hasCredentials()) {
oAuth.verifyRequest(this.req, this.metadataBackend, callback);
} else {
callback(null, false);
}
};
OAuthAuth.prototype.getCredentials = function() {
return oAuth.parseTokens(this.req);
};
OAuthAuth.prototype.hasCredentials = function() {
if (this.isOAuthRequest === null) {
var passed_tokens = oAuth.parseTokens(this.req);
this.isOAuthRequest = !_.isEmpty(passed_tokens);
}
return this.isOAuthRequest;
};
module.exports = OAuthAuth;
module.exports.backend = oAuth;

29
app/controllers/cache_status_controller.js Normal file
View File

@ -0,0 +1,29 @@
'use strict';
var _ = require('underscore');
function CacheStatusController(tableCache) {
this.tableCache = tableCache;
}
CacheStatusController.prototype.route = function (app) {
app.get(global.settings.base_url + '/cachestatus', this.handleCacheStatus.bind(this));
};
CacheStatusController.prototype.handleCacheStatus = function (req, res) {
var tableCacheValues = this.tableCache.values();
var totalExplainKeys = tableCacheValues.length;
var totalExplainHits = _.reduce(tableCacheValues, function(memo, res) {
return memo + res.hits;
}, 0);
res.send({
explain: {
pid: process.pid,
hits: totalExplainHits,
keys : totalExplainKeys
}
});
};
module.exports = CacheStatusController;

207
app/controllers/copy_controller.js Normal file
View File

@ -0,0 +1,207 @@
'use strict';
const userMiddleware = require('../middlewares/user');
const errorMiddleware = require('../middlewares/error');
const authorizationMiddleware = require('../middlewares/authorization');
const connectionParamsMiddleware = require('../middlewares/connection-params');
const { initializeProfilerMiddleware } = require('../middlewares/profiler');
const rateLimitsMiddleware = require('../middlewares/rate-limit');
const dbQuotaMiddleware = require('../middlewares/db-quota');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
const errorHandlerFactory = require('../services/error_handler_factory');
const StreamCopy = require('../services/stream_copy');
const StreamCopyMetrics = require('../services/stream_copy_metrics');
const zlib = require('zlib');
const { PassThrough } = require('stream');
function CopyController(metadataBackend, userDatabaseService, userLimitsService, logger) {
this.metadataBackend = metadataBackend;
this.userDatabaseService = userDatabaseService;
this.userLimitsService = userLimitsService;
this.logger = logger;
}
CopyController.prototype.route = function (app) {
const { base_url } = global.settings;
const copyFromMiddlewares = endpointGroup => {
return [
initializeProfilerMiddleware('copyfrom'),
userMiddleware(this.metadataBackend),
rateLimitsMiddleware(this.userLimitsService, endpointGroup),
authorizationMiddleware(this.metadataBackend),
connectionParamsMiddleware(this.userDatabaseService),
validateCopyQuery(),
dbQuotaMiddleware(),
handleCopyFrom(this.logger),
errorHandler(),
errorMiddleware()
];
};
const copyToMiddlewares = endpointGroup => {
return [
initializeProfilerMiddleware('copyto'),
userMiddleware(this.metadataBackend),
rateLimitsMiddleware(this.userLimitsService, endpointGroup),
authorizationMiddleware(this.metadataBackend),
connectionParamsMiddleware(this.userDatabaseService),
validateCopyQuery(),
handleCopyTo(this.logger),
errorHandler(),
errorMiddleware()
];
};
app.post(`${base_url}/sql/copyfrom`, copyFromMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_FROM));
app.get(`${base_url}/sql/copyto`, copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO));
};
function handleCopyTo (logger) {
return function handleCopyToMiddleware (req, res, next) {
const sql = req.query.q;
const { userDbParams, user } = res.locals;
const filename = req.query.filename || 'carto-sql-copyto.dmp';
// it is not sure, nginx may choose not to compress the body
// but we want to know it and save it in the metrics
// https://github.com/CartoDB/CartoDB-SQL-API/issues/515
const isGzip = req.get('accept-encoding') && req.get('accept-encoding').includes('gzip');
const streamCopy = new StreamCopy(sql, userDbParams);
const metrics = new StreamCopyMetrics(logger, 'copyto', sql, user, isGzip);
res.header("Content-Disposition", `attachment; filename=${encodeURIComponent(filename)}`);
res.header("Content-Type", "application/octet-stream");
streamCopy.getPGStream(StreamCopy.ACTION_TO, (err, pgstream) => {
if (err) {
return next(err);
}
pgstream
.on('data', data => metrics.addSize(data.length))
.on('error', err => {
metrics.end(null, err);
pgstream.unpipe(res);
return next(err);
})
.on('end', () => metrics.end( streamCopy.getRowCount(StreamCopy.ACTION_TO) ))
.pipe(res)
.on('close', () => {
const err = new Error('Connection closed by client');
pgstream.emit('cancelQuery', err);
pgstream.emit('error', err);
})
.on('error', err => {
pgstream.emit('error', err);
});
});
};
}
function handleCopyFrom (logger) {
return function handleCopyFromMiddleware (req, res, next) {
const sql = req.query.q;
const { userDbParams, user, dbRemainingQuota } = res.locals;
const isGzip = req.get('content-encoding') === 'gzip';
const COPY_FROM_MAX_POST_SIZE = global.settings.copy_from_max_post_size || 2 * 1024 * 1024 * 1024; // 2 GB
const COPY_FROM_MAX_POST_SIZE_PRETTY = global.settings.copy_from_max_post_size_pretty || '2 GB';
const streamCopy = new StreamCopy(sql, userDbParams);
const metrics = new StreamCopyMetrics(logger, 'copyfrom', sql, user, isGzip);
streamCopy.getPGStream(StreamCopy.ACTION_FROM, (err, pgstream) => {
if (err) {
return next(err);
}
req
.on('data', data => isGzip ? metrics.addGzipSize(data.length) : undefined)
.on('error', err => {
metrics.end(null, err);
pgstream.emit('error', err);
})
.on('close', () => {
const err = new Error('Connection closed by client');
pgstream.emit('cancelQuery', err);
pgstream.emit('error', err);
})
.pipe(isGzip ? zlib.createGunzip() : new PassThrough())
.on('error', err => {
err.message = `Error while gunzipping: ${err.message}`;
metrics.end(null, err);
pgstream.emit('error', err);
})
.on('data', data => {
metrics.addSize(data.length);
if(metrics.size > dbRemainingQuota) {
const quotaError = new Error('DB Quota exceeded');
pgstream.emit('cancelQuery', err);
pgstream.emit('error', quotaError);
}
if((metrics.gzipSize || metrics.size) > COPY_FROM_MAX_POST_SIZE) {
const maxPostSizeError = new Error(
`COPY FROM maximum POST size of ${COPY_FROM_MAX_POST_SIZE_PRETTY} exceeded`
);
pgstream.emit('cancelQuery', err);
pgstream.emit('error', maxPostSizeError);
}
})
.pipe(pgstream)
.on('error', err => {
metrics.end(null, err);
req.unpipe(pgstream);
return next(err);
})
.on('end', () => {
metrics.end( streamCopy.getRowCount(StreamCopy.ACTION_FROM) );
const { time, rows } = metrics;
if (!rows) {
return next(new Error("No rows copied"));
}
res.send({
time,
total_rows: rows
});
});
});
};
}
function validateCopyQuery () {
return function validateCopyQueryMiddleware (req, res, next) {
const sql = req.query.q;
if (!sql) {
return next(new Error("SQL is missing"));
}
if (!sql.toUpperCase().startsWith("COPY ")) {
return next(new Error("SQL must start with COPY"));
}
next();
};
}
function errorHandler () {
return function errorHandlerMiddleware (err, req, res, next) {
if (res.headersSent) {
console.error("EXCEPTION REPORT: " + err.stack);
const errorHandler = errorHandlerFactory(err);
res.write(JSON.stringify(errorHandler.getResponse()));
res.end();
} else {
return next(err);
}
};
}
module.exports = CopyController;

14
app/controllers/generic_controller.js Normal file
View File

@ -0,0 +1,14 @@
'use strict';
function GenericController() {
}
GenericController.prototype.route = function (app) {
app.options('*', this.handleRequest.bind(this));
};
GenericController.prototype.handleRequest = function(req, res) {
res.end();
};
module.exports = GenericController;

35
app/controllers/health_check_controller.js Normal file
View File

@ -0,0 +1,35 @@
'use strict';
var HealthCheck = require('../monitoring/health_check');
function HealthCheckController() {
this.healthCheck = new HealthCheck(global.settings.disabled_file);
}
HealthCheckController.prototype.route = function (app) {
app.get(global.settings.base_url + '/health', this.handleHealthCheck.bind(this));
};
HealthCheckController.prototype.handleHealthCheck = function (req, res) {
var healthConfig = global.settings.health || {};
if (!!healthConfig.enabled) {
var startTime = Date.now();
this.healthCheck.check(function(err) {
var ok = !err;
var response = {
enabled: true,
ok: ok,
elapsed: Date.now() - startTime
};
if (err) {
response.err = err.message;
}
res.status(ok ? 200 : 503).send(response);
});
} else {
res.status(200).send({enabled: false, ok: true});
}
};
module.exports = HealthCheckController;

253
app/controllers/job_controller.js Normal file
View File

@ -0,0 +1,253 @@
'use strict';
const util = require('util');
const bodyParserMiddleware = require('../middlewares/body-parser');
const userMiddleware = require('../middlewares/user');
const { initializeProfilerMiddleware, finishProfilerMiddleware } = require('../middlewares/profiler');
const authorizationMiddleware = require('../middlewares/authorization');
const connectionParamsMiddleware = require('../middlewares/connection-params');
const errorMiddleware = require('../middlewares/error');
const rateLimitsMiddleware = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
function JobController(metadataBackend, userDatabaseService, jobService, statsdClient, userLimitsService) {
this.metadataBackend = metadataBackend;
this.userDatabaseService = userDatabaseService;
this.jobService = jobService;
this.statsdClient = statsdClient;
this.userLimitsService = userLimitsService;
}
module.exports = JobController;
JobController.prototype.route = function (app) {
const { base_url } = global.settings;
const jobMiddlewares = composeJobMiddlewares(
this.metadataBackend,
this.userDatabaseService,
this.jobService,
this.statsdClient,
this.userLimitsService
);
app.get(
`${base_url}/jobs-wip`,
bodyParserMiddleware(),
listWorkInProgressJobs(this.jobService),
sendResponse(),
errorMiddleware()
);
app.post(
`${base_url}/sql/job`,
bodyParserMiddleware(),
checkBodyPayloadSize(),
jobMiddlewares('create', createJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_CREATE)
);
app.get(
`${base_url}/sql/job/:job_id`,
bodyParserMiddleware(),
jobMiddlewares('retrieve', getJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_GET)
);
app.delete(
`${base_url}/sql/job/:job_id`,
bodyParserMiddleware(),
jobMiddlewares('cancel', cancelJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_DELETE)
);
};
function composeJobMiddlewares (metadataBackend, userDatabaseService, jobService, statsdClient, userLimitsService) {
return function jobMiddlewares (action, jobMiddleware, endpointGroup) {
const forceToBeMaster = true;
return [
initializeProfilerMiddleware('job'),
userMiddleware(metadataBackend),
rateLimitsMiddleware(userLimitsService, endpointGroup),
authorizationMiddleware(metadataBackend, forceToBeMaster),
connectionParamsMiddleware(userDatabaseService),
jobMiddleware(jobService),
setServedByDBHostHeader(),
finishProfilerMiddleware(),
logJobResult(action),
incrementSuccessMetrics(statsdClient),
sendResponse(),
incrementErrorMetrics(statsdClient),
errorMiddleware()
];
};
}
function cancelJob (jobService) {
return function cancelJobMiddleware (req, res, next) {
const { job_id } = req.params;
jobService.cancel(job_id, (err, job) => {
if (req.profiler) {
req.profiler.done('cancelJob');
}
if (err) {
return next(err);
}
res.body = job.serialize();
next();
});
};
}
function getJob (jobService) {
return function getJobMiddleware (req, res, next) {
const { job_id } = req.params;
jobService.get(job_id, (err, job) => {
if (req.profiler) {
req.profiler.done('getJob');
}
if (err) {
return next(err);
}
res.body = job.serialize();
next();
});
};
}
function createJob (jobService) {
return function createJobMiddleware (req, res, next) {
const params = Object.assign({}, req.query, req.body);
var data = {
user: res.locals.user,
query: params.query,
host: res.locals.userDbParams.host,
port: global.settings.db_batch_port || res.locals.userDbParams.port,
pass: res.locals.userDbParams.pass,
dbname: res.locals.userDbParams.dbname,
dbuser: res.locals.userDbParams.user
};
jobService.create(data, (err, job) => {
if (req.profiler) {
req.profiler.done('createJob');
}
if (err) {
return next(err);
}
res.locals.job_id = job.job_id;
res.statusCode = 201;
res.body = job.serialize();
next();
});
};
}
function listWorkInProgressJobs (jobService) {
return function listWorkInProgressJobsMiddleware (req, res, next) {
jobService.listWorkInProgressJobs((err, list) => {
if (err) {
return next(err);
}
res.body = list;
next();
});
};
}
function checkBodyPayloadSize () {
return function checkBodyPayloadSizeMiddleware(req, res, next) {
const payload = JSON.stringify(req.body);
if (payload.length > MAX_LIMIT_QUERY_SIZE_IN_BYTES) {
return next(new Error(getMaxSizeErrorMessage(payload)), res);
}
next();
};
}
const ONE_KILOBYTE_IN_BYTES = 1024;
const MAX_LIMIT_QUERY_SIZE_IN_KB = 16;
const MAX_LIMIT_QUERY_SIZE_IN_BYTES = MAX_LIMIT_QUERY_SIZE_IN_KB * ONE_KILOBYTE_IN_BYTES;
function getMaxSizeErrorMessage(sql) {
return util.format([
'Your payload is too large: %s bytes. Max size allowed is %s bytes (%skb).',
'Are you trying to import data?.',
'Please, check out import api http://docs.cartodb.com/cartodb-platform/import-api/'
].join(' '),
sql.length,
MAX_LIMIT_QUERY_SIZE_IN_BYTES,
Math.round(MAX_LIMIT_QUERY_SIZE_IN_BYTES / ONE_KILOBYTE_IN_BYTES)
);
}
module.exports.MAX_LIMIT_QUERY_SIZE_IN_BYTES = MAX_LIMIT_QUERY_SIZE_IN_BYTES;
module.exports.getMaxSizeErrorMessage = getMaxSizeErrorMessage;
function setServedByDBHostHeader () {
return function setServedByDBHostHeaderMiddleware (req, res, next) {
const { userDbParams } = res.locals;
if (userDbParams.host) {
res.header('X-Served-By-DB-Host', res.locals.userDbParams.host);
}
next();
};
}
function logJobResult (action) {
return function logJobResultMiddleware (req, res, next) {
if (process.env.NODE_ENV !== 'test') {
console.info(JSON.stringify({
type: 'sql_api_batch_job',
username: res.locals.user,
action: action,
job_id: req.params.job_id || res.locals.job_id
}));
}
next();
};
}
const METRICS_PREFIX = 'sqlapi.job';
function incrementSuccessMetrics (statsdClient) {
return function incrementSuccessMetricsMiddleware (req, res, next) {
if (statsdClient !== undefined) {
statsdClient.increment(`${METRICS_PREFIX}.success`);
}
next();
};
}
function incrementErrorMetrics (statsdClient) {
return function incrementErrorMetricsMiddleware (err, req, res, next) {
if (statsdClient !== undefined) {
statsdClient.increment(`${METRICS_PREFIX}.error`);
}
next(err);
};
}
function sendResponse () {
return function sendResponseMiddleware (req, res) {
res.status(res.statusCode || 200).send(res.body);
};
}

275
app/controllers/query_controller.js Normal file
View File

@ -0,0 +1,275 @@
'use strict';
var _ = require('underscore');
var step = require('step');
var PSQL = require('cartodb-psql');
var CachedQueryTables = require('../services/cached-query-tables');
const pgEntitiesAccessValidator = require('../services/pg-entities-access-validator');
var queryMayWrite = require('../utils/query_may_write');
var formats = require('../models/formats');
var sanitize_filename = require('../utils/filename_sanitizer');
var getContentDisposition = require('../utils/content_disposition');
const bodyParserMiddleware = require('../middlewares/body-parser');
const userMiddleware = require('../middlewares/user');
const errorMiddleware = require('../middlewares/error');
const authorizationMiddleware = require('../middlewares/authorization');
const connectionParamsMiddleware = require('../middlewares/connection-params');
const timeoutLimitsMiddleware = require('../middlewares/timeout-limits');
const { initializeProfilerMiddleware } = require('../middlewares/profiler');
const rateLimitsMiddleware = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware;
var ONE_YEAR_IN_SECONDS = 31536000; // 1 year time to live by default
function QueryController(metadataBackend, userDatabaseService, tableCache, statsd_client, userLimitsService) {
this.metadataBackend = metadataBackend;
this.statsd_client = statsd_client;
this.userDatabaseService = userDatabaseService;
this.queryTables = new CachedQueryTables(tableCache);
this.userLimitsService = userLimitsService;
}
QueryController.prototype.route = function (app) {
const { base_url } = global.settings;
const forceToBeMaster = false;
const queryMiddlewares = () => {
return [
bodyParserMiddleware(),
initializeProfilerMiddleware('query'),
userMiddleware(this.metadataBackend),
rateLimitsMiddleware(this.userLimitsService, RATE_LIMIT_ENDPOINTS_GROUPS.QUERY),
authorizationMiddleware(this.metadataBackend, forceToBeMaster),
connectionParamsMiddleware(this.userDatabaseService),
timeoutLimitsMiddleware(this.metadataBackend),
this.handleQuery.bind(this),
errorMiddleware()
];
};
app.all(`${base_url}/sql`, queryMiddlewares());
app.all(`${base_url}/sql.:f`, queryMiddlewares());
};
// jshint maxcomplexity:21
QueryController.prototype.handleQuery = function (req, res, next) {
var self = this;
// extract input
var body = (req.body) ? req.body : {};
// clone so don't modify req.params or req.body so oauth is not broken
var params = _.extend({}, req.query, body);
var sql = params.q;
var limit = parseInt(params.rows_per_page);
var offset = parseInt(params.page);
var orderBy = params.order_by;
var sortOrder = params.sort_order;
var requestedFormat = params.format;
var format = _.isArray(requestedFormat) ? _.last(requestedFormat) : requestedFormat;
var requestedFilename = params.filename;
var filename = requestedFilename;
var requestedSkipfields = params.skipfields;
const { user: username, userDbParams: dbopts, authDbParams, userLimits, authorizationLevel } = res.locals;
var skipfields;
var dp = params.dp; // decimal point digits (defaults to 6)
var gn = "the_geom"; // TODO: read from configuration FILE
req.aborted = false;
req.on("close", function() {
if (req.formatter && _.isFunction(req.formatter.cancel)) {
req.formatter.cancel();
}
req.aborted = true; // TODO: there must be a builtin way to check this
});
function checkAborted(step) {
if ( req.aborted ) {
var err = new Error("Request aborted during " + step);
// We'll use status 499, same as ngnix in these cases
// see http://en.wikipedia.org/wiki/List_of_HTTP_status_codes#4xx_Client_Error
err.http_status = 499;
throw err;
}
}
try {
// sanitize and apply defaults to input
dp = (dp === "" || _.isUndefined(dp)) ? '6' : dp;
format = (format === "" || _.isUndefined(format)) ? 'json' : format.toLowerCase();
filename = (filename === "" || _.isUndefined(filename)) ? 'cartodb-query' : sanitize_filename(filename);
sql = (sql === "" || _.isUndefined(sql)) ? null : sql;
limit = (!_.isNaN(limit)) ? limit : null;
offset = (!_.isNaN(offset)) ? offset * limit : null;
// Accept both comma-separated string or array of comma-separated strings
if ( requestedSkipfields ) {
if ( _.isString(requestedSkipfields) ) {
skipfields = requestedSkipfields.split(',');
} else if ( _.isArray(requestedSkipfields) ) {
skipfields = [];
_.each(requestedSkipfields, function(ele) {
skipfields = skipfields.concat(ele.split(','));
});
}
} else {
skipfields = [];
}
//if ( -1 === supportedFormats.indexOf(format) )
if ( ! formats.hasOwnProperty(format) ) {
throw new Error("Invalid format: " + format);
}
if (!_.isString(sql)) {
throw new Error("You must indicate a sql query");
}
var formatter;
if ( req.profiler ) {
req.profiler.done('init');
}
// 1. Get the list of tables affected by the query
// 2. Setup headers
// 3. Send formatted results back
// 4. Handle error
step(
function queryExplain() {
var next = this;
checkAborted('queryExplain');
var pg = new PSQL(authDbParams);
var skipCache = authorizationLevel === 'master';
self.queryTables.getAffectedTablesFromQuery(pg, sql, skipCache, function(err, result) {
if (err) {
var errorMessage = (err && err.message) || 'unknown error';
console.error("Error on query explain '%s': %s", sql, errorMessage);
}
return next(null, result);
});
},
function setHeaders(err, affectedTables) {
if (err) {
throw err;
}
var mayWrite = queryMayWrite(sql);
if ( req.profiler ) {
req.profiler.done('queryExplain');
}
checkAborted('setHeaders');
if(!pgEntitiesAccessValidator.validate(affectedTables, authorizationLevel)) {
const syntaxError = new SyntaxError("system tables are forbidden");
syntaxError.http_status = 403;
throw(syntaxError);
}
var FormatClass = formats[format];
formatter = new FormatClass();
req.formatter = formatter;
// configure headers for given format
var use_inline = !requestedFormat && !requestedFilename;
res.header("Content-Disposition", getContentDisposition(formatter, filename, use_inline));
res.header("Content-Type", formatter.getContentType());
// set cache headers
var cachePolicy = req.query.cache_policy;
if (cachePolicy === 'persist') {
res.header('Cache-Control', 'public,max-age=' + ONE_YEAR_IN_SECONDS);
} else {
var maxAge = (mayWrite) ? 0 : ONE_YEAR_IN_SECONDS;
res.header('Cache-Control', 'no-cache,max-age='+maxAge+',must-revalidate,public');
}
// Only set an X-Cache-Channel for responses we want Varnish to cache.
var skipNotUpdatedAtTables = true;
if (!!affectedTables && affectedTables.getTables(skipNotUpdatedAtTables).length > 0 && !mayWrite) {
res.header('X-Cache-Channel', affectedTables.getCacheChannel(skipNotUpdatedAtTables));
res.header('Surrogate-Key', affectedTables.key(skipNotUpdatedAtTables).join(' '));
}
if(!!affectedTables) {
res.header('Last-Modified',
new Date(affectedTables.getLastUpdatedAt(Number(new Date()))).toUTCString());
}
return null;
},
function generateFormat(err){
if (err) {
throw err;
}
checkAborted('generateFormat');
// TODO: drop this, fix UI!
sql = new PSQL.QueryWrapper(sql).orderBy(orderBy, sortOrder).window(limit, offset).query();
var opts = {
username: username,
dbopts: dbopts,
sink: res,
gn: gn,
dp: dp,
skipfields: skipfields,
sql: sql,
filename: filename,
bufferedRows: global.settings.bufferedRows,
callback: params.callback,
abortChecker: checkAborted,
timeout: userLimits.timeout
};
if ( req.profiler ) {
opts.profiler = req.profiler;
opts.beforeSink = function() {
req.profiler.done('beforeSink');
res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
};
}
if (dbopts.host) {
res.header('X-Served-By-DB-Host', dbopts.host);
}
formatter.sendResponse(opts, this);
},
function errorHandle(err){
formatter = null;
if (err) {
next(err);
}
if ( req.profiler ) {
req.profiler.sendStats();
}
if (self.statsd_client) {
if ( err ) {
self.statsd_client.increment('sqlapi.query.error');
} else {
self.statsd_client.increment('sqlapi.query.success');
}
}
}
);
} catch (err) {
next(err);
if (self.statsd_client) {
self.statsd_client.increment('sqlapi.query.error');
}
}
};
module.exports = QueryController;

18
app/controllers/version_controller.js Normal file
View File

@ -0,0 +1,18 @@
'use strict';
var version = {
cartodb_sql_api: require(__dirname + '/../../package.json').version
};
function VersionController() {
}
VersionController.prototype.route = function (app) {
app.get(global.settings.base_url + '/version', this.handleVersion.bind(this));
};
VersionController.prototype.handleVersion = function (req, res) {
res.send(version);
};
module.exports = VersionController;

125
app/middlewares/authorization.js Normal file
View File

@ -0,0 +1,125 @@
'use strict';
const AuthApi = require('../auth/auth_api');
const basicAuth = require('basic-auth');
module.exports = function authorization (metadataBackend, forceToBeMaster = false) {
return function authorizationMiddleware (req, res, next) {
const { user } = res.locals;
const credentials = getCredentialsFromRequest(req);
if (!userMatches(credentials, user)) {
if (req.profiler) {
req.profiler.done('authorization');
}
return next(new Error('permission denied'));
}
res.locals.api_key = credentials.apiKeyToken;
const params = Object.assign({ metadataBackend }, res.locals, req.query, req.body);
const authApi = new AuthApi(req, params);
authApi.verifyCredentials(function (err, authorizationLevel) {
if (req.profiler) {
req.profiler.done('authorization');
}
if (err) {
return next(err);
}
res.locals.authorizationLevel = authorizationLevel;
if (forceToBeMaster && authorizationLevel !== 'master') {
return next(new Error('permission denied'));
}
res.set('vary', 'Authorization'); //Honor Authorization header when caching.
next();
});
};
};
const credentialsGetters = [
getCredentialsFromHeaderAuthorization,
getCredentialsFromRequestQueryString,
getCredentialsFromRequestBody,
];
function getCredentialsFromRequest (req) {
let credentials = null;
for (var getter of credentialsGetters) {
credentials = getter(req);
if (apiKeyTokenFound(credentials)) {
break;
}
}
return credentials;
}
function getCredentialsFromHeaderAuthorization(req) {
const { pass, name } = basicAuth(req) || {};
if (pass !== undefined && name !== undefined) {
return {
apiKeyToken: pass,
user: name
};
}
return false;
}
function getCredentialsFromRequestQueryString(req) {
if (req.query.api_key) {
return {
apiKeyToken: req.query.api_key
};
}
if (req.query.map_key) {
return {
apiKeyToken: req.query.map_key
};
}
return false;
}
function getCredentialsFromRequestBody(req) {
if (req.body && req.body.api_key) {
return {
apiKeyToken: req.body.api_key
};
}
if (req.body && req.body.map_key) {
return {
apiKeyToken: req.body.map_key
};
}
return false;
}
function apiKeyTokenFound(credentials) {
if (typeof credentials === 'boolean') {
return credentials;
}
if (credentials.apiKeyToken !== undefined) {
return true;
}
return false;
}
function userMatches (credentials, user) {
return !(credentials.user !== undefined && credentials.user !== user);
}

146
app/middlewares/body-parser.js Normal file
View File

@ -0,0 +1,146 @@
'use strict';
/*!
* Connect - bodyParser
* Copyright(c) 2010 Sencha Inc.
* Copyright(c) 2011 TJ Holowaychuk
* MIT Licensed
*/
/**
* Module dependencies.
*/
var qs = require('qs');
var multer = require('multer');
/**
* Extract the mime type from the given request's
* _Content-Type_ header.
*
* @param {IncomingMessage} req
* @return {String}
* @api private
*/
function mime(req) {
var str = req.headers['content-type'] || '';
return str.split(';')[0];
}
/**
* Parse request bodies.
*
* By default _application/json_, _application/x-www-form-urlencoded_,
* and _multipart/form-data_ are supported, however you may map `connect.bodyParser.parse[contentType]`
* to a function receiving `(req, options, callback)`.
*
* Examples:
*
* connect.createServer(
* connect.bodyParser()
* , function(req, res) {
* res.end('viewing user ' + req.body.user.name);
* }
* );
*
* $ curl -d 'user[name]=tj' http://localhost/
* $ curl -d '{"user":{"name":"tj"}}' -H "Content-Type: application/json" http://localhost/
*
* Multipart req.files:
*
* As a security measure files are stored in a separate object, stored
* as `req.files`. This prevents attacks that may potentially alter
* filenames, and depending on the application gain access to restricted files.
*
* Multipart configuration:
*
* The `options` passed are provided to each parser function.
* The _multipart/form-data_ parser merges these with formidable's
* IncomingForm object, allowing you to tweak the upload directory,
* size limits, etc. For example you may wish to retain the file extension
* and change the upload directory:
*
* server.use(bodyParser({ uploadDir: '/www/mysite.com/uploads' }));
*
* View [node-formidable](https://github.com/felixge/node-formidable) for more information.
*
* If you wish to use formidable directly within your app, and do not
* desire this behaviour for multipart requests simply remove the
* parser:
*
* delete connect.bodyParser.parse['multipart/form-data'];
*
* Or
*
* delete express.bodyParser.parse['multipart/form-data'];
*
* @param {Object} options
* @return {Function}
* @api public
*/
exports = module.exports = function bodyParser(options){
options = options || {};
return function bodyParser(req, res, next) {
if (req.body) {
return next();
}
req.body = {};
if ('GET' === req.method || 'HEAD' === req.method) {
return next();
}
var parser = exports.parse[mime(req)];
if (parser) {
parser(req, options, next);
} else {
next();
}
};
};
/**
* Parsers.
*/
exports.parse = {};
/**
* Parse application/x-www-form-urlencoded.
*/
exports.parse['application/x-www-form-urlencoded'] = function(req, options, fn){
var buf = '';
req.setEncoding('utf8');
req.on('data', function(chunk){ buf += chunk; });
req.on('end', function(){
try {
req.body = buf.length ? qs.parse(buf) : {};
fn();
} catch (err){
fn(err);
}
});
};
/**
* Parse application/json.
*/
exports.parse['application/json'] = function(req, options, fn){
var buf = '';
req.setEncoding('utf8');
req.on('data', function(chunk){ buf += chunk; });
req.on('end', function(){
try {
req.body = buf.length ? JSON.parse(buf) : {};
fn();
} catch (err){
fn(err);
}
});
};
var multipartMiddleware = multer({ limits: { fieldSize: Infinity } });
exports.parse['multipart/form-data'] = multipartMiddleware.none();

23
app/middlewares/connection-params.js Normal file
View File

@ -0,0 +1,23 @@
'use strict';
module.exports = function connectionParams (userDatabaseService) {
return function connectionParamsMiddleware (req, res, next) {
const { user, api_key: apikeyToken, authorizationLevel } = res.locals;
userDatabaseService.getConnectionParams(user, apikeyToken, authorizationLevel,
function (err, userDbParams, authDbParams) {
if (req.profiler) {
req.profiler.done('getConnectionParams');
}
if (err) {
return next(err);
}
res.locals.userDbParams = userDbParams;
res.locals.authDbParams = authDbParams;
next();
});
};
};

16
app/middlewares/cors.js Normal file
View File

@ -0,0 +1,16 @@
'use strict';
module.exports = function cors(extraHeaders) {
return function(req, res, next) {
var baseHeaders = 'X-Requested-With, X-Prototype-Version, X-CSRF-Token, Authorization';
if(extraHeaders) {
baseHeaders += ', ' + extraHeaders;
}
res.header('Access-Control-Allow-Origin', '*');
res.header('Access-Control-Allow-Headers', baseHeaders);
next();
};
};

26
app/middlewares/db-quota.js Normal file
View File

@ -0,0 +1,26 @@
'use strict';
const PSQL = require('cartodb-psql');
const remainingQuotaQuery = 'SELECT _CDB_UserQuotaInBytes() - CDB_UserDataSize(current_schema()) AS remaining_quota';
module.exports = function dbQuota () {
return function dbQuotaMiddleware (req, res, next) {
const { userDbParams } = res.locals;
const pg = new PSQL(userDbParams);
pg.connect((err, client, done) => {
if (err) {
return next(err);
}
client.query(remainingQuotaQuery, (err, result) => {
if(err) {
return next(err);
}
const remainingQuota = result.rows[0].remaining_quota;
res.locals.dbRemainingQuota = remainingQuota;
done();
next();
});
});
};
};

91
app/middlewares/error.js Normal file
View File

@ -0,0 +1,91 @@
'use strict';
const errorHandlerFactory = require('../services/error_handler_factory');
const MAX_ERROR_STRING_LENGTH = 1024;
module.exports = function error() {
return function errorMiddleware(err, req, res, next) {
const errorHandler = errorHandlerFactory(err);
let errorResponse = errorHandler.getResponse();
if (global.settings.environment === 'development') {
errorResponse.stack = err.stack;
}
if (global.settings.environment !== 'test') {
// TODO: email this Exception report
console.error("EXCEPTION REPORT: " + err.stack);
}
// Force inline content disposition
res.header("Content-Disposition", 'inline');
if (req && req.profiler) {
req.profiler.done('finish');
res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
}
setErrorHeader(errorHandler, res);
res.header('Content-Type', 'application/json; charset=utf-8');
res.status(getStatusError(errorHandler, req));
if (req.query && req.query.callback) {
res.jsonp(errorResponse);
} else {
res.json(errorResponse);
}
if (req && req.profiler) {
res.req.profiler.sendStats();
}
next();
};
};
function getStatusError(errorHandler, req) {
let statusError = errorHandler.http_status;
// JSONP has to return 200 status error
if (req && req.query && req.query.callback) {
statusError = 200;
}
return statusError;
}
function setErrorHeader(errorHandler, res) {
const errorsLog = {
context: errorHandler.context,
detail: errorHandler.detail,
hint: errorHandler.hint,
statusCode: errorHandler.http_status,
message: errorHandler.message
};
res.set('X-SQLAPI-Errors', stringifyForLogs(errorsLog));
}
/**
* Remove problematic nested characters
* from object for logs RegEx
*
* @param {Object} object
*/
function stringifyForLogs(object) {
Object.keys(object).map(key => {
if (typeof object[key] === 'string') {
object[key] = object[key]
.substring(0, MAX_ERROR_STRING_LENGTH)
.replace(/[^a-zA-Z0-9]/g, ' ');
} else if (typeof object[key] === 'object') {
stringifyForLogs(object[key]);
} else if (object[key] instanceof Array) {
for (let element of object[key]) {
stringifyForLogs(element);
}
}
});
return JSON.stringify(object);
}

24
app/middlewares/profiler.js Normal file
View File

@ -0,0 +1,24 @@
'use strict';
module.exports.initializeProfilerMiddleware = function initializeProfiler (label) {
return function initializeProfilerMiddleware (req, res, next) {
if (req.profiler) {
req.profiler.start(`sqlapi.${label}`);
}
next();
};
};
module.exports.finishProfilerMiddleware = function finishProfiler () {
return function finishProfilerMiddleware (req, res, next) {
if (req.profiler) {
req.profiler.end();
req.profiler.sendStats();
res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
}
next();
};
};

61
app/middlewares/rate-limit.js Normal file
View File

@ -0,0 +1,61 @@
'use strict';
const RATE_LIMIT_ENDPOINTS_GROUPS = {
QUERY: 'query',
JOB_CREATE: 'job_create',
JOB_GET: 'job_get',
JOB_DELETE: 'job_delete',
COPY_FROM: 'copy_from',
COPY_TO: 'copy_to'
};
function rateLimit(userLimits, endpointGroup = null) {
if (!isRateLimitEnabled(endpointGroup)) {
return function rateLimitDisabledMiddleware(req, res, next) { next(); };
}
return function rateLimitMiddleware(req, res, next) {
userLimits.getRateLimit(res.locals.user, endpointGroup, function(err, userRateLimit) {
if (err) {
return next(err);
}
if (!userRateLimit) {
return next();
}
const [isBlocked, limit, remaining, retry, reset] = userRateLimit;
res.set({
'Carto-Rate-Limit-Limit': limit,
'Carto-Rate-Limit-Remaining': remaining,
'Carto-Rate-Limit-Reset': reset
});
if (isBlocked) {
// retry is floor rounded in seconds by redis-cell
res.set('Retry-After', retry + 1);
const rateLimitError = new Error(
'You are over platform\'s limits. Please contact us to know more details'
);
rateLimitError.http_status = 429;
rateLimitError.context = 'limit';
rateLimitError.detail = 'rate-limit';
return next(rateLimitError);
}
return next();
});
};
}
function isRateLimitEnabled(endpointGroup) {
return global.settings.ratelimits.rateLimitsEnabled &&
endpointGroup &&
global.settings.ratelimits.endpoints[endpointGroup];
}
module.exports = rateLimit;
module.exports.RATE_LIMIT_ENDPOINTS_GROUPS = RATE_LIMIT_ENDPOINTS_GROUPS;

25
app/middlewares/timeout-limits.js Normal file
View File

@ -0,0 +1,25 @@
'use strict';
module.exports = function timeoutLimits (metadataBackend) {
return function timeoutLimitsMiddleware (req, res, next) {
const { user, authorizationLevel } = res.locals;
metadataBackend.getUserTimeoutRenderLimits(user, function (err, timeoutRenderLimit) {
if (req.profiler) {
req.profiler.done('getUserTimeoutLimits');
}
if (err) {
return next(err);
}
const userLimits = {
timeout: (authorizationLevel === 'master') ? timeoutRenderLimit.render : timeoutRenderLimit.renderPublic
};
res.locals.userLimits = userLimits;
next();
});
};
};

38
app/middlewares/user.js Normal file
View File

@ -0,0 +1,38 @@
'use strict';
const CdbRequest = require('../models/cartodb_request');
module.exports = function user(metadataBackend) {
const cdbRequest = new CdbRequest();
return function userMiddleware (req, res, next) {
res.locals.user = getUserNameFromRequest(req, cdbRequest);
checkUserExists(metadataBackend, res.locals.user, function(err, userExists) {
if (err || !userExists) {
const error = new Error('Unauthorized');
error.type = 'auth';
error.subtype = 'user-not-found';
error.http_status = 404;
error.message = errorUserNotFoundMessageTemplate(res.locals.user);
next(error);
}
return next();
});
};
};
function getUserNameFromRequest(req, cdbRequest) {
return cdbRequest.userByReq(req);
}
function checkUserExists(metadataBackend, userName, callback) {
metadataBackend.getUserId(userName, function(err) {
callback(err, !err);
});
}
function errorUserNotFoundMessageTemplate(user) {
return `Sorry, we can't find CARTO user '${user}'. Please check that you have entered the correct domain.`;
}

166
app/models/bin_encoder.js Normal file
View File

@ -0,0 +1,166 @@
'use strict';
function ArrayBufferSer(type, data, options) {
if(type === undefined) {
throw "ArrayBufferSer should be created with a type";
}
this.options = options || {};
this._initFunctions();
this.headerSize = 8;
this.data = data;
this.type = type = Math.min(type, ArrayBufferSer.BUFFER);
var size = this._sizeFor(this.headerSize, data);
this.buffer = new Buffer(this.headerSize + size);
this.buffer.writeUInt32BE(type, 0); // this could be one byte but for byte padding is better to be 4 bytes
this.buffer.writeUInt32BE(size, 4);
this.offset = this.headerSize;
var w = this.writeFn[type];
var i;
if(!this.options.delta) {
for(i = 0; i < data.length; ++i) {
this[w](data[i]);
}
} else {
this[w](data[0]);
for(i = 1; i < data.length; ++i) {
this[w](data[i] - data[i - 1]);
}
}
}
//
// constants
//
ArrayBufferSer.INT8 = 1;
ArrayBufferSer.UINT8 = 2;
ArrayBufferSer.UINT8_CLAMP = 3;
ArrayBufferSer.INT16 = 4;
ArrayBufferSer.UINT16 = 5;
ArrayBufferSer.INT32 = 6;
ArrayBufferSer.UINT32 = 7;
ArrayBufferSer.FLOAT32 = 8;
//ArrayBufferSer.FLOAT64 = 9; not supported
ArrayBufferSer.STRING = 10;
ArrayBufferSer.BUFFER = 11;
ArrayBufferSer.MAX_PADDING = ArrayBufferSer.INT32;
ArrayBufferSer.typeNames = {
'int8': ArrayBufferSer.INT8,
'uint8': ArrayBufferSer.UINT8,
'uintclamp': ArrayBufferSer.UINT8_CLAMP,
'int16': ArrayBufferSer.INT16,
'uint16': ArrayBufferSer.UINT16,
'int32': ArrayBufferSer.INT32,
'uint32': ArrayBufferSer.UINT32,
'float32': ArrayBufferSer.FLOAT32,
'string': ArrayBufferSer.STRING,
'buffer': ArrayBufferSer.BUFFER
};
ArrayBufferSer.prototype = {
// 0 not used
sizes: [NaN, 1, 1, 1, 2, 2, 4, 4, 4, 8],
_paddingFor: function(off, type) {
var s = this.sizes[type];
if(s) {
var r = off % s;
return r === 0 ? 0 : s - r;
}
return 0;
},
_sizeFor: function(offset, t) {
var self = this;
var s = this.sizes[this.type];
if(s) {
return s*t.length;
}
s = 0;
if(this.type === ArrayBufferSer.STRING) {
// calculate size with padding
t.forEach(function(arr) {
var pad = self._paddingFor(offset, ArrayBufferSer.MAX_PADDING);
s += pad;
offset += pad;
var len = (self.headerSize + arr.length*2);
s += len;
offset += len;
});
} else {
t.forEach(function(arr) {
var pad = self._paddingFor(offset, ArrayBufferSer.MAX_PADDING);
s += pad;
offset += pad;
s += arr.getSize();
offset += arr.getSize();
});
}
return s;
},
getDataSize: function() {
return this._sizeFor(0, this.data);
},
getSize: function() {
return this.headerSize + this._sizeFor(this.headerSize, this.data);
},
writeFn: [
'',
'writeInt8',
'writeUInt8',
'writeUInt8Clamp',
'writeInt16LE',
'writeUInt16LE',
'writeUInt32LE',
'writeUInt32LE',
'writeFloatLE',
'writeDoubleLE',
'writeString',
'writteBuffer'
],
_initFunctions: function() {
var self = this;
this.writeFn.forEach(function(fn) {
if(self[fn] === undefined) {
self[fn] = function(d) {
self.buffer[fn](d, self.offset);
self.offset += self.sizes[self.type];
};
}
});
},
writeUInt8Clamp: function(c) {
this.buffer.writeUInt8(Math.min(255, c), this.offset);
this.offset += 1;
},
writeString: function(s) {
var arr = [];
for(var i = 0, len = s.length; i < len; ++i) {
arr.push(s.charCodeAt(i));
}
var str = new ArrayBufferSer(ArrayBufferSer.UINT16, arr);
this.writteBuffer(str);
},
writteBuffer: function(b) {
this.offset += this._paddingFor(this.offset, ArrayBufferSer.MAX_PADDING);
// copy header
b.buffer.copy(this.buffer, this.offset);
this.offset += b.buffer.length;
}
};
module.exports = ArrayBufferSer;

41
app/models/cartodb_request.js Normal file
View File

@ -0,0 +1,41 @@
'use strict';
/**
* this module provides cartodb-specific interpretation
* of request headers
*/
function CartodbRequest() {
}
module.exports = CartodbRequest;
/**
* If the request contains the user use it, if not guess from the host
*/
CartodbRequest.prototype.userByReq = function(req) {
if (req.params.user) {
return req.params.user;
}
return userByHostName(req.headers.host);
};
var re_userFromHost = new RegExp(
global.settings.user_from_host || '^([^\\.]+)\\.' // would extract "strk" from "strk.cartodb.com"
);
function userByHostName(host) {
var mat = host.match(re_userFromHost);
if (!mat) {
console.error("ERROR: user pattern '" + re_userFromHost + "' does not match hostname '" + host + "'");
return;
}
if (mat.length !== 2) {
console.error(
"ERROR: pattern '" + re_userFromHost + "' gave unexpected matches against '" + host + "': " + mat
);
return;
}
return mat[1];
}

18
app/models/formats/README Normal file
View File

@ -0,0 +1,18 @@
Format classes are required to expose a constructor with no arguments,
a getFileExtension() and a sendResponse(opts, callback) method.
The ``opts`` parameter contains:
sink Output stream to send the reponse to
sql SQL query requested by the user
skipfields Comma separate list of fields to skip from output
really only needed with "SELECT *" queries
gn Name of the geometry column (for formats requiring one)
dp Number of decimal points of precision for geometries (if used)
database Name of the database to connect to
user_id Identifier of the user
filename Name to use for attachment disposition
The ``callback`` parameter is a function that is invoked when the
format object finished with sending the result to the sink.
If an error occurs the callback is invoked with an Error argument.

22
app/models/formats/index.js Normal file
View File

@ -0,0 +1,22 @@
'use strict';
var fs = require("fs");
var formats = {};
function formatFilesWithPath(dir) {
var formatDir = __dirname + '/' + dir;
return fs.readdirSync(formatDir).map(function(formatFile) {
return formatDir + '/' + formatFile;
});
}
var formatFilesPaths = []
.concat(formatFilesWithPath('ogr'))
.concat(formatFilesWithPath('pg'));
formatFilesPaths.forEach(function(file) {
var format = require(file);
formats[format.prototype.id] = format;
});
module.exports = formats;

346
app/models/formats/ogr.js Normal file
View File

@ -0,0 +1,346 @@
'use strict';
var crypto = require('crypto');
var step = require('step');
var fs = require('fs');
var _ = require('underscore');
var PSQL = require('cartodb-psql');
var spawn = require('child_process').spawn;
// Keeps track of what's waiting baking for export
var bakingExports = {};
function OgrFormat(id) {
this.id = id;
}
OgrFormat.prototype = {
id: "ogr",
is_file: true,
getQuery: function(/*sql, options*/) {
return null; // dont execute the query
},
transform: function(/*result, options, callback*/) {
throw "should not be called for file formats";
},
getContentType: function(){ return this._contentType; },
getFileExtension: function(){ return this._fileExtension; },
getKey: function(options) {
return [this.id,
options.dbopts.dbname,
options.dbopts.user,
options.gn,
this.generateMD5(options.filename),
this.generateMD5(options.sql)].concat(options.skipfields).join(':');
},
generateMD5: function (data){
var hash = crypto.createHash('md5');
hash.update(data);
return hash.digest('hex');
}
};
// Internal function usable by all OGR-driven outputs
OgrFormat.prototype.toOGR = function(options, out_format, out_filename, callback) {
//var gcol = options.gn;
var sql = options.sql;
var skipfields = options.skipfields;
var out_layername = options.filename;
var dbopts = options.dbopts;
var ogr2ogr = global.settings.ogr2ogrCommand || 'ogr2ogr';
var dbhost = dbopts.host;
var dbport = dbopts.port;
var dbuser = dbopts.user;
var dbpass = dbopts.pass;
var dbname = dbopts.dbname;
var timeout = options.timeout;
var that = this;
var columns = [];
var geocol;
var pg;
// Drop ending semicolon (ogr doens't like it)
sql = sql.replace(/;\s*$/, '');
const theGeomFirst = (fieldA, fieldB) => {
if (fieldA.name === 'the_geom') {
return -1;
}
if (fieldB.name === 'the_geom') {
return 1;
}
return 0;
};
step (
function fetchColumns() {
var colsql = 'SELECT * FROM (' + sql + ') as _cartodbsqlapi LIMIT 0';
pg = new PSQL(dbopts);
pg.query(colsql, this);
},
function findSRS(err, result) {
if (err) {
throw err;
}
var needSRS = that._needSRS;
columns = result.fields
// skip columns
.filter(field => skipfields.indexOf(field.name) === -1)
// put "the_geom" first (if exists)
.sort(theGeomFirst)
// get first geometry to calculate SRID ("the_geom" if exists)
.map(field => {
if (needSRS && !geocol && pg.typeName(field.dataTypeID) === 'geometry') {
geocol = field.name;
}
return field;
})
// apply quotes to columns
.map(field => out_format === 'CSV' ? pg.quoteIdentifier(field.name)+'::text' : pg.quoteIdentifier(field.name));
if ( ! needSRS || ! geocol ) {
return null;
}
var next = this;
var qgeocol = pg.quoteIdentifier(geocol);
var sridsql = 'SELECT ST_Srid(' + qgeocol + ') as srid, GeometryType(' +
qgeocol + ') as type FROM (' + sql + ') as _cartodbsqlapi WHERE ' +
qgeocol + ' is not null limit 1';
pg.query(sridsql, function(err, result) {
if ( err ) { next(err); return; }
if ( result.rows.length ) {
var srid = result.rows[0].srid;
var type = result.rows[0].type;
next(null, srid, type);
} else {
// continue as srid and geom type are not critical when there are no results
next(null);
}
});
},
function spawnDumper(err, srid, type) {
if (err) {
throw err;
}
var next = this;
var ogrsql = 'SELECT ' + columns.join(',') + ' FROM (' + sql + ') as _cartodbsqlapi';
var ogrargs = [
'-f', out_format,
'-lco', 'RESIZE=YES',
'-lco', 'ENCODING=UTF-8',
'-lco', 'LINEFORMAT=CRLF',
out_filename,
"PG:host=" + dbhost + " port=" + dbport + " user=" + dbuser + " dbname=" + dbname + " password=" + dbpass,
'-sql', ogrsql
];
if ( srid ) {
ogrargs.push('-a_srs', 'EPSG:'+srid);
}
if ( type ) {
ogrargs.push('-nlt', type);
}
if (options.cmd_params){
ogrargs = ogrargs.concat(options.cmd_params);
}
ogrargs.push('-nln', out_layername);
// TODO: research if `exec` could fit better than `spawn`
var child = spawn(ogr2ogr, ogrargs);
var timedOut = false;
var ogrTimeout;
if (timeout > 0) {
ogrTimeout = setTimeout(function () {
timedOut = true;
child.kill();
}, timeout);
}
child.on('error', function (err) {
clearTimeout(ogrTimeout);
next(err);
});
var stderrData = [];
child.stderr.setEncoding('utf8');
child.stderr.on('data', function (data) {
stderrData.push(data);
});
child.on('exit', function(code) {
clearTimeout(ogrTimeout);
if (timedOut) {
return next(new Error('statement timeout'));
}
if (code !== 0) {
var errMessage = 'ogr2ogr command return code ' + code;
if (stderrData.length > 0) {
errMessage += ', Error: ' + stderrData.join('\n');
}
return next(new Error(errMessage));
}
return next();
});
},
function finish(err) {
callback(err, out_filename);
}
);
};
OgrFormat.prototype.toOGR_SingleFile = function(options, fmt, callback) {
var dbname = options.dbopts.dbname;
var user_id = options.dbopts.user;
var gcol = options.gcol;
var sql = options.sql;
var skipfields = options.skipfields;
var ext = this._fileExtension;
var layername = options.filename;
var tmpdir = global.settings.tmpDir || '/tmp';
var reqKey = [
fmt,
dbname,
user_id,
gcol,
this.generateMD5(layername),
this.generateMD5(sql)
].concat(skipfields).join(':');
var outdirpath = tmpdir + '/sqlapi-' + process.pid + '-' + reqKey;
var dumpfile = outdirpath + ':cartodb-query.' + ext;
// TODO: following tests:
// - fetch query with no "the_geom" column
this.toOGR(options, fmt, dumpfile, callback);
};
OgrFormat.prototype.sendResponse = function(opts, callback) {
//var next = callback;
var reqKey = this.getKey(opts);
var qElem = new ExportRequest(opts.sink, callback, opts.beforeSink);
var baking = bakingExports[reqKey];
if ( baking ) {
baking.req.push( qElem );
} else {
baking = bakingExports[reqKey] = { req: [ qElem ] };
this.generate(opts, function(err, dumpfile) {
if ( opts.profiler ) {
opts.profiler.done('generate');
}
step (
function sendResults() {
var nextPipe = function(finish) {
var r = baking.req.shift();
if ( ! r ) { finish(null); return; }
r.sendFile(err, dumpfile, function() {
nextPipe(finish);
});
};
if ( ! err ) {
nextPipe(this);
} else {
_.each(baking.req, function(r) {
r.cb(err);
});
return true;
}
},
function cleanup(/*err*/) {
delete bakingExports[reqKey];
// unlink dump file (sync to avoid race condition)
console.log("removing", dumpfile);
try { fs.unlinkSync(dumpfile); }
catch (e) {
if ( e.code !== 'ENOENT' ) {
console.log("Could not unlink dumpfile " + dumpfile + ": " + e);
}
}
}
);
});
}
};
// TODO: put in an ExportRequest.js ----- {
function ExportRequest(ostream, callback, beforeSink) {
this.cb = callback;
this.beforeSink = beforeSink;
this.ostream = ostream;
this.istream = null;
this.canceled = false;
var that = this;
this.ostream.on('close', function() {
//console.log("Request close event, qElem.stream is " + qElem.stream);
that.canceled = true;
if ( that.istream ) {
that.istream.destroy();
}
});
}
ExportRequest.prototype.sendFile = function (err, filename, callback) {
var that = this;
if ( ! this.canceled ) {
//console.log("Creating readable stream out of dumpfile");
this.istream = fs.createReadStream(filename)
.on('open', function(/*fd*/) {
if ( that.beforeSink ) {
that.beforeSink();
}
that.istream.pipe(that.ostream);
callback();
})
.on('error', function(e) {
console.log("Can't send response: " + e);
that.ostream.end();
callback();
});
} else {
//console.log("Response was canceled, not streaming the file");
callback();
}
this.cb();
};
//------ }
module.exports = OgrFormat;

16
app/models/formats/ogr/csv.js Normal file
View File

@ -0,0 +1,16 @@
'use strict';
var ogr = require('./../ogr');
function CsvFormat() {}
CsvFormat.prototype = new ogr('csv');
CsvFormat.prototype._contentType = "text/csv; charset=utf-8; header=present";
CsvFormat.prototype._fileExtension = "csv";
CsvFormat.prototype.generate = function(options, callback) {
this.toOGR_SingleFile(options, 'CSV', callback);
};
module.exports = CsvFormat;

25
app/models/formats/ogr/geopackage.js Normal file
View File

@ -0,0 +1,25 @@
'use strict';
var ogr = require('./../ogr');
function GeoPackageFormat() {}
GeoPackageFormat.prototype = new ogr('gpkg');
GeoPackageFormat.prototype._contentType = "application/x-sqlite3; charset=utf-8";
GeoPackageFormat.prototype._fileExtension = "gpkg";
// As of GDAL 1.10.1 SRID detection is bogus, so we use
// our own method. See:
// http://trac.osgeo.org/gdal/ticket/5131
// http://trac.osgeo.org/gdal/ticket/5287
// http://github.com/CartoDB/CartoDB-SQL-API/issues/110
// http://github.com/CartoDB/CartoDB-SQL-API/issues/116
// Bug was fixed in GDAL 1.10.2
GeoPackageFormat.prototype._needSRS = true;
GeoPackageFormat.prototype.generate = function(options, callback) {
options.cmd_params = ['-lco', 'FID=cartodb_id'];
this.toOGR_SingleFile(options, 'GPKG', callback);
};
module.exports = GeoPackageFormat;

24
app/models/formats/ogr/kml.js Normal file
View File

@ -0,0 +1,24 @@
'use strict';
var ogr = require('./../ogr');
function KmlFormat() {}
KmlFormat.prototype = new ogr('kml');
KmlFormat.prototype._contentType = "application/kml; charset=utf-8";
KmlFormat.prototype._fileExtension = "kml";
// As of GDAL 1.10.1 SRID detection is bogus, so we use
// our own method. See:
// http://trac.osgeo.org/gdal/ticket/5131
// http://trac.osgeo.org/gdal/ticket/5287
// http://github.com/CartoDB/CartoDB-SQL-API/issues/110
// http://github.com/CartoDB/CartoDB-SQL-API/issues/116
// Bug was fixed in GDAL 1.10.2
KmlFormat.prototype._needSRS = true;
KmlFormat.prototype.generate = function(options, callback) {
this.toOGR_SingleFile(options, 'KML', callback);
};
module.exports = KmlFormat;

135
app/models/formats/ogr/shp.js Normal file
View File

@ -0,0 +1,135 @@
'use strict';
var step = require('step');
var fs = require('fs');
var spawn = require('child_process').spawn;
var ogr = require('./../ogr');
function ShpFormat() {
}
ShpFormat.prototype = new ogr('shp');
ShpFormat.prototype._contentType = "application/zip; charset=utf-8";
ShpFormat.prototype._fileExtension = "zip";
// As of GDAL 1.10 SRID detection is bogus, so we use
// our own method. See:
// http://trac.osgeo.org/gdal/ticket/5131
// http://trac.osgeo.org/gdal/ticket/5287
// http://github.com/CartoDB/CartoDB-SQL-API/issues/110
// http://github.com/CartoDB/CartoDB-SQL-API/issues/116
ShpFormat.prototype._needSRS = true;
ShpFormat.prototype.generate = function(options, callback) {
this.toSHP(options, callback);
};
ShpFormat.prototype.toSHP = function (options, callback) {
var dbname = options.database;
var user_id = options.user_id;
var gcol = options.gn;
var sql = options.sql;
var skipfields = options.skipfields;
var filename = options.filename;
var fmtObj = this;
var zip = global.settings.zipCommand || 'zip';
var zipOptions = '-qrj';
var tmpdir = global.settings.tmpDir || '/tmp';
var reqKey = [ 'shp', dbname, user_id, gcol, this.generateMD5(sql) ].concat(skipfields).join(':');
var outdirpath = tmpdir + '/sqlapi-' + process.pid + '-' + reqKey;
var zipfile = outdirpath + '.zip';
var shapefile = outdirpath + '/' + filename + '.shp';
// TODO: following tests:
// - fetch query with no "the_geom" column
step (
function createOutDir() {
fs.mkdir(outdirpath, 0o777, this);
},
function spawnDumper(err) {
if (err) {
throw err;
}
fmtObj.toOGR(options, 'ESRI Shapefile', shapefile, this);
},
function doZip(err) {
if (err) {
throw err;
}
var next = this;
var child = spawn(zip, [zipOptions, zipfile, outdirpath ]);
child.on('error', function (err) {
next(new Error('Error executing zip command, ' + err));
});
var stderrData = [];
child.stderr.setEncoding('utf8');
child.stderr.on('data', function (data) {
stderrData.push(data);
});
child.on('exit', function(code) {
if (code !== 0) {
var errMessage = 'Zip command return code ' + code;
if (stderrData.length) {
errMessage += ', Error: ' + stderrData.join('\n');
}
return next(new Error(errMessage));
}
return next();
});
},
function cleanupDir(topError) {
var next = this;
// Unlink the dir content
var unlinkall = function(dir, files, finish) {
var f = files.shift();
if ( ! f ) { finish(null); return; }
var fn = dir + '/' + f;
fs.unlink(fn, function(err) {
if ( err ) {
console.log("Unlinking " + fn + ": " + err);
finish(err);
} else {
unlinkall(dir, files, finish);
}
});
};
fs.readdir(outdirpath, function(err, files) {
if ( err ) {
if ( err.code !== 'ENOENT' ) {
next(new Error([topError, err].join('\n')));
} else {
next(topError);
}
} else {
unlinkall(outdirpath, files, function(/*err*/) {
fs.rmdir(outdirpath, function(err) {
if ( err ) {
console.log("Removing dir " + outdirpath + ": " + err);
}
next(topError, zipfile);
});
});
}
});
},
function finalStep(err, zipfile) {
callback(err, zipfile);
}
);
};
module.exports = ShpFormat;

25
app/models/formats/ogr/spatialite.js Normal file
View File

@ -0,0 +1,25 @@
'use strict';
var ogr = require('./../ogr');
function SpatiaLiteFormat() {}
SpatiaLiteFormat.prototype = new ogr('spatialite');
SpatiaLiteFormat.prototype._contentType = "application/x-sqlite3; charset=utf-8";
SpatiaLiteFormat.prototype._fileExtension = "sqlite";
// As of GDAL 1.10.1 SRID detection is bogus, so we use
// our own method. See:
// http://trac.osgeo.org/gdal/ticket/5131
// http://trac.osgeo.org/gdal/ticket/5287
// http://github.com/CartoDB/CartoDB-SQL-API/issues/110
// http://github.com/CartoDB/CartoDB-SQL-API/issues/116
// Bug was fixed in GDAL 1.10.2
SpatiaLiteFormat.prototype._needSRS = true;
SpatiaLiteFormat.prototype.generate = function(options, callback) {
this.toOGR_SingleFile(options, 'SQLite', callback);
options.cmd_params = ['SPATIALITE=yes'];
};
module.exports = SpatiaLiteFormat;

164
app/models/formats/pg.js Normal file
View File

@ -0,0 +1,164 @@
'use strict';
var step = require('step');
var PSQL = require('cartodb-psql');
function PostgresFormat(id) {
this.id = id;
}
PostgresFormat.prototype = {
getQuery: function(sql/*, options*/) {
return sql;
},
getContentType: function(){
return this._contentType;
},
getFileExtension: function() {
return this.id;
}
};
PostgresFormat.prototype.handleQueryRow = function(row, result) {
result.addRow(row);
};
PostgresFormat.prototype.handleQueryRowWithSkipFields = function(row, result) {
var sf = this.opts.skipfields;
for ( var j=0; j<sf.length; ++j ) {
delete row[sf[j]];
}
this.handleQueryRow(row, result);
};
PostgresFormat.prototype.handleNotice = function(msg, result) {
if ( ! result.notices ) {
result.notices = [];
}
for (var i=0; i<msg.length; i++) {
result.notices.push(msg[i]);
}
};
PostgresFormat.prototype.handleQueryEnd = function(result) {
this.queryCanceller = undefined;
if ( this.error ) {
this.callback(this.error);
return;
}
if ( this.opts.profiler ) {
this.opts.profiler.done('gotRows');
}
this.opts.total_time = (Date.now() - this.start_time)/1000;
// Drop field description for skipped fields
if (this.hasSkipFields) {
var sf = this.opts.skipfields;
var newfields = [];
for ( var j=0; j<result.fields.length; ++j ) {
var f = result.fields[j];
if ( sf.indexOf(f.name) === -1 ) {
newfields.push(f);
}
}
result.fields = newfields;
}
var that = this;
step (
function packageResult() {
if ( that.opts.abortChecker ) {
that.opts.abortChecker('packageResult');
}
that.transform(result, that.opts, this);
},
function sendResults(err, out){
if (err) {
throw err;
}
// return to browser
if ( out ) {
if ( that.opts.beforeSink ) {
that.opts.beforeSink();
}
that.opts.sink.send(out);
} else {
console.error("No output from transform, doing nothing ?!");
}
},
function errorHandle(err){
that.callback(err);
}
);
};
PostgresFormat.prototype.sendResponse = function(opts, callback) {
if ( this.callback ) {
callback(new Error("Invalid double call to .sendResponse on a pg formatter"));
return;
}
this.callback = callback;
this.opts = opts;
this.hasSkipFields = opts.skipfields.length;
var sql = this.getQuery(opts.sql, {
gn: opts.gn,
dp: opts.dp,
skipfields: opts.skipfields
});
var that = this;
this.start_time = Date.now();
this.client = new PSQL(opts.dbopts);
this.client.eventedQuery(sql, function(err, query, queryCanceller) {
that.queryCanceller = queryCanceller;
if (err) {
callback(err);
return;
}
if ( that.opts.profiler ) {
that.opts.profiler.done('eventedQuery');
}
if (that.hasSkipFields) {
query.on('row', that.handleQueryRowWithSkipFields.bind(that));
} else {
query.on('row', that.handleQueryRow.bind(that));
}
query.on('end', that.handleQueryEnd.bind(that));
query.on('error', function(err) {
that.error = err;
if (err.message && err.message.match(/row too large, was \d* bytes/i)) {
return console.error(JSON.stringify({
username: opts.username,
type: 'row_size_limit_exceeded',
error: err.message
}));
}
that.handleQueryEnd();
});
query.on('notice', function(msg) {
that.handleNotice(msg, query._result);
});
});
};
PostgresFormat.prototype.cancel = function() {
if (this.queryCanceller) {
this.queryCanceller.call();
}
};
module.exports = PostgresFormat;

87
app/models/formats/pg/arraybuffer.js Normal file
View File

@ -0,0 +1,87 @@
'use strict';
var _ = require('underscore');
var pg = require('./../pg');
var ArrayBufferSer = require("../../bin_encoder");
function BinaryFormat() {}
BinaryFormat.prototype = new pg('arraybuffer');
BinaryFormat.prototype._contentType = "application/octet-stream";
BinaryFormat.prototype._extractTypeFromName = function(name) {
var g = name.match(/.*__(uintclamp|uint|int|float)(8|16|32)/i);
if(g && g.length === 3) {
var typeName = g[1] + g[2];
return ArrayBufferSer.typeNames[typeName];
}
};
// jshint maxcomplexity:12
BinaryFormat.prototype.transform = function(result, options, callback) {
var total_rows = result.rowCount;
var rows = result.rows;
// get headers
if(!total_rows) {
callback(null, new Buffer(0));
return;
}
var headersNames = Object.keys(rows[0]);
var headerTypes = [];
if(_.contains(headersNames, 'the_geom')) {
callback(new Error("geometry types are not supported"), null);
return;
}
try {
var i;
var t;
// get header types (and guess from name)
for(i = 0; i < headersNames.length; ++i) {
r = rows[0];
n = headersNames[i];
if(typeof(r[n]) === 'string') {
headerTypes.push(ArrayBufferSer.STRING);
} else if(typeof(r[n]) === 'object') {
t = this._extractTypeFromName(n);
t = t || ArrayBufferSer.FLOAT32;
headerTypes.push(ArrayBufferSer.BUFFER + t);
} else {
t = this._extractTypeFromName(n);
headerTypes.push(t || ArrayBufferSer.FLOAT32);
}
}
// pack the data
var header = new ArrayBufferSer(ArrayBufferSer.STRING, headersNames);
var data = [header];
for(i = 0; i < headersNames.length; ++i) {
var d = [];
var n = headersNames[i];
for(var r = 0; r < total_rows; ++r) {
var row = rows[r][n];
if(headerTypes[i] > ArrayBufferSer.BUFFER) {
row = new ArrayBufferSer(headerTypes[i] - ArrayBufferSer.BUFFER, row);
}
d.push(row);
}
var b = new ArrayBufferSer(headerTypes[i], d);
data.push(b);
}
// create the final buffer
var all = new ArrayBufferSer(ArrayBufferSer.BUFFER, data);
callback(null, all.buffer);
} catch(e) {
callback(e, null);
}
};
module.exports = BinaryFormat;

120
app/models/formats/pg/geojson.js Normal file
View File

@ -0,0 +1,120 @@
'use strict';
var _ = require('underscore');
var pg = require('./../pg');
const errorHandlerFactory = require('../../../services/error_handler_factory');
function GeoJsonFormat() {
this.buffer = '';
}
GeoJsonFormat.prototype = new pg('geojson');
GeoJsonFormat.prototype._contentType = "application/json; charset=utf-8";
GeoJsonFormat.prototype.getQuery = function(sql, options) {
var gn = options.gn;
var dp = options.dp;
return 'SELECT *, ST_AsGeoJSON(' + gn + ',' + dp + ') as the_geom FROM (' + sql + ') as foo';
};
GeoJsonFormat.prototype.startStreaming = function() {
this.total_rows = 0;
if (this.opts.beforeSink) {
this.opts.beforeSink();
}
if (this.opts.callback) {
this.buffer += this.opts.callback + '(';
}
this.buffer += '{"type": "FeatureCollection", "features": [';
this._streamingStarted = true;
};
GeoJsonFormat.prototype.handleQueryRow = function(row) {
if ( ! this._streamingStarted ) {
this.startStreaming();
}
var geojson = [
'{',
'"type":"Feature",',
'"geometry":' + row[this.opts.gn] + ',',
'"properties":'
];
delete row[this.opts.gn];
delete row.the_geom_webmercator;
geojson.push(JSON.stringify(row));
geojson.push('}');
this.buffer += (this.total_rows++ ? ',' : '') + geojson.join('');
if (this.total_rows % (this.opts.bufferedRows || 1000)) {
this.opts.sink.write(this.buffer);
this.buffer = '';
}
};
GeoJsonFormat.prototype.handleQueryEnd = function(/*result*/) {
if (this.error && !this._streamingStarted) {
this.callback(this.error);
return;
}
if ( this.opts.profiler ) {
this.opts.profiler.done('gotRows');
}
if ( ! this._streamingStarted ) {
this.startStreaming();
}
this.buffer += ']'; // end of features
if (this.error) {
this.buffer += ',"error":' + JSON.stringify(errorHandlerFactory(this.error).getResponse().error);
}
this.buffer += '}'; // end of root object
if (this.opts.callback) {
this.buffer += ')';
}
this.opts.sink.write(this.buffer);
this.opts.sink.end();
this.buffer = '';
this.callback();
};
function _toGeoJSON(data, gn, callback){
try {
var out = {
type: "FeatureCollection",
features: []
};
_.each(data.rows, function(ele){
var _geojson = {
type: "Feature",
properties: { },
geometry: { }
};
_geojson.geometry = JSON.parse(ele[gn]);
delete ele[gn];
delete ele.the_geom_webmercator; // TODO: use skipfields
_geojson.properties = ele;
out.features.push(_geojson);
});
// return payload
callback(null, out);
} catch (err) {
callback(err,null);
}
}
module.exports = GeoJsonFormat;
module.exports.toGeoJSON = _toGeoJSON;

174
app/models/formats/pg/json.js Normal file
View File

@ -0,0 +1,174 @@
'use strict';
var _ = require('underscore');
var pg = require('./../pg');
const errorHandlerFactory = require('../../../services/error_handler_factory');
function JsonFormat() {
this.buffer = '';
this.lastKnownResult = {};
}
JsonFormat.prototype = new pg('json');
JsonFormat.prototype._contentType = "application/json; charset=utf-8";
// jshint maxcomplexity:9
JsonFormat.prototype.formatResultFields = function(flds) {
flds = flds || [];
var nfields = {};
for (var i=0; i<flds.length; ++i) {
var f = flds[i];
var cname = this.client.typeName(f.dataTypeID);
var tname;
if ( ! cname ) {
tname = 'unknown(' + f.dataTypeID + ')';
} else {
if ( cname.match('bool') ) {
tname = 'boolean';
}
else if ( cname.match(/int|float|numeric/) ) {
tname = 'number';
}
else if ( cname.match(/text|char|unknown/) ) {
tname = 'string';
}
else if ( cname.match(/date|time/) ) {
tname = 'date';
}
else {
tname = cname;
}
if ( tname && cname.match(/^_/) ) {
tname += '[]';
}
}
nfields[f.name] = { type: tname };
}
return nfields;
};
JsonFormat.prototype.startStreaming = function() {
this.total_rows = 0;
if (this.opts.beforeSink) {
this.opts.beforeSink();
}
if (this.opts.callback) {
this.buffer += this.opts.callback + '(';
}
this.buffer += '{"rows":[';
this._streamingStarted = true;
};
JsonFormat.prototype.handleQueryRow = function(row, result) {
if ( ! this._streamingStarted ) {
this.startStreaming();
}
this.lastKnownResult = result;
this.buffer += (this.total_rows++ ? ',' : '') + JSON.stringify(row, function (key, value) {
if (value !== value) {
return 'NaN';
}
if (value === Infinity) {
return 'Infinity';
}
if (value === -Infinity) {
return '-Infinity';
}
return value;
});
if (this.total_rows % (this.opts.bufferedRows || 1000)) {
this.opts.sink.write(this.buffer);
this.buffer = '';
}
};
// jshint maxcomplexity:13
JsonFormat.prototype.handleQueryEnd = function(result) {
if (this.error && !this._streamingStarted) {
this.callback(this.error);
return;
}
if ( this.opts.profiler ) {
this.opts.profiler.done('gotRows');
}
if ( ! this._streamingStarted ) {
this.startStreaming();
}
this.opts.total_time = (Date.now() - this.start_time)/1000;
result = result || this.lastKnownResult || {};
// Drop field description for skipped fields
if (this.hasSkipFields) {
var newfields = [];
var sf = this.opts.skipfields;
for (var i = 0; i < result.fields.length; i++) {
var f = result.fields[i];
if ( sf.indexOf(f.name) === -1 ) {
newfields.push(f);
}
}
result.fields = newfields;
}
var total_time = (Date.now() - this.start_time)/1000;
var out = [
'],', // end of "rows" array
'"time":', JSON.stringify(total_time),
',"fields":', JSON.stringify(this.formatResultFields(result.fields)),
',"total_rows":', JSON.stringify(result.rowCount || this.total_rows)
];
if (this.error) {
out.push(',"error":', JSON.stringify(errorHandlerFactory(this.error).getResponse().error));
}
if ( result.notices && result.notices.length > 0 ) {
var notices = {},
severities = [];
_.each(result.notices, function(notice) {
var severity = notice.severity.toLowerCase() + 's';
if (!notices[severity]) {
severities.push(severity);
notices[severity] = [];
}
notices[severity].push(notice.message);
});
_.each(severities, function(severity) {
out.push(',');
out.push(JSON.stringify(severity));
out.push(':');
out.push(JSON.stringify(notices[severity]));
});
}
out.push('}');
this.buffer += out.join('');
if (this.opts.callback) {
this.buffer += ')';
}
this.opts.sink.write(this.buffer);
this.opts.sink.end();
this.buffer = '';
this.callback();
};
module.exports = JsonFormat;

166
app/models/formats/pg/svg.js Normal file
View File

@ -0,0 +1,166 @@
'use strict';
var pg = require('./../pg');
var svg_width = 1024.0;
var svg_height = 768.0;
var svg_ratio = svg_width/svg_height;
var radius = 5; // in pixels (based on svg_width and svg_height)
var stroke_width = 1; // in pixels (based on svg_width and svg_height)
var stroke_color = 'black';
// fill settings affect polygons and points (circles)
var fill_opacity = 0.5; // 0.0 is fully transparent, 1.0 is fully opaque
// unused if fill_color='none'
var fill_color = 'none'; // affects polygons and circles
function SvgFormat() {
this.totalRows = 0;
this.bbox = null; // will be computed during the results scan
this.buffer = '';
this._streamingStarted = false;
}
SvgFormat.prototype = new pg('svg');
SvgFormat.prototype._contentType = "image/svg+xml; charset=utf-8";
SvgFormat.prototype.getQuery = function(sql, options) {
var gn = options.gn;
var dp = options.dp;
return 'WITH source AS ( ' + sql + '), extent AS ( ' +
' SELECT ST_Extent(' + gn + ') AS e FROM source ' +
'), extent_info AS ( SELECT e, ' +
'st_xmin(e) as ex0, st_ymax(e) as ey0, ' +
'st_xmax(e)-st_xmin(e) as ew, ' +
'st_ymax(e)-st_ymin(e) as eh FROM extent )' +
', trans AS ( SELECT CASE WHEN ' +
'eh = 0 THEN ' + svg_width +
'/ COALESCE(NULLIF(ew,0),' + svg_width +') WHEN ' +
svg_ratio + ' <= (ew / eh) THEN (' +
svg_width + '/ew ) ELSE (' +
svg_height + '/eh ) END as s ' +
', ex0 as x0, ey0 as y0 FROM extent_info ) ' +
'SELECT st_TransScale(e, -x0, -y0, s, s)::box2d as ' +
gn + '_box, ST_Dimension(' + gn + ') as ' + gn +
'_dimension, ST_AsSVG(ST_TransScale(' + gn + ', ' +
'-x0, -y0, s, s), 0, ' + dp + ') as ' + gn +
//+ ', ex0, ey0, ew, eh, s ' // DEBUG ONLY +
' FROM trans, extent_info, source' +
' ORDER BY the_geom_dimension ASC';
};
SvgFormat.prototype.startStreaming = function() {
if (this.opts.beforeSink) {
this.opts.beforeSink();
}
var header = [
'<?xml version="1.0" standalone="no"?>',
'<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">'
];
var rootTag = '<svg ';
if ( this.bbox ) {
// expand box by "radius" + "stroke-width"
// TODO: use a Box2d class for these ops
var growby = radius + stroke_width;
this.bbox.xmin -= growby;
this.bbox.ymin -= growby;
this.bbox.xmax += growby;
this.bbox.ymax += growby;
this.bbox.width = this.bbox.xmax - this.bbox.xmin;
this.bbox.height = this.bbox.ymax - this.bbox.ymin;
rootTag += 'viewBox="' + this.bbox.xmin + ' ' + (-this.bbox.ymax) + ' ' +
this.bbox.width + ' ' + this.bbox.height + '" ';
}
rootTag += 'style="fill-opacity:' + fill_opacity + '; stroke:' + stroke_color + '; ' +
'stroke-width:' + stroke_width + '; fill:' + fill_color + '" ';
rootTag += 'xmlns="http://www.w3.org/2000/svg" version="1.1">\n';
header.push(rootTag);
this.opts.sink.write(header.join('\n'));
this._streamingStarted = true;
};
// jshint maxcomplexity:11
SvgFormat.prototype.handleQueryRow = function(row) {
this.totalRows++;
if ( ! row.hasOwnProperty(this.opts.gn) ) {
this.error = new Error('column "' + this.opts.gn + '" does not exist');
}
var g = row[this.opts.gn];
if ( ! g ) {
return;
} // null or empty
// jshint ignore:start
var gdims = row[this.opts.gn + '_dimension'];
// TODO: add an identifier, if any of "cartodb_id", "oid", "id", "gid" are found
// TODO: add "class" attribute to help with styling ?
if ( gdims == '0' ) {
this.buffer += '<circle r="' + radius + '" ' + g + ' />\n';
} else if ( gdims == '1' ) {
// Avoid filling closed linestrings
this.buffer += '<path ' + ( fill_color !== 'none' ? 'fill="none" ' : '' ) + 'd="' + g + '" />\n';
} else if ( gdims == '2' ) {
this.buffer += '<path d="' + g + '" />\n';
}
// jshint ignore:end
if ( ! this.bbox ) {
// Parse layer extent: "BOX(x y, X Y)"
// NOTE: the name of the extent field is
// determined by the same code adding the
// ST_AsSVG call (in queryResult)
//
var bbox = row[this.opts.gn + '_box'];
bbox = bbox.match(/BOX\(([^ ]*) ([^ ,]*),([^ ]*) ([^)]*)\)/);
this.bbox = {
xmin: parseFloat(bbox[1]),
ymin: parseFloat(bbox[2]),
xmax: parseFloat(bbox[3]),
ymax: parseFloat(bbox[4])
};
}
if (!this._streamingStarted && this.bbox) {
this.startStreaming();
}
if (this._streamingStarted && (this.totalRows % (this.opts.bufferedRows || 1000))) {
this.opts.sink.write(this.buffer);
this.buffer = '';
}
};
SvgFormat.prototype.handleQueryEnd = function() {
if ( this.error && !this._streamingStarted) {
this.callback(this.error);
return;
}
if ( this.opts.profiler ) {
this.opts.profiler.done('gotRows');
}
if (!this._streamingStarted) {
this.startStreaming();
}
// rootTag close
this.buffer += '</svg>\n';
this.opts.sink.write(this.buffer);
this.opts.sink.end();
this.callback();
};
module.exports = SvgFormat;

138
app/models/formats/pg/topojson.js Normal file
View File

@ -0,0 +1,138 @@
'use strict';
var pg = require('./../pg');
var _ = require('underscore');
var geojson = require('./geojson');
var TopoJSON = require('topojson');
function TopoJsonFormat() {
this.features = [];
}
TopoJsonFormat.prototype = new pg('topojson');
TopoJsonFormat.prototype.getQuery = function(sql, options) {
return geojson.prototype.getQuery(sql, options) + ' where ' + options.gn + ' is not null';
};
TopoJsonFormat.prototype.handleQueryRow = function(row) {
var _geojson = {
type: "Feature"
};
_geojson.geometry = JSON.parse(row[this.opts.gn]);
delete row[this.opts.gn];
delete row.the_geom_webmercator;
_geojson.properties = row;
this.features.push(_geojson);
};
TopoJsonFormat.prototype.handleQueryEnd = function() {
if (this.error) {
this.callback(this.error);
return;
}
if ( this.opts.profiler ) {
this.opts.profiler.done('gotRows');
}
var topology = TopoJSON.topology(this.features, {
"quantization": 1e4,
"force-clockwise": true,
"property-filter": function(d) {
return d;
}
});
this.features = [];
var stream = this.opts.sink;
var jsonpCallback = this.opts.callback;
var bufferedRows = this.opts.bufferedRows;
var buffer = '';
var immediately = global.setImmediate || process.nextTick;
function streamObjectSubtree(obj, key, done) {
buffer += '"' + key + '":';
var isObject = _.isObject(obj[key]),
isArray = _.isArray(obj[key]),
isIterable = isArray || isObject;
if (isIterable) {
buffer += isArray ? '[' : '{';
var subtreeKeys = Object.keys(obj[key]);
var pos = 0;
function streamNext() {
immediately(function() {
var subtreeKey = subtreeKeys.shift();
if (!isArray) {
buffer += '"' + subtreeKey + '":';
}
buffer += JSON.stringify(obj[key][subtreeKey]);
if (pos++ % (bufferedRows || 1000)) {
stream.write(buffer);
buffer = '';
}
if (subtreeKeys.length > 0) {
delete obj[key][subtreeKey];
buffer += ',';
streamNext();
} else {
buffer += isArray ? ']' : '}';
stream.write(buffer);
buffer = '';
done();
}
});
}
streamNext();
} else {
buffer += JSON.stringify(obj[key]);
done();
}
}
if (jsonpCallback) {
buffer += jsonpCallback + '(';
}
buffer += '{';
var keys = Object.keys(topology);
function sendResponse() {
immediately(function () {
var key = keys.shift();
function done() {
if (keys.length > 0) {
delete topology[key];
buffer += ',';
sendResponse();
} else {
buffer += '}';
if (jsonpCallback) {
buffer += ')';
}
stream.write(buffer);
stream.end();
topology = null;
}
}
streamObjectSubtree(topology, key, done);
});
}
sendResponse();
this.callback();
};
TopoJsonFormat.prototype.cancel = function() {
if (this.queryCanceller) {
this.queryCanceller.call();
}
};
module.exports = TopoJsonFormat;

34
app/monitoring/health_check.js Normal file
View File

@ -0,0 +1,34 @@
'use strict';
var step = require('step'),
fs = require('fs');
function HealthCheck(disableFile) {
this.disableFile = disableFile;
}
module.exports = HealthCheck;
HealthCheck.prototype.check = function(callback) {
var self = this;
step(
function getManualDisable() {
fs.readFile(self.disableFile, this);
},
function handleDisabledFile(err, data) {
var next = this;
if (err) {
return next();
}
if (!!data) {
err = new Error(data);
err.http_status = 503;
throw err;
}
},
function handleResult(err) {
callback(err);
}
);
};

285
app/postgresql/error_codes.js Normal file
View File

@ -0,0 +1,285 @@
'use strict';
var _ = require('underscore');
// reference http://www.postgresql.org/docs/9.3/static/errcodes-appendix.html
// Used `^([A-Z0-9]*)\s(.*)` -> `"$1": "$2"` to create the JS object
var codeToCondition = {
// Class 00 — Successful Completion
"00000": "successful_completion",
// Class 01 — Warning
"01000": "warning",
"0100C": "dynamic_result_sets_returned",
"01008": "implicit_zero_bit_padding",
"01003": "null_value_eliminated_in_set_function",
"01007": "privilege_not_granted",
"01006": "privilege_not_revoked",
"01004": "string_data_right_truncation",
"01P01": "deprecated_feature",
// Class 02 — No Data (this is also a warning class per the SQL standard)
"02000": "no_data",
"02001": "no_additional_dynamic_result_sets_returned",
// Class 03 — SQL Statement Not Yet Complete
"03000": "sql_statement_not_yet_complete",
// Class 08 — Connection Exception
"08000": "connection_exception",
"08003": "connection_does_not_exist",
"08006": "connection_failure",
"08001": "sqlclient_unable_to_establish_sqlconnection",
"08004": "sqlserver_rejected_establishment_of_sqlconnection",
"08007": "transaction_resolution_unknown",
"08P01": "protocol_violation",
// Class 09 — Triggered Action Exception
"09000": "triggered_action_exception",
// Class 0A — Feature Not Supported
"0A000": "feature_not_supported",
// Class 0B — Invalid Transaction Initiation
"0B000": "invalid_transaction_initiation",
// Class 0F — Locator Exception
"0F000": "locator_exception",
"0F001": "invalid_locator_specification",
// Class 0L — Invalid Grantor
"0L000": "invalid_grantor",
"0LP01": "invalid_grant_operation",
// Class 0P — Invalid Role Specification
"0P000": "invalid_role_specification",
// Class 0Z — Diagnostics Exception
"0Z000": "diagnostics_exception",
"0Z002": "stacked_diagnostics_accessed_without_active_handler",
// Class 20 — Case Not Found
"20000": "case_not_found",
// Class 21 — Cardinality Violation
"21000": "cardinality_violation",
// Class 22 — Data Exception
"22000": "data_exception",
"2202E": "array_subscript_error",
"22021": "character_not_in_repertoire",
"22008": "datetime_field_overflow",
"22012": "division_by_zero",
"22005": "error_in_assignment",
"2200B": "escape_character_conflict",
"22022": "indicator_overflow",
"22015": "interval_field_overflow",
"2201E": "invalid_argument_for_logarithm",
"22014": "invalid_argument_for_ntile_function",
"22016": "invalid_argument_for_nth_value_function",
"2201F": "invalid_argument_for_power_function",
"2201G": "invalid_argument_for_width_bucket_function",
"22018": "invalid_character_value_for_cast",
"22007": "invalid_datetime_format",
"22019": "invalid_escape_character",
"2200D": "invalid_escape_octet",
"22025": "invalid_escape_sequence",
"22P06": "nonstandard_use_of_escape_character",
"22010": "invalid_indicator_parameter_value",
"22023": "invalid_parameter_value",
"2201B": "invalid_regular_expression",
"2201W": "invalid_row_count_in_limit_clause",
"2201X": "invalid_row_count_in_result_offset_clause",
"22009": "invalid_time_zone_displacement_value",
"2200C": "invalid_use_of_escape_character",
"2200G": "most_specific_type_mismatch",
"22004": "null_value_not_allowed",
"22002": "null_value_no_indicator_parameter",
"22003": "numeric_value_out_of_range",
"22026": "string_data_length_mismatch",
"22001": "string_data_right_truncation",
"22011": "substring_error",
"22027": "trim_error",
"22024": "unterminated_c_string",
"2200F": "zero_length_character_string",
"22P01": "floating_point_exception",
"22P02": "invalid_text_representation",
"22P03": "invalid_binary_representation",
"22P04": "bad_copy_file_format",
"22P05": "untranslatable_character",
"2200L": "not_an_xml_document",
"2200M": "invalid_xml_document",
"2200N": "invalid_xml_content",
"2200S": "invalid_xml_comment",
"2200T": "invalid_xml_processing_instruction",
// Class 23 — Integrity Constraint Violation
"23000": "integrity_constraint_violation",
"23001": "restrict_violation",
"23502": "not_null_violation",
"23503": "foreign_key_violation",
"23505": "unique_violation",
"23514": "check_violation",
"23P01": "exclusion_violation",
// Class 24 — Invalid Cursor State
"24000": "invalid_cursor_state",
// Class 25 — Invalid Transaction State
"25000": "invalid_transaction_state",
"25001": "active_sql_transaction",
"25002": "branch_transaction_already_active",
"25008": "held_cursor_requires_same_isolation_level",
"25003": "inappropriate_access_mode_for_branch_transaction",
"25004": "inappropriate_isolation_level_for_branch_transaction",
"25005": "no_active_sql_transaction_for_branch_transaction",
"25006": "read_only_sql_transaction",
"25007": "schema_and_data_statement_mixing_not_supported",
"25P01": "no_active_sql_transaction",
"25P02": "in_failed_sql_transaction",
// Class 26 — Invalid SQL Statement Name
"26000": "invalid_sql_statement_name",
// Class 27 — Triggered Data Change Violation
"27000": "triggered_data_change_violation",
// Class 28 — Invalid Authorization Specification
"28000": "invalid_authorization_specification",
"28P01": "invalid_password",
// Class 2B — Dependent Privilege Descriptors Still Exist
"2B000": "dependent_privilege_descriptors_still_exist",
"2BP01": "dependent_objects_still_exist",
// Class 2D — Invalid Transaction Termination
"2D000": "invalid_transaction_termination",
// Class 2F — SQL Routine Exception
"2F000": "sql_routine_exception",
"2F005": "function_executed_no_return_statement",
"2F002": "modifying_sql_data_not_permitted",
"2F003": "prohibited_sql_statement_attempted",
"2F004": "reading_sql_data_not_permitted",
// Class 34 — Invalid Cursor Name
"34000": "invalid_cursor_name",
// Class 38 — External Routine Exception
"38000": "external_routine_exception",
"38001": "containing_sql_not_permitted",
"38002": "modifying_sql_data_not_permitted",
"38003": "prohibited_sql_statement_attempted",
"38004": "reading_sql_data_not_permitted",
// Class 39 — External Routine Invocation Exception
"39000": "external_routine_invocation_exception",
"39001": "invalid_sqlstate_returned",
"39004": "null_value_not_allowed",
"39P01": "trigger_protocol_violated",
"39P02": "srf_protocol_violated",
// Class 3B — Savepoint Exception
"3B000": "savepoint_exception",
"3B001": "invalid_savepoint_specification",
// Class 3D — Invalid Catalog Name
"3D000": "invalid_catalog_name",
// Class 3F — Invalid Schema Name
"3F000": "invalid_schema_name",
// Class 40 — Transaction Rollback
"40000": "transaction_rollback",
"40002": "transaction_integrity_constraint_violation",
"40001": "serialization_failure",
"40003": "statement_completion_unknown",
"40P01": "deadlock_detected",
// Class 42 — Syntax Error or Access Rule Violation
"42000": "syntax_error_or_access_rule_violation",
"42601": "syntax_error",
"42501": "insufficient_privilege",
"42846": "cannot_coerce",
"42803": "grouping_error",
"42P20": "windowing_error",
"42P19": "invalid_recursion",
"42830": "invalid_foreign_key",
"42602": "invalid_name",
"42622": "name_too_long",
"42939": "reserved_name",
"42804": "datatype_mismatch",
"42P18": "indeterminate_datatype",
"42P21": "collation_mismatch",
"42P22": "indeterminate_collation",
"42809": "wrong_object_type",
"42703": "undefined_column",
"42883": "undefined_function",
"42P01": "undefined_table",
"42P02": "undefined_parameter",
"42704": "undefined_object",
"42701": "duplicate_column",
"42P03": "duplicate_cursor",
"42P04": "duplicate_database",
"42723": "duplicate_function",
"42P05": "duplicate_prepared_statement",
"42P06": "duplicate_schema",
"42P07": "duplicate_table",
"42712": "duplicate_alias",
"42710": "duplicate_object",
"42702": "ambiguous_column",
"42725": "ambiguous_function",
"42P08": "ambiguous_parameter",
"42P09": "ambiguous_alias",
"42P10": "invalid_column_reference",
"42611": "invalid_column_definition",
"42P11": "invalid_cursor_definition",
"42P12": "invalid_database_definition",
"42P13": "invalid_function_definition",
"42P14": "invalid_prepared_statement_definition",
"42P15": "invalid_schema_definition",
"42P16": "invalid_table_definition",
"42P17": "invalid_object_definition",
// Class 44 — WITH CHECK OPTION Violation
"44000": "with_check_option_violation",
// Class 53 — Insufficient Resources
"53000": "insufficient_resources",
"53100": "disk_full",
"53200": "out_of_memory",
"53300": "too_many_connections",
"53400": "configuration_limit_exceeded",
// Class 54 — Program Limit Exceeded
"54000": "program_limit_exceeded",
"54001": "statement_too_complex",
"54011": "too_many_columns",
"54023": "too_many_arguments",
// Class 55 — Object Not In Prerequisite State
"55000": "object_not_in_prerequisite_state",
"55006": "object_in_use",
"55P02": "cant_change_runtime_param",
"55P03": "lock_not_available",
// Class 57 — Operator Intervention
"57000": "operator_intervention",
"57014": "query_canceled",
"57P01": "admin_shutdown",
"57P02": "crash_shutdown",
"57P03": "cannot_connect_now",
"57P04": "database_dropped",
// Class 58 — System Error (errors external to PostgreSQL itself)
"58000": "system_error",
"58030": "io_error",
"58P01": "undefined_file",
"58P02": "duplicate_file",
// Class F0 — Configuration File Error
"F0000": "config_file_error",
"F0001": "lock_file_exists",
// Class HV — Foreign Data Wrapper Error (SQL/MED)
"HV000": "fdw_error",
"HV005": "fdw_column_name_not_found",
"HV002": "fdw_dynamic_parameter_value_needed",
"HV010": "fdw_function_sequence_error",
"HV021": "fdw_inconsistent_descriptor_information",
"HV024": "fdw_invalid_attribute_value",
"HV007": "fdw_invalid_column_name",
"HV008": "fdw_invalid_column_number",
"HV004": "fdw_invalid_data_type",
"HV006": "fdw_invalid_data_type_descriptors",
"HV091": "fdw_invalid_descriptor_field_identifier",
"HV00B": "fdw_invalid_handle",
"HV00C": "fdw_invalid_option_index",
"HV00D": "fdw_invalid_option_name",
"HV090": "fdw_invalid_string_length_or_buffer_length",
"HV00A": "fdw_invalid_string_format",
"HV009": "fdw_invalid_use_of_null_pointer",
"HV014": "fdw_too_many_handles",
"HV001": "fdw_out_of_memory",
"HV00P": "fdw_no_schemas",
"HV00J": "fdw_option_name_not_found",
"HV00K": "fdw_reply_handle",
"HV00Q": "fdw_schema_not_found",
"HV00R": "fdw_table_not_found",
"HV00L": "fdw_unable_to_create_execution",
"HV00M": "fdw_unable_to_create_reply",
"HV00N": "fdw_unable_to_establish_connection",
// Class P0 — PL/pgSQL Error
"P0000": "plpgsql_error",
"P0001": "raise_exception",
"P0002": "no_data_found",
"P0003": "too_many_rows",
// Class XX — Internal Error
"XX000": "internal_error",
"XX001": "data_corrupted",
"XX002": "index_corrupted"
};
module.exports.codeToCondition = codeToCondition;
module.exports.conditionToCode = _.invert(codeToCondition);

226
app/server.js Normal file
View File

@ -0,0 +1,226 @@
'use strict';
// CartoDB SQL API
//
// all requests expect the following URL args:
// - `sql` {String} SQL to execute
//
// for private (read/write) queries:
// - OAuth. Must have proper OAuth 1.1 headers. For OAuth 1.1 spec see Google
//
// eg. /api/v1/?sql=SELECT 1 as one (with a load of OAuth headers or URL arguments)
//
// for public (read only) queries:
// - sql only, provided the subdomain exists in CartoDB and the table's sharing options are public
//
// eg. vizzuality.cartodb.com/api/v1/?sql=SELECT * from my_table
//
var express = require('express');
var Profiler = require('./stats/profiler-proxy');
var _ = require('underscore');
var fs = require('fs');
var mkdirp = require('mkdirp');
var TableCacheFactory = require('./utils/table_cache_factory');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var UserDatabaseService = require('./services/user_database_service');
var UserLimitsService = require('./services/user_limits');
var JobPublisher = require('../batch/pubsub/job-publisher');
var JobQueue = require('../batch/job_queue');
var JobBackend = require('../batch/job_backend');
var JobCanceller = require('../batch/job_canceller');
var JobService = require('../batch/job_service');
const Logger = require('./services/logger');
var cors = require('./middlewares/cors');
var GenericController = require('./controllers/generic_controller');
var QueryController = require('./controllers/query_controller');
var CopyController = require('./controllers/copy_controller');
var JobController = require('./controllers/job_controller');
var CacheStatusController = require('./controllers/cache_status_controller');
var HealthCheckController = require('./controllers/health_check_controller');
var VersionController = require('./controllers/version_controller');
var batchFactory = require('../batch');
process.env.PGAPPNAME = process.env.PGAPPNAME || 'cartodb_sqlapi';
// override Date.toJSON
require('./utils/date_to_json');
// jshint maxcomplexity:9
function App(statsClient) {
var app = express();
var redisPool = new RedisPool({
name: 'sql-api',
host: global.settings.redis_host,
port: global.settings.redis_port,
max: global.settings.redisPool,
idleTimeoutMillis: global.settings.redisIdleTimeoutMillis,
reapIntervalMillis: global.settings.redisReapIntervalMillis
});
var metadataBackend = cartodbRedis({ pool: redisPool });
// Set default configuration
global.settings.db_pubuser = global.settings.db_pubuser || "publicuser";
global.settings.bufferedRows = global.settings.bufferedRows || 1000;
global.settings.ratelimits = Object.assign(
{
rateLimitsEnabled: false,
endpoints: {
query: false,
job_create: false,
job_get: false,
job_delete: false
}
},
global.settings.ratelimits
);
global.settings.tmpDir = global.settings.tmpDir || '/tmp';
if (!fs.existsSync(global.settings.tmpDir)) {
mkdirp.sync(global.settings.tmpDir);
}
var tableCache = new TableCacheFactory().build(global.settings);
// Size based on https://github.com/CartoDB/cartodb.js/blob/3.15.2/src/geo/layer_definition.js#L72
var SQL_QUERY_BODY_LOG_MAX_LENGTH = 2000;
app.getSqlQueryFromRequestBody = function(req) {
var sqlQuery = req.body && req.body.q;
if (!sqlQuery) {
return '';
}
if (sqlQuery.length > SQL_QUERY_BODY_LOG_MAX_LENGTH) {
sqlQuery = sqlQuery.substring(0, SQL_QUERY_BODY_LOG_MAX_LENGTH) + ' [...]';
}
return JSON.stringify({q: sqlQuery});
};
if ( global.log4js ) {
var loggerOpts = {
buffer: true,
// log4js provides a tokens solution as expess but in does not provide the request/response in the callback.
// Thus it is not possible to extract relevant information from them.
// This is a workaround to be able to access request/response.
format: function(req, res, format) {
var logFormat = global.settings.log_format ||
':remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type]';
logFormat = logFormat.replace(/:sql/, app.getSqlQueryFromRequestBody(req));
return format(logFormat);
}
};
app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level:'info'})));
}
app.use(cors());
// Use step-profiler
app.use(function bootstrap$prepareRequestResponse(req, res, next) {
res.locals = res.locals || {};
if (global.settings.api_hostname) {
res.header('X-Served-By-Host', global.settings.api_hostname);
}
var profile = global.settings.useProfiler;
req.profiler = new Profiler({
profile: profile,
statsd_client: statsClient
});
next();
});
// Set connection timeout
if ( global.settings.hasOwnProperty('node_socket_timeout') ) {
var timeout = parseInt(global.settings.node_socket_timeout);
app.use(function(req, res, next) {
req.connection.setTimeout(timeout);
next();
});
}
app.enable('jsonp callback');
app.set("trust proxy", true);
app.disable('x-powered-by');
app.disable('etag');
// basic routing
var userDatabaseService = new UserDatabaseService(metadataBackend);
const userLimitsServiceOptions = {
limits: {
rateLimitsEnabled: global.settings.ratelimits.rateLimitsEnabled
}
};
const userLimitsService = new UserLimitsService(metadataBackend, userLimitsServiceOptions);
const dataIngestionLogger = new Logger(global.settings.dataIngestionLogPath, 'data-ingestion');
app.dataIngestionLogger = dataIngestionLogger;
var jobPublisher = new JobPublisher(redisPool);
var jobQueue = new JobQueue(metadataBackend, jobPublisher);
var jobBackend = new JobBackend(metadataBackend, jobQueue);
var jobCanceller = new JobCanceller();
var jobService = new JobService(jobBackend, jobCanceller);
var genericController = new GenericController();
genericController.route(app);
var queryController = new QueryController(
metadataBackend,
userDatabaseService,
tableCache,
statsClient,
userLimitsService
);
queryController.route(app);
var copyController = new CopyController(
metadataBackend,
userDatabaseService,
userLimitsService,
dataIngestionLogger
);
copyController.route(app);
var jobController = new JobController(
metadataBackend,
userDatabaseService,
jobService,
statsClient,
userLimitsService
);
jobController.route(app);
var cacheStatusController = new CacheStatusController(tableCache);
cacheStatusController.route(app);
var healthCheckController = new HealthCheckController();
healthCheckController.route(app);
var versionController = new VersionController();
versionController.route(app);
var isBatchProcess = process.argv.indexOf('--no-batch') === -1;
if (global.settings.environment !== 'test' && isBatchProcess) {
var batchName = global.settings.api_hostname || 'batch';
app.batch = batchFactory(
metadataBackend, redisPool, batchName, statsClient, global.settings.batch_log_filename
);
app.batch.start();
}
return app;
}
module.exports = App;

44
app/services/cached-query-tables.js Normal file
View File

@ -0,0 +1,44 @@
'use strict';
var QueryTables = require('cartodb-query-tables');
var generateMD5 = require('../utils/md5');
function CachedQueryTables(tableCache) {
this.tableCache = tableCache;
}
module.exports = CachedQueryTables;
CachedQueryTables.prototype.getAffectedTablesFromQuery = function(pg, sql, skipCache, callback) {
var self = this;
var cacheKey = sqlCacheKey(pg.username(), sql);
var cachedResult;
if (!skipCache) {
cachedResult = this.tableCache.peek(cacheKey);
}
if (cachedResult) {
cachedResult.hits++;
return callback(null, cachedResult.result);
} else {
QueryTables.getAffectedTablesFromQuery(pg, sql, function(err, result) {
if (err) {
return callback(err);
}
self.tableCache.set(cacheKey, {
result: result,
hits: 0
});
return callback(null, result);
});
}
};
function sqlCacheKey(user, sql) {
return user + ':' + generateMD5(sql);
}

36
app/services/error_handler.js Normal file
View File

@ -0,0 +1,36 @@
'use strict';
class ErrorHandler extends Error {
constructor({ message, context, detail, hint, http_status, name }) {
super(message);
this.http_status = this.getHttpStatus(http_status);
this.context = context;
this.detail = detail;
this.hint = hint;
if (name) {
this.name = name;
}
}
getResponse() {
return {
error: [this.message],
context: this.context,
detail: this.detail,
hint: this.hint
};
}
getHttpStatus(http_status = 400) {
if (this.message.includes('permission denied')) {
return 403;
}
return http_status;
}
}
module.exports = ErrorHandler;

41
app/services/error_handler_factory.js Normal file
View File

@ -0,0 +1,41 @@
'use strict';
const ErrorHandler = require('./error_handler');
const { codeToCondition } = require('../postgresql/error_codes');
module.exports = function ErrorHandlerFactory (err) {
if (isTimeoutError(err)) {
return createTimeoutError();
} else {
return createGenericError(err);
}
};
function isTimeoutError(err) {
return err.message && (
err.message.indexOf('statement timeout') > -1 ||
err.message.indexOf('RuntimeError: Execution of function interrupted by signal') > -1 ||
err.message.indexOf('canceling statement due to user request') > -1
);
}
function createTimeoutError() {
return new ErrorHandler({
message: 'You are over platform\'s limits: SQL query timeout error.' +
' Refactor your query before running again or contact CARTO support for more details.',
context: 'limit',
detail: 'datasource',
http_status: 429
});
}
function createGenericError(err) {
return new ErrorHandler({
message: err.message,
context: err.context,
detail: err.detail,
hint: err.hint,
http_status: err.http_status,
name: codeToCondition[err.code] || err.name
});
}

38
app/services/logger.js Normal file
View File

@ -0,0 +1,38 @@
'use strict';
const bunyan = require('bunyan');
class Logger {
constructor (path, name) {
const stream = {
level: process.env.NODE_ENV === 'test' ? 'fatal' : 'info'
};
if (path) {
stream.path = path;
} else {
stream.stream = process.stdout;
}
this.path = path;
this.logger = bunyan.createLogger({
name,
streams: [stream]
});
}
info (log, message) {
this.logger.info(log, message);
}
warn (log, message) {
this.logger.warn(log, message);
}
reopenFileStreams () {
console.log('Reloading log file', this.path);
this.logger.reopenFileStreams();
}
}
module.exports = Logger;

63
app/services/pg-entities-access-validator.js Normal file
View File

@ -0,0 +1,63 @@
'use strict';
const FORBIDDEN_ENTITIES = {
carto: ['*'],
cartodb: [
'cdb_analysis_catalog',
'cdb_conf',
'cdb_tablemetadata'
],
pg_catalog: ['*'],
information_schema: ['*'],
public: ['spatial_ref_sys'],
topology: [
'layer',
'topology'
]
};
const Validator = {
validate(affectedTables, authorizationLevel) {
let hardValidationResult = true;
let softValidationResult = true;
if (!!affectedTables && affectedTables.tables) {
if (global.settings.validatePGEntitiesAccess) {
hardValidationResult = this.hardValidation(affectedTables.tables);
}
if (authorizationLevel !== 'master') {
softValidationResult = this.softValidation(affectedTables.tables);
}
}
return hardValidationResult && softValidationResult;
},
hardValidation(tables) {
for (let table of tables) {
if (FORBIDDEN_ENTITIES[table.schema_name] && FORBIDDEN_ENTITIES[table.schema_name].length &&
(
FORBIDDEN_ENTITIES[table.schema_name][0] === '*' ||
FORBIDDEN_ENTITIES[table.schema_name].includes(table.table_name)
)
) {
return false;
}
}
return true;
},
softValidation(tables) {
for (let table of tables) {
if (table.table_name.match(/\bpg_/)) {
return false;
}
}
return true;
}
};
module.exports = Validator;

77
app/services/stream_copy.js Normal file
View File

@ -0,0 +1,77 @@
'use strict';
const PSQL = require('cartodb-psql');
const copyTo = require('pg-copy-streams').to;
const copyFrom = require('pg-copy-streams').from;
const { Client } = require('pg');
const ACTION_TO = 'to';
const ACTION_FROM = 'from';
const DEFAULT_TIMEOUT = "'5h'";
module.exports = class StreamCopy {
constructor(sql, userDbParams) {
const dbParams = Object.assign({}, userDbParams, {
port: global.settings.db_batch_port || userDbParams.port
});
this.pg = new PSQL(dbParams);
this.sql = sql;
this.stream = null;
this.timeout = global.settings.copy_timeout || DEFAULT_TIMEOUT;
}
static get ACTION_TO() {
return ACTION_TO;
}
static get ACTION_FROM() {
return ACTION_FROM;
}
getPGStream(action, cb) {
this.pg.connect((err, client, done) => {
if (err) {
return cb(err);
}
client.query('SET statement_timeout=' + this.timeout, (err) => {
if (err) {
return cb(err);
}
const streamMaker = action === ACTION_TO ? copyTo : copyFrom;
this.stream = streamMaker(this.sql);
const pgstream = client.query(this.stream);
pgstream
.on('end', () => {
if(action === ACTION_TO) {
pgstream.connection.stream.resume();
}
done();
})
.on('error', err => done(err))
.on('cancelQuery', err => {
if(action === ACTION_TO) {
// See https://www.postgresql.org/docs/9.5/static/protocol-flow.html#PROTOCOL-COPY
const cancelingClient = new Client(client.connectionParameters);
cancelingClient.cancel(client, pgstream);
// see https://node-postgres.com/api/pool#releasecallback
return done(err);
} else if (action === ACTION_FROM) {
client.connection.sendCopyFail('CARTO SQL API: Connection closed by client');
}
});
cb(null, pgstream);
});
});
}
getRowCount() {
return this.stream.rowCount;
}
};

85
app/services/stream_copy_metrics.js Normal file
View File

@ -0,0 +1,85 @@
'use strict';
const { getFormatFromCopyQuery } = require('../utils/query_info');
module.exports = class StreamCopyMetrics {
constructor(logger, type, sql, user, isGzip = false) {
this.logger = logger;
this.type = type;
this.format = getFormatFromCopyQuery(sql);
this.isGzip = isGzip;
this.username = user;
this.size = 0;
this.gzipSize = 0;
this.rows = 0;
this.startTime = new Date();
this.endTime = null;
this.time = null;
this.success = true;
this.error = null;
this.ended = false;
}
addSize(size) {
this.size += size;
}
addGzipSize(size) {
this.gzipSize += size;
}
end(rows = null, error = null) {
if (this.ended) {
return;
}
this.ended = true;
if (Number.isInteger(rows)) {
this.rows = rows;
}
if (error instanceof Error) {
this.error = error;
}
this.endTime = new Date();
this.time = (this.endTime.getTime() - this.startTime.getTime()) / 1000;
this._log(
this.startTime.toISOString(),
this.isGzip && this.gzipSize ? this.gzipSize : null,
this.error ? this.error.message : null
);
}
_log(timestamp, gzipSize = null, errorMessage = null) {
let logData = {
type: this.type,
format: this.format,
size: this.size,
rows: this.rows,
gzip: this.isGzip,
'cdb-user': this.username,
time: this.time,
timestamp
};
if (gzipSize) {
logData.gzipSize = gzipSize;
}
if (errorMessage) {
logData.error = errorMessage;
this.success = false;
}
logData.success = this.success;
this.logger.info(logData);
}
};

107
app/services/user_database_service.js Normal file
View File

@ -0,0 +1,107 @@
'use strict';
function isApiKeyFound(apikey) {
return apikey.type !== null &&
apikey.user !== null &&
apikey.databasePassword !== null &&
apikey.databaseRole !== null;
}
function UserDatabaseService(metadataBackend) {
this.metadataBackend = metadataBackend;
}
function errorUserNotFoundMessageTemplate (user) {
return `Sorry, we can't find CARTO user '${user}'. Please check that you have entered the correct domain.`;
}
function isOauthAuthorization({ apikeyToken, authorizationLevel }) {
return (authorizationLevel === 'master') && !apikeyToken;
}
/**
* Callback is invoked with `dbParams` and `authDbParams`.
* `dbParams` depends on AuthApi verification so it might return a public user with just SELECT permission, where
* `authDbParams` will always return connection params as AuthApi had authorized the connection.
* That might be useful when you have to run a query with and without permissions.
*
* @param {AuthApi} authApi
* @param {String} cdbUsername
* @param {Function} callback (err, dbParams, authDbParams)
*/
UserDatabaseService.prototype.getConnectionParams = function (username, apikeyToken, authorizationLevel, callback) {
this.metadataBackend.getAllUserDBParams(username, (err, dbParams) => {
if (err) {
err.http_status = 404;
err.message = errorUserNotFoundMessageTemplate(username);
return callback(err);
}
const commonDBConfiguration = {
port: global.settings.db_port,
host: dbParams.dbhost,
dbname: dbParams.dbname,
};
this.metadataBackend.getMasterApikey(username, (err, masterApikey) => {
if (err) {
err.http_status = 404;
err.message = errorUserNotFoundMessageTemplate(username);
return callback(err);
}
if (!isApiKeyFound(masterApikey)) {
const apiKeyNotFoundError = new Error('Unauthorized');
apiKeyNotFoundError.type = 'auth';
apiKeyNotFoundError.subtype = 'api-key-not-found';
apiKeyNotFoundError.http_status = 401;
return callback(apiKeyNotFoundError);
}
const masterDBConfiguration = Object.assign({
user: masterApikey.databaseRole,
pass: masterApikey.databasePassword
},
commonDBConfiguration);
if (isOauthAuthorization({ apikeyToken, authorizationLevel})) {
return callback(null, masterDBConfiguration, masterDBConfiguration);
}
// Default Api key fallback
apikeyToken = apikeyToken || 'default_public';
this.metadataBackend.getApikey(username, apikeyToken, (err, apikey) => {
if (err) {
err.http_status = 404;
err.message = errorUserNotFoundMessageTemplate(username);
return callback(err);
}
if (!isApiKeyFound(apikey)) {
const apiKeyNotFoundError = new Error('Unauthorized');
apiKeyNotFoundError.type = 'auth';
apiKeyNotFoundError.subtype = 'api-key-not-found';
apiKeyNotFoundError.http_status = 401;
return callback(apiKeyNotFoundError);
}
const DBConfiguration = Object.assign({
user: apikey.databaseRole,
pass: apikey.databasePassword
},
commonDBConfiguration);
callback(null, DBConfiguration, masterDBConfiguration);
});
});
});
};
module.exports = UserDatabaseService;

27
app/services/user_limits.js Normal file
View File

@ -0,0 +1,27 @@
'use strict';
/**
* UserLimits
* @param {cartodb-redis} metadataBackend
* @param {object} options
*/
class UserLimits {
constructor(metadataBackend, options = {}) {
this.metadataBackend = metadataBackend;
this.options = options;
this.preprareRateLimit();
}
preprareRateLimit() {
if (this.options.limits.rateLimitsEnabled) {
this.metadataBackend.loadRateLimitsScript();
}
}
getRateLimit(user, endpointGroup, callback) {
this.metadataBackend.getRateLimit(user, 'sql', endpointGroup, callback);
}
}
module.exports = UserLimits;

75
app/stats/client.js Normal file
View File

@ -0,0 +1,75 @@
'use strict';
var _ = require('underscore');
var debug = require('debug')('windshaft:stats_client');
var StatsD = require('node-statsd').StatsD;
module.exports = {
/**
* Returns an StatsD instance or an stub object that replicates the StatsD public interface so there is no need to
* keep checking if the stats_client is instantiated or not.
*
* The first call to this method implies all future calls will use the config specified in the very first call.
*
* TODO: It's far from ideal to use make this a singleton, improvement desired.
* We proceed this way to be able to use StatsD from several places sharing one single StatsD instance.
*
* @param config Configuration for StatsD, if undefined it will return an stub
* @returns {StatsD|Object}
*/
getInstance: function(config) {
if (!this.instance) {
var instance;
if (config) {
instance = new StatsD(config);
instance.last_error = { msg: '', count: 0 };
instance.socket.on('error', function (err) {
var last_err = instance.last_error;
var last_msg = last_err.msg;
var this_msg = '' + err;
if (this_msg !== last_msg) {
debug("statsd client socket error: " + err);
instance.last_error.count = 1;
instance.last_error.msg = this_msg;
} else {
++last_err.count;
if (!last_err.interval) {
instance.last_error.interval = setInterval(function () {
var count = instance.last_error.count;
if (count > 1) {
debug("last statsd client socket error repeated " + count + " times");
instance.last_error.count = 1;
clearInterval(instance.last_error.interval);
instance.last_error.interval = null;
}
}, 1000);
}
}
});
} else {
var stubFunc = function (stat, value, sampleRate, callback) {
if (_.isFunction(callback)) {
callback(null, 0);
}
};
instance = {
timing: stubFunc,
increment: stubFunc,
decrement: stubFunc,
gauge: stubFunc,
unique: stubFunc,
set: stubFunc,
sendAll: stubFunc,
send: stubFunc
};
}
this.instance = instance;
}
return this.instance;
}
};

55
app/stats/profiler-proxy.js Normal file
View File

@ -0,0 +1,55 @@
'use strict';
var Profiler = require('step-profiler');
/**
* Proxy to encapsulate node-step-profiler module so there is no need to check if there is an instance
*/
function ProfilerProxy(opts) {
this.profile = !!opts.profile;
this.profiler = null;
if (!!opts.profile) {
this.profiler = new Profiler({statsd_client: opts.statsd_client});
}
}
ProfilerProxy.prototype.done = function(what) {
if (this.profile) {
this.profiler.done(what);
}
};
ProfilerProxy.prototype.end = function() {
if (this.profile) {
this.profiler.end();
}
};
ProfilerProxy.prototype.start = function(what) {
if (this.profile) {
this.profiler.start(what);
}
};
ProfilerProxy.prototype.add = function(what) {
if (this.profile) {
this.profiler.add(what || {});
}
};
ProfilerProxy.prototype.sendStats = function() {
if (this.profile) {
this.profiler.sendStats();
}
};
ProfilerProxy.prototype.toString = function() {
return this.profile ? this.profiler.toString() : "";
};
ProfilerProxy.prototype.toJSONString = function() {
return this.profile ? this.profiler.toJSONString() : "{}";
};
module.exports = ProfilerProxy;

5
app/utils/cache_key_generator.js Normal file
View File

@ -0,0 +1,5 @@
'use strict';
module.exports = function generateCacheKey(database, affectedTables) {
return database + ":" + affectedTables.join(',');
};

8
app/utils/content_disposition.js Normal file
View File

@ -0,0 +1,8 @@
'use strict';
module.exports = function getContentDisposition(formatter, filename, inline) {
var ext = formatter.getFileExtension();
var time = new Date().toUTCString();
return ( inline ? 'inline' : 'attachment' ) + '; filename=' + filename + '.' + ext + '; ' +
'modification-date="' + time + '";';
};

19
app/utils/date_to_json.js Normal file
View File

@ -0,0 +1,19 @@
'use strict';
// jshint ignore:start
function pad(n) {
return n < 10 ? '0' + n : n;
}
Date.prototype.toJSON = function() {
var s = this.getFullYear() + '-' + pad(this.getMonth() + 1) + '-' + pad(this.getDate()) + 'T' +
pad(this.getHours()) + ':' + pad(this.getMinutes()) + ':' + pad(this.getSeconds());
var offset = this.getTimezoneOffset();
if (offset === 0) {
s += 'Z';
} else {
s += ( offset < 0 ? '+' : '-' ) + pad(Math.abs(offset / 60)) + pad(Math.abs(offset % 60));
}
return s;
};
// jshint ignore:end

9
app/utils/filename_sanitizer.js Normal file
View File

@ -0,0 +1,9 @@
'use strict';
var path = require('path');
module.exports = function sanitize_filename(filename) {
filename = path.basename(filename, path.extname(filename));
filename = filename.replace(/[;()\[\]<>'"\s]/g, '_');
return filename;
};

9
app/utils/md5.js Normal file
View File

@ -0,0 +1,9 @@
'use strict';
var crypto = require('crypto');
module.exports = function generateMD5(data){
var hash = crypto.createHash('md5');
hash.update(data);
return hash.digest('hex');
};

49
app/utils/no_cache.js Normal file
View File

@ -0,0 +1,49 @@
'use strict';
/**
* This module provides an object with the interface of an LRU cache
* but that actually does not store anything.
*
* See https://github.com/isaacs/node-lru-cache/tree/v2.5.0
*/
function NoCache() {
}
module.exports = NoCache;
NoCache.prototype.set = function (/* key, value */) {
return true;
};
NoCache.prototype.get = function (/* key */) {
return undefined;
};
NoCache.prototype.peek = function (/* key */) {
return undefined;
};
NoCache.prototype.del = function (/* key */) {
return undefined;
};
NoCache.prototype.reset = function () {
return undefined;
};
NoCache.prototype.has = function (/* key */) {
return false;
};
NoCache.prototype.forEach = function (/* fn, thisp */) {
return undefined;
};
NoCache.prototype.keys = function () {
return [];
};
NoCache.prototype.values = function () {
return [];
};

31
app/utils/query_info.js Normal file
View File

@ -0,0 +1,31 @@
'use strict';
const COPY_FORMATS = ['TEXT', 'CSV', 'BINARY'];
module.exports = {
getFormatFromCopyQuery(copyQuery) {
let format = 'TEXT'; // Postgres default format
copyQuery = copyQuery.toUpperCase();
if (!copyQuery.startsWith("COPY ")) {
return false;
}
if(copyQuery.includes(' WITH') && copyQuery.includes('FORMAT ')) {
const regex = /\bFORMAT\s+(\w+)/;
const result = regex.exec(copyQuery);
if (result && result.length === 2) {
if (COPY_FORMATS.includes(result[1])) {
format = result[1];
format = format.toUpperCase();
} else {
format = false;
}
}
}
return format;
}
};

14
app/utils/query_may_write.js Normal file
View File

@ -0,0 +1,14 @@
'use strict';
var sqlQueryMayWriteRegex = new RegExp("\\b(alter|insert|update|delete|create|drop|reindex|truncate|refresh)\\b", "i");
/**
* This is a fuzzy check, the return could be true even if the query doesn't really write anything. But you can be
* pretty sure of a false return.
*
* @param sql The SQL statement to check against
* @returns {boolean} Return true of the given query may write to the database
*/
module.exports = function queryMayWrite(sql) {
return sqlQueryMayWriteRegex.test(sql);
};

32
app/utils/table_cache_factory.js Normal file
View File

@ -0,0 +1,32 @@
'use strict';
var LRU = require('lru-cache');
var NoCache = require('./no_cache');
/**
* This module abstracts the creation of a tableCache,
* depending on the configuration passed along
*/
function TableCacheFactory() {
}
module.exports = TableCacheFactory;
TableCacheFactory.prototype.build = function (settings) {
var enabled = settings.tableCacheEnabled || false;
var tableCache = null;
if(enabled) {
tableCache = LRU({
// store no more than these many items in the cache
max: settings.tableCacheMax || 8192,
// consider entries expired after these many milliseconds (10 minutes by default)
maxAge: settings.tableCacheMaxAge || 1000*60*10
});
} else {
tableCache = new NoCache();
}
return tableCache;
};

122
batch/README.md Normal file
View File

@ -0,0 +1,122 @@
# Batch Queries
This document describes features from Batch Queries, it also details some internals that might be useful for maintainers
and developers.
## Redis data structures
### Jobs definition
Redis Hash: `batch:jobs:{UUID}`.
Redis DB: 5.
It stores the job definition, the user, and some metadata like the final status, the failure reason, and so.
### Job queues
Redis List: `batch:queue:{username}`.
Redis DB: 5.
It stores a pending list of jobs per user. It points to a job definition with the `{UUID}`.
### Job notifications
Redis Pub/Sub channel: `batch:users`.
Redis DB: 0.
In order to notify new jobs, it uses a Pub/Sub channel were the username for the queued job is published.
## Job types
Format for the currently supported query types, and what they are missing in terms of features.
### Simple
```json
{
"query": "update ..."
}
```
Does not support main fallback queries. Ideally it should support something like:
```json
{
"query": "update ...",
"onsuccess": "select 'general success fallback'",
"onerror": "select 'general error fallback'"
}
```
### Multiple
```json
{
"query": [
"update ...",
"select ... into ..."
]
}
```
Does not support main fallback queries. Ideally it should support something like:
```json
{
"query": [
"update ...",
"select ... into ..."
],
"onsuccess": "select 'general success fallback'",
"onerror": "select 'general error fallback'"
}
```
### Fallback
```json
{
"query": {
"query": [
{
"query": "select 1",
"onsuccess": "select 'success fallback query 1'",
"onerror": "select 'error fallback query 1'"
},
{
"query": "select 2",
"onerror": "select 'error fallback query 2'"
}
],
"onsuccess": "select 'general success fallback'",
"onerror": "select 'general error fallback'"
}
}
```
It's weird to have two nested `query` attributes. Also, it's not possible to mix _plain_ with _fallback_ ones.
Ideally it should support something like:
```json
{
"query": [
{
"query": "select 1",
"onsuccess": "select 'success fallback query 1'",
"onerror": "select 'error fallback query 1'"
},
"select 2"
],
"onsuccess": "select 'general success fallback'",
"onerror": "select 'general error fallback'"
}
}
```
Where you don't need a nested `query` attribute, it's just an array as in Multiple job type, and you can mix objects and
plain queries.

16
batch/batch-logger.js Normal file
View File

@ -0,0 +1,16 @@
'use strict';
const Logger = require('../app/services/logger');
class BatchLogger extends Logger {
constructor (path, name) {
super(path, name);
}
log (job) {
return job.log(this.logger);
}
}
module.exports = BatchLogger;

217
batch/batch.js Normal file
View File

@ -0,0 +1,217 @@
'use strict';
var util = require('util');
var EventEmitter = require('events').EventEmitter;
var debug = require('./util/debug')('batch');
var queue = require('queue-async');
var HostScheduler = require('./scheduler/host-scheduler');
var EMPTY_QUEUE = true;
var MINUTE = 60 * 1000;
var SCHEDULE_INTERVAL = 1 * MINUTE;
function Batch(name, userDatabaseMetadataService, jobSubscriber, jobQueue, jobRunner, jobService, redisPool, logger) {
EventEmitter.call(this);
this.name = name || 'batch';
this.userDatabaseMetadataService = userDatabaseMetadataService;
this.jobSubscriber = jobSubscriber;
this.jobQueue = jobQueue;
this.jobRunner = jobRunner;
this.jobService = jobService;
this.logger = logger;
this.hostScheduler = new HostScheduler(this.name, { run: this.processJob.bind(this) }, redisPool);
// map: user => jobId. Will be used for draining jobs.
this.workInProgressJobs = {};
}
util.inherits(Batch, EventEmitter);
module.exports = Batch;
Batch.prototype.start = function () {
var self = this;
var onJobHandler = createJobHandler(self.name, self.userDatabaseMetadataService, self.hostScheduler);
self.jobQueue.scanQueues(function (err, queues) {
if (err) {
return self.emit('error', err);
}
queues.forEach(onJobHandler);
self._startScheduleInterval(onJobHandler);
self.jobSubscriber.subscribe(onJobHandler, function (err) {
if (err) {
return self.emit('error', err);
}
self.emit('ready');
});
});
};
function createJobHandler (name, userDatabaseMetadataService, hostScheduler) {
return function onJobHandler(user) {
userDatabaseMetadataService.getUserMetadata(user, function (err, userDatabaseMetadata) {
if (err) {
return debug('Could not get host user=%s from %s. Reason: %s', user, name, err.message);
}
var host = userDatabaseMetadata.host;
debug('[%s] onJobHandler(%s, %s)', name, user, host);
hostScheduler.add(host, user, function(err) {
if (err) {
return debug(
'Could not schedule host=%s user=%s from %s. Reason: %s', host, user, name, err.message
);
}
});
});
};
}
Batch.prototype._startScheduleInterval = function (onJobHandler) {
var self = this;
self.scheduleInterval = setInterval(function () {
self.jobQueue.getQueues(function (err, queues) {
if (err) {
return debug('Could not get queues from %s. Reason: %s', self.name, err.message);
}
queues.forEach(onJobHandler);
});
}, SCHEDULE_INTERVAL);
};
Batch.prototype._stopScheduleInterval = function () {
if (this.scheduleInterval) {
clearInterval(this.scheduleInterval);
}
};
Batch.prototype.processJob = function (user, callback) {
var self = this;
self.jobQueue.dequeue(user, function (err, jobId) {
if (err) {
return callback(new Error('Could not get job from "' + user + '". Reason: ' + err.message), !EMPTY_QUEUE);
}
if (!jobId) {
debug('Queue empty user=%s', user);
return callback(null, EMPTY_QUEUE);
}
self._processWorkInProgressJob(user, jobId, function (err, job) {
if (err) {
debug(err);
if (err.name === 'JobNotRunnable') {
return callback(null, !EMPTY_QUEUE);
}
return callback(err, !EMPTY_QUEUE);
}
debug(
'[%s] Job=%s status=%s user=%s (failed_reason=%s)',
self.name, jobId, job.data.status, user, job.failed_reason
);
self.logger.log(job);
return callback(null, !EMPTY_QUEUE);
});
});
};
Batch.prototype._processWorkInProgressJob = function (user, jobId, callback) {
var self = this;
self.setWorkInProgressJob(user, jobId, function (errSet) {
if (errSet) {
debug(new Error('Could not add job to work-in-progress list. Reason: ' + errSet.message));
}
self.jobRunner.run(jobId, function (err, job) {
self.clearWorkInProgressJob(user, jobId, function (errClear) {
if (errClear) {
debug(new Error('Could not clear job from work-in-progress list. Reason: ' + errClear.message));
}
return callback(err, job);
});
});
});
};
Batch.prototype.drain = function (callback) {
var self = this;
var workingUsers = this.getWorkInProgressUsers();
var batchQueues = queue(workingUsers.length);
workingUsers.forEach(function (user) {
batchQueues.defer(self._drainJob.bind(self), user);
});
batchQueues.awaitAll(function (err) {
if (err) {
debug('Something went wrong draining', err);
} else {
debug('Drain complete');
}
callback();
});
};
Batch.prototype._drainJob = function (user, callback) {
var self = this;
var job_id = this.getWorkInProgressJob(user);
if (!job_id) {
return process.nextTick(function () {
return callback();
});
}
this.jobService.drain(job_id, function (err) {
if (err && err.name === 'CancelNotAllowedError') {
return callback();
}
if (err) {
return callback(err);
}
self.jobQueue.enqueueFirst(user, job_id, callback);
});
};
Batch.prototype.stop = function (callback) {
this.removeAllListeners();
this._stopScheduleInterval();
this.jobSubscriber.unsubscribe(callback);
};
/* Work in progress jobs */
Batch.prototype.setWorkInProgressJob = function(user, jobId, callback) {
this.workInProgressJobs[user] = jobId;
this.jobService.addWorkInProgressJob(user, jobId, callback);
};
Batch.prototype.getWorkInProgressJob = function(user) {
return this.workInProgressJobs[user];
};
Batch.prototype.clearWorkInProgressJob = function(user, jobId, callback) {
delete this.workInProgressJobs[user];
this.jobService.clearWorkInProgressJob(user, jobId, callback);
};
Batch.prototype.getWorkInProgressUsers = function() {
return Object.keys(this.workInProgressJobs);
};

39
batch/index.js Normal file
View File

@ -0,0 +1,39 @@
'use strict';
var JobRunner = require('./job_runner');
var QueryRunner = require('./query_runner');
var JobCanceller = require('./job_canceller');
var JobSubscriber = require('./pubsub/job-subscriber');
var UserDatabaseMetadataService = require('./user_database_metadata_service');
var JobPublisher = require('./pubsub/job-publisher');
var JobQueue = require('./job_queue');
var JobBackend = require('./job_backend');
var JobService = require('./job_service');
var BatchLogger = require('./batch-logger');
var Batch = require('./batch');
module.exports = function batchFactory (metadataBackend, redisPool, name, statsdClient, loggerPath) {
var userDatabaseMetadataService = new UserDatabaseMetadataService(metadataBackend);
var jobSubscriber = new JobSubscriber(redisPool);
var jobPublisher = new JobPublisher(redisPool);
var jobQueue = new JobQueue(metadataBackend, jobPublisher);
var jobBackend = new JobBackend(metadataBackend, jobQueue);
var queryRunner = new QueryRunner(userDatabaseMetadataService);
var jobCanceller = new JobCanceller();
var jobService = new JobService(jobBackend, jobCanceller);
var jobRunner = new JobRunner(jobService, jobQueue, queryRunner, metadataBackend, statsdClient);
var logger = new BatchLogger(loggerPath, 'batch-queries');
return new Batch(
name,
userDatabaseMetadataService,
jobSubscriber,
jobQueue,
jobRunner,
jobService,
redisPool,
logger
);
};

290
batch/job_backend.js Normal file
View File

@ -0,0 +1,290 @@
'use strict';
var REDIS_PREFIX = 'batch:jobs:';
var REDIS_DB = 5;
var JobStatus = require('./job_status');
var queue = require('queue-async');
var debug = require('./util/debug')('job-backend');
function JobBackend(metadataBackend, jobQueue) {
this.metadataBackend = metadataBackend;
this.jobQueue = jobQueue;
this.maxNumberOfQueuedJobs = global.settings.batch_max_queued_jobs || 64;
this.inSecondsJobTTLAfterFinished = global.settings.finished_jobs_ttl_in_seconds || 2 * 3600; // 2 hours
this.hostname = global.settings.api_hostname || 'batch';
}
function toRedisParams(job) {
var redisParams = [REDIS_PREFIX + job.job_id];
var obj = JSON.parse(JSON.stringify(job));
delete obj.job_id;
for (var property in obj) {
if (obj.hasOwnProperty(property)) {
redisParams.push(property);
if (property === 'query' && typeof obj[property] !== 'string') {
redisParams.push(JSON.stringify(obj[property]));
} else {
redisParams.push(obj[property]);
}
}
}
return redisParams;
}
function toObject(job_id, redisParams, redisValues) {
var obj = {};
redisParams.shift(); // job_id value
redisParams.pop(); // WARN: weird function pushed by metadataBackend
for (var i = 0; i < redisParams.length; i++) {
// TODO: this should be moved to job model
if (redisParams[i] === 'query') {
try {
obj[redisParams[i]] = JSON.parse(redisValues[i]);
} catch (e) {
obj[redisParams[i]] = redisValues[i];
}
} else if (redisValues[i]) {
obj[redisParams[i]] = redisValues[i];
}
}
obj.job_id = job_id; // adds redisKey as object property
return obj;
}
function isJobFound(redisValues) {
return !!(redisValues[0] && redisValues[1] && redisValues[2] && redisValues[3] && redisValues[4]);
}
function getNotFoundError(job_id) {
var notFoundError = new Error('Job with id ' + job_id + ' not found');
notFoundError.name = 'NotFoundError';
return notFoundError;
}
JobBackend.prototype.get = function (job_id, callback) {
if (!job_id) {
return callback(getNotFoundError(job_id));
}
var self = this;
var redisParams = [
REDIS_PREFIX + job_id,
'user',
'status',
'query',
'created_at',
'updated_at',
'host',
'failed_reason',
'fallback_status',
'host',
'port',
'pass',
'dbname',
'dbuser'
];
self.metadataBackend.redisCmd(REDIS_DB, 'HMGET', redisParams , function (err, redisValues) {
if (err) {
return callback(err);
}
if (!isJobFound(redisValues)) {
return callback(getNotFoundError(job_id));
}
var jobData = toObject(job_id, redisParams, redisValues);
callback(null, jobData);
});
};
JobBackend.prototype.create = function (job, callback) {
var self = this;
this.jobQueue.size(job.user, function(err, size) {
if (err) {
return callback(new Error('Failed to create job, could not determine user queue size'));
}
if (size >= self.maxNumberOfQueuedJobs) {
return callback(new Error(
'Failed to create job. ' +
'Max number of jobs (' + self.maxNumberOfQueuedJobs + ') queued reached'
));
}
self.get(job.job_id, function (err) {
if (err && err.name !== 'NotFoundError') {
return callback(err);
}
self.save(job, function (err, jobSaved) {
if (err) {
return callback(err);
}
self.jobQueue.enqueue(job.user, job.job_id, function (err) {
if (err) {
return callback(err);
}
return callback(null, jobSaved);
});
});
});
});
};
JobBackend.prototype.update = function (job, callback) {
var self = this;
self.get(job.job_id, function (err) {
if (err) {
return callback(err);
}
self.save(job, callback);
});
};
JobBackend.prototype.save = function (job, callback) {
var self = this;
var redisParams = toRedisParams(job);
self.metadataBackend.redisCmd(REDIS_DB, 'HMSET', redisParams , function (err) {
if (err) {
return callback(err);
}
self.setTTL(job, function (err) {
if (err) {
return callback(err);
}
self.get(job.job_id, function (err, job) {
if (err) {
return callback(err);
}
callback(null, job);
});
});
});
};
var WORK_IN_PROGRESS_JOB = {
DB: 5,
PREFIX_USER: 'batch:wip:user:',
USER_INDEX_KEY: 'batch:wip:users'
};
JobBackend.prototype.addWorkInProgressJob = function (user, jobId, callback) {
var userWIPKey = WORK_IN_PROGRESS_JOB.PREFIX_USER + user;
debug('add job %s to user %s (%s)', jobId, user, userWIPKey);
this.metadataBackend.redisMultiCmd(WORK_IN_PROGRESS_JOB.DB, [
['SADD', WORK_IN_PROGRESS_JOB.USER_INDEX_KEY, user],
['RPUSH', userWIPKey, jobId]
], callback);
};
JobBackend.prototype.clearWorkInProgressJob = function (user, jobId, callback) {
var self = this;
var DB = WORK_IN_PROGRESS_JOB.DB;
var userWIPKey = WORK_IN_PROGRESS_JOB.PREFIX_USER + user;
var params = [userWIPKey, 0, jobId];
self.metadataBackend.redisCmd(DB, 'LREM', params, function (err) {
if (err) {
return callback(err);
}
params = [userWIPKey, 0, -1];
self.metadataBackend.redisCmd(DB, 'LRANGE', params, function (err, workInProgressJobs) {
if (err) {
return callback(err);
}
debug('user %s has work in progress jobs %j', user, workInProgressJobs);
if (workInProgressJobs.length < 0) {
return callback();
}
debug('delete user %s from index', user);
params = [WORK_IN_PROGRESS_JOB.USER_INDEX_KEY, user];
self.metadataBackend.redisCmd(DB, 'SREM', params, function (err) {
if (err) {
return callback(err);
}
return callback();
});
});
});
};
JobBackend.prototype.listWorkInProgressJobByUser = function (user, callback) {
var userWIPKey = WORK_IN_PROGRESS_JOB.PREFIX_USER + user;
var params = [userWIPKey, 0, -1];
this.metadataBackend.redisCmd(WORK_IN_PROGRESS_JOB.DB, 'LRANGE', params, callback);
};
JobBackend.prototype.listWorkInProgressJobs = function (callback) {
var self = this;
var DB = WORK_IN_PROGRESS_JOB.DB;
var params = [WORK_IN_PROGRESS_JOB.USER_INDEX_KEY];
this.metadataBackend.redisCmd(DB, 'SMEMBERS', params, function (err, workInProgressUsers) {
if (err) {
return callback(err);
}
if (workInProgressUsers < 1) {
return callback(null, {});
}
debug('found %j work in progress users', workInProgressUsers);
var usersQueue = queue(4);
workInProgressUsers.forEach(function (user) {
usersQueue.defer(self.listWorkInProgressJobByUser.bind(self), user);
});
usersQueue.awaitAll(function (err, userWorkInProgressJobs) {
if (err) {
return callback(err);
}
var workInProgressJobs = workInProgressUsers.reduce(function (users, user, index) {
users[user] = userWorkInProgressJobs[index];
debug('found %j work in progress jobs for user %s', userWorkInProgressJobs[index], user);
return users;
}, {});
callback(null, workInProgressJobs);
});
});
};
JobBackend.prototype.setTTL = function (job, callback) {
var self = this;
var redisKey = REDIS_PREFIX + job.job_id;
if (!JobStatus.isFinal(job.status)) {
return callback();
}
self.metadataBackend.redisCmd(REDIS_DB, 'EXPIRE', [ redisKey, this.inSecondsJobTTLAfterFinished ], callback);
};
module.exports = JobBackend;

78
batch/job_canceller.js Normal file
View File

@ -0,0 +1,78 @@
'use strict';
var PSQL = require('cartodb-psql');
function JobCanceller() {
}
module.exports = JobCanceller;
JobCanceller.prototype.cancel = function (job, callback) {
const dbConfiguration = {
host: job.data.host,
port: job.data.port,
dbname: job.data.dbname,
user: job.data.dbuser,
pass: job.data.pass,
};
doCancel(job.data.job_id, dbConfiguration, callback);
};
function doCancel(job_id, dbConfiguration, callback) {
var pg = new PSQL(dbConfiguration);
getQueryPID(pg, job_id, function (err, pid) {
if (err) {
return callback(err);
}
if (!pid) {
return callback();
}
doCancelQuery(pg, pid, function (err, isCancelled) {
if (err) {
return callback(err);
}
if (!isCancelled) {
return callback(new Error('Query has not been cancelled'));
}
callback();
});
});
}
function getQueryPID(pg, job_id, callback) {
var getPIDQuery = "SELECT pid FROM pg_stat_activity WHERE query LIKE '/* " + job_id + " */%'";
pg.query(getPIDQuery, function(err, result) {
if (err) {
return callback(err);
}
if (!result.rows[0] || !result.rows[0].pid) {
// query is not running actually, but we have to callback w/o error to cancel the job anyway.
return callback();
}
callback(null, result.rows[0].pid);
});
}
function doCancelQuery(pg, pid, callback) {
var cancelQuery = 'SELECT pg_cancel_backend(' + pid + ')';
pg.query(cancelQuery, function (err, result) {
if (err) {
return callback(err);
}
var isCancelled = result.rows[0].pg_cancel_backend;
callback(null, isCancelled);
});
}

165
batch/job_queue.js Normal file
View File

@ -0,0 +1,165 @@
'use strict';
var debug = require('./util/debug')('queue');
var queueAsync = require('queue-async');
function JobQueue(metadataBackend, jobPublisher) {
this.metadataBackend = metadataBackend;
this.jobPublisher = jobPublisher;
}
module.exports = JobQueue;
var QUEUE = {
DB: 5,
PREFIX: 'batch:queue:',
INDEX: 'batch:indexes:queue'
};
module.exports.QUEUE = QUEUE;
JobQueue.prototype.enqueue = function (user, jobId, callback) {
debug('JobQueue.enqueue user=%s, jobId=%s', user, jobId);
this.metadataBackend.redisMultiCmd(QUEUE.DB, [
[ 'LPUSH', QUEUE.PREFIX + user, jobId ],
[ 'SADD', QUEUE.INDEX, user ]
], function (err) {
if (err) {
return callback(err);
}
this.jobPublisher.publish(user);
callback();
}.bind(this));
};
JobQueue.prototype.size = function (user, callback) {
this.metadataBackend.redisCmd(QUEUE.DB, 'LLEN', [ QUEUE.PREFIX + user ], callback);
};
JobQueue.prototype.dequeue = function (user, callback) {
var dequeueScript = [
'local job_id = redis.call("RPOP", KEYS[1])',
'if redis.call("LLEN", KEYS[1]) == 0 then',
' redis.call("SREM", KEYS[2], ARGV[1])',
'end',
'return job_id'
].join('\n');
var redisParams = [
dequeueScript, //lua source code
2, // Two "keys" to pass
QUEUE.PREFIX + user, //KEYS[1], the key of the queue
QUEUE.INDEX, //KEYS[2], the key of the index
user // ARGV[1] - value of the element to remove from the index
];
this.metadataBackend.redisCmd(QUEUE.DB, 'EVAL', redisParams, function (err, jobId) {
debug('JobQueue.dequeued user=%s, jobId=%s', user, jobId);
return callback(err, jobId);
});
};
JobQueue.prototype.enqueueFirst = function (user, jobId, callback) {
debug('JobQueue.enqueueFirst user=%s, jobId=%s', user, jobId);
this.metadataBackend.redisMultiCmd(QUEUE.DB, [
[ 'RPUSH', QUEUE.PREFIX + user, jobId ],
[ 'SADD', QUEUE.INDEX, user ]
], function (err) {
if (err) {
return callback(err);
}
this.jobPublisher.publish(user);
callback();
}.bind(this));
};
JobQueue.prototype.getQueues = function (callback) {
this.metadataBackend.redisCmd(QUEUE.DB, 'SMEMBERS', [ QUEUE.INDEX ], function (err, queues) {
if (err) {
return callback(err);
}
callback(null, queues);
});
};
JobQueue.prototype.scanQueues = function (callback) {
var self = this;
self.scan(function (err, queues) {
if (err) {
return callback(err);
}
self.addToQueueIndex(queues, function (err) {
if (err) {
return callback(err);
}
callback(null, queues);
});
});
};
JobQueue.prototype.scan = function (callback) {
var self = this;
var initialCursor = ['0'];
var users = {};
self._scan(initialCursor, users, function(err, users) {
if (err) {
return callback(err);
}
callback(null, Object.keys(users));
});
};
JobQueue.prototype._scan = function (cursor, users, callback) {
var self = this;
var redisParams = [cursor[0], 'MATCH', QUEUE.PREFIX + '*'];
self.metadataBackend.redisCmd(QUEUE.DB, 'SCAN', redisParams, function (err, currentCursor) {
if (err) {
return callback(null, users);
}
var queues = currentCursor[1];
if (queues) {
queues.forEach(function (queue) {
var user = queue.substr(QUEUE.PREFIX.length);
users[user] = true;
});
}
var hasMore = currentCursor[0] !== '0';
if (!hasMore) {
return callback(null, users);
}
self._scan(currentCursor, users, callback);
});
};
JobQueue.prototype.addToQueueIndex = function (users, callback) {
var self = this;
var usersQueues = queueAsync(users.length);
users.forEach(function (user) {
usersQueues.defer(function (user, callback) {
self.metadataBackend.redisCmd(QUEUE.DB, 'SADD', [ QUEUE.INDEX, user], callback);
}, user);
});
usersQueues.awaitAll(function (err) {
if (err) {
return callback(err);
}
callback(null);
});
};

144
batch/job_runner.js Normal file
View File

@ -0,0 +1,144 @@
'use strict';
var errorCodes = require('../app/postgresql/error_codes').codeToCondition;
var jobStatus = require('./job_status');
var Profiler = require('step-profiler');
var _ = require('underscore');
var REDIS_LIMITS = {
DB: 5,
PREFIX: 'limits:batch:' // + username
};
function JobRunner(jobService, jobQueue, queryRunner, metadataBackend, statsdClient) {
this.jobService = jobService;
this.jobQueue = jobQueue;
this.queryRunner = queryRunner;
this.metadataBackend = metadataBackend;
this.statsdClient = statsdClient;
}
JobRunner.prototype.run = function (job_id, callback) {
var self = this;
var profiler = new Profiler({ statsd_client: self.statsdClient });
profiler.start('sqlapi.batch.job');
self.jobService.get(job_id, function (err, job) {
if (err) {
return callback(err);
}
self.getQueryStatementTimeout(job.data.user, function(err, timeout) {
if (err) {
return callback(err);
}
var query = job.getNextQuery();
if (_.isObject(query)) {
if (Number.isFinite(query.timeout) && query.timeout > 0) {
timeout = Math.min(timeout, query.timeout);
}
query = query.query;
}
try {
job.setStatus(jobStatus.RUNNING);
} catch (err) {
return callback(err);
}
self.jobService.save(job, function (err, job) {
if (err) {
return callback(err);
}
profiler.done('running');
self._run(job, query, timeout, profiler, callback);
});
});
});
};
JobRunner.prototype.getQueryStatementTimeout = function(username, callback) {
var timeout = 12 * 3600 * 1000;
if (Number.isFinite(global.settings.batch_query_timeout)) {
timeout = global.settings.batch_query_timeout;
}
var batchLimitsKey = REDIS_LIMITS.PREFIX + username;
this.metadataBackend.redisCmd(REDIS_LIMITS.DB, 'HGET', [batchLimitsKey, 'timeout'], function(err, timeoutLimit) {
if (timeoutLimit !== null && Number.isFinite(+timeoutLimit)) {
timeout = +timeoutLimit;
}
return callback(null, timeout);
});
};
JobRunner.prototype._run = function (job, query, timeout, profiler, callback) {
var self = this;
const dbparams = {
pass: job.data.pass,
user: job.data.dbuser,
dbname: job.data.dbname,
port: job.data.port,
host: job.data.host
};
self.queryRunner.run(job.data.job_id, query, job.data.user, timeout, dbparams, function (err /*, result */) {
if (err) {
if (!err.code) {
return callback(err);
}
// if query has been cancelled then it's going to get the current
// job status saved by query_canceller
if (cancelledByUser(err)) {
return self.jobService.get(job.data.job_id, callback);
}
}
try {
if (err) {
profiler.done('failed');
job.setStatus(jobStatus.FAILED, err.message);
} else {
profiler.done('success');
job.setStatus(jobStatus.DONE);
}
} catch (err) {
return callback(err);
}
self.jobService.save(job, function (err, job) {
if (err) {
return callback(err);
}
profiler.done('done');
profiler.end();
profiler.sendStats();
if (!job.hasNextQuery()) {
return callback(null, job);
}
self.jobQueue.enqueueFirst(job.data.user, job.data.job_id, function (err) {
if (err) {
return callback(err);
}
callback(null, job);
});
});
});
};
function cancelledByUser(err) {
return errorCodes[err.code.toString()] === 'query_canceled' && err.message.match(/user.*request/);
}
module.exports = JobRunner;

136
batch/job_service.js Normal file
View File

@ -0,0 +1,136 @@
'use strict';
var debug = require('./util/debug')('job-service');
var JobFactory = require('./models/job_factory');
var jobStatus = require('./job_status');
function JobService(jobBackend, jobCanceller) {
this.jobBackend = jobBackend;
this.jobCanceller = jobCanceller;
}
module.exports = JobService;
JobService.prototype.get = function (job_id, callback) {
this.jobBackend.get(job_id, function (err, data) {
if (err) {
return callback(err);
}
var job;
try {
job = JobFactory.create(data);
} catch (err) {
return callback(err);
}
callback(null, job);
});
};
JobService.prototype.create = function (data, callback) {
try {
var job = JobFactory.create(data);
job.validate();
this.jobBackend.create(job.data, function (err) {
if (err) {
return callback(err);
}
callback(null, job);
});
} catch (err) {
return callback(err);
}
};
JobService.prototype.save = function (job, callback) {
var self = this;
try {
job.validate();
} catch (err) {
return callback(err);
}
self.jobBackend.update(job.data, function (err, data) {
if (err) {
return callback(err);
}
try {
job = JobFactory.create(data);
} catch (err) {
return callback(err);
}
callback(null, job);
});
};
JobService.prototype.cancel = function (job_id, callback) {
var self = this;
self.get(job_id, function (err, job) {
if (err) {
return callback(err);
}
var isPending = job.isPending();
try {
job.setStatus(jobStatus.CANCELLED);
} catch (err) {
return callback(err);
}
if (isPending) {
return self.save(job, callback);
}
self.jobCanceller.cancel(job, function (err) {
if (err) {
return callback(err);
}
self.save(job, callback);
});
});
};
JobService.prototype.drain = function (job_id, callback) {
var self = this;
self.get(job_id, function (err, job) {
if (err) {
return callback(err);
}
self.jobCanceller.cancel(job, function (err) {
if (err) {
debug('There was an error while draining job %s, %s ', job_id, err);
return callback(err);
}
try {
job.setStatus(jobStatus.PENDING);
} catch (err) {
return callback(err);
}
self.jobBackend.update(job.data, callback);
});
});
};
JobService.prototype.addWorkInProgressJob = function (user, jobId, callback) {
this.jobBackend.addWorkInProgressJob(user, jobId, callback);
};
JobService.prototype.clearWorkInProgressJob = function (user, jobId, callback) {
this.jobBackend.clearWorkInProgressJob(user, jobId, callback);
};
JobService.prototype.listWorkInProgressJobs = function (callback) {
this.jobBackend.listWorkInProgressJobs(callback);
};

23
batch/job_status.js Normal file
View File

@ -0,0 +1,23 @@
'use strict';
var JOB_STATUS_ENUM = {
PENDING: 'pending',
RUNNING: 'running',
DONE: 'done',
CANCELLED: 'cancelled',
FAILED: 'failed',
SKIPPED: 'skipped',
UNKNOWN: 'unknown'
};
module.exports = JOB_STATUS_ENUM;
var finalStatus = [
JOB_STATUS_ENUM.CANCELLED,
JOB_STATUS_ENUM.DONE,
JOB_STATUS_ENUM.FAILED,
JOB_STATUS_ENUM.UNKNOWN
];
module.exports.isFinal = function(status) {
return finalStatus.indexOf(status) !== -1;
};

74
batch/leader/locker.js Normal file
View File

@ -0,0 +1,74 @@
'use strict';
var RedisDistlockLocker = require('./provider/redis-distlock');
var debug = require('../util/debug')('leader-locker');
var EventEmitter = require('events').EventEmitter;
var util = require('util');
var LOCK = {
TTL: 5000
};
function Locker(locker, ttl) {
EventEmitter.call(this);
this.locker = locker;
this.ttl = (Number.isFinite(ttl) && ttl > 0) ? ttl : LOCK.TTL;
this.renewInterval = this.ttl / 5;
this.intervalIds = {};
}
util.inherits(Locker, EventEmitter);
module.exports = Locker;
Locker.prototype.lock = function(resource, callback) {
var self = this;
debug('Locker.lock(%s, %d)', resource, this.ttl);
this.locker.lock(resource, this.ttl, function (err, lock) {
if (!err) {
self.startRenewal(resource);
}
return callback(err, lock);
});
};
Locker.prototype.unlock = function(resource, callback) {
var self = this;
debug('Locker.unlock(%s)', resource);
this.locker.unlock(resource, function(err) {
self.stopRenewal(resource);
return callback(err);
});
};
Locker.prototype.startRenewal = function(resource) {
var self = this;
if (!this.intervalIds.hasOwnProperty(resource)) {
this.intervalIds[resource] = setInterval(function() {
debug('Trying to extend lock resource=%s', resource);
self.locker.lock(resource, self.ttl, function(err, _lock) {
if (err) {
self.emit('error', err, resource);
return self.stopRenewal(resource);
}
if (_lock) {
debug('Extended lock resource=%s', resource);
}
});
}, this.renewInterval);
}
};
Locker.prototype.stopRenewal = function(resource) {
if (this.intervalIds.hasOwnProperty(resource)) {
clearInterval(this.intervalIds[resource]);
delete this.intervalIds[resource];
}
};
module.exports.create = function createLocker(type, config) {
if (type !== 'redis-distlock') {
throw new Error('Invalid type Locker type. Valid types are: "redis-distlock"');
}
var locker = new RedisDistlockLocker(config.pool);
return new Locker(locker, config.ttl);
};

111
batch/leader/provider/redis-distlock.js Normal file
View File

@ -0,0 +1,111 @@
'use strict';
var REDIS_DISTLOCK = {
DB: 5,
PREFIX: 'batch:locks:'
};
var Redlock = require('redlock');
var debug = require('../../util/debug')('leader:redis-distlock');
function RedisDistlockLocker(redisPool) {
this.pool = redisPool;
this.redlock = new Redlock([{}], {
// see http://redis.io/topics/distlock
driftFactor: 0.01, // time in ms
// the max number of times Redlock will attempt to lock a resource before failing
retryCount: 3,
// the time in ms between attempts
retryDelay: 100
});
this._locks = {};
}
module.exports = RedisDistlockLocker;
module.exports.type = 'redis-distlock';
function resourceId(resource) {
return REDIS_DISTLOCK.PREFIX + resource;
}
RedisDistlockLocker.prototype.lock = function(resource, ttl, callback) {
var self = this;
debug('RedisDistlockLocker.lock(%s, %d)', resource, ttl);
var lockId = resourceId(resource);
var lock = this._getLock(lockId);
function acquireCallback(err, _lock) {
if (err) {
return callback(err);
}
self._setLock(lockId, _lock);
return callback(null, _lock);
}
if (lock) {
return this._tryExtend(lock, ttl, function(err, _lock) {
if (err) {
return self._tryAcquire(lockId, ttl, acquireCallback);
}
return callback(null, _lock);
});
} else {
return this._tryAcquire(lockId, ttl, acquireCallback);
}
};
RedisDistlockLocker.prototype.unlock = function(resource, callback) {
var self = this;
var lock = this._getLock(resourceId(resource));
if (lock) {
this.pool.acquire(REDIS_DISTLOCK.DB, function (err, client) {
if (err) {
return callback(err);
}
self.redlock.servers = [client];
return self.redlock.unlock(lock, function(err) {
self.pool.release(REDIS_DISTLOCK.DB, client);
return callback(err);
});
});
}
};
RedisDistlockLocker.prototype._getLock = function(resource) {
if (this._locks.hasOwnProperty(resource)) {
return this._locks[resource];
}
return null;
};
RedisDistlockLocker.prototype._setLock = function(resource, lock) {
this._locks[resource] = lock;
};
RedisDistlockLocker.prototype._tryExtend = function(lock, ttl, callback) {
var self = this;
this.pool.acquire(REDIS_DISTLOCK.DB, function (err, client) {
if (err) {
return callback(err);
}
self.redlock.servers = [client];
return lock.extend(ttl, function(err, _lock) {
self.pool.release(REDIS_DISTLOCK.DB, client);
return callback(err, _lock);
});
});
};
RedisDistlockLocker.prototype._tryAcquire = function(resource, ttl, callback) {
var self = this;
this.pool.acquire(REDIS_DISTLOCK.DB, function (err, client) {
if (err) {
return callback(err);
}
self.redlock.servers = [client];
return self.redlock.lock(resource, ttl, function(err, _lock) {
self.pool.release(REDIS_DISTLOCK.DB, client);
return callback(err, _lock);
});
});
};

144
batch/maintenance/host-user-queue-mover.js Normal file
View File

@ -0,0 +1,144 @@
'use strict';
var asyncQ = require('queue-async');
var debug = require('../util/debug')('queue-mover');
var forever = require('../util/forever');
var QUEUE = {
OLD: {
DB: 5,
PREFIX: 'batch:queues:' // host
},
NEW: {
DB: 5,
PREFIX: 'batch:queue:' // user
}
};
function HostUserQueueMover(jobQueue, jobService, locker, redisPool) {
this.jobQueue = jobQueue;
this.jobService = jobService;
this.locker = locker;
this.pool = redisPool;
}
module.exports = HostUserQueueMover;
HostUserQueueMover.prototype.moveOldJobs = function(callback) {
var self = this;
this.getOldQueues(function(err, hosts) {
var async = asyncQ(4);
hosts.forEach(function(host) {
async.defer(self.moveOldQueueJobs.bind(self), host);
});
async.awaitAll(function (err) {
if (err) {
debug('Something went wrong moving jobs', err);
} else {
debug('Finished moving all jobs');
}
callback();
});
});
};
HostUserQueueMover.prototype.moveOldQueueJobs = function(host, callback) {
var self = this;
// do forever, it does not throw a stack overflow
forever(
function (next) {
self.locker.lock(host, function(err) {
// we didn't get the lock for the host
if (err) {
debug('Could not lock host=%s. Reason: %s', host, err.message);
return next(err);
}
debug('Locked host=%s', host);
self.processNextJob(host, next);
});
},
function (err) {
if (err) {
debug(err.name === 'EmptyQueue' ? err.message : err);
}
self.locker.unlock(host, callback);
}
);
};
//this.metadataBackend.redisCmd(QUEUE.DB, 'RPOP', [ QUEUE.PREFIX + user ], callback);
HostUserQueueMover.prototype.processNextJob = function (host, callback) {
var self = this;
this.pool.acquire(QUEUE.OLD.DB, function(err, client) {
if (err) {
return callback(err);
}
client.lpop(QUEUE.OLD.PREFIX + host, function(err, jobId) {
self.pool.release(QUEUE.OLD.DB, client);
debug('Found jobId=%s at queue=%s', jobId, host);
if (!jobId) {
var emptyQueueError = new Error('Empty queue');
emptyQueueError.name = 'EmptyQueue';
return callback(emptyQueueError);
}
self.jobService.get(jobId, function(err, job) {
if (err) {
debug(err);
return callback();
}
if (job) {
return self.jobQueue.enqueueFirst(job.data.user, jobId, function() {
return callback();
});
}
return callback();
});
});
});
};
HostUserQueueMover.prototype.getOldQueues = function(callback) {
var initialCursor = ['0'];
var hosts = {};
var self = this;
this.pool.acquire(QUEUE.OLD.DB, function(err, client) {
if (err) {
return callback(err);
}
self._getOldQueues(client, initialCursor, hosts, function(err, hosts) {
self.pool.release(QUEUE.DB, client);
return callback(err, Object.keys(hosts));
});
});
};
HostUserQueueMover.prototype._getOldQueues = function (client, cursor, hosts, callback) {
var self = this;
var redisParams = [cursor[0], 'MATCH', QUEUE.OLD.PREFIX + '*'];
client.scan(redisParams, function(err, currentCursor) {
if (err) {
return callback(null, hosts);
}
var queues = currentCursor[1];
if (queues) {
queues.forEach(function (queue) {
var user = queue.substr(QUEUE.OLD.PREFIX.length);
hosts[user] = true;
});
}
var hasMore = currentCursor[0] !== '0';
if (!hasMore) {
return callback(null, hosts);
}
self._getOldQueues(client, currentCursor, hosts, callback);
});
};

122
batch/models/job_base.js Normal file
View File

@ -0,0 +1,122 @@
'use strict';
var util = require('util');
var uuid = require('node-uuid');
var JobStateMachine = require('./job_state_machine');
var jobStatus = require('../job_status');
var mandatoryProperties = [
'job_id',
'status',
'query',
'created_at',
'updated_at',
'host',
'user'
];
function JobBase(data) {
JobStateMachine.call(this);
var now = new Date().toISOString();
this.data = data;
if (!this.data.job_id) {
this.data.job_id = uuid.v4();
}
if (!this.data.created_at) {
this.data.created_at = now;
}
if (!this.data.updated_at) {
this.data.updated_at = now;
}
}
util.inherits(JobBase, JobStateMachine);
module.exports = JobBase;
// should be implemented by childs
JobBase.prototype.getNextQuery = function () {
throw new Error('Unimplemented method');
};
JobBase.prototype.hasNextQuery = function () {
return !!this.getNextQuery();
};
JobBase.prototype.isPending = function () {
return this.data.status === jobStatus.PENDING;
};
JobBase.prototype.isRunning = function () {
return this.data.status === jobStatus.RUNNING;
};
JobBase.prototype.isDone = function () {
return this.data.status === jobStatus.DONE;
};
JobBase.prototype.isCancelled = function () {
return this.data.status === jobStatus.CANCELLED;
};
JobBase.prototype.isFailed = function () {
return this.data.status === jobStatus.FAILED;
};
JobBase.prototype.isUnknown = function () {
return this.data.status === jobStatus.UNKNOWN;
};
JobBase.prototype.setQuery = function (query) {
var now = new Date().toISOString();
if (!this.isPending()) {
throw new Error('Job is not pending, it cannot be updated');
}
this.data.updated_at = now;
this.data.query = query;
};
JobBase.prototype.setStatus = function (finalStatus, errorMesssage) {
var now = new Date().toISOString();
var initialStatus = this.data.status;
var isValid = this.isValidTransition(initialStatus, finalStatus);
if (!isValid) {
throw new Error('Cannot set status from ' + initialStatus + ' to ' + finalStatus);
}
this.data.updated_at = now;
this.data.status = finalStatus;
if (finalStatus === jobStatus.FAILED && errorMesssage) {
this.data.failed_reason = errorMesssage;
}
};
JobBase.prototype.validate = function () {
for (var i = 0; i < mandatoryProperties.length; i++) {
if (!this.data[mandatoryProperties[i]]) {
throw new Error('property "' + mandatoryProperties[i] + '" is mandatory');
}
}
};
JobBase.prototype.serialize = function () {
var data = JSON.parse(JSON.stringify(this.data));
delete data.host;
delete data.dbuser;
delete data.port;
delete data.dbname;
delete data.pass;
return data;
};
JobBase.prototype.log = function(/*logger*/) {
return false;
};

26
batch/models/job_factory.js Normal file
View File

@ -0,0 +1,26 @@
'use strict';
var JobSimple = require('./job_simple');
var JobMultiple = require('./job_multiple');
var JobFallback = require('./job_fallback');
var Models = [ JobSimple, JobMultiple, JobFallback ];
function JobFactory() {
}
module.exports = JobFactory;
JobFactory.create = function (data) {
if (!data.query) {
throw new Error('You must indicate a valid SQL');
}
for (var i = 0; i < Models.length; i++) {
if (Models[i].is(data.query)) {
return new Models[i](data);
}
}
throw new Error('there is no job class for the provided query');
};

279
batch/models/job_fallback.js Normal file
View File

@ -0,0 +1,279 @@
'use strict';
var util = require('util');
var JobBase = require('./job_base');
var JobStatus = require('../job_status');
var QueryFallback = require('./query/query_fallback');
var MainFallback = require('./query/main_fallback');
var QueryFactory = require('./query/query_factory');
function JobFallback(jobDefinition) {
JobBase.call(this, jobDefinition);
this.init();
this.queries = [];
for (var i = 0; i < this.data.query.query.length; i++) {
this.queries[i] = QueryFactory.create(this.data, i);
}
if (MainFallback.is(this.data)) {
this.fallback = new MainFallback();
}
}
util.inherits(JobFallback, JobBase);
module.exports = JobFallback;
// 1. from user: {
// query: {
// query: [{
// query: 'select ...',
// onsuccess: 'select ..'
// }],
// onerror: 'select ...'
// }
// }
//
// 2. from redis: {
// status: 'pending',
// fallback_status: 'pending'
// query: {
// query: [{
// query: 'select ...',
// onsuccess: 'select ..'
// status: 'pending',
// fallback_status: 'pending',
// }],
// onerror: 'select ...'
// }
// }
JobFallback.is = function (query) {
if (!query.query) {
return false;
}
if (!Array.isArray(query.query)) {
return false;
}
for (var i = 0; i < query.query.length; i++) {
if (!QueryFallback.is(query.query[i])) {
return false;
}
}
return true;
};
JobFallback.prototype.init = function () {
for (var i = 0; i < this.data.query.query.length; i++) {
if (shouldInitStatus(this.data.query.query[i])){
this.data.query.query[i].status = JobStatus.PENDING;
}
if (shouldInitQueryFallbackStatus(this.data.query.query[i])) {
this.data.query.query[i].fallback_status = JobStatus.PENDING;
}
}
if (shouldInitStatus(this.data)) {
this.data.status = JobStatus.PENDING;
}
if (shouldInitFallbackStatus(this.data)) {
this.data.fallback_status = JobStatus.PENDING;
}
};
function shouldInitStatus(jobOrQuery) {
return !jobOrQuery.status;
}
function shouldInitQueryFallbackStatus(query) {
return (query.onsuccess || query.onerror) && !query.fallback_status;
}
function shouldInitFallbackStatus(job) {
return (job.query.onsuccess || job.query.onerror) && !job.fallback_status;
}
JobFallback.prototype.getNextQueryFromQueries = function () {
for (var i = 0; i < this.queries.length; i++) {
if (this.queries[i].hasNextQuery(this.data)) {
return this.queries[i].getNextQuery(this.data);
}
}
};
JobFallback.prototype.hasNextQueryFromQueries = function () {
return !!this.getNextQueryFromQueries();
};
JobFallback.prototype.getNextQueryFromFallback = function () {
if (this.fallback && this.fallback.hasNextQuery(this.data)) {
return this.fallback.getNextQuery(this.data);
}
};
JobFallback.prototype.getNextQuery = function () {
var query = this.getNextQueryFromQueries();
if (!query) {
query = this.getNextQueryFromFallback();
}
return query;
};
JobFallback.prototype.setQuery = function (query) {
if (!JobFallback.is(query)) {
throw new Error('You must indicate a valid SQL');
}
JobFallback.super_.prototype.setQuery.call(this, query);
};
JobFallback.prototype.setStatus = function (status, errorMesssage) {
var now = new Date().toISOString();
var hasChanged = this.setQueryStatus(status, this.data, errorMesssage);
hasChanged = this.setJobStatus(status, this.data, hasChanged, errorMesssage);
hasChanged = this.setFallbackStatus(status, this.data, hasChanged);
if (!hasChanged.isValid) {
throw new Error('Cannot set status to ' + status);
}
this.data.updated_at = now;
};
JobFallback.prototype.setQueryStatus = function (status, job, errorMesssage) {
return this.queries.reduce(function (hasChanged, query) {
var result = query.setStatus(status, this.data, hasChanged, errorMesssage);
return result.isValid ? result : hasChanged;
}.bind(this), { isValid: false, appliedToFallback: false });
};
JobFallback.prototype.setJobStatus = function (status, job, hasChanged, errorMesssage) {
var result = {
isValid: false,
appliedToFallback: false
};
status = this.shiftStatus(status, hasChanged);
result.isValid = this.isValidTransition(job.status, status);
if (result.isValid) {
job.status = status;
if (status === JobStatus.FAILED && errorMesssage && !hasChanged.appliedToFallback) {
job.failed_reason = errorMesssage;
}
}
return result.isValid ? result : hasChanged;
};
JobFallback.prototype.setFallbackStatus = function (status, job, hasChanged) {
var result = hasChanged;
if (this.fallback && !this.hasNextQueryFromQueries()) {
result = this.fallback.setStatus(status, job, hasChanged);
}
return result.isValid ? result : hasChanged;
};
JobFallback.prototype.shiftStatus = function (status, hasChanged) {
// jshint maxcomplexity: 7
if (hasChanged.appliedToFallback) {
if (!this.hasNextQueryFromQueries() && (status === JobStatus.DONE || status === JobStatus.FAILED)) {
status = this.getLastFinishedStatus();
} else if (status === JobStatus.DONE || status === JobStatus.FAILED){
status = JobStatus.PENDING;
}
} else if (this.hasNextQueryFromQueries() && status !== JobStatus.RUNNING) {
status = JobStatus.PENDING;
}
return status;
};
JobFallback.prototype.getLastFinishedStatus = function () {
return this.queries.reduce(function (lastFinished, query) {
var status = query.getStatus(this.data);
return this.isFinalStatus(status) ? status : lastFinished;
}.bind(this), JobStatus.DONE);
};
JobFallback.prototype.log = function(logger) {
if (!isFinished(this)) {
return false;
}
var queries = this.data.query.query;
for (var i = 0; i < queries.length; i++) {
var query = queries[i];
var logEntry = {
created: this.data.created_at,
waiting: elapsedTime(this.data.created_at, query.started_at),
time: query.started_at,
endtime: query.ended_at,
username: this.data.user,
dbhost: this.data.host,
job: this.data.job_id,
status: query.status,
elapsed: elapsedTime(query.started_at, query.ended_at)
};
var queryId = query.id;
var tag = 'query';
if (queryId) {
logEntry.query_id = queryId;
var node = parseQueryId(queryId);
if (node) {
logEntry.analysis = node.analysisId;
logEntry.node = node.nodeId;
logEntry.type = node.nodeType;
tag = 'analysis';
}
}
logger.info(logEntry, tag);
}
return true;
};
function isFinished (job) {
return JobStatus.isFinal(job.data.status) &&
(!job.data.fallback_status || JobStatus.isFinal(job.data.fallback_status));
}
function parseQueryId (queryId) {
var data = queryId.split(':');
if (data.length === 3) {
return {
analysisId: data[0],
nodeId: data[1],
nodeType: data[2]
};
}
return null;
}
function elapsedTime (started_at, ended_at) {
if (!started_at || !ended_at) {
return;
}
var start = new Date(started_at);
var end = new Date(ended_at);
return end.getTime() - start.getTime();
}

91
batch/models/job_multiple.js Normal file
View File

@ -0,0 +1,91 @@
'use strict';
var util = require('util');
var JobBase = require('./job_base');
var jobStatus = require('../job_status');
function JobMultiple(jobDefinition) {
JobBase.call(this, jobDefinition);
this.init();
}
util.inherits(JobMultiple, JobBase);
module.exports = JobMultiple;
JobMultiple.is = function (query) {
if (!Array.isArray(query)) {
return false;
}
// 1. From user: ['select * from ...', 'select * from ...']
// 2. From redis: [ { query: 'select * from ...', status: 'pending' },
// { query: 'select * from ...', status: 'pending' } ]
for (var i = 0; i < query.length; i++) {
if (typeof query[i] !== 'string') {
if (typeof query[i].query !== 'string') {
return false;
}
}
}
return true;
};
JobMultiple.prototype.init = function () {
if (!this.data.status) {
this.data.status = jobStatus.PENDING;
}
for (var i = 0; i < this.data.query.length; i++) {
if (!this.data.query[i].query && !this.data.query[i].status) {
this.data.query[i] = {
query: this.data.query[i],
status: jobStatus.PENDING
};
}
}
};
JobMultiple.prototype.getNextQuery = function () {
for (var i = 0; i < this.data.query.length; i++) {
if (this.data.query[i].status === jobStatus.PENDING) {
return this.data.query[i].query;
}
}
};
JobMultiple.prototype.setQuery = function (query) {
if (!JobMultiple.is(query)) {
throw new Error('You must indicate a valid SQL');
}
JobMultiple.super_.prototype.setQuery.call(this, query);
};
JobMultiple.prototype.setStatus = function (finalStatus, errorMesssage) {
var initialStatus = this.data.status;
// if transition is to "done" and there are more queries to run
// then job status must be "pending" instead of "done"
// else job status transition to done (if "running")
if (finalStatus === jobStatus.DONE && this.hasNextQuery()) {
JobMultiple.super_.prototype.setStatus.call(this, jobStatus.PENDING);
} else {
JobMultiple.super_.prototype.setStatus.call(this, finalStatus, errorMesssage);
}
for (var i = 0; i < this.data.query.length; i++) {
var isValid = JobMultiple.super_.prototype.isValidTransition(this.data.query[i].status, finalStatus);
if (isValid) {
this.data.query[i].status = finalStatus;
if (finalStatus === jobStatus.FAILED && errorMesssage) {
this.data.query[i].failed_reason = errorMesssage;
}
return;
}
}
throw new Error('Cannot set status from ' + initialStatus + ' to ' + finalStatus);
};

34
batch/models/job_simple.js Normal file
View File

@ -0,0 +1,34 @@
'use strict';
var util = require('util');
var JobBase = require('./job_base');
var jobStatus = require('../job_status');
function JobSimple(jobDefinition) {
JobBase.call(this, jobDefinition);
if (!this.data.status) {
this.data.status = jobStatus.PENDING;
}
}
util.inherits(JobSimple, JobBase);
module.exports = JobSimple;
JobSimple.is = function (query) {
return typeof query === 'string';
};
JobSimple.prototype.getNextQuery = function () {
if (this.isPending()) {
return this.data.query;
}
};
JobSimple.prototype.setQuery = function (query) {
if (!JobSimple.is(query)) {
throw new Error('You must indicate a valid SQL');
}
JobSimple.super_.prototype.setQuery.call(this, query);
};

39
batch/models/job_state_machine.js Normal file
View File

@ -0,0 +1,39 @@
'use strict';
var assert = require('assert');
var JobStatus = require('../job_status');
var validStatusTransitions = [
[JobStatus.PENDING, JobStatus.RUNNING],
[JobStatus.PENDING, JobStatus.CANCELLED],
[JobStatus.PENDING, JobStatus.UNKNOWN],
[JobStatus.PENDING, JobStatus.SKIPPED],
[JobStatus.RUNNING, JobStatus.DONE],
[JobStatus.RUNNING, JobStatus.FAILED],
[JobStatus.RUNNING, JobStatus.CANCELLED],
[JobStatus.RUNNING, JobStatus.PENDING],
[JobStatus.RUNNING, JobStatus.UNKNOWN]
];
function JobStateMachine () {
}
module.exports = JobStateMachine;
JobStateMachine.prototype.isValidTransition = function (initialStatus, finalStatus) {
var transition = [ initialStatus, finalStatus ];
for (var i = 0; i < validStatusTransitions.length; i++) {
try {
assert.deepEqual(transition, validStatusTransitions[i]);
return true;
} catch (e) {
continue;
}
}
return false;
};
JobStateMachine.prototype.isFinalStatus = function (status) {
return JobStatus.isFinal(status);
};

78
batch/models/query/fallback.js Normal file
View File

@ -0,0 +1,78 @@
'use strict';
var util = require('util');
var QueryBase = require('./query_base');
var jobStatus = require('../../job_status');
function Fallback(index) {
QueryBase.call(this, index);
}
util.inherits(Fallback, QueryBase);
module.exports = Fallback;
Fallback.is = function (query) {
if (query.onsuccess || query.onerror) {
return true;
}
return false;
};
Fallback.prototype.getNextQuery = function (job) {
if (this.hasOnSuccess(job)) {
return this.getOnSuccess(job);
}
if (this.hasOnError(job)) {
return this.getOnError(job);
}
};
Fallback.prototype.getOnSuccess = function (job) {
if (job.query.query[this.index].status === jobStatus.DONE &&
job.query.query[this.index].fallback_status === jobStatus.PENDING) {
var onsuccessQuery = job.query.query[this.index].onsuccess;
if (onsuccessQuery) {
onsuccessQuery = onsuccessQuery.replace(/<%=\s*job_id\s*%>/g, job.job_id);
}
return onsuccessQuery;
}
};
Fallback.prototype.hasOnSuccess = function (job) {
return !!this.getOnSuccess(job);
};
Fallback.prototype.getOnError = function (job) {
if (job.query.query[this.index].status === jobStatus.FAILED &&
job.query.query[this.index].fallback_status === jobStatus.PENDING) {
var onerrorQuery = job.query.query[this.index].onerror;
if (onerrorQuery) {
onerrorQuery = onerrorQuery.replace(/<%=\s*job_id\s*%>/g, job.job_id);
onerrorQuery = onerrorQuery.replace(/<%=\s*error_message\s*%>/g, job.query.query[this.index].failed_reason);
}
return onerrorQuery;
}
};
Fallback.prototype.hasOnError = function (job) {
return !!this.getOnError(job);
};
Fallback.prototype.setStatus = function (status, job, errorMessage) {
var isValid = false;
isValid = this.isValidTransition(job.query.query[this.index].fallback_status, status);
if (isValid) {
job.query.query[this.index].fallback_status = status;
if (status === jobStatus.FAILED && errorMessage) {
job.query.query[this.index].failed_reason = errorMessage;
}
}
return isValid;
};
Fallback.prototype.getStatus = function (job) {
return job.query.query[this.index].fallback_status;
};

74
batch/models/query/main_fallback.js Normal file
View File

@ -0,0 +1,74 @@
'use strict';
var util = require('util');
var QueryBase = require('./query_base');
var jobStatus = require('../../job_status');
function MainFallback() {
QueryBase.call(this);
}
util.inherits(MainFallback, QueryBase);
module.exports = MainFallback;
MainFallback.is = function (job) {
if (job.query.onsuccess || job.query.onerror) {
return true;
}
return false;
};
MainFallback.prototype.getNextQuery = function (job) {
if (this.hasOnSuccess(job)) {
return this.getOnSuccess(job);
}
if (this.hasOnError(job)) {
return this.getOnError(job);
}
};
MainFallback.prototype.getOnSuccess = function (job) {
if (job.status === jobStatus.DONE && job.fallback_status === jobStatus.PENDING) {
return job.query.onsuccess;
}
};
MainFallback.prototype.hasOnSuccess = function (job) {
return !!this.getOnSuccess(job);
};
MainFallback.prototype.getOnError = function (job) {
if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.PENDING) {
return job.query.onerror;
}
};
MainFallback.prototype.hasOnError = function (job) {
return !!this.getOnError(job);
};
MainFallback.prototype.setStatus = function (status, job, previous) {
var isValid = false;
var appliedToFallback = false;
if (previous.isValid && !previous.appliedToFallback) {
if (this.isFinalStatus(status) && !this.hasNextQuery(job)) {
isValid = this.isValidTransition(job.fallback_status, jobStatus.SKIPPED);
if (isValid) {
job.fallback_status = jobStatus.SKIPPED;
appliedToFallback = true;
}
}
} else if (!previous.isValid) {
isValid = this.isValidTransition(job.fallback_status, status);
if (isValid) {
job.fallback_status = status;
appliedToFallback = true;
}
}
return { isValid: isValid, appliedToFallback: appliedToFallback };
};

57
batch/models/query/query.js Normal file
View File

@ -0,0 +1,57 @@
'use strict';
var util = require('util');
var QueryBase = require('./query_base');
var jobStatus = require('../../job_status');
function Query(index) {
QueryBase.call(this, index);
}
util.inherits(Query, QueryBase);
module.exports = Query;
Query.is = function (query) {
if (query.query && typeof query.query === 'string') {
return true;
}
return false;
};
Query.prototype.getNextQuery = function (job) {
if (job.query.query[this.index].status === jobStatus.PENDING) {
var query = {
query: job.query.query[this.index].query
};
if (Number.isFinite(job.query.query[this.index].timeout)) {
query.timeout = job.query.query[this.index].timeout;
}
return query;
}
};
Query.prototype.setStatus = function (status, job, errorMesssage) {
var isValid = false;
isValid = this.isValidTransition(job.query.query[this.index].status, status);
if (isValid) {
job.query.query[this.index].status = status;
if (status === jobStatus.RUNNING) {
job.query.query[this.index].started_at = new Date().toISOString();
}
if (this.isFinalStatus(status)) {
job.query.query[this.index].ended_at = new Date().toISOString();
}
if (status === jobStatus.FAILED && errorMesssage) {
job.query.query[this.index].failed_reason = errorMesssage;
}
}
return isValid;
};
Query.prototype.getStatus = function (job) {
return job.query.query[this.index].status;
};

31
batch/models/query/query_base.js Normal file
View File

@ -0,0 +1,31 @@
'use strict';
var util = require('util');
var JobStateMachine = require('../job_state_machine');
function QueryBase(index) {
JobStateMachine.call(this);
this.index = index;
}
util.inherits(QueryBase, JobStateMachine);
module.exports = QueryBase;
// should be implemented
QueryBase.prototype.setStatus = function () {
throw new Error('Unimplemented method');
};
// should be implemented
QueryBase.prototype.getNextQuery = function () {
throw new Error('Unimplemented method');
};
QueryBase.prototype.hasNextQuery = function (job) {
return !!this.getNextQuery(job);
};
QueryBase.prototype.getStatus = function () {
throw new Error('Unimplemented method');
};

16
batch/models/query/query_factory.js Normal file
View File

@ -0,0 +1,16 @@
'use strict';
var QueryFallback = require('./query_fallback');
function QueryFactory() {
}
module.exports = QueryFactory;
QueryFactory.create = function (job, index) {
if (QueryFallback.is(job.query.query[index])) {
return new QueryFallback(job, index);
}
throw new Error('there is no query class for the provided query');
};

75
batch/models/query/query_fallback.js Normal file
View File

@ -0,0 +1,75 @@
'use strict';
var util = require('util');
var QueryBase = require('./query_base');
var Query = require('./query');
var Fallback = require('./fallback');
var jobStatus = require('../../job_status');
function QueryFallback(job, index) {
QueryBase.call(this, index);
this.init(job, index);
}
util.inherits(QueryFallback, QueryBase);
QueryFallback.is = function (query) {
if (Query.is(query)) {
return true;
}
return false;
};
QueryFallback.prototype.init = function (job, index) {
this.query = new Query(index);
if (Fallback.is(job.query.query[index])) {
this.fallback = new Fallback(index);
}
};
QueryFallback.prototype.getNextQuery = function (job) {
if (this.query.hasNextQuery(job)) {
return this.query.getNextQuery(job);
}
if (this.fallback && this.fallback.hasNextQuery(job)) {
return this.fallback.getNextQuery(job);
}
};
QueryFallback.prototype.setStatus = function (status, job, previous, errorMesssage) {
// jshint maxcomplexity: 9
var isValid = false;
var appliedToFallback = false;
if (previous.isValid && !previous.appliedToFallback) {
if (status === jobStatus.FAILED || status === jobStatus.CANCELLED) {
this.query.setStatus(jobStatus.SKIPPED, job, errorMesssage);
if (this.fallback) {
this.fallback.setStatus(jobStatus.SKIPPED, job);
}
}
} else if (!previous.isValid) {
isValid = this.query.setStatus(status, job, errorMesssage);
if (this.fallback) {
if (!isValid) {
isValid = this.fallback.setStatus(status, job, errorMesssage);
appliedToFallback = true;
} else if (isValid && this.isFinalStatus(status) && !this.fallback.hasNextQuery(job)) {
this.fallback.setStatus(jobStatus.SKIPPED, job);
}
}
}
return { isValid: isValid, appliedToFallback: appliedToFallback };
};
QueryFallback.prototype.getStatus = function (job) {
return this.query.getStatus(job);
};
module.exports = QueryFallback;

4
batch/pubsub/channel.js Normal file
View File

@ -0,0 +1,4 @@
module.exports = {
DB: 0,
NAME: 'batch:users'
};

31
batch/pubsub/job-publisher.js Normal file
View File

@ -0,0 +1,31 @@
'use strict';
var Channel = require('./channel');
var debug = require('./../util/debug')('pubsub:publisher');
var error = require('./../util/debug')('pubsub:publisher:error');
function JobPublisher(pool) {
this.pool = pool;
}
JobPublisher.prototype.publish = function (user) {
var self = this;
this.pool.acquire(Channel.DB, function (err, client) {
if (err) {
return error('Error adquiring redis client: ' + err.message);
}
client.publish(Channel.NAME, user, function (err) {
self.pool.release(Channel.DB, client);
if (err) {
return error('Error publishing to ' + Channel.NAME + ':' + user + ', ' + err.message);
}
debug('publish to ' + Channel.NAME + ':' + user);
});
});
};
module.exports = JobPublisher;

54
batch/pubsub/job-subscriber.js Normal file
View File

@ -0,0 +1,54 @@
'use strict';
var Channel = require('./channel');
var debug = require('./../util/debug')('pubsub:subscriber');
var error = require('./../util/debug')('pubsub:subscriber:error');
function JobSubscriber(pool) {
this.pool = pool;
}
module.exports = JobSubscriber;
JobSubscriber.prototype.subscribe = function (onJobHandler, callback) {
var self = this;
self.pool.acquire(Channel.DB, function(err, client) {
if (err) {
if (callback) {
callback(err);
}
return error('Error adquiring redis client: ' + err.message);
}
self.client = client;
client.removeAllListeners('message');
client.unsubscribe(Channel.NAME);
client.subscribe(Channel.NAME);
client.on('message', function (channel, user) {
debug('message received in channel=%s from user=%s', channel, user);
onJobHandler(user);
});
client.on('error', function () {
self.unsubscribe();
self.pool.release(Channel.DB, client);
self.subscribe(onJobHandler);
});
if (callback) {
callback();
}
});
};
JobSubscriber.prototype.unsubscribe = function (callback) {
if (this.client && this.client.connected) {
this.client.unsubscribe(Channel.NAME, callback);
} else {
if (callback) {
return callback(null);
}
}
};

54
batch/query_runner.js Normal file
View File

@ -0,0 +1,54 @@
'use strict';
var PSQL = require('cartodb-psql');
var debug = require('./util/debug')('query-runner');
function QueryRunner(userDatabaseMetadataService) {
this.userDatabaseMetadataService = userDatabaseMetadataService;
}
module.exports = QueryRunner;
function hasDBParams (dbparams) {
return (dbparams.user && dbparams.host && dbparams.port && dbparams.dbname && dbparams.pass);
}
QueryRunner.prototype.run = function (job_id, sql, user, timeout, dbparams, callback) {
if (hasDBParams(dbparams)) {
return this._run(dbparams, job_id, sql, timeout, callback);
}
const dbConfigurationError = new Error('Batch Job DB misconfiguration');
return callback(dbConfigurationError);
};
QueryRunner.prototype._run = function (dbparams, job_id, sql, timeout, callback) {
var pg = new PSQL(dbparams);
pg.query('SET statement_timeout=' + timeout, function (err) {
if(err) {
return callback(err);
}
// mark query to allow to users cancel their queries
sql = '/* ' + job_id + ' */ ' + sql;
debug('Running query [timeout=%d] %s', timeout, sql);
pg.eventedQuery(sql, function (err, query) {
if (err) {
return callback(err);
}
query.on('error', callback);
query.on('end', function (result) {
// only if result is present then query is done sucessfully otherwise an error has happened
// and it was handled by error listener
if (result) {
callback(null, result);
}
});
});
});
};

11
batch/scheduler/capacity/fixed.js Normal file
View File

@ -0,0 +1,11 @@
'use strict';
function FixedCapacity(capacity) {
this.capacity = Math.max(1, capacity);
}
module.exports = FixedCapacity;
FixedCapacity.prototype.getCapacity = function(callback) {
return callback(null, this.capacity);
};

Some files were not shown because too many files have changed in this diff Show More