Merge branch 'master' of github.com:CartoDB/CartoDB-SQL-API
This commit is contained in:
Daniel García Aubert
commit
bf0a030100
4
.gitignore
vendored
4
.gitignore
vendored
@ -1,5 +1,5 @@
|
||||
config/environments/*.js
|
||||
logs/*.log
|
||||
logs
|
||||
pids/*.pid
|
||||
*.sock
|
||||
test/tmp/*
|
||||
@ -12,4 +12,4 @@ test/acceptance/oauth/venv/*
|
||||
coverage/
|
||||
npm-debug.log
|
||||
log/*.log
|
||||
yarn.lock
|
||||
yarn.lock
|
||||
|
||||
1
NEWS.md
1
NEWS.md
@ -5,6 +5,7 @@ Released 2019-mm-dd
|
||||
|
||||
Breaking changes:
|
||||
- Removed "base_url" definition in configuration files. Added new property "routes". See [`cec1c60`](https://github.com/CartoDB/CartoDB-SQL-API/commit/cec1c609c04d134f45ee74a81f82c07a19bf11bd) for more details.
|
||||
- Remove development client and munin tool, both abandoned years ago.
|
||||
|
||||
Announcements:
|
||||
|
||||
|
||||
@ -1,104 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
function ArrayBufferSer(arrayBuffer) {
|
||||
this.buffer = arrayBuffer;
|
||||
this.offset = 0;
|
||||
this.sections = this.readArray();
|
||||
this.headers = this.sections[0];
|
||||
this._headersIndex = {};
|
||||
for(var i = 0; i < this.headers.length; ++i) {
|
||||
this._headersIndex[this.headers[i]] = i;
|
||||
}
|
||||
if(this.sections.length > 1) {
|
||||
this.length = this.sections[1].length;
|
||||
}
|
||||
}
|
||||
|
||||
ArrayBufferSer.INT8 = 1;
|
||||
ArrayBufferSer.UINT8 = 2;
|
||||
ArrayBufferSer.UINT8_CLAMP = 3;
|
||||
ArrayBufferSer.INT16 = 4;
|
||||
ArrayBufferSer.UINT16 = 5;
|
||||
ArrayBufferSer.INT32 = 6;
|
||||
ArrayBufferSer.UINT32 = 7;
|
||||
ArrayBufferSer.FLOAT32 = 8;
|
||||
//ArrayBufferSer.FLOAT64 = 9; not supported
|
||||
ArrayBufferSer.STRING = 10;
|
||||
ArrayBufferSer.BUFFER = 11;
|
||||
|
||||
|
||||
ArrayBufferSer.prototype = {
|
||||
|
||||
sizes: [NaN, 1, 1, 1, 2, 2, 4, 4, 4, 8],
|
||||
|
||||
types: [
|
||||
null,
|
||||
Int8Array,
|
||||
Uint8Array,
|
||||
Uint8ClampedArray,
|
||||
Int16Array,
|
||||
Uint16Array,
|
||||
Int32Array,
|
||||
Uint32Array,
|
||||
Float32Array,
|
||||
Float64Array
|
||||
],
|
||||
|
||||
get: function(columnName) {
|
||||
var i = this._headersIndex[columnName]
|
||||
if(i != undefined) {
|
||||
return this.sections[i + 1]
|
||||
}
|
||||
return;
|
||||
},
|
||||
|
||||
_paddingFor: function(offset, type) {
|
||||
var s = this.sizes[type]
|
||||
if(s) {
|
||||
var r = offset % s;
|
||||
return r == 0 ? 0 : s - r;
|
||||
}
|
||||
return 0;
|
||||
},
|
||||
|
||||
readUInt32: function() {
|
||||
var i = new DataView(this.buffer).getUint32(this.offset);
|
||||
this.offset += 4;
|
||||
return i
|
||||
},
|
||||
|
||||
readArray: function() {
|
||||
var type = this.readUInt32();
|
||||
var size = this.readUInt32();
|
||||
if(type < ArrayBufferSer.STRING) {
|
||||
var a = new this.types[type](this.buffer, this.offset, size/this.sizes[type]);
|
||||
this.offset += size;
|
||||
return a;
|
||||
} else if(type == ArrayBufferSer.STRING) {
|
||||
var target = this.offset + size;
|
||||
var b = [];
|
||||
while(this.offset < target) {
|
||||
this.offset += this._paddingFor(this.offset, ArrayBufferSer.INT32);
|
||||
var arr = this.readArray();
|
||||
if(arr) {
|
||||
var str = '';
|
||||
for(var i = 0; i < arr.length; ++i) {
|
||||
str += String.fromCharCode(arr[i]);
|
||||
}
|
||||
b.push(str);
|
||||
}
|
||||
// parse sttring
|
||||
}
|
||||
return b;
|
||||
} else if(type == ArrayBufferSer.BUFFER) {
|
||||
var b = [];
|
||||
var target = this.offset + size;
|
||||
while(this.offset < target) {
|
||||
this.offset += this._paddingFor(this.offset, ArrayBufferSer.INT32);
|
||||
b.push(this.readArray());
|
||||
}
|
||||
return b;
|
||||
}
|
||||
}
|
||||
|
||||
};
|
||||
@ -1,32 +0,0 @@
|
||||
|
||||
<html>
|
||||
<body>
|
||||
<script src="bin_decoder.js"></script>
|
||||
<script>
|
||||
function get(url, callback) {
|
||||
var req = new XMLHttpRequest();
|
||||
req.onreadystatechange = function() {
|
||||
if (req.readyState == 4){
|
||||
if (req.status == 200){
|
||||
callback(req);
|
||||
} else {
|
||||
callback(null);
|
||||
}
|
||||
}
|
||||
};
|
||||
req.open("GET", url, true)
|
||||
req.responseType = 'arraybuffer';
|
||||
req.send(null)
|
||||
return req;
|
||||
}
|
||||
|
||||
var q = 'select%20owner,%20array_agg(lat::integer)%20as%20arr_lat__int8,%20array_agg(long)%20as%20arr_long%20%20%20from%20antenas%20%20group%20by%20owner,%20lat,%20long%20limit%2020';
|
||||
|
||||
get('http://development.localhost.lan:8080/api/v1/sql?q=' + q+ '&format=bin', function(req) {
|
||||
var b = new ArrayBufferSer(req.response);
|
||||
console.log(b.length);
|
||||
window.b = b;
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@ -59,7 +59,7 @@ module.exports.db_port = '5432';
|
||||
module.exports.db_batch_port = '5432';
|
||||
module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours
|
||||
module.exports.batch_query_timeout = 5 * 1000; // 5 seconds in milliseconds
|
||||
module.exports.batch_log_filename = 'logs/batch-queries.log';
|
||||
//module.exports.batch_log_filename = 'logs/batch-queries.log';
|
||||
module.exports.copy_timeout = "'5h'";
|
||||
module.exports.copy_from_max_post_size = 2 * 1024 * 1024 * 1024 // 2 GB;
|
||||
module.exports.copy_from_max_post_size_pretty = '2 GB';
|
||||
@ -149,7 +149,7 @@ module.exports.ratelimits = {
|
||||
}
|
||||
|
||||
module.exports.validatePGEntitiesAccess = false;
|
||||
module.exports.dataIngestionLogPath = 'logs/data-ingestion.log';
|
||||
//module.exports.dataIngestionLogPath = 'logs/data-ingestion.log';
|
||||
module.exports.logQueries = true;
|
||||
module.exports.maxQueriesLogLength = 1024;
|
||||
|
||||
|
||||
216
tools/cdbsql
216
tools/cdbsql
@ -1,216 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
// Command line tool for CartoDB SQL API
|
||||
//
|
||||
// https://github.com/Vizzuality/CartoDB-SQL-API
|
||||
//
|
||||
|
||||
var http = require('http');
|
||||
|
||||
var nodevers = process.versions.node.split('.');
|
||||
|
||||
// NOTE: readline is also available in 0.4 but doesn't work
|
||||
var hasReadline = parseInt(nodevers[0]) > 0 || parseInt(nodevers[1]) >= 8;
|
||||
//console.log('Node version ' + nodevers.join(',') + ( hasReadline ? ' has' : ' does not have' ) + ' readline support');
|
||||
|
||||
var readline = hasReadline ? require('readline') : null;
|
||||
var tty = require('tty');
|
||||
|
||||
var me = process.argv[1];
|
||||
|
||||
process.argv.shift(); // this will be "node" (argv[0])
|
||||
process.argv.shift(); // this will be "benchmark.js" (argv[1])
|
||||
|
||||
var batch_mode = false;
|
||||
var format = 'json';
|
||||
var username = 'dev';
|
||||
var domain = 'localhost';
|
||||
var port = 8080;
|
||||
var api_version = 2;
|
||||
var api_key = process.env['CDBSQL_APIKEY'];
|
||||
var sql;
|
||||
var decimal_places;
|
||||
var echo_queries = false;
|
||||
var skipfields;
|
||||
var verbose = 0;
|
||||
|
||||
function usage(exit_code) {
|
||||
console.log("Usage: " + me + " [OPTIONS] <query>");
|
||||
if ( hasReadline ) {
|
||||
console.log(" " + me + " [OPTIONS]");
|
||||
}
|
||||
console.log("Options:");
|
||||
console.log(" -v verbose operations (off)");
|
||||
console.log(" --help print this help");
|
||||
console.log(" --user <string> cartodb username (" + username + ")");
|
||||
console.log(" --domain <string> service domain (" + domain + ")");
|
||||
console.log(" --port <num> service tcp port number (" + port + ")");
|
||||
console.log(" --api-version <num> API version (" + api_version +")");
|
||||
console.log(" --key <string> API authentication key (CDBSQL_APIKEY)");
|
||||
console.log(" --format <string> Response format (json)");
|
||||
console.log(" --dp <num> Decimal places in geojson format (unspecified)");
|
||||
console.log(" --skipfields <string> Comma-separated list of fields to skip (none)");
|
||||
console.log(" --echo-queries echo commands sent to server");
|
||||
if ( hasReadline ) {
|
||||
console.log(" --batch Send all read queries at once (off)");
|
||||
}
|
||||
process.exit(exit_code);
|
||||
}
|
||||
|
||||
var arg;
|
||||
while ( arg = process.argv.shift() ) {
|
||||
if ( arg == '-v' ) {
|
||||
++verbose;
|
||||
}
|
||||
else if ( arg == '--help' ) {
|
||||
usage(0);
|
||||
}
|
||||
else if ( arg == '--key' ) {
|
||||
api_key = process.argv.shift();
|
||||
}
|
||||
else if ( arg == '--domain' ) {
|
||||
domain = process.argv.shift();
|
||||
}
|
||||
else if ( arg == '--user' ) {
|
||||
username = process.argv.shift();
|
||||
}
|
||||
else if ( arg == '--port' ) {
|
||||
port = process.argv.shift();
|
||||
}
|
||||
else if ( arg == '--api-version' ) {
|
||||
api_version = process.argv.shift();
|
||||
}
|
||||
else if ( arg == '--format' ) {
|
||||
format = process.argv.shift();
|
||||
}
|
||||
else if ( arg == '--dp' ) {
|
||||
decimal_places = process.argv.shift();
|
||||
}
|
||||
else if ( arg == '--batch' ) {
|
||||
batch_mode = true;
|
||||
}
|
||||
else if ( arg == '--skipfields' ) {
|
||||
skipfields = process.argv.shift();
|
||||
}
|
||||
else if ( arg == '--echo-queries' ) {
|
||||
echo_queries = true;
|
||||
}
|
||||
else if ( ! sql ) {
|
||||
sql = arg;
|
||||
}
|
||||
else {
|
||||
usage(1);
|
||||
}
|
||||
}
|
||||
|
||||
var hostname = username + '.' + domain;
|
||||
|
||||
if ( ! tty.isatty(process.stdin) || ! tty.isatty(process.stdout) ) {
|
||||
batch_mode = true;
|
||||
}
|
||||
|
||||
if ( ! sql ) {
|
||||
|
||||
if ( ! batch_mode ) {
|
||||
|
||||
if ( readline ) {
|
||||
|
||||
var rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
|
||||
rl.setPrompt(hostname + '> ');
|
||||
rl.prompt();
|
||||
|
||||
sql = '';
|
||||
rl.on('line', function(line) {
|
||||
sql += line;
|
||||
// TODO: some sanity checking, like trim the line or check if it ends with semicolon
|
||||
if ( sql.length ) {
|
||||
processQuery(sql, function() {
|
||||
sql = '';
|
||||
rl.prompt();
|
||||
});
|
||||
} else rl.prompt();
|
||||
}).on('close', function() {
|
||||
if ( sql.length ) {
|
||||
console.warn("Unprocessed sql left: [" + sql + "]");
|
||||
}
|
||||
console.log("Good bye");
|
||||
}).on('SIGCONT', function() {
|
||||
// this is needed so not to exit on stop/resume
|
||||
rl.prompt();
|
||||
});
|
||||
} else {
|
||||
usage(1);
|
||||
}
|
||||
} else { // batch mode
|
||||
process.stdin.resume();
|
||||
process.stdin.setEncoding('utf8');
|
||||
sql = '';
|
||||
process.stdin.on('data', function(chunk) {
|
||||
// TODO: some sanity checking, like trim the line or check if it ends with semicolon
|
||||
processQuery(chunk);
|
||||
});
|
||||
}
|
||||
} else {
|
||||
processQuery(sql);
|
||||
}
|
||||
|
||||
// -- Perform the request
|
||||
|
||||
function processQuery(sql, callback)
|
||||
{
|
||||
|
||||
var post_data = 'q=' + encodeURIComponent(sql);
|
||||
|
||||
var opt = {
|
||||
host: hostname,
|
||||
port: port,
|
||||
path: '/api/v' + api_version + '/sql?format=' + encodeURIComponent(format),
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
'Content-Length': post_data.length
|
||||
}
|
||||
};
|
||||
|
||||
if ( api_key ) opt.path += '&api_key=' + api_key;
|
||||
if ( typeof(decimal_places) != 'undefined' ) opt.path += '&dp=' + decimal_places;
|
||||
if ( typeof(skipfields) != 'undefined' ) opt.path += '&skipfields=' + encodeURIComponent(skipfields);
|
||||
|
||||
var body = '';
|
||||
var request = 'http://' + opt.host + ':' + opt.port + opt.path;
|
||||
if ( verbose ) {
|
||||
console.log("Posting " + post_data + " to ", request);
|
||||
}
|
||||
|
||||
var req = http.request(opt, function(res) {
|
||||
//console.log("Response status: " + res.statusCode);
|
||||
res.on('data', function(chunk) {
|
||||
body += chunk;
|
||||
//console.log("data: "); console.dir(json);
|
||||
});
|
||||
|
||||
res.on('end', function() {
|
||||
console.log("Request:", request);
|
||||
var sqlprint = sql.length > 100 ? sql.substring(0, 100) + ' ... [truncated ' + (sql.length-100) + ' bytes]' : sql;
|
||||
sqlprint = sqlprint.split('\n').join(' ');
|
||||
if ( echo_queries ) console.log("Query:", sqlprint);
|
||||
console.log("Response status: " + res.statusCode);
|
||||
console.log('Response body:');
|
||||
console.dir(body);
|
||||
if ( callback ) callback();
|
||||
});
|
||||
|
||||
}).on('error', function(e) {
|
||||
console.log("Request:", request);
|
||||
console.log("Error: " + e.message);
|
||||
if ( callback ) callback();
|
||||
});
|
||||
|
||||
req.write(post_data);
|
||||
req.end();
|
||||
|
||||
}
|
||||
@ -1,19 +0,0 @@
|
||||
MUNIN_PLUGINS_DIR=/etc/munin/plugins
|
||||
MUNIN_PLUGINS_CONFIG_DIR=/etc/munin/plugin-conf.d
|
||||
PWD=$(shell pwd)
|
||||
|
||||
all: cdbsqlapi.conf
|
||||
|
||||
cdbsqlapi.conf: cdbsqlapi.conf.in
|
||||
sed 's#@PWD@#$(PWD)#' < $< > $@
|
||||
|
||||
install-munin-plugin-conf: cdbsqlapi.conf
|
||||
install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/cdbsqlapi.conf
|
||||
|
||||
install-munin-plugin: cdbsqlapi
|
||||
install -m 755 $< $(MUNIN_PLUGINS_DIR)/cdbsqlapi
|
||||
|
||||
install: install-munin-plugin install-munin-plugin-conf
|
||||
|
||||
clean:
|
||||
rm -f cdbsqlapi.conf
|
||||
@ -1,74 +0,0 @@
|
||||
#!/bin/sh
|
||||
|
||||
envnik=$(basename "${SQLAPI_ENVIRONMENT}" .js)
|
||||
|
||||
if test "$1" = "config"; then
|
||||
echo "graph_title fd usage (${envnik})"
|
||||
cat <<"EOM"
|
||||
graph_vlabel number of file descriptors
|
||||
graph_category sqlapi
|
||||
graph_scale no
|
||||
procs.label Number of worker processes
|
||||
pgsql.label PostgreSQL connections (max)
|
||||
redis.label Redis connections (max)
|
||||
http.label Incoming http requests (max)
|
||||
nfd.label Total file descriptors (max)
|
||||
EOM
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if test x"$1" != x; then
|
||||
SQLAPI_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js
|
||||
fi
|
||||
|
||||
if test -z "$SQLAPI_ENVIRONMENT"; then
|
||||
echo "Usage: $0 [<environment>]" >&2
|
||||
echo " or: [SQLAPI_ENVIRONMENT=<environment>] $0" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
http_port=$(echo "console.log(require('${SQLAPI_ENVIRONMENT}').node_port)" | node) || exit 1
|
||||
pgsql_port=$(echo "console.log(require('${SQLAPI_ENVIRONMENT}').db_port)" | node) || exit 1
|
||||
redis_port=$(echo "console.log(require('${SQLAPI_ENVIRONMENT}').redis_port)" | node) || exit 1
|
||||
|
||||
pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}')
|
||||
nworkers=$(echo "${pids}" | wc -l)
|
||||
pids=$(echo "${pids}" | paste -sd ' ')
|
||||
|
||||
if test -z "${pids}"; then
|
||||
echo "No processes found listening on tcp port '${http_port}'" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tmpreport="/tmp/checkfd.$$.txt"
|
||||
|
||||
lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}"
|
||||
|
||||
maxdb=0
|
||||
maxredis=0
|
||||
maxhttp=0
|
||||
maxtot=0
|
||||
|
||||
for pid in ${pids}; do
|
||||
|
||||
cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l);
|
||||
if test $cnt -gt $maxdb; then maxdb=$cnt; fi
|
||||
|
||||
cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l);
|
||||
if test $cnt -gt $maxredis; then maxredis=$cnt; fi
|
||||
|
||||
cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l);
|
||||
if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi
|
||||
|
||||
cnt=$(grep "${pid}" "${tmpreport}" | wc -l);
|
||||
if test $cnt -gt $maxtot; then maxtot=$cnt; fi
|
||||
|
||||
done
|
||||
|
||||
echo "procs.value ${nworkers}"
|
||||
echo "pgsql.value ${maxdb}"
|
||||
echo "redis.value ${maxredis}"
|
||||
echo "http.value ${maxhttp}"
|
||||
echo "nfd.value ${maxtot}"
|
||||
|
||||
rm -f "${tmpreport}"
|
||||
@ -1,5 +0,0 @@
|
||||
# Configuration file for munin plugin
|
||||
|
||||
[cdbsqlapi]
|
||||
user root
|
||||
env.SQLAPI_ENVIRONMENT @PWD@/../../config/environments/production.js
|
||||
Loading…
Reference in New Issue
Block a user