Merge branch 'master' of github.com:CartoDB/CartoDB-SQL-API

This commit is contained in:
Daniel García Aubert 2019-10-08 10:05:44 +02:00
commit bf0a030100
10 changed files with 5 additions and 454 deletions

2
.gitignore vendored
View File

@ -1,5 +1,5 @@
config/environments/*.js config/environments/*.js
logs/*.log logs
pids/*.pid pids/*.pid
*.sock *.sock
test/tmp/* test/tmp/*

View File

@ -5,6 +5,7 @@ Released 2019-mm-dd
Breaking changes: Breaking changes:
- Removed "base_url" definition in configuration files. Added new property "routes". See [`cec1c60`](https://github.com/CartoDB/CartoDB-SQL-API/commit/cec1c609c04d134f45ee74a81f82c07a19bf11bd) for more details. - Removed "base_url" definition in configuration files. Added new property "routes". See [`cec1c60`](https://github.com/CartoDB/CartoDB-SQL-API/commit/cec1c609c04d134f45ee74a81f82c07a19bf11bd) for more details.
- Remove development client and munin tool, both abandoned years ago.
Announcements: Announcements:

View File

@ -1,104 +0,0 @@
'use strict';
function ArrayBufferSer(arrayBuffer) {
this.buffer = arrayBuffer;
this.offset = 0;
this.sections = this.readArray();
this.headers = this.sections[0];
this._headersIndex = {};
for(var i = 0; i < this.headers.length; ++i) {
this._headersIndex[this.headers[i]] = i;
}
if(this.sections.length > 1) {
this.length = this.sections[1].length;
}
}
ArrayBufferSer.INT8 = 1;
ArrayBufferSer.UINT8 = 2;
ArrayBufferSer.UINT8_CLAMP = 3;
ArrayBufferSer.INT16 = 4;
ArrayBufferSer.UINT16 = 5;
ArrayBufferSer.INT32 = 6;
ArrayBufferSer.UINT32 = 7;
ArrayBufferSer.FLOAT32 = 8;
//ArrayBufferSer.FLOAT64 = 9; not supported
ArrayBufferSer.STRING = 10;
ArrayBufferSer.BUFFER = 11;
ArrayBufferSer.prototype = {
sizes: [NaN, 1, 1, 1, 2, 2, 4, 4, 4, 8],
types: [
null,
Int8Array,
Uint8Array,
Uint8ClampedArray,
Int16Array,
Uint16Array,
Int32Array,
Uint32Array,
Float32Array,
Float64Array
],
get: function(columnName) {
var i = this._headersIndex[columnName]
if(i != undefined) {
return this.sections[i + 1]
}
return;
},
_paddingFor: function(offset, type) {
var s = this.sizes[type]
if(s) {
var r = offset % s;
return r == 0 ? 0 : s - r;
}
return 0;
},
readUInt32: function() {
var i = new DataView(this.buffer).getUint32(this.offset);
this.offset += 4;
return i
},
readArray: function() {
var type = this.readUInt32();
var size = this.readUInt32();
if(type < ArrayBufferSer.STRING) {
var a = new this.types[type](this.buffer, this.offset, size/this.sizes[type]);
this.offset += size;
return a;
} else if(type == ArrayBufferSer.STRING) {
var target = this.offset + size;
var b = [];
while(this.offset < target) {
this.offset += this._paddingFor(this.offset, ArrayBufferSer.INT32);
var arr = this.readArray();
if(arr) {
var str = '';
for(var i = 0; i < arr.length; ++i) {
str += String.fromCharCode(arr[i]);
}
b.push(str);
}
// parse sttring
}
return b;
} else if(type == ArrayBufferSer.BUFFER) {
var b = [];
var target = this.offset + size;
while(this.offset < target) {
this.offset += this._paddingFor(this.offset, ArrayBufferSer.INT32);
b.push(this.readArray());
}
return b;
}
}
};

View File

@ -1,32 +0,0 @@
<html>
<body>
<script src="bin_decoder.js"></script>
<script>
function get(url, callback) {
var req = new XMLHttpRequest();
req.onreadystatechange = function() {
if (req.readyState == 4){
if (req.status == 200){
callback(req);
} else {
callback(null);
}
}
};
req.open("GET", url, true)
req.responseType = 'arraybuffer';
req.send(null)
return req;
}
var q = 'select%20owner,%20array_agg(lat::integer)%20as%20arr_lat__int8,%20array_agg(long)%20as%20arr_long%20%20%20from%20antenas%20%20group%20by%20owner,%20lat,%20long%20limit%2020';
get('http://development.localhost.lan:8080/api/v1/sql?q=' + q+ '&format=bin', function(req) {
var b = new ArrayBufferSer(req.response);
console.log(b.length);
window.b = b;
});
</script>
</body>
</html>

View File

@ -59,7 +59,7 @@ module.exports.db_port = '5432';
module.exports.db_batch_port = '5432'; module.exports.db_batch_port = '5432';
module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours
module.exports.batch_query_timeout = 5 * 1000; // 5 seconds in milliseconds module.exports.batch_query_timeout = 5 * 1000; // 5 seconds in milliseconds
module.exports.batch_log_filename = 'logs/batch-queries.log'; //module.exports.batch_log_filename = 'logs/batch-queries.log';
module.exports.copy_timeout = "'5h'"; module.exports.copy_timeout = "'5h'";
module.exports.copy_from_max_post_size = 2 * 1024 * 1024 * 1024 // 2 GB; module.exports.copy_from_max_post_size = 2 * 1024 * 1024 * 1024 // 2 GB;
module.exports.copy_from_max_post_size_pretty = '2 GB'; module.exports.copy_from_max_post_size_pretty = '2 GB';
@ -149,7 +149,7 @@ module.exports.ratelimits = {
} }
module.exports.validatePGEntitiesAccess = false; module.exports.validatePGEntitiesAccess = false;
module.exports.dataIngestionLogPath = 'logs/data-ingestion.log'; //module.exports.dataIngestionLogPath = 'logs/data-ingestion.log';
module.exports.logQueries = true; module.exports.logQueries = true;
module.exports.maxQueriesLogLength = 1024; module.exports.maxQueriesLogLength = 1024;

View File

View File

@ -1,216 +0,0 @@
#!/usr/bin/env node
// Command line tool for CartoDB SQL API
//
// https://github.com/Vizzuality/CartoDB-SQL-API
//
var http = require('http');
var nodevers = process.versions.node.split('.');
// NOTE: readline is also available in 0.4 but doesn't work
var hasReadline = parseInt(nodevers[0]) > 0 || parseInt(nodevers[1]) >= 8;
//console.log('Node version ' + nodevers.join(',') + ( hasReadline ? ' has' : ' does not have' ) + ' readline support');
var readline = hasReadline ? require('readline') : null;
var tty = require('tty');
var me = process.argv[1];
process.argv.shift(); // this will be "node" (argv[0])
process.argv.shift(); // this will be "benchmark.js" (argv[1])
var batch_mode = false;
var format = 'json';
var username = 'dev';
var domain = 'localhost';
var port = 8080;
var api_version = 2;
var api_key = process.env['CDBSQL_APIKEY'];
var sql;
var decimal_places;
var echo_queries = false;
var skipfields;
var verbose = 0;
function usage(exit_code) {
console.log("Usage: " + me + " [OPTIONS] <query>");
if ( hasReadline ) {
console.log(" " + me + " [OPTIONS]");
}
console.log("Options:");
console.log(" -v verbose operations (off)");
console.log(" --help print this help");
console.log(" --user <string> cartodb username (" + username + ")");
console.log(" --domain <string> service domain (" + domain + ")");
console.log(" --port <num> service tcp port number (" + port + ")");
console.log(" --api-version <num> API version (" + api_version +")");
console.log(" --key <string> API authentication key (CDBSQL_APIKEY)");
console.log(" --format <string> Response format (json)");
console.log(" --dp <num> Decimal places in geojson format (unspecified)");
console.log(" --skipfields <string> Comma-separated list of fields to skip (none)");
console.log(" --echo-queries echo commands sent to server");
if ( hasReadline ) {
console.log(" --batch Send all read queries at once (off)");
}
process.exit(exit_code);
}
var arg;
while ( arg = process.argv.shift() ) {
if ( arg == '-v' ) {
++verbose;
}
else if ( arg == '--help' ) {
usage(0);
}
else if ( arg == '--key' ) {
api_key = process.argv.shift();
}
else if ( arg == '--domain' ) {
domain = process.argv.shift();
}
else if ( arg == '--user' ) {
username = process.argv.shift();
}
else if ( arg == '--port' ) {
port = process.argv.shift();
}
else if ( arg == '--api-version' ) {
api_version = process.argv.shift();
}
else if ( arg == '--format' ) {
format = process.argv.shift();
}
else if ( arg == '--dp' ) {
decimal_places = process.argv.shift();
}
else if ( arg == '--batch' ) {
batch_mode = true;
}
else if ( arg == '--skipfields' ) {
skipfields = process.argv.shift();
}
else if ( arg == '--echo-queries' ) {
echo_queries = true;
}
else if ( ! sql ) {
sql = arg;
}
else {
usage(1);
}
}
var hostname = username + '.' + domain;
if ( ! tty.isatty(process.stdin) || ! tty.isatty(process.stdout) ) {
batch_mode = true;
}
if ( ! sql ) {
if ( ! batch_mode ) {
if ( readline ) {
var rl = readline.createInterface({
input: process.stdin,
output: process.stdout,
});
rl.setPrompt(hostname + '> ');
rl.prompt();
sql = '';
rl.on('line', function(line) {
sql += line;
// TODO: some sanity checking, like trim the line or check if it ends with semicolon
if ( sql.length ) {
processQuery(sql, function() {
sql = '';
rl.prompt();
});
} else rl.prompt();
}).on('close', function() {
if ( sql.length ) {
console.warn("Unprocessed sql left: [" + sql + "]");
}
console.log("Good bye");
}).on('SIGCONT', function() {
// this is needed so not to exit on stop/resume
rl.prompt();
});
} else {
usage(1);
}
} else { // batch mode
process.stdin.resume();
process.stdin.setEncoding('utf8');
sql = '';
process.stdin.on('data', function(chunk) {
// TODO: some sanity checking, like trim the line or check if it ends with semicolon
processQuery(chunk);
});
}
} else {
processQuery(sql);
}
// -- Perform the request
function processQuery(sql, callback)
{
var post_data = 'q=' + encodeURIComponent(sql);
var opt = {
host: hostname,
port: port,
path: '/api/v' + api_version + '/sql?format=' + encodeURIComponent(format),
method: 'POST',
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': post_data.length
}
};
if ( api_key ) opt.path += '&api_key=' + api_key;
if ( typeof(decimal_places) != 'undefined' ) opt.path += '&dp=' + decimal_places;
if ( typeof(skipfields) != 'undefined' ) opt.path += '&skipfields=' + encodeURIComponent(skipfields);
var body = '';
var request = 'http://' + opt.host + ':' + opt.port + opt.path;
if ( verbose ) {
console.log("Posting " + post_data + " to ", request);
}
var req = http.request(opt, function(res) {
//console.log("Response status: " + res.statusCode);
res.on('data', function(chunk) {
body += chunk;
//console.log("data: "); console.dir(json);
});
res.on('end', function() {
console.log("Request:", request);
var sqlprint = sql.length > 100 ? sql.substring(0, 100) + ' ... [truncated ' + (sql.length-100) + ' bytes]' : sql;
sqlprint = sqlprint.split('\n').join(' ');
if ( echo_queries ) console.log("Query:", sqlprint);
console.log("Response status: " + res.statusCode);
console.log('Response body:');
console.dir(body);
if ( callback ) callback();
});
}).on('error', function(e) {
console.log("Request:", request);
console.log("Error: " + e.message);
if ( callback ) callback();
});
req.write(post_data);
req.end();
}

View File

@ -1,19 +0,0 @@
MUNIN_PLUGINS_DIR=/etc/munin/plugins
MUNIN_PLUGINS_CONFIG_DIR=/etc/munin/plugin-conf.d
PWD=$(shell pwd)
all: cdbsqlapi.conf
cdbsqlapi.conf: cdbsqlapi.conf.in
sed 's#@PWD@#$(PWD)#' < $< > $@
install-munin-plugin-conf: cdbsqlapi.conf
install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/cdbsqlapi.conf
install-munin-plugin: cdbsqlapi
install -m 755 $< $(MUNIN_PLUGINS_DIR)/cdbsqlapi
install: install-munin-plugin install-munin-plugin-conf
clean:
rm -f cdbsqlapi.conf

View File

@ -1,74 +0,0 @@
#!/bin/sh
envnik=$(basename "${SQLAPI_ENVIRONMENT}" .js)
if test "$1" = "config"; then
echo "graph_title fd usage (${envnik})"
cat <<"EOM"
graph_vlabel number of file descriptors
graph_category sqlapi
graph_scale no
procs.label Number of worker processes
pgsql.label PostgreSQL connections (max)
redis.label Redis connections (max)
http.label Incoming http requests (max)
nfd.label Total file descriptors (max)
EOM
exit 0
fi
if test x"$1" != x; then
SQLAPI_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js
fi
if test -z "$SQLAPI_ENVIRONMENT"; then
echo "Usage: $0 [<environment>]" >&2
echo " or: [SQLAPI_ENVIRONMENT=<environment>] $0" >&2
exit 1
fi
http_port=$(echo "console.log(require('${SQLAPI_ENVIRONMENT}').node_port)" | node) || exit 1
pgsql_port=$(echo "console.log(require('${SQLAPI_ENVIRONMENT}').db_port)" | node) || exit 1
redis_port=$(echo "console.log(require('${SQLAPI_ENVIRONMENT}').redis_port)" | node) || exit 1
pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}')
nworkers=$(echo "${pids}" | wc -l)
pids=$(echo "${pids}" | paste -sd ' ')
if test -z "${pids}"; then
echo "No processes found listening on tcp port '${http_port}'" >&2
exit 1
fi
tmpreport="/tmp/checkfd.$$.txt"
lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}"
maxdb=0
maxredis=0
maxhttp=0
maxtot=0
for pid in ${pids}; do
cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l);
if test $cnt -gt $maxdb; then maxdb=$cnt; fi
cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l);
if test $cnt -gt $maxredis; then maxredis=$cnt; fi
cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l);
if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi
cnt=$(grep "${pid}" "${tmpreport}" | wc -l);
if test $cnt -gt $maxtot; then maxtot=$cnt; fi
done
echo "procs.value ${nworkers}"
echo "pgsql.value ${maxdb}"
echo "redis.value ${maxredis}"
echo "http.value ${maxhttp}"
echo "nfd.value ${maxtot}"
rm -f "${tmpreport}"

View File

@ -1,5 +0,0 @@
# Configuration file for munin plugin
[cdbsqlapi]
user root
env.SQLAPI_ENVIRONMENT @PWD@/../../config/environments/production.js