diff --git a/.gitignore b/.gitignore index bc686d66..0d0f591b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ config/environments/*.js -logs/*.log +logs pids/*.pid *.sock test/tmp/* @@ -12,4 +12,4 @@ test/acceptance/oauth/venv/* coverage/ npm-debug.log log/*.log -yarn.lock \ No newline at end of file +yarn.lock diff --git a/NEWS.md b/NEWS.md index f89f7585..0ec5ecae 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,7 @@ Released 2019-mm-dd Breaking changes: - Removed "base_url" definition in configuration files. Added new property "routes". See [`cec1c60`](https://github.com/CartoDB/CartoDB-SQL-API/commit/cec1c609c04d134f45ee74a81f82c07a19bf11bd) for more details. +- Remove development client and munin tool, both abandoned years ago. Announcements: diff --git a/client/bin_decoder.js b/client/bin_decoder.js deleted file mode 100644 index 81ac1262..00000000 --- a/client/bin_decoder.js +++ /dev/null @@ -1,104 +0,0 @@ -'use strict'; - -function ArrayBufferSer(arrayBuffer) { - this.buffer = arrayBuffer; - this.offset = 0; - this.sections = this.readArray(); - this.headers = this.sections[0]; - this._headersIndex = {}; - for(var i = 0; i < this.headers.length; ++i) { - this._headersIndex[this.headers[i]] = i; - } - if(this.sections.length > 1) { - this.length = this.sections[1].length; - } -} - -ArrayBufferSer.INT8 = 1; -ArrayBufferSer.UINT8 = 2; -ArrayBufferSer.UINT8_CLAMP = 3; -ArrayBufferSer.INT16 = 4; -ArrayBufferSer.UINT16 = 5; -ArrayBufferSer.INT32 = 6; -ArrayBufferSer.UINT32 = 7; -ArrayBufferSer.FLOAT32 = 8; -//ArrayBufferSer.FLOAT64 = 9; not supported -ArrayBufferSer.STRING = 10; -ArrayBufferSer.BUFFER = 11; - - -ArrayBufferSer.prototype = { - - sizes: [NaN, 1, 1, 1, 2, 2, 4, 4, 4, 8], - - types: [ - null, - Int8Array, - Uint8Array, - Uint8ClampedArray, - Int16Array, - Uint16Array, - Int32Array, - Uint32Array, - Float32Array, - Float64Array - ], - - get: function(columnName) { - var i = this._headersIndex[columnName] - if(i != undefined) { - return this.sections[i + 1] - } - return; - }, - - _paddingFor: function(offset, type) { - var s = this.sizes[type] - if(s) { - var r = offset % s; - return r == 0 ? 0 : s - r; - } - return 0; - }, - - readUInt32: function() { - var i = new DataView(this.buffer).getUint32(this.offset); - this.offset += 4; - return i - }, - - readArray: function() { - var type = this.readUInt32(); - var size = this.readUInt32(); - if(type < ArrayBufferSer.STRING) { - var a = new this.types[type](this.buffer, this.offset, size/this.sizes[type]); - this.offset += size; - return a; - } else if(type == ArrayBufferSer.STRING) { - var target = this.offset + size; - var b = []; - while(this.offset < target) { - this.offset += this._paddingFor(this.offset, ArrayBufferSer.INT32); - var arr = this.readArray(); - if(arr) { - var str = ''; - for(var i = 0; i < arr.length; ++i) { - str += String.fromCharCode(arr[i]); - } - b.push(str); - } - // parse sttring - } - return b; - } else if(type == ArrayBufferSer.BUFFER) { - var b = []; - var target = this.offset + size; - while(this.offset < target) { - this.offset += this._paddingFor(this.offset, ArrayBufferSer.INT32); - b.push(this.readArray()); - } - return b; - } - } - -}; diff --git a/client/test.html b/client/test.html deleted file mode 100644 index 683aa97b..00000000 --- a/client/test.html +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 2f8cddf3..ccfe6ad7 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -59,7 +59,7 @@ module.exports.db_port = '5432'; module.exports.db_batch_port = '5432'; module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours module.exports.batch_query_timeout = 5 * 1000; // 5 seconds in milliseconds -module.exports.batch_log_filename = 'logs/batch-queries.log'; +//module.exports.batch_log_filename = 'logs/batch-queries.log'; module.exports.copy_timeout = "'5h'"; module.exports.copy_from_max_post_size = 2 * 1024 * 1024 * 1024 // 2 GB; module.exports.copy_from_max_post_size_pretty = '2 GB'; @@ -149,7 +149,7 @@ module.exports.ratelimits = { } module.exports.validatePGEntitiesAccess = false; -module.exports.dataIngestionLogPath = 'logs/data-ingestion.log'; +//module.exports.dataIngestionLogPath = 'logs/data-ingestion.log'; module.exports.logQueries = true; module.exports.maxQueriesLogLength = 1024; diff --git a/logs/.gitneedsme b/logs/.gitneedsme deleted file mode 100644 index e69de29b..00000000 diff --git a/tools/cdbsql b/tools/cdbsql deleted file mode 100755 index ca27899d..00000000 --- a/tools/cdbsql +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env node - -// Command line tool for CartoDB SQL API -// -// https://github.com/Vizzuality/CartoDB-SQL-API -// - -var http = require('http'); - -var nodevers = process.versions.node.split('.'); - -// NOTE: readline is also available in 0.4 but doesn't work -var hasReadline = parseInt(nodevers[0]) > 0 || parseInt(nodevers[1]) >= 8; -//console.log('Node version ' + nodevers.join(',') + ( hasReadline ? ' has' : ' does not have' ) + ' readline support'); - -var readline = hasReadline ? require('readline') : null; -var tty = require('tty'); - -var me = process.argv[1]; - -process.argv.shift(); // this will be "node" (argv[0]) -process.argv.shift(); // this will be "benchmark.js" (argv[1]) - -var batch_mode = false; -var format = 'json'; -var username = 'dev'; -var domain = 'localhost'; -var port = 8080; -var api_version = 2; -var api_key = process.env['CDBSQL_APIKEY']; -var sql; -var decimal_places; -var echo_queries = false; -var skipfields; -var verbose = 0; - -function usage(exit_code) { - console.log("Usage: " + me + " [OPTIONS] "); - if ( hasReadline ) { - console.log(" " + me + " [OPTIONS]"); - } - console.log("Options:"); - console.log(" -v verbose operations (off)"); - console.log(" --help print this help"); - console.log(" --user cartodb username (" + username + ")"); - console.log(" --domain service domain (" + domain + ")"); - console.log(" --port service tcp port number (" + port + ")"); - console.log(" --api-version API version (" + api_version +")"); - console.log(" --key API authentication key (CDBSQL_APIKEY)"); - console.log(" --format Response format (json)"); - console.log(" --dp Decimal places in geojson format (unspecified)"); - console.log(" --skipfields Comma-separated list of fields to skip (none)"); - console.log(" --echo-queries echo commands sent to server"); - if ( hasReadline ) { - console.log(" --batch Send all read queries at once (off)"); - } - process.exit(exit_code); -} - -var arg; -while ( arg = process.argv.shift() ) { - if ( arg == '-v' ) { - ++verbose; - } - else if ( arg == '--help' ) { - usage(0); - } - else if ( arg == '--key' ) { - api_key = process.argv.shift(); - } - else if ( arg == '--domain' ) { - domain = process.argv.shift(); - } - else if ( arg == '--user' ) { - username = process.argv.shift(); - } - else if ( arg == '--port' ) { - port = process.argv.shift(); - } - else if ( arg == '--api-version' ) { - api_version = process.argv.shift(); - } - else if ( arg == '--format' ) { - format = process.argv.shift(); - } - else if ( arg == '--dp' ) { - decimal_places = process.argv.shift(); - } - else if ( arg == '--batch' ) { - batch_mode = true; - } - else if ( arg == '--skipfields' ) { - skipfields = process.argv.shift(); - } - else if ( arg == '--echo-queries' ) { - echo_queries = true; - } - else if ( ! sql ) { - sql = arg; - } - else { - usage(1); - } -} - -var hostname = username + '.' + domain; - -if ( ! tty.isatty(process.stdin) || ! tty.isatty(process.stdout) ) { - batch_mode = true; -} - -if ( ! sql ) { - - if ( ! batch_mode ) { - - if ( readline ) { - - var rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); - - rl.setPrompt(hostname + '> '); - rl.prompt(); - - sql = ''; - rl.on('line', function(line) { - sql += line; - // TODO: some sanity checking, like trim the line or check if it ends with semicolon - if ( sql.length ) { - processQuery(sql, function() { - sql = ''; - rl.prompt(); - }); - } else rl.prompt(); - }).on('close', function() { - if ( sql.length ) { - console.warn("Unprocessed sql left: [" + sql + "]"); - } - console.log("Good bye"); - }).on('SIGCONT', function() { - // this is needed so not to exit on stop/resume - rl.prompt(); - }); - } else { - usage(1); - } - } else { // batch mode - process.stdin.resume(); - process.stdin.setEncoding('utf8'); - sql = ''; - process.stdin.on('data', function(chunk) { - // TODO: some sanity checking, like trim the line or check if it ends with semicolon - processQuery(chunk); - }); - } -} else { - processQuery(sql); -} - -// -- Perform the request - -function processQuery(sql, callback) -{ - - var post_data = 'q=' + encodeURIComponent(sql); - - var opt = { - host: hostname, - port: port, - path: '/api/v' + api_version + '/sql?format=' + encodeURIComponent(format), - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - 'Content-Length': post_data.length - } - }; - - if ( api_key ) opt.path += '&api_key=' + api_key; - if ( typeof(decimal_places) != 'undefined' ) opt.path += '&dp=' + decimal_places; - if ( typeof(skipfields) != 'undefined' ) opt.path += '&skipfields=' + encodeURIComponent(skipfields); - - var body = ''; - var request = 'http://' + opt.host + ':' + opt.port + opt.path; - if ( verbose ) { - console.log("Posting " + post_data + " to ", request); - } - - var req = http.request(opt, function(res) { - //console.log("Response status: " + res.statusCode); - res.on('data', function(chunk) { - body += chunk; - //console.log("data: "); console.dir(json); - }); - - res.on('end', function() { - console.log("Request:", request); - var sqlprint = sql.length > 100 ? sql.substring(0, 100) + ' ... [truncated ' + (sql.length-100) + ' bytes]' : sql; - sqlprint = sqlprint.split('\n').join(' '); - if ( echo_queries ) console.log("Query:", sqlprint); - console.log("Response status: " + res.statusCode); - console.log('Response body:'); - console.dir(body); - if ( callback ) callback(); - }); - - }).on('error', function(e) { - console.log("Request:", request); - console.log("Error: " + e.message); - if ( callback ) callback(); - }); - - req.write(post_data); - req.end(); - -} diff --git a/tools/munin/Makefile b/tools/munin/Makefile deleted file mode 100644 index 70b70917..00000000 --- a/tools/munin/Makefile +++ /dev/null @@ -1,19 +0,0 @@ -MUNIN_PLUGINS_DIR=/etc/munin/plugins -MUNIN_PLUGINS_CONFIG_DIR=/etc/munin/plugin-conf.d -PWD=$(shell pwd) - -all: cdbsqlapi.conf - -cdbsqlapi.conf: cdbsqlapi.conf.in - sed 's#@PWD@#$(PWD)#' < $< > $@ - -install-munin-plugin-conf: cdbsqlapi.conf - install -m 644 $< $(MUNIN_PLUGINS_CONFIG_DIR)/cdbsqlapi.conf - -install-munin-plugin: cdbsqlapi - install -m 755 $< $(MUNIN_PLUGINS_DIR)/cdbsqlapi - -install: install-munin-plugin install-munin-plugin-conf - -clean: - rm -f cdbsqlapi.conf diff --git a/tools/munin/cdbsqlapi b/tools/munin/cdbsqlapi deleted file mode 100755 index 77d77afa..00000000 --- a/tools/munin/cdbsqlapi +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/sh - -envnik=$(basename "${SQLAPI_ENVIRONMENT}" .js) - -if test "$1" = "config"; then -echo "graph_title fd usage (${envnik})" -cat <<"EOM" -graph_vlabel number of file descriptors -graph_category sqlapi -graph_scale no -procs.label Number of worker processes -pgsql.label PostgreSQL connections (max) -redis.label Redis connections (max) -http.label Incoming http requests (max) -nfd.label Total file descriptors (max) -EOM -exit 0 -fi - -if test x"$1" != x; then - SQLAPI_ENVIRONMENT=$(cd $(dirname $0); pwd)/../../config/environments/${1}.js -fi - -if test -z "$SQLAPI_ENVIRONMENT"; then - echo "Usage: $0 []" >&2 - echo " or: [SQLAPI_ENVIRONMENT=] $0" >&2 - exit 1 -fi - -http_port=$(echo "console.log(require('${SQLAPI_ENVIRONMENT}').node_port)" | node) || exit 1 -pgsql_port=$(echo "console.log(require('${SQLAPI_ENVIRONMENT}').db_port)" | node) || exit 1 -redis_port=$(echo "console.log(require('${SQLAPI_ENVIRONMENT}').redis_port)" | node) || exit 1 - -pids=$(lsof -i :${http_port} | grep LISTEN | awk '{print $2}') -nworkers=$(echo "${pids}" | wc -l) -pids=$(echo "${pids}" | paste -sd ' ') - -if test -z "${pids}"; then - echo "No processes found listening on tcp port '${http_port}'" >&2 - exit 1 -fi - -tmpreport="/tmp/checkfd.$$.txt" - -lsof -Pp $(echo "${pids}" | tr ' ' ',') > "${tmpreport}" - -maxdb=0 -maxredis=0 -maxhttp=0 -maxtot=0 - -for pid in ${pids}; do - - cnt=$(grep "${pid}" "${tmpreport}" | grep ":${pgsql_port} " | wc -l); - if test $cnt -gt $maxdb; then maxdb=$cnt; fi - - cnt=$(grep "${pid}" "${tmpreport}" | grep ":${redis_port} " | wc -l); - if test $cnt -gt $maxredis; then maxredis=$cnt; fi - - cnt=$(grep "${pid}" "${tmpreport}" | grep ":${http_port}-" | grep -v "LISTEN" | wc -l); - if test $cnt -gt $maxhttp; then maxhttp=$cnt; fi - - cnt=$(grep "${pid}" "${tmpreport}" | wc -l); - if test $cnt -gt $maxtot; then maxtot=$cnt; fi - -done - -echo "procs.value ${nworkers}" -echo "pgsql.value ${maxdb}" -echo "redis.value ${maxredis}" -echo "http.value ${maxhttp}" -echo "nfd.value ${maxtot}" - -rm -f "${tmpreport}" diff --git a/tools/munin/cdbsqlapi.conf.in b/tools/munin/cdbsqlapi.conf.in deleted file mode 100644 index 021ea369..00000000 --- a/tools/munin/cdbsqlapi.conf.in +++ /dev/null @@ -1,5 +0,0 @@ -# Configuration file for munin plugin - -[cdbsqlapi] -user root -env.SQLAPI_ENVIRONMENT @PWD@/../../config/environments/production.js