cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into avoid-scan-to-discover-queues

Browse Source
This commit is contained in:
Daniel García Aubert 2017-03-31 14:31:40 +02:00
commit da2a98cdb5
6 changed files with 126 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
NEWS.md
View File

@ -1,9 +1,16 @@
#Changelog
## 1.43.2
## 1.44.1
Released 2017-mm-dd
## 1.44.0
Released 2017-03-30
Announcements:
* Active GC interval for Node.js >=v6.
## 1.43.1
Released 2017-01-16

38
app.js
View File

@ -11,6 +11,7 @@
*/
var fs = require('fs');
var path = require('path');
var os = require('os');
var argv = require('yargs')
.usage('Usage: $0 <environment> [options]')
@ -80,7 +81,17 @@ if ( ! global.settings.base_url ) {
var version = require("./package").version;
var server = require('./app/server')();
var StatsClient = require('./app/stats/client');
if (global.settings.statsd) {
// Perform keyword substitution in statsd
if (global.settings.statsd.prefix) {
var hostToken = os.hostname().split('.').reverse().join('.');
global.settings.statsd.prefix = global.settings.statsd.prefix.replace(/:host/, hostToken);
}
}
var statsClient = StatsClient.getInstance(global.settings.statsd);
var server = require('./app/server')(statsClient);
var listener = server.listen(global.settings.node_port, global.settings.node_host);
listener.on('listening', function() {
console.info("Using Node.js %s", process.version);
@ -119,3 +130,28 @@ process.on('SIGTERM', function () {
process.exit(0);
});
});
function isGteMinVersion(version, minVersion) {
var versionMatch = /[a-z]?([0-9]*)/.exec(version);
if (versionMatch) {
var majorVersion = parseInt(versionMatch[1], 10);
if (Number.isFinite(majorVersion)) {
return majorVersion >= minVersion;
}
}
return false;
}
if (global.gc && isGteMinVersion(process.version, 6)) {
var gcInterval = Number.isFinite(global.settings.gc_interval) ?
global.settings.gc_interval :
10000;
if (gcInterval > 0) {
setInterval(function gcForcedCycle() {
var start = Date.now();
global.gc();
statsClient.timing('sqlapi.gc', Date.now() - start);
}, gcInterval);
}
}

53
app/server.js
View File

@ -16,9 +16,7 @@
var express = require('express');
var bodyParser = require('./middlewares/body-parser');
var os = require('os');
var Profiler = require('./stats/profiler-proxy');
var StatsD = require('node-statsd').StatsD;
var _ = require('underscore');
var LRU = require('lru-cache');
@ -49,8 +47,8 @@ process.env.PGAPPNAME = process.env.PGAPPNAME || 'cartodb_sqlapi';
// override Date.toJSON
require('./utils/date_to_json');
// jshint maxcomplexity:12
function App() {
// jshint maxcomplexity:9
function App(statsClient) {
var app = express();
@ -107,45 +105,6 @@ function App() {
app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level:'info'})));
}
// Initialize statsD client if requested
var statsd_client;
if ( global.settings.statsd ) {
// Perform keyword substitution in statsd
if ( global.settings.statsd.prefix ) {
var host_token = os.hostname().split('.').reverse().join('.');
global.settings.statsd.prefix = global.settings.statsd.prefix.replace(/:host/, host_token);
}
statsd_client = new StatsD(global.settings.statsd);
statsd_client.last_error = { msg:'', count:0 };
statsd_client.socket.on('error', function(err) {
var last_err = statsd_client.last_error;
var last_msg = last_err.msg;
var this_msg = ''+err;
if ( this_msg !== last_msg ) {
console.error("statsd client socket error: " + err);
statsd_client.last_error.count = 1;
statsd_client.last_error.msg = this_msg;
} else {
++last_err.count;
if ( ! last_err.interval ) {
//console.log("Installing interval");
statsd_client.last_error.interval = setInterval(function() {
var count = statsd_client.last_error.count;
if ( count > 1 ) {
console.error("last statsd client socket error repeated " + count + " times");
statsd_client.last_error.count = 1;
//console.log("Clearing interval");
clearInterval(statsd_client.last_error.interval);
statsd_client.last_error.interval = null;
}
}, 1000);
}
}
});
}
app.use(cors());
// Use step-profiler
@ -159,7 +118,7 @@ function App() {
var profile = global.settings.useProfiler;
req.profiler = new Profiler({
profile: profile,
statsd_client: statsd_client
statsd_client: statsClient
});
next();
});
@ -193,10 +152,10 @@ function App() {
var genericController = new GenericController();
genericController.route(app);
var queryController = new QueryController(userDatabaseService, tableCache, statsd_client);
var queryController = new QueryController(userDatabaseService, tableCache, statsClient);
queryController.route(app);
var jobController = new JobController(userDatabaseService, jobService, statsd_client);
var jobController = new JobController(userDatabaseService, jobService, statsClient);
jobController.route(app);
var cacheStatusController = new CacheStatusController(tableCache);
@ -213,7 +172,7 @@ function App() {
if (global.settings.environment !== 'test' && isBatchProcess) {
var batchName = global.settings.api_hostname || 'batch';
app.batch = batchFactory(
metadataBackend, redisPool, batchName, statsd_client, global.settings.batch_log_filename
metadataBackend, redisPool, batchName, statsClient, global.settings.batch_log_filename
);
app.batch.start();
}

73
app/stats/client.js Normal file
View File

@ -0,0 +1,73 @@
var _ = require('underscore');
var debug = require('debug')('windshaft:stats_client');
var StatsD = require('node-statsd').StatsD;
module.exports = {
/**
* Returns an StatsD instance or an stub object that replicates the StatsD public interface so there is no need to
* keep checking if the stats_client is instantiated or not.
*
* The first call to this method implies all future calls will use the config specified in the very first call.
*
* TODO: It's far from ideal to use make this a singleton, improvement desired.
* We proceed this way to be able to use StatsD from several places sharing one single StatsD instance.
*
* @param config Configuration for StatsD, if undefined it will return an stub
* @returns {StatsD|Object}
*/
getInstance: function(config) {
if (!this.instance) {
var instance;
if (config) {
instance = new StatsD(config);
instance.last_error = { msg: '', count: 0 };
instance.socket.on('error', function (err) {
var last_err = instance.last_error;
var last_msg = last_err.msg;
var this_msg = '' + err;
if (this_msg !== last_msg) {
debug("statsd client socket error: " + err);
instance.last_error.count = 1;
instance.last_error.msg = this_msg;
} else {
++last_err.count;
if (!last_err.interval) {
instance.last_error.interval = setInterval(function () {
var count = instance.last_error.count;
if (count > 1) {
debug("last statsd client socket error repeated " + count + " times");
instance.last_error.count = 1;
clearInterval(instance.last_error.interval);
instance.last_error.interval = null;
}
}, 1000);
}
}
});
} else {
var stubFunc = function (stat, value, sampleRate, callback) {
if (_.isFunction(callback)) {
callback(null, 0);
}
};
instance = {
timing: stubFunc,
increment: stubFunc,
decrement: stubFunc,
gauge: stubFunc,
unique: stubFunc,
set: stubFunc,
sendAll: stubFunc,
send: stubFunc
};
}
this.instance = instance;
}
return this.instance;
}
};

2
npm-shrinkwrap.json generated
View File

@ -1,6 +1,6 @@
{
"name": "cartodb_sql_api",
"version": "1.43.2",
"version": "1.44.1",
"dependencies": {
"bintrees": {
"version": "1.0.1",

2
package.json
View File

@ -5,7 +5,7 @@
"keywords": [
"cartodb"
],
"version": "1.43.2",
"version": "1.44.1",
"repository": {
"type": "git",
"url": "git://github.com/CartoDB/CartoDB-SQL-API.git"