762a240890
- Log system revamp: - Logs to stdout, disabled while testing - Use header `X-Request-Id`, or create a new `uuid` when no present, to identyfy log entries - Be able to set log level from env variable `LOG_LEVEL`, useful while testing: `LOG_LEVEL=info npm test`; even more human-readable: `LOG_LEVEL=info npm t | ./node_modules/.bin/pino-pretty` - Be able to reduce the footprint in the final log file depending on the environment - Use one logger for every service: Queries, Batch Queries (Jobs), and Data Ingestion (CopyTo/CopyFrom) - Stop using headers such as: `X-SQL-API-Log`, `X-SQL-API-Profiler`, and `X-SQL-API-Errors` as a way to log info. - Be able to tag requests with labels as an easier way to provide business metrics - Metro: Add log-collector utility (`metro`), it will be moved to its own repository. Attaching it here fro development purposes. Try it with the following command `LOG_LEVEL=info npm t | node metro` - Metro: Creates `metrics-collector.js` a stream to update Prometheus' counters and histograms and exposes them via Express' app (`:9145/metrics`). Use the ones defined in `grok_exporter` Announcements: - Profiler is always set. No need to check its existence anymore - Unify profiler usage for every endpoint Bug fixes: - Avoid hung requests while fetching user identifier
164 lines
6.0 KiB
Plaintext
164 lines
6.0 KiB
Plaintext
// Time in milliseconds to force GC cycle.
|
|
// Disable by using <=0 value.
|
|
module.exports.gc_interval = 10000;
|
|
module.exports.routes = {
|
|
// Each entry corresponds with an express' router.
|
|
// You must define at least one path. However, middlewares are optional.
|
|
api: [{
|
|
// Required: path where other "routers" or "controllers" will be attached to.
|
|
paths: [
|
|
// In case the path has a :user param the username will be the one specified in the URL,
|
|
// otherwise it will fallback to extract the username from the host header.
|
|
'/api/:version',
|
|
'/user/:user/api/:version',
|
|
],
|
|
// Optional: attach middlewares at the begining of the router
|
|
// to perform custom operations.
|
|
middlewares: [
|
|
function noop () {
|
|
return function noopMiddleware (req, res, next) {
|
|
next();
|
|
}
|
|
}
|
|
],
|
|
sql: [{
|
|
// Required
|
|
paths: [
|
|
'/sql'
|
|
],
|
|
// Optional
|
|
middlewares: []
|
|
}]
|
|
}]
|
|
};
|
|
// If useProfiler is true every response will be served with an
|
|
// X-SQLAPI-Profile header containing elapsed timing for various
|
|
// steps taken for producing the response.
|
|
module.exports.useProfiler = true;
|
|
// Regular expression pattern to extract username
|
|
// from hostname. Must have a single grabbing block.
|
|
module.exports.user_from_host = '^(.*)\\.cartodb\\.com$';
|
|
module.exports.node_port = 8080;
|
|
module.exports.node_host = '127.0.0.1';
|
|
// idle socket timeout, in miliseconds
|
|
module.exports.node_socket_timeout = 600000;
|
|
module.exports.environment = 'production';
|
|
// Supported labels: 'user_id' (read from redis)
|
|
module.exports.db_base_name = 'cartodb_user_<%= user_id %>_db';
|
|
// Supported labels: 'user_id' (read from redis)
|
|
module.exports.db_user = 'cartodb_user_<%= user_id %>';
|
|
// Supported labels: 'user_id', 'user_password' (both read from redis)
|
|
module.exports.db_user_pass = '<%= user_password %>'
|
|
// Name of the anonymous PostgreSQL user
|
|
module.exports.db_pubuser = 'publicuser';
|
|
// Password for the anonymous PostgreSQL user
|
|
module.exports.db_pubuser_pass = 'public';
|
|
module.exports.db_host = 'localhost';
|
|
module.exports.db_port = '6432';
|
|
module.exports.db_batch_port = '5432';
|
|
module.exports.finished_jobs_ttl_in_seconds = 2 * 3600; // 2 hours
|
|
module.exports.batch_query_timeout = 12 * 3600 * 1000; // 12 hours in milliseconds
|
|
module.exports.copy_timeout = "'5h'";
|
|
module.exports.copy_from_max_post_size = 2 * 1024 * 1024 * 1024 // 2 GB;
|
|
module.exports.copy_from_max_post_size_pretty = '2 GB';
|
|
module.exports.copy_from_minimum_input_speed = 0; // 1 byte per second
|
|
module.exports.copy_from_maximum_slow_input_speed_interval = 15 // 15 seconds
|
|
// Max number of queued jobs a user can have at a given time
|
|
module.exports.batch_max_queued_jobs = 64;
|
|
// Capacity strategy to use.
|
|
// It allows to tune how many queries run at a db host at the same time.
|
|
// Options: 'fixed', 'http-simple', 'http-load'
|
|
module.exports.batch_capacity_strategy = 'fixed';
|
|
// Applies when strategy='fixed'.
|
|
// Number of simultaneous users running queries in the same host.
|
|
// It will use 1 as min.
|
|
// Default 4.
|
|
module.exports.batch_capacity_fixed_amount = 4;
|
|
// Applies when strategy='http-simple' or strategy='http-load'.
|
|
// HTTP endpoint to check db host load.
|
|
// Helps to decide the number of simultaneous users running queries in that host.
|
|
// 'http-simple' will use 'available_cores' to decide the number.
|
|
// 'http-load' will use 'cores' and 'relative_load' to decide the number.
|
|
// It will use 1 as min.
|
|
// If no template is provided it will default to 'fixed' strategy.
|
|
module.exports.batch_capacity_http_url_template = 'http://<%= dbhost %>:9999/load';
|
|
// Max database connections in the pool
|
|
// Subsequent connections will wait for a free slot.i
|
|
// NOTE: not used by OGR-mediated accesses
|
|
module.exports.db_pool_size = 500;
|
|
// Milliseconds before a connection is removed from pool
|
|
module.exports.db_pool_idleTimeout = 30000;
|
|
// Milliseconds between idle client checking
|
|
module.exports.db_pool_reapInterval = 1000;
|
|
// max number of bytes for a row, when exceeded the query will throw an error
|
|
//module.exports.db_max_row_size = 10 * 1024 * 1024;
|
|
// allows to use an object to connect with node-postgres instead of a connection string
|
|
module.exports.db_use_config_object = true;
|
|
// requires enabling db_use_config_object=true
|
|
// allows to enable/disable keep alive for database connections
|
|
// by default is not enabled
|
|
module.exports.db_keep_alive = {
|
|
enabled: true,
|
|
initialDelay: 5000 // Not used yet
|
|
};
|
|
module.exports.redis_host = '127.0.0.1';
|
|
module.exports.redis_port = 6379;
|
|
module.exports.redisPool = 50;
|
|
module.exports.redisIdleTimeoutMillis = 10000;
|
|
module.exports.redisReapIntervalMillis = 1000;
|
|
module.exports.redisLog = false;
|
|
|
|
// Temporary directory, make sure it is writable by server user
|
|
module.exports.tmpDir = '/tmp';
|
|
// change ogr2ogr command or path
|
|
module.exports.ogr2ogrCommand = 'ogr2ogr';
|
|
// change zip command or path
|
|
module.exports.zipCommand = 'zip';
|
|
// Optional statsd support
|
|
module.exports.statsd = {
|
|
host: 'localhost',
|
|
port: 8125,
|
|
prefix: 'dev.:host.',
|
|
cacheDns: true
|
|
// support all allowed node-statsd options
|
|
};
|
|
module.exports.health = {
|
|
enabled: true,
|
|
username: 'development',
|
|
query: 'select 1'
|
|
};
|
|
module.exports.oauth = {
|
|
allowedHosts: ['carto.com', 'cartodb.com']
|
|
};
|
|
module.exports.disabled_file = 'pids/disabled';
|
|
|
|
module.exports.ratelimits = {
|
|
// whether it should rate limit endpoints (global configuration)
|
|
rateLimitsEnabled: false,
|
|
// whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
|
|
endpoints: {
|
|
query: false,
|
|
job_create: false,
|
|
job_get: false,
|
|
job_delete: false,
|
|
copy_from: false,
|
|
copy_to: false
|
|
}
|
|
}
|
|
|
|
module.exports.validatePGEntitiesAccess = false;
|
|
module.exports.logQueries = true;
|
|
module.exports.maxQueriesLogLength = 1024;
|
|
|
|
module.exports.cache = {
|
|
ttl: 60 * 60 * 24 * 365, // one year in seconds
|
|
fallbackTtl: 60 * 5 // five minutes in seconds
|
|
};
|
|
|
|
module.exports.pubSubMetrics = {
|
|
enabled: true,
|
|
project_id: 'avid-wavelet-844',
|
|
credentials: '',
|
|
topic: 'raw-metric-events'
|
|
};
|