cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #966 from CartoDB/unify-connection-pool-config

Browse Source 
Unify connection pool config
This commit is contained in:
Daniel G. Aubert 2018-05-30 14:31:39 +02:00 committed by GitHub
commit 12b91e7671
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 147 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
config/environments/development.js.example
View File

@ -115,32 +115,18 @@ var config = {
// Supported labels: 'user_id', 'user_password' (both read from redis)
,postgres_auth_pass: '<%= user_password %>'
,postgres: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
type: "postgis",
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 5432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
/* experimental
geometry_field: "the_geom",
extent: "-180,-90,180,90",
srid: 4326,
*/
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
max_size: 500
pool: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
,mapnik_version: undefined
,mapnik_tile_format: 'png8:m=h'
@ -159,15 +145,7 @@ var config = {
//If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
//PostGIS 2.4 is required for this to work
//If disabled it will use Mapnik MVT generation
usePostGIS: false,
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
usePostGIS: false
},
mapnik: {
// The size of the pool of internal mapnik backend
@ -221,6 +199,29 @@ var config = {
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
postgis: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 5432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
},
limits: {
// Time in milliseconds a render request can take before it fails, some notes:
// - 0 means no render limit
@ -234,25 +235,6 @@ var config = {
cacheOnTimeout: true
},
geojson: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
},
// SQL queries will be wrapped with ST_ClipByBox2D
// Returning the portion of a geometry falling within a rectangle
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
// geometries will be simplified using ST_RemoveRepeatedPoints
// which cost is no more expensive than snapping and results are
// much closer to the original geometry
removeRepeatedPoints: false // this requires postgis >=2.2
},
// If enabled Mapnik will reuse the features retrieved from the database
// instead of requesting them once per style inside a layer
'cache-features': true,
@ -279,16 +261,7 @@ var config = {
src: __dirname + '/../../assets/default-placeholder.png'
}
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
torque: {}
}
// anything analyses related
,analysis: {

80
config/environments/production.js.example
View File

@ -115,26 +115,18 @@ var config = {
// Supported labels: 'user_id', 'user_password' (both read from redis)
,postgres_auth_pass: '<%= user_password %>'
,postgres: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 6432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
port: 5432,
pool: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
,mapnik_version: undefined
,mapnik_tile_format: 'png8:m=h'
@ -215,6 +207,29 @@ var config = {
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
postgis: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 5432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
},
limits: {
// Time in milliseconds a render request can take before it fails, some notes:
// - 0 means no render limit
@ -228,26 +243,6 @@ var config = {
cacheOnTimeout: true
},
geojson: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
},
// SQL queries will be wrapped with ST_ClipByBox2D
// Returning the portion of a geometry falling within a rectangle
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
// geometries will be simplified using ST_RemoveRepeatedPoints
// which cost is no more expensive than snapping and results are
// much closer to the original geometry
removeRepeatedPoints: false // this requires postgis >=2.2
},
// If enabled Mapnik will reuse the features retrieved from the database
// instead of requesting them once per style inside a layer
'cache-features': true,
@ -274,16 +269,7 @@ var config = {
src: __dirname + '/../../assets/default-placeholder.png'
}
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
torque: {}
}
// anything analyses related
,analysis: {

90
config/environments/staging.js.example
View File

@ -115,26 +115,18 @@ var config = {
// Supported labels: 'user_id', 'user_password' (both read from redis)
,postgres_auth_pass: '<%= user_password %>'
,postgres: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 6432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
max_size: 500
port: 5432,
pool: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
,mapnik_version: undefined
,mapnik_tile_format: 'png8:m=h'
@ -153,15 +145,7 @@ var config = {
//If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
//PostGIS 2.4 is required for this to work
//If disabled it will use Mapnik MVT generation
usePostGIS: false,
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
usePostGIS: false
},
mapnik: {
// The size of the pool of internal mapnik backend
@ -215,6 +199,29 @@ var config = {
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
postgis: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 5432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
},
limits: {
// Time in milliseconds a render request can take before it fails, some notes:
// - 0 means no render limit
@ -228,26 +235,6 @@ var config = {
cacheOnTimeout: true
},
geojson: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
},
// SQL queries will be wrapped with ST_ClipByBox2D
// Returning the portion of a geometry falling within a rectangle
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
// geometries will be simplified using ST_RemoveRepeatedPoints
// which cost is no more expensive than snapping and results are
// much closer to the original geometry
removeRepeatedPoints: false // this requires postgis >=2.2
},
// If enabled Mapnik will reuse the features retrieved from the database
// instead of requesting them once per style inside a layer
'cache-features': true,
@ -274,16 +261,7 @@ var config = {
src: __dirname + '/../../assets/default-placeholder.png'
}
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
torque: {}
}
// anything analyses related
,analysis: {

76
config/environments/test.js.example
View File

@ -120,20 +120,14 @@ var config = {
password: "public",
host: '127.0.0.1',
port: 5432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
max_size: 500
pool: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
,mapnik_version: ''
,mapnik_tile_format: 'png8:m=h'
@ -214,6 +208,29 @@ var config = {
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
postgis: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 5432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
},
limits: {
// Time in milliseconds a render request can take before it fails, some notes:
// - 0 means no render limit
@ -227,26 +244,6 @@ var config = {
cacheOnTimeout: true
},
geojson: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
},
// SQL queries will be wrapped with ST_ClipByBox2D
// Returning the portion of a geometry falling within a rectangle
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
// geometries will be simplified using ST_RemoveRepeatedPoints
// which cost is no more expensive than snapping and results are
// much closer to the original geometry
removeRepeatedPoints: false // this requires postgis >=2.2
},
// If enabled Mapnik will reuse the features retrieved from the database
// instead of requesting them once per style inside a layer
'cache-features': true,
@ -270,16 +267,7 @@ var config = {
src: __dirname + '/../../assets/default-placeholder.png'
}
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
torque: {}
}
// anything analyses related
,analysis: {

4
lib/cartodb/api/api-router.js
View File

@ -311,9 +311,9 @@ function createRendererFactory ({ redisPool, serverOptions, environmentOptions }
mapnik: serverOptions.renderer.mapnik
},
http: serverOptions.renderer.http,
mvt: serverOptions.renderer.mvt
mvt: serverOptions.renderer.mvt,
torque: serverOptions.renderer.torque
});
return rendererFactory;
}

26
lib/cartodb/server_options.js
View File

@ -16,6 +16,15 @@ var rendererConfig = _.defaults(global.environment.renderer || {}, {
snapToGrid: false,
clipByBox2d: false,
metrics: false,
postgis: {
simplify_geometries: false,
extent: '-20037508.3,-20037508.3,20037508.3,20037508.3',
row_limit: 65535,
persist_connection: false,
use_overviews: true,
max_size: 500,
twkb_encoding: true
},
limits: {}
},
http: {}
@ -118,7 +127,7 @@ module.exports = {
// TODO: allow to specify in configuration
srid: 3857
},
datasource: global.environment.postgres,
datasource: rendererConfig.mapnik.postgis || global.environment.postgres || {},
cachedir: global.environment.millstone.cache_basedir,
use_workers: rendererConfig.mapnik.useCartocssWorkers || false,
mapnik_version: global.environment.mapnik_version,
@ -131,18 +140,9 @@ module.exports = {
statsInterval: rendererConfig.statsInterval
},
renderer: {
mvt: rendererConfig.mvt,
mapnik: _.defaults(rendererConfig.mapnik, {
geojson: {
dbPoolParams: {
size: 16,
idleTimeout: 3000,
reapInterval: 1000
},
clipByBox2d: false
}
}),
torque: rendererConfig.torque,
mvt: Object.assign({ dbPoolParams: global.environment.postgres.pool }, rendererConfig.mvt),
mapnik: rendererConfig.mapnik,
torque: Object.assign({ dbPoolParams: global.environment.postgres.pool }, rendererConfig.torque),
http: rendererConfig.http
},