Merge pull request #966 from CartoDB/unify-connection-pool-config

Unify connection pool config

config/environments/development.js.example

@@ -115,32 +115,18 @@ var config = {
     // Supported labels: 'user_id', 'user_password' (both read from redis)
     ,postgres_auth_pass: '<%= user_password %>'
     ,postgres: {
-        // Parameters to pass to datasource plugin of mapnik
-        // See http://github.com/mapnik/mapnik/wiki/PostGIS
-        type: "postgis",
         user: "publicuser",
         password: "public",
         host: '127.0.0.1',
         port: 5432,
-        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        pool: {
-        /* experimental
+            // maximum number of resources to create at any given time
-        geometry_field: "the_geom",
+            size: 16,
-        extent: "-180,-90,180,90",
+            // max milliseconds a resource can go unused before it should be destroyed
-        srid: 4326,
+            idleTimeout: 3000,
-        */
+            // frequency to check for idle resources
-        // max number of rows to return when querying data, 0 means no limit
+            reapInterval: 1000
-        row_limit: 65535,
+        }
-        simplify_geometries: true,
-        use_overviews: true, // use overviews to retrieve raster
-        /*
-         * Set persist_connection to false if you want
-         * database connections to be closed on renderer
-         * expiration (1 minute after last use).
-         * Setting to true (the default) would never
-         * close any connection for the server's lifetime
-         */
-        persist_connection: false,
-        max_size: 500
     }
     ,mapnik_version: undefined
     ,mapnik_tile_format: 'png8:m=h'
@@ -159,15 +145,7 @@ var config = {
         //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
         //PostGIS 2.4 is required for this to work
         //If disabled it will use Mapnik MVT generation
-        usePostGIS: false,
+        usePostGIS: false
-        dbPoolParams: {
-              // maximum number of resources to create at any given time
-              size: 16,
-              // max milliseconds a resource can go unused before it should be destroyed
-              idleTimeout: 3000,
-              // frequency to check for idle resources
-              reapInterval: 1000
-          }
       },
       mapnik: {
           // The size of the pool of internal mapnik backend
@@ -221,6 +199,29 @@ var config = {
           // It will only work if snapToGrid is enabled
           clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
 
+          postgis: {
+              // Parameters to pass to datasource plugin of mapnik
+              // See http://github.com/mapnik/mapnik/wiki/PostGIS
+              user: "publicuser",
+              password: "public",
+              host: '127.0.0.1',
+              port: 5432,
+              extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+              // max number of rows to return when querying data, 0 means no limit
+              row_limit: 65535,
+              /*
+               * Set persist_connection to false if you want
+               * database connections to be closed on renderer
+               * expiration (1 minute after last use).
+               * Setting to true (the default) would never
+               * close any connection for the server's lifetime
+               */
+               persist_connection: false,
+               simplify_geometries: true,
+               use_overviews: true, // use overviews to retrieve raster
+               max_size: 500
+          },
+
           limits: {
               // Time in milliseconds a render request can take before it fails, some notes:
               //  - 0 means no render limit
@@ -234,25 +235,6 @@ var config = {
               cacheOnTimeout: true
           },
 
-          geojson: {
-                dbPoolParams: {
-                      // maximum number of resources to create at any given time
-                      size: 16,
-                      // max milliseconds a resource can go unused before it should be destroyed
-                      idleTimeout: 3000,
-                      // frequency to check for idle resources
-                      reapInterval: 1000
-                },
-
-              // SQL queries will be wrapped with ST_ClipByBox2D
-              // Returning the portion of a geometry falling within a rectangle
-              // It will only work if snapToGrid is enabled
-              clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
-              // geometries will be simplified using ST_RemoveRepeatedPoints
-              // which cost is no more expensive than snapping and results are
-              // much closer to the original geometry
-              removeRepeatedPoints: false // this requires postgis >=2.2
-          },
           // If enabled Mapnik will reuse the features retrieved from the database
           // instead of requesting them once per style inside a layer
           'cache-features': true,
@@ -279,16 +261,7 @@ var config = {
               src: __dirname + '/../../assets/default-placeholder.png'
           }
       },
-      torque: {
+      torque: {}
-          dbPoolParams: {
-              // maximum number of resources to create at any given time
-              size: 16,
-              // max milliseconds a resource can go unused before it should be destroyed
-              idleTimeout: 3000,
-              // frequency to check for idle resources
-              reapInterval: 1000
-          }
-      }
     }
     // anything analyses related
     ,analysis: {

config/environments/production.js.example

@@ -115,26 +115,18 @@ var config = {
     // Supported labels: 'user_id', 'user_password' (both read from redis)
     ,postgres_auth_pass: '<%= user_password %>'
     ,postgres: {
-        // Parameters to pass to datasource plugin of mapnik
-        // See http://github.com/mapnik/mapnik/wiki/PostGIS
         user: "publicuser",
         password: "public",
         host: '127.0.0.1',
-        port: 6432,
+        port: 5432,
-        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        pool: {
-        // max number of rows to return when querying data, 0 means no limit
+            // maximum number of resources to create at any given time
-        row_limit: 65535,
+            size: 16,
-        /*
+            // max milliseconds a resource can go unused before it should be destroyed
-         * Set persist_connection to false if you want
+            idleTimeout: 3000,
-         * database connections to be closed on renderer
+            // frequency to check for idle resources
-         * expiration (1 minute after last use).
+            reapInterval: 1000
-         * Setting to true (the default) would never
+        }
-         * close any connection for the server's lifetime
-         */
-        persist_connection: false,
-        simplify_geometries: true,
-        use_overviews: true, // use overviews to retrieve raster
-        max_size: 500
     }
     ,mapnik_version: undefined
     ,mapnik_tile_format: 'png8:m=h'
@@ -215,6 +207,29 @@ var config = {
           // It will only work if snapToGrid is enabled
           clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
 
+          postgis: {
+              // Parameters to pass to datasource plugin of mapnik
+              // See http://github.com/mapnik/mapnik/wiki/PostGIS
+              user: "publicuser",
+              password: "public",
+              host: '127.0.0.1',
+              port: 5432,
+              extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+              // max number of rows to return when querying data, 0 means no limit
+              row_limit: 65535,
+              /*
+               * Set persist_connection to false if you want
+               * database connections to be closed on renderer
+               * expiration (1 minute after last use).
+               * Setting to true (the default) would never
+               * close any connection for the server's lifetime
+               */
+               persist_connection: false,
+               simplify_geometries: true,
+               use_overviews: true, // use overviews to retrieve raster
+               max_size: 500
+          },
+
           limits: {
               // Time in milliseconds a render request can take before it fails, some notes:
               //  - 0 means no render limit
@@ -228,26 +243,6 @@ var config = {
               cacheOnTimeout: true
           },
 
-          geojson: {
-                dbPoolParams: {
-                      // maximum number of resources to create at any given time
-                      size: 16,
-                      // max milliseconds a resource can go unused before it should be destroyed
-                      idleTimeout: 3000,
-                      // frequency to check for idle resources
-                      reapInterval: 1000
-                },
-
-              // SQL queries will be wrapped with ST_ClipByBox2D
-              // Returning the portion of a geometry falling within a rectangle
-              // It will only work if snapToGrid is enabled
-              clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
-              // geometries will be simplified using ST_RemoveRepeatedPoints
-              // which cost is no more expensive than snapping and results are
-              // much closer to the original geometry
-              removeRepeatedPoints: false // this requires postgis >=2.2
-          },
-
           // If enabled Mapnik will reuse the features retrieved from the database
           // instead of requesting them once per style inside a layer
           'cache-features': true,
@@ -274,16 +269,7 @@ var config = {
               src: __dirname + '/../../assets/default-placeholder.png'
           }
       },
-      torque: {
+      torque: {}
-          dbPoolParams: {
-              // maximum number of resources to create at any given time
-              size: 16,
-              // max milliseconds a resource can go unused before it should be destroyed
-              idleTimeout: 3000,
-              // frequency to check for idle resources
-              reapInterval: 1000
-          }
-      }
     }
     // anything analyses related
     ,analysis: {

config/environments/staging.js.example

@@ -115,26 +115,18 @@ var config = {
     // Supported labels: 'user_id', 'user_password' (both read from redis)
     ,postgres_auth_pass: '<%= user_password %>'
     ,postgres: {
-        // Parameters to pass to datasource plugin of mapnik
-        // See http://github.com/mapnik/mapnik/wiki/PostGIS
         user: "publicuser",
         password: "public",
         host: '127.0.0.1',
-        port: 6432,
+        port: 5432,
-        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        pool: {
-        // max number of rows to return when querying data, 0 means no limit
+            // maximum number of resources to create at any given time
-        row_limit: 65535,
+            size: 16,
-        simplify_geometries: true,
+            // max milliseconds a resource can go unused before it should be destroyed
-        use_overviews: true, // use overviews to retrieve raster
+            idleTimeout: 3000,
-        /*
+            // frequency to check for idle resources
-         * Set persist_connection to false if you want
+            reapInterval: 1000
-         * database connections to be closed on renderer
+        }
-         * expiration (1 minute after last use).
-         * Setting to true (the default) would never
-         * close any connection for the server's lifetime
-         */
-        persist_connection: false,
-        max_size: 500
     }
     ,mapnik_version: undefined
     ,mapnik_tile_format: 'png8:m=h'
@@ -153,15 +145,7 @@ var config = {
         //If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
         //PostGIS 2.4 is required for this to work
         //If disabled it will use Mapnik MVT generation
-        usePostGIS: false,
+        usePostGIS: false
-        dbPoolParams: {
-              // maximum number of resources to create at any given time
-              size: 16,
-              // max milliseconds a resource can go unused before it should be destroyed
-              idleTimeout: 3000,
-              // frequency to check for idle resources
-              reapInterval: 1000
-          }
       },
       mapnik: {
           // The size of the pool of internal mapnik backend
@@ -215,6 +199,29 @@ var config = {
           // It will only work if snapToGrid is enabled
           clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
 
+          postgis: {
+              // Parameters to pass to datasource plugin of mapnik
+              // See http://github.com/mapnik/mapnik/wiki/PostGIS
+              user: "publicuser",
+              password: "public",
+              host: '127.0.0.1',
+              port: 5432,
+              extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+              // max number of rows to return when querying data, 0 means no limit
+              row_limit: 65535,
+              /*
+               * Set persist_connection to false if you want
+               * database connections to be closed on renderer
+               * expiration (1 minute after last use).
+               * Setting to true (the default) would never
+               * close any connection for the server's lifetime
+               */
+               persist_connection: false,
+               simplify_geometries: true,
+               use_overviews: true, // use overviews to retrieve raster
+               max_size: 500
+          },
+
           limits: {
               // Time in milliseconds a render request can take before it fails, some notes:
               //  - 0 means no render limit
@@ -228,26 +235,6 @@ var config = {
               cacheOnTimeout: true
           },
 
-          geojson: {
-                dbPoolParams: {
-                      // maximum number of resources to create at any given time
-                      size: 16,
-                      // max milliseconds a resource can go unused before it should be destroyed
-                      idleTimeout: 3000,
-                      // frequency to check for idle resources
-                      reapInterval: 1000
-                },
-
-              // SQL queries will be wrapped with ST_ClipByBox2D
-              // Returning the portion of a geometry falling within a rectangle
-              // It will only work if snapToGrid is enabled
-              clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
-              // geometries will be simplified using ST_RemoveRepeatedPoints
-              // which cost is no more expensive than snapping and results are
-              // much closer to the original geometry
-              removeRepeatedPoints: false // this requires postgis >=2.2
-          },
-
           // If enabled Mapnik will reuse the features retrieved from the database
           // instead of requesting them once per style inside a layer
           'cache-features': true,
@@ -274,16 +261,7 @@ var config = {
               src: __dirname + '/../../assets/default-placeholder.png'
           }
       },
-      torque: {
+      torque: {}
-          dbPoolParams: {
-              // maximum number of resources to create at any given time
-              size: 16,
-              // max milliseconds a resource can go unused before it should be destroyed
-              idleTimeout: 3000,
-              // frequency to check for idle resources
-              reapInterval: 1000
-          }
-      }
     }
     // anything analyses related
     ,analysis: {

config/environments/test.js.example

@@ -120,20 +120,14 @@ var config = {
         password: "public",
         host: '127.0.0.1',
         port: 5432,
-        extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+        pool: {
-        // max number of rows to return when querying data, 0 means no limit
+            // maximum number of resources to create at any given time
-        row_limit: 65535,
+            size: 16,
-        simplify_geometries: true,
+            // max milliseconds a resource can go unused before it should be destroyed
-        use_overviews: true, // use overviews to retrieve raster
+            idleTimeout: 3000,
-        /*
+            // frequency to check for idle resources
-         * Set persist_connection to false if you want
+            reapInterval: 1000
-         * database connections to be closed on renderer
+        }
-         * expiration (1 minute after last use).
-         * Setting to true (the default) would never
-         * close any connection for the server's lifetime
-         */
-        persist_connection: false,
-        max_size: 500
     }
     ,mapnik_version: ''
     ,mapnik_tile_format: 'png8:m=h'
@@ -214,6 +208,29 @@ var config = {
           // It will only work if snapToGrid is enabled
           clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
 
+          postgis: {
+              // Parameters to pass to datasource plugin of mapnik
+              // See http://github.com/mapnik/mapnik/wiki/PostGIS
+              user: "publicuser",
+              password: "public",
+              host: '127.0.0.1',
+              port: 5432,
+              extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
+              // max number of rows to return when querying data, 0 means no limit
+              row_limit: 65535,
+              /*
+               * Set persist_connection to false if you want
+               * database connections to be closed on renderer
+               * expiration (1 minute after last use).
+               * Setting to true (the default) would never
+               * close any connection for the server's lifetime
+               */
+               persist_connection: false,
+               simplify_geometries: true,
+               use_overviews: true, // use overviews to retrieve raster
+               max_size: 500
+          },
+
           limits: {
               // Time in milliseconds a render request can take before it fails, some notes:
               //  - 0 means no render limit
@@ -227,26 +244,6 @@ var config = {
               cacheOnTimeout: true
           },
 
-          geojson: {
-                dbPoolParams: {
-                      // maximum number of resources to create at any given time
-                      size: 16,
-                      // max milliseconds a resource can go unused before it should be destroyed
-                      idleTimeout: 3000,
-                      // frequency to check for idle resources
-                      reapInterval: 1000
-                },
-
-              // SQL queries will be wrapped with ST_ClipByBox2D
-              // Returning the portion of a geometry falling within a rectangle
-              // It will only work if snapToGrid is enabled
-              clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
-              // geometries will be simplified using ST_RemoveRepeatedPoints
-              // which cost is no more expensive than snapping and results are
-              // much closer to the original geometry
-              removeRepeatedPoints: false // this requires postgis >=2.2
-          },
-
           // If enabled Mapnik will reuse the features retrieved from the database
           // instead of requesting them once per style inside a layer
           'cache-features': true,
@@ -270,16 +267,7 @@ var config = {
               src: __dirname + '/../../assets/default-placeholder.png'
           }
       },
-      torque: {
+      torque: {}
-          dbPoolParams: {
-              // maximum number of resources to create at any given time
-              size: 16,
-              // max milliseconds a resource can go unused before it should be destroyed
-              idleTimeout: 3000,
-              // frequency to check for idle resources
-              reapInterval: 1000
-          }
-      }
     }
     // anything analyses related
     ,analysis: {

lib/cartodb/api/api-router.js

@@ -311,9 +311,9 @@ function createRendererFactory ({ redisPool, serverOptions, environmentOptions }
             mapnik: serverOptions.renderer.mapnik
         },
         http: serverOptions.renderer.http,
-        mvt: serverOptions.renderer.mvt
+        mvt: serverOptions.renderer.mvt,
+        torque: serverOptions.renderer.torque
     });
 
-
     return rendererFactory;
 }

lib/cartodb/server_options.js

@@ -16,6 +16,15 @@ var rendererConfig = _.defaults(global.environment.renderer || {}, {
         snapToGrid: false,
         clipByBox2d: false,
         metrics: false,
+        postgis: {
+            simplify_geometries: false,
+            extent: '-20037508.3,-20037508.3,20037508.3,20037508.3',
+            row_limit: 65535,
+            persist_connection: false,
+            use_overviews: true,
+            max_size: 500,
+            twkb_encoding: true
+        },
         limits: {}
     },
     http: {}
@@ -118,7 +127,7 @@ module.exports = {
             // TODO: allow to specify in configuration
             srid: 3857
         },
-        datasource: global.environment.postgres,
+        datasource: rendererConfig.mapnik.postgis || global.environment.postgres || {},
         cachedir: global.environment.millstone.cache_basedir,
         use_workers: rendererConfig.mapnik.useCartocssWorkers || false,
         mapnik_version: global.environment.mapnik_version,
@@ -131,18 +140,9 @@ module.exports = {
         statsInterval: rendererConfig.statsInterval
     },
     renderer: {
-        mvt: rendererConfig.mvt,
+        mvt: Object.assign({ dbPoolParams: global.environment.postgres.pool }, rendererConfig.mvt),
-        mapnik: _.defaults(rendererConfig.mapnik, {
+        mapnik: rendererConfig.mapnik,
-            geojson: {
+        torque: Object.assign({ dbPoolParams: global.environment.postgres.pool }, rendererConfig.torque),
-                dbPoolParams: {
-                    size: 16,
-                    idleTimeout: 3000,
-                    reapInterval: 1000
-                },
-                clipByBox2d: false
-            }
-        }),
-        torque: rendererConfig.torque,
         http: rendererConfig.http
     },