cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into remove-auth-fallback

Browse Source
This commit is contained in:
Eneko Lakasta 2018-05-30 17:53:24 +02:00
commit 6414cd52c0
106 changed files with 4559 additions and 2772 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
NEWS.md
View File

@ -1,16 +1,44 @@
# Changelog
## 6.1.1
## 6.2.0
Released 2018-mm-dd
New features:
- CI tests with Ubuntu Xenial + PostgreSQL 10.1 and Ubuntu Precise + PostgreSQL 9.5
- Upgrades Windshaft to [4.8.1](https://github.com/CartoDB/Windshaft/blob/4.8.1/NEWS.md#version-481) which includes:
- Update internal deps.
- A fix in mapnik-vector-tile to avoid grouping together properties with the same value but a different type.
- Performance improvements in the marker symbolizer (local cache, avoid building the collision matrix when possible).
- MVT: Disable simplify_distance to avoid multiple simplifications.
- Fix a bug with zero length lines not being rendered when using the marker symbolizer.
- Upgrades Camshaft to [0.61.10](https://github.com/CartoDB/camshaft/releases/tag/0.61.10):
- Use Dollar-Quoted String Constants to avoid Syntax Error while running moran analyses.
- Update other deps:
- body-parser: 1.18.3
- cartodb-psql: 0.11.0
- dot: 1.1.2
- express: 4.16.3
- lru-cache: 4.1.3
- node-statsd: 0.1.1,
- queue-async: 1.1.0
- request: 2.87.0
- semver: 5.5.0
- step: 1.0.0
- yargs: 11.1.0
- Update devel deps:
- istanbul: 0.4.5
- jshint: 2.9.5
- mocha: 3.5.3
- moment: 2.22.1
- nock: 9.2.6
- strftime: 0.10.0
- Optional instantiation metadata stats (https://github.com/CartoDB/Windshaft-cartodb/pull/952)
Bug Fixes:
- Validates tile coordinates (z/x/y) from request params to be a valid integer value.
- Upgrades Windshaft to 4.7.1, which includes a fix in mapnik-vector-tile to avoid grouping together properties with the same value but different type.
- Static maps fails for unsupported formats
- Handling errors extracting the column type on dataviews
- Fix `meta.stats.estimatedFeatureCount` for aggregations and queries with tokens
## 6.1.0
Released 2018-04-16

2
app.js
View File

@ -100,8 +100,6 @@ if ( global.environment.log_filename ) {
global.log4js.configure(log4jsConfig);
global.logger = global.log4js.getLogger();
global.environment.api_hostname = require('os').hostname().split('.')[0];
// Include cartodb_windshaft only _after_ the "global" variable is set
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/28
var cartodbWindshaft = require('./lib/cartodb/server');

140
config/environments/development.js.example
View File

@ -15,16 +15,49 @@ var config = {
// Base URLs for the APIs
//
// See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
//
// See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
,routes: {
v1: {
paths: [
'/api/v1',
'/user/:user/api/v1',
],
// Base url for the Detached Maps API
// "/api/v1/map" is the new API,
map: {
paths: [
'/map',
]
},
// Base url for the Templated Maps API
// "/api/v1/map/named" is the new API,
// "/tiles/template" is for compatibility with versions up to 1.6.x
,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)'
template: {
paths: [
'/map/named'
]
}
},
// For compatibility with versions up to 1.6.x
v0: {
paths: [
'/tiles'
],
// Base url for the Detached Maps API
// "maps" is the the new API,
// "tiles/layergroup" is for compatibility with versions up to 1.6.x
,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
// "/tiles/layergroup" is for compatibility with versions up to 1.6.x
map: {
paths: [
'/layergroup'
]
},
// Base url for the Templated Maps API
// "/tiles/template" is for compatibility with versions up to 1.6.x
template: {
paths: [
'/template'
]
}
}
}
// Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
//
@ -67,32 +100,18 @@ var config = {
// Supported labels: 'user_id', 'user_password' (both read from redis)
,postgres_auth_pass: '<%= user_password %>'
,postgres: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
type: "postgis",
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 5432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
/* experimental
geometry_field: "the_geom",
extent: "-180,-90,180,90",
srid: 4326,
*/
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
max_size: 500
pool: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
,mapnik_version: undefined
,mapnik_tile_format: 'png8:m=h'
@ -111,15 +130,7 @@ var config = {
//If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
//PostGIS 2.4 is required for this to work
//If disabled it will use Mapnik MVT generation
usePostGIS: false,
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
usePostGIS: false
},
mapnik: {
// The size of the pool of internal mapnik backend
@ -173,6 +184,29 @@ var config = {
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
postgis: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 5432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
},
limits: {
// Time in milliseconds a render request can take before it fails, some notes:
// - 0 means no render limit
@ -186,25 +220,6 @@ var config = {
cacheOnTimeout: true
},
geojson: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
},
// SQL queries will be wrapped with ST_ClipByBox2D
// Returning the portion of a geometry falling within a rectangle
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
// geometries will be simplified using ST_RemoveRepeatedPoints
// which cost is no more expensive than snapping and results are
// much closer to the original geometry
removeRepeatedPoints: false // this requires postgis >=2.2
},
// If enabled Mapnik will reuse the features retrieved from the database
// instead of requesting them once per style inside a layer
'cache-features': true,
@ -231,16 +246,7 @@ var config = {
src: __dirname + '/../../assets/default-placeholder.png'
}
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
torque: {}
}
// anything analyses related
,analysis: {

127
config/environments/production.js.example
View File

@ -15,16 +15,49 @@ var config = {
// Base URLs for the APIs
//
// See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
//
// See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
,routes: {
v1: {
paths: [
'/api/v1',
'/user/:user/api/v1',
],
// Base url for the Detached Maps API
// "/api/v1/map" is the new API,
map: {
paths: [
'/map',
]
},
// Base url for the Templated Maps API
// "/api/v1/map/named" is the new API,
// "/tiles/template" is for compatibility with versions up to 1.6.x
,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)'
template: {
paths: [
'/map/named'
]
}
},
// For compatibility with versions up to 1.6.x
v0: {
paths: [
'/tiles'
],
// Base url for the Detached Maps API
// "maps" is the the new API,
// "tiles/layergroup" is for compatibility with versions up to 1.6.x
,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
// "/tiles/layergroup" is for compatibility with versions up to 1.6.x
map: {
paths: [
'/layergroup'
]
},
// Base url for the Templated Maps API
// "/tiles/template" is for compatibility with versions up to 1.6.x
template: {
paths: [
'/template'
]
}
}
}
// Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
//
@ -67,26 +100,18 @@ var config = {
// Supported labels: 'user_id', 'user_password' (both read from redis)
,postgres_auth_pass: '<%= user_password %>'
,postgres: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 6432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
port: 5432,
pool: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
,mapnik_version: undefined
,mapnik_tile_format: 'png8:m=h'
@ -167,6 +192,29 @@ var config = {
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
postgis: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 5432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
},
limits: {
// Time in milliseconds a render request can take before it fails, some notes:
// - 0 means no render limit
@ -180,26 +228,6 @@ var config = {
cacheOnTimeout: true
},
geojson: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
},
// SQL queries will be wrapped with ST_ClipByBox2D
// Returning the portion of a geometry falling within a rectangle
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
// geometries will be simplified using ST_RemoveRepeatedPoints
// which cost is no more expensive than snapping and results are
// much closer to the original geometry
removeRepeatedPoints: false // this requires postgis >=2.2
},
// If enabled Mapnik will reuse the features retrieved from the database
// instead of requesting them once per style inside a layer
'cache-features': true,
@ -226,16 +254,7 @@ var config = {
src: __dirname + '/../../assets/default-placeholder.png'
}
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
torque: {}
}
// anything analyses related
,analysis: {

139
config/environments/staging.js.example
View File

@ -15,16 +15,49 @@ var config = {
// Base URLs for the APIs
//
// See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
//
// Base url for the Templated Maps API
// "/api/v1/maps/named" is the new API,
// "/tiles/template" is for compatibility with versions up to 1.6.x
,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)'
// See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
,routes: {
v1: {
paths: [
'/api/v1',
'/user/:user/api/v1',
],
// Base url for the Detached Maps API
// "/api/v1/map" is the new API,
map: {
paths: [
'/map',
]
},
// Base url for the Templated Maps API
// "/api/v1/map/named" is the new API,
template: {
paths: [
'/map/named'
]
}
},
// For compatibility with versions up to 1.6.x
v0: {
paths: [
'/tiles'
],
// Base url for the Detached Maps API
// "/api/v1/maps" is the the new API,
// "/tiles/layergroup" is for compatibility with versions up to 1.6.x
,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
map: {
paths: [
'/layergroup'
]
},
// Base url for the Templated Maps API
// "/tiles/template" is for compatibility with versions up to 1.6.x
template: {
paths: [
'/template'
]
}
}
}
// Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
//
@ -67,26 +100,18 @@ var config = {
// Supported labels: 'user_id', 'user_password' (both read from redis)
,postgres_auth_pass: '<%= user_password %>'
,postgres: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 6432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
max_size: 500
port: 5432,
pool: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
,mapnik_version: undefined
,mapnik_tile_format: 'png8:m=h'
@ -105,15 +130,7 @@ var config = {
//If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
//PostGIS 2.4 is required for this to work
//If disabled it will use Mapnik MVT generation
usePostGIS: false,
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
usePostGIS: false
},
mapnik: {
// The size of the pool of internal mapnik backend
@ -167,6 +184,29 @@ var config = {
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
postgis: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 5432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
},
limits: {
// Time in milliseconds a render request can take before it fails, some notes:
// - 0 means no render limit
@ -180,26 +220,6 @@ var config = {
cacheOnTimeout: true
},
geojson: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
},
// SQL queries will be wrapped with ST_ClipByBox2D
// Returning the portion of a geometry falling within a rectangle
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
// geometries will be simplified using ST_RemoveRepeatedPoints
// which cost is no more expensive than snapping and results are
// much closer to the original geometry
removeRepeatedPoints: false // this requires postgis >=2.2
},
// If enabled Mapnik will reuse the features retrieved from the database
// instead of requesting them once per style inside a layer
'cache-features': true,
@ -226,16 +246,7 @@ var config = {
src: __dirname + '/../../assets/default-placeholder.png'
}
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
torque: {}
}
// anything analyses related
,analysis: {

121
config/environments/test.js.example
View File

@ -16,15 +16,48 @@ var config = {
// Base URLs for the APIs
//
// See https://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
//
,routes: {
v1: {
paths: [
'/api/v1',
'/user/:user/api/v1',
],
// Base url for the Detached Maps API
// "/api/v1/map" is the new API,
map: {
paths: [
'/map',
]
},
// Base url for the Templated Maps API
// "/api/v1/map/named" is the new API,
// "/tiles/template" is for compatibility with versions up to 1.6.x
,base_url_templated: '(?:/api/v1/map/named|/user/:user/api/v1/map/named|/tiles/template)'
template: {
paths: [
'/map/named'
]
}
},
// For compatibility with versions up to 1.6.x
v0: {
paths: [
'/tiles'
],
// Base url for the Detached Maps API
// "maps" is the the new API,
// "tiles/layergroup" is for compatibility with versions up to 1.6.x
,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)'
// "/tiles/layergroup" is for compatibility with versions up to 1.6.x
map: {
paths: [
'/layergroup'
]
},
// Base url for the Templated Maps API
// "/tiles/template" is for compatibility with versions up to 1.6.x
template: {
paths: [
'/template'
]
}
}
}
// Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
//
@ -72,20 +105,14 @@ var config = {
password: "public",
host: '127.0.0.1',
port: 5432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
max_size: 500
pool: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
,mapnik_version: ''
,mapnik_tile_format: 'png8:m=h'
@ -166,6 +193,29 @@ var config = {
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
postgis: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "publicuser",
password: "public",
host: '127.0.0.1',
port: 5432,
extent: "-20037508.3,-20037508.3,20037508.3,20037508.3",
// max number of rows to return when querying data, 0 means no limit
row_limit: 65535,
/*
* Set persist_connection to false if you want
* database connections to be closed on renderer
* expiration (1 minute after last use).
* Setting to true (the default) would never
* close any connection for the server's lifetime
*/
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
},
limits: {
// Time in milliseconds a render request can take before it fails, some notes:
// - 0 means no render limit
@ -179,26 +229,6 @@ var config = {
cacheOnTimeout: true
},
geojson: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
},
// SQL queries will be wrapped with ST_ClipByBox2D
// Returning the portion of a geometry falling within a rectangle
// It will only work if snapToGrid is enabled
clipByBox2d: false, // this requires postgis >=2.2 and geos >=3.5
// geometries will be simplified using ST_RemoveRepeatedPoints
// which cost is no more expensive than snapping and results are
// much closer to the original geometry
removeRepeatedPoints: false // this requires postgis >=2.2
},
// If enabled Mapnik will reuse the features retrieved from the database
// instead of requesting them once per style inside a layer
'cache-features': true,
@ -222,16 +252,7 @@ var config = {
src: __dirname + '/../../assets/default-placeholder.png'
}
},
torque: {
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
}
torque: {}
}
// anything analyses related
,analysis: {

319
lib/cartodb/api/api-router.js Normal file
View File

@ -0,0 +1,319 @@
const { Router: router } = require('express');
const RedisPool = require('redis-mpool');
const cartodbRedis = require('cartodb-redis');
const windshaft = require('windshaft');
const PgConnection = require('../backends/pg_connection');
const AnalysisBackend = require('../backends/analysis');
const AnalysisStatusBackend = require('../backends/analysis-status');
const DataviewBackend = require('../backends/dataview');
const TemplateMaps = require('../backends/template_maps.js');
const PgQueryRunner = require('../backends/pg_query_runner');
const StatsBackend = require('../backends/stats');
const AuthBackend = require('../backends/auth');
const UserLimitsBackend = require('../backends/user-limits');
const OverviewsMetadataBackend = require('../backends/overviews-metadata');
const FilterStatsApi = require('../backends/filter-stats');
const TablesExtentBackend = require('../backends/tables-extent');
const LayergroupAffectedTablesCache = require('../cache/layergroup_affected_tables');
const SurrogateKeysCache = require('../cache/surrogate_keys_cache');
const VarnishHttpCacheBackend = require('../cache/backend/varnish_http');
const FastlyCacheBackend = require('../cache/backend/fastly');
const NamedMapProviderCache = require('../cache/named_map_provider_cache');
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
const MapConfigNamedLayersAdapter = require('../models/mapconfig/adapter/mapconfig-named-layers-adapter');
const MapConfigBufferSizeAdapter = require('../models/mapconfig/adapter/mapconfig-buffer-size-adapter');
const AnalysisMapConfigAdapter = require('../models/mapconfig/adapter/analysis-mapconfig-adapter');
const MapConfigOverviewsAdapter = require('../models/mapconfig/adapter/mapconfig-overviews-adapter');
const TurboCartoAdapter = require('../models/mapconfig/adapter/turbo-carto-adapter');
const DataviewsWidgetsAdapter = require('../models/mapconfig/adapter/dataviews-widgets-adapter');
const AggregationMapConfigAdapter = require('../models/mapconfig/adapter/aggregation-mapconfig-adapter');
const MapConfigAdapter = require('../models/mapconfig/adapter');
const ResourceLocator = require('../models/resource-locator');
const LayergroupMetadata = require('../utils/layergroup-metadata');
const RendererStatsReporter = require('../stats/reporter/renderer');
const initializeStatusCode = require('./middlewares/initialize-status-code');
const logger = require('./middlewares/logger');
const bodyParser = require('body-parser');
const servedByHostHeader = require('./middlewares/served-by-host-header');
const stats = require('./middlewares/stats');
const lzmaMiddleware = require('./middlewares/lzma');
const cors = require('./middlewares/cors');
const user = require('./middlewares/user');
const sendResponse = require('./middlewares/send-response');
const syntaxError = require('./middlewares/syntax-error');
const errorMiddleware = require('./middlewares/error-middleware');
const MapRouter = require('./map/map-router');
const TemplateRouter = require('./template/template-router');
module.exports = class ApiRouter {
constructor ({ serverOptions, environmentOptions }) {
this.serverOptions = serverOptions;
const redisOptions = Object.assign({
name: 'windshaft-server',
unwatchOnRelease: false,
noReadyCheck: true
}, environmentOptions.redis);
const redisPool = new RedisPool(redisOptions);
redisPool.on('status', function (status) {
var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.';
global.statsClient.gauge(keyPrefix + 'count', status.count);
global.statsClient.gauge(keyPrefix + 'unused', status.unused);
global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
});
const metadataBackend = cartodbRedis({ pool: redisPool });
const pgConnection = new PgConnection(metadataBackend);
const mapStore = new windshaft.storage.MapStore({
pool: redisPool,
expire_time: serverOptions.grainstore.default_layergroup_ttl
});
const rendererFactory = createRendererFactory({ redisPool, serverOptions, environmentOptions });
const rendererCacheOpts = Object.assign({
ttl: 60000, // 60 seconds TTL by default
statsInterval: 60000 // reports stats every milliseconds defined here
}, serverOptions.renderCache || {});
const rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts);
const rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval);
rendererStatsReporter.start();
const tileBackend = new windshaft.backend.Tile(rendererCache);
const attributesBackend = new windshaft.backend.Attributes();
const previewBackend = new windshaft.backend.Preview(rendererCache);
const mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
const mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
const surrogateKeysCacheBackends = createSurrogateKeysCacheBackends(serverOptions);
const surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends);
const templateMaps = createTemplateMaps({ redisPool, surrogateKeysCache });
const analysisStatusBackend = new AnalysisStatusBackend();
const analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
const dataviewBackend = new DataviewBackend(analysisBackend);
const statsBackend = new StatsBackend();
const userLimitsBackend = new UserLimitsBackend(metadataBackend, {
limits: {
cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
render: serverOptions.renderer.mapnik.limits.render || 0,
rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled
}
});
const authBackend = new AuthBackend(pgConnection, metadataBackend, mapStore, templateMaps);
const layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
if (process.env.NODE_ENV === 'test') {
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
}
const pgQueryRunner = new PgQueryRunner(pgConnection);
const overviewsMetadataBackend = new OverviewsMetadataBackend(pgQueryRunner);
const filterStatsBackend = new FilterStatsApi(pgQueryRunner);
const tablesExtentBackend = new TablesExtentBackend(pgQueryRunner);
const mapConfigAdapter = new MapConfigAdapter(
new MapConfigNamedLayersAdapter(templateMaps, pgConnection),
new MapConfigBufferSizeAdapter(),
new SqlWrapMapConfigAdapter(),
new DataviewsWidgetsAdapter(),
new AnalysisMapConfigAdapter(analysisBackend),
new AggregationMapConfigAdapter(pgConnection),
new MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend),
new TurboCartoAdapter()
);
const resourceLocator = new ResourceLocator(global.environment);
const layergroupMetadata = new LayergroupMetadata(resourceLocator);
const namedMapProviderCache = new NamedMapProviderCache(
templateMaps,
pgConnection,
metadataBackend,
userLimitsBackend,
mapConfigAdapter,
layergroupAffectedTablesCache
);
['update', 'delete'].forEach(function(eventType) {
templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
});
const collaborators = {
analysisStatusBackend,
attributesBackend,
dataviewBackend,
previewBackend,
tileBackend,
pgConnection,
mapStore,
userLimitsBackend,
layergroupAffectedTablesCache,
authBackend,
surrogateKeysCache,
templateMaps,
mapBackend,
metadataBackend,
mapConfigAdapter,
statsBackend,
layergroupMetadata,
namedMapProviderCache,
tablesExtentBackend
};
this.mapRouter = new MapRouter({ collaborators });
this.templateRouter = new TemplateRouter({ collaborators });
}
register (app) {
// FIXME: we need a better way to reset cache while running tests
if (process.env.NODE_ENV === 'test') {
app.layergroupAffectedTablesCache = this.layergroupAffectedTablesCache;
}
Object.keys(this.serverOptions.routes).forEach(apiVersion => {
const routes = this.serverOptions.routes[apiVersion];
const apiRouter = router({ mergeParams: true });
apiRouter.use(logger(this.serverOptions));
apiRouter.use(initializeStatusCode());
apiRouter.use(bodyParser.json());
apiRouter.use(servedByHostHeader());
apiRouter.use(stats({
enabled: this.serverOptions.useProfiler,
statsClient: global.statsClient
}));
apiRouter.use(lzmaMiddleware());
apiRouter.use(cors());
apiRouter.use(user());
this.templateRouter.register(apiRouter, routes.template.paths);
this.mapRouter.register(apiRouter, routes.map.paths);
apiRouter.use(sendResponse());
apiRouter.use(syntaxError());
apiRouter.use(errorMiddleware());
const apiPaths = routes.paths;
apiPaths.forEach(path => app.use(path, apiRouter));
});
}
};
function createTemplateMaps ({ redisPool, surrogateKeysCache }) {
const templateMaps = new TemplateMaps(redisPool, {
max_user_templates: global.environment.maxUserTemplates
});
function invalidateNamedMap (owner, templateName) {
var startTime = Date.now();
surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) {
var logMessage = JSON.stringify({
username: owner,
type: 'named_map_invalidation',
elapsed: Date.now() - startTime,
error: !!err ? JSON.stringify(err.message) : undefined
});
if (err) {
global.logger.warn(logMessage);
} else {
global.logger.info(logMessage);
}
});
}
['update', 'delete'].forEach(function(eventType) {
templateMaps.on(eventType, invalidateNamedMap);
});
return templateMaps;
}
function createSurrogateKeysCacheBackends(serverOptions) {
var cacheBackends = [];
if (serverOptions.varnish_purge_enabled) {
cacheBackends.push(
new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port)
);
}
if (serverOptions.fastly &&
!!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) {
cacheBackends.push(
new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId)
);
}
return cacheBackends;
}
const timeoutErrorTilePath = __dirname + '/../../../assets/render-timeout-fallback.png';
const timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
function createRendererFactory ({ redisPool, serverOptions, environmentOptions }) {
var onTileErrorStrategy;
if (environmentOptions.enabledFeatures.onTileErrorStrategy !== false) {
onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) {
function isRenderTimeoutError (err) {
return err.message === 'Render timed out';
}
function isDatasourceTimeoutError (err) {
return err.message && err.message.match(/canceling statement due to statement timeout/i);
}
function isTimeoutError (err) {
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
}
function isRasterFormat (format) {
return format === 'png' || format === 'jpg';
}
if (isTimeoutError(err) && isRasterFormat(format)) {
return callback(null, timeoutErrorTile, {
'Content-Type': 'image/png',
}, {});
} else {
return callback(err, tile, headers, stats);
}
};
}
const rendererFactory = new windshaft.renderer.Factory({
onTileErrorStrategy: onTileErrorStrategy,
mapnik: {
redisPool: redisPool,
grainstore: serverOptions.grainstore,
mapnik: serverOptions.renderer.mapnik
},
http: serverOptions.renderer.http,
mvt: serverOptions.renderer.mvt,
torque: serverOptions.renderer.torque
});
return rendererFactory;
}

48
lib/cartodb/controllers/analyses.js → lib/cartodb/api/map/analyses-catalog-controller.js
View File

@ -1,44 +1,39 @@
const PSQL = require('cartodb-psql');
const cors = require('../middleware/cors');
const user = require('../middleware/user');
const cleanUpQueryParams = require('../middleware/clean-up-query-params');
const credentials = require('../middleware/credentials');
const authorize = require('../middleware/authorize');
const dbConnSetup = require('../middleware/db-conn-setup');
const rateLimit = require('../middleware/rate-limit');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const authorize = require('../middlewares/authorize');
const dbConnSetup = require('../middlewares/db-conn-setup');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const cacheControlHeader = require('../middleware/cache-control-header');
const sendResponse = require('../middleware/send-response');
const dbParamsFromResLocals = require('../utils/database-params');
const cacheControlHeader = require('../middlewares/cache-control-header');
const dbParamsFromResLocals = require('../../utils/database-params');
function AnalysesController(pgConnection, authApi, userLimitsApi) {
module.exports = class AnalysesController {
constructor (pgConnection, authBackend, userLimitsBackend) {
this.pgConnection = pgConnection;
this.authApi = authApi;
this.userLimitsApi = userLimitsApi;
}
this.authBackend = authBackend;
this.userLimitsBackend = userLimitsBackend;
}
module.exports = AnalysesController;
register (mapRouter) {
mapRouter.get('/analyses/catalog', this.middlewares());
}
AnalysesController.prototype.register = function (app) {
const { base_url_mapconfig: mapconfigBasePath } = app;
app.get(
`${mapconfigBasePath}/analyses/catalog`,
cors(),
user(),
middlewares () {
return [
credentials(),
authorize(this.authApi),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG),
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG),
cleanUpQueryParams(),
createPGClient(),
getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }),
getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }),
prepareResponse(),
cacheControlHeader({ ttl: 10, revalidate: true }),
sendResponse(),
unauthorizedError()
);
];
}
};
function createPGClient () {
@ -107,6 +102,7 @@ function prepareResponse () {
return -1;
});
res.statusCode = 200;
res.body = { catalog: analysisCatalog };
next();

59
lib/cartodb/api/map/analysis-layergroup-controller.js Normal file
View File

@ -0,0 +1,59 @@
const layergroupToken = require('../middlewares/layergroup-token');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
const authorize = require('../middlewares/authorize');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const dbParamsFromResLocals = require('../../utils/database-params');
module.exports = class AnalysisLayergroupController {
constructor (analysisStatusBackend, pgConnection, userLimitsBackend, authBackend) {
this.analysisStatusBackend = analysisStatusBackend;
this.pgConnection = pgConnection;
this.userLimitsBackend = userLimitsBackend;
this.authBackend = authBackend;
}
register (mapRouter) {
mapRouter.get('/:token/analysis/node/:nodeId', this.middlewares());
}
middlewares () {
return [
layergroupToken(),
credentials(),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS),
cleanUpQueryParams(),
analysisNodeStatus(this.analysisStatusBackend)
];
}
};
function analysisNodeStatus (analysisStatusBackend) {
return function analysisNodeStatusMiddleware(req, res, next) {
const { nodeId } = req.params;
const dbParams = dbParamsFromResLocals(res.locals);
analysisStatusBackend.getNodeStatus(nodeId, dbParams, (err, nodeStatus, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET NODE STATUS';
return next(err);
}
res.set({
'Cache-Control': 'public,max-age=5',
'Last-Modified': new Date().toUTCString()
});
res.statusCode = 200;
res.body = nodeStatus;
next();
});
};
}

212
lib/cartodb/api/map/anonymous-map-controller.js Normal file
View File

@ -0,0 +1,212 @@
const windshaft = require('windshaft');
const MapConfig = windshaft.model.MapConfig;
const Datasource = windshaft.model.Datasource;
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
const authorize = require('../middlewares/authorize');
const initProfiler = require('../middlewares/init-profiler');
const checkJsonContentType = require('../middlewares/check-json-content-type');
const incrementMapViewCount = require('../middlewares/increment-map-view-count');
const augmentLayergroupData = require('../middlewares/augment-layergroup-data');
const cacheControlHeader = require('../middlewares/cache-control-header');
const cacheChannelHeader = require('../middlewares/cache-channel-header');
const surrogateKeyHeader = require('../middlewares/surrogate-key-header');
const lastModifiedHeader = require('../middlewares/last-modified-header');
const lastUpdatedTimeLayergroup = require('../middlewares/last-updated-time-layergroup');
const layerStats = require('../middlewares/layer-stats');
const layergroupIdHeader = require('../middlewares/layergroup-id-header');
const layergroupMetadata = require('../middlewares/layergroup-metadata');
const mapError = require('../middlewares/map-error');
const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
module.exports = class AnonymousMapController {
/**
* @param {AuthBackend} authBackend
* @param {PgConnection} pgConnection
* @param {TemplateMaps} templateMaps
* @param {MapBackend} mapBackend
* @param metadataBackend
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsBackend} userLimitsBackend
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @param {MapConfigAdapter} mapConfigAdapter
* @param {StatsBackend} statsBackend
* @constructor
*/
constructor (
pgConnection,
templateMaps,
mapBackend,
metadataBackend,
surrogateKeysCache,
userLimitsBackend,
layergroupAffectedTables,
mapConfigAdapter,
statsBackend,
authBackend,
layergroupMetadata
) {
this.pgConnection = pgConnection;
this.templateMaps = templateMaps;
this.mapBackend = mapBackend;
this.metadataBackend = metadataBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsBackend = userLimitsBackend;
this.layergroupAffectedTables = layergroupAffectedTables;
this.mapConfigAdapter = mapConfigAdapter;
this.statsBackend = statsBackend;
this.authBackend = authBackend;
this.layergroupMetadata = layergroupMetadata;
}
register (mapRouter) {
mapRouter.options('/');
mapRouter.get('/', this.middlewares());
mapRouter.post('/', this.middlewares());
}
middlewares () {
const isTemplateInstantiation = false;
const useTemplateHash = false;
const includeQuery = true;
const label = 'ANONYMOUS LAYERGROUP';
const addContext = true;
return [
credentials(),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS),
cleanUpQueryParams(['aggregation']),
initProfiler(isTemplateInstantiation),
checkJsonContentType(),
checkCreateLayergroup(),
prepareAdapterMapConfig(this.mapConfigAdapter),
createLayergroup (
this.mapBackend,
this.userLimitsBackend,
this.pgConnection,
this.layergroupAffectedTables
),
incrementMapViewCount(this.metadataBackend),
augmentLayergroupData(),
cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader({ now: true }),
lastUpdatedTimeLayergroup(),
layerStats(this.pgConnection, this.statsBackend),
layergroupIdHeader(this.templateMaps, useTemplateHash),
layergroupMetadata(this.layergroupMetadata, includeQuery),
mapError({ label, addContext })
];
}
};
function checkCreateLayergroup () {
return function checkCreateLayergroupMiddleware (req, res, next) {
if (req.method === 'GET') {
const { config } = req.query;
if (!config) {
return next(new Error('layergroup GET needs a "config" parameter'));
}
try {
req.body = JSON.parse(config);
} catch (err) {
return next(err);
}
}
req.profiler.done('checkCreateLayergroup');
return next();
};
}
function prepareAdapterMapConfig (mapConfigAdapter) {
return function prepareAdapterMapConfigMiddleware(req, res, next) {
const requestMapConfig = req.body;
const { user, api_key } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
const context = {
analysisConfiguration: {
user,
db: {
host: dbhost,
port: dbport,
dbname: dbname,
user: dbuser,
pass: dbpassword
},
batch: {
username: user,
apiKey: api_key
}
}
};
mapConfigAdapter.getMapConfig(user, requestMapConfig, params, context, (err, requestMapConfig) => {
req.profiler.done('anonymous.getMapConfig');
if (err) {
return next(err);
}
req.body = requestMapConfig;
res.locals.context = context;
next();
});
};
}
function createLayergroup (mapBackend, userLimitsBackend, pgConnection, affectedTablesCache) {
return function createLayergroupMiddleware (req, res, next) {
const requestMapConfig = req.body;
const { context } = res.locals;
const { user, cache_buster, api_key } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const params = {
cache_buster, api_key,
dbuser, dbname, dbpassword, dbhost, dbport
};
const datasource = context.datasource || Datasource.EmptyDatasource();
const mapConfig = new MapConfig(requestMapConfig, datasource);
const mapConfigProvider = new CreateLayergroupMapConfigProvider(
mapConfig,
user,
userLimitsBackend,
pgConnection,
affectedTablesCache,
params
);
res.locals.mapConfig = mapConfig;
res.locals.analysesResults = context.analysesResults;
const mapParams = { dbuser, dbname, dbpassword, dbhost, dbport };
mapBackend.createLayergroup(mapConfig, mapParams, mapConfigProvider, (err, layergroup) => {
req.profiler.done('createLayergroup');
if (err) {
return next(err);
}
res.statusCode = 200;
res.body = layergroup;
res.locals.mapConfigProvider = mapConfigProvider;
next();
});
};
}

58
lib/cartodb/controllers/layergroup/attributes.js → lib/cartodb/api/map/attributes-layergroup-controller.js
View File

@ -1,54 +1,50 @@
const cors = require('../../middleware/cors');
const user = require('../../middleware/user');
const layergroupToken = require('../../middleware/layergroup-token');
const cleanUpQueryParams = require('../../middleware/clean-up-query-params');
const credentials = require('../../middleware/credentials');
const dbConnSetup = require('../../middleware/db-conn-setup');
const authorize = require('../../middleware/authorize');
const rateLimit = require('../../middleware/rate-limit');
const layergroupToken = require('../middlewares/layergroup-token');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
const authorize = require('../middlewares/authorize');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider');
const cacheControlHeader = require('../../middleware/cache-control-header');
const cacheChannelHeader = require('../../middleware/cache-channel-header');
const surrogateKeyHeader = require('../../middleware/surrogate-key-header');
const lastModifiedHeader = require('../../middleware/last-modified-header');
const sendResponse = require('../../middleware/send-response');
const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider');
const cacheControlHeader = require('../middlewares/cache-control-header');
const cacheChannelHeader = require('../middlewares/cache-channel-header');
const surrogateKeyHeader = require('../middlewares/surrogate-key-header');
const lastModifiedHeader = require('../middlewares/last-modified-header');
module.exports = class AttribitesController {
module.exports = class AttributesLayergroupController {
constructor (
attributesBackend,
pgConnection,
mapStore,
userLimitsApi,
userLimitsBackend,
layergroupAffectedTablesCache,
authApi,
authBackend,
surrogateKeysCache
) {
this.attributesBackend = attributesBackend;
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.userLimitsApi = userLimitsApi;
this.userLimitsBackend = userLimitsBackend;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.authApi = authApi;
this.authBackend = authBackend;
this.surrogateKeysCache = surrogateKeysCache;
}
register (app) {
const { base_url_mapconfig: mapConfigBasePath } = app;
register (mapRouter) {
mapRouter.get('/:token/:layer/attributes/:fid', this.middlewares());
}
app.get(
`${mapConfigBasePath}/:token/:layer/attributes/:fid`,
cors(),
user(),
middlewares () {
return [
layergroupToken(),
credentials(),
authorize(this.authApi),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES),
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES),
cleanUpQueryParams(),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.userLimitsBackend,
this.pgConnection,
this.layergroupAffectedTablesCache
),
@ -56,9 +52,8 @@ module.exports = class AttribitesController {
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
lastModifiedHeader()
];
}
};
@ -85,6 +80,7 @@ function getFeatureAttributes (attributesBackend) {
return next(err);
}
res.statusCode = 200;
res.body = tile;
next();

142
lib/cartodb/api/map/dataview-layergroup-controller.js Normal file
View File

@ -0,0 +1,142 @@
const layergroupToken = require('../middlewares/layergroup-token');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
const authorize = require('../middlewares/authorize');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider');
const cacheControlHeader = require('../middlewares/cache-control-header');
const cacheChannelHeader = require('../middlewares/cache-channel-header');
const surrogateKeyHeader = require('../middlewares/surrogate-key-header');
const lastModifiedHeader = require('../middlewares/last-modified-header');
const ALLOWED_DATAVIEW_QUERY_PARAMS = [
'filters', // json
'own_filter', // 0, 1
'no_filters', // 0, 1
'bbox', // w,s,e,n
'start', // number
'end', // number
'column_type', // string
'bins', // number
'aggregation', //string
'offset', // number
'q', // widgets search
'categories', // number
];
module.exports = class DataviewLayergroupController {
constructor (
dataviewBackend,
pgConnection,
mapStore,
userLimitsBackend,
layergroupAffectedTablesCache,
authBackend,
surrogateKeysCache
) {
this.dataviewBackend = dataviewBackend;
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.userLimitsBackend = userLimitsBackend;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.authBackend = authBackend;
this.surrogateKeysCache = surrogateKeysCache;
}
register (mapRouter) {
// Undocumented/non-supported API endpoint methods.
// Use at your own peril.
mapRouter.get('/:token/dataview/:dataviewName', this.middlewares({
action: 'get',
rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW
}));
mapRouter.get('/:token/:layer/widget/:dataviewName', this.middlewares({
action: 'get',
rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW
}));
mapRouter.get('/:token/dataview/:dataviewName/search', this.middlewares({
action: 'search',
rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH
}));
mapRouter.get('/:token/:layer/widget/:dataviewName/search', this.middlewares({
action: 'search',
rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH
}));
}
middlewares ({ action, rateLimitGroup }) {
return [
layergroupToken(),
credentials(),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsBackend, rateLimitGroup),
cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsBackend,
this.pgConnection,
this.layergroupAffectedTablesCache
),
action === 'search' ? dataviewSearch(this.dataviewBackend) : getDataview(this.dataviewBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader()
];
}
};
function getDataview (dataviewBackend) {
return function getDataviewMiddleware (req, res, next) {
const { user, mapConfigProvider } = res.locals;
const { dataviewName } = req.params;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const params = Object.assign({ dataviewName, dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET DATAVIEW';
return next(err);
}
res.statusCode = 200;
res.body = dataview;
next();
});
};
}
function dataviewSearch (dataviewBackend) {
return function dataviewSearchMiddleware (req, res, next) {
const { user, mapConfigProvider } = res.locals;
const { dataviewName } = req.params;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET DATAVIEW SEARCH';
return next(err);
}
res.statusCode = 200;
res.body = searchResult;
next();
});
};
}

129
lib/cartodb/api/map/map-router.js Normal file
View File

@ -0,0 +1,129 @@
const { Router: router } = require('express');
const AnalysisLayergroupController = require('./analysis-layergroup-controller');
const AttributesLayergroupController = require('./attributes-layergroup-controller');
const DataviewLayergroupController = require('./dataview-layergroup-controller');
const PreviewLayergroupController = require('./preview-layergroup-controller');
const TileLayergroupController = require('./tile-layergroup-controller');
const AnonymousMapController = require('./anonymous-map-controller');
const PreviewTemplateController = require('./preview-template-controller');
const AnalysesCatalogController = require('./analyses-catalog-controller');
module.exports = class MapRouter {
constructor ({ collaborators }) {
const {
analysisStatusBackend,
attributesBackend,
dataviewBackend,
previewBackend,
tileBackend,
pgConnection,
mapStore,
userLimitsBackend,
layergroupAffectedTablesCache,
authBackend,
surrogateKeysCache,
templateMaps,
mapBackend,
metadataBackend,
mapConfigAdapter,
statsBackend,
layergroupMetadata,
namedMapProviderCache,
tablesExtentBackend
} = collaborators;
this.analysisLayergroupController = new AnalysisLayergroupController(
analysisStatusBackend,
pgConnection,
userLimitsBackend,
authBackend
);
this.attributesLayergroupController = new AttributesLayergroupController(
attributesBackend,
pgConnection,
mapStore,
userLimitsBackend,
layergroupAffectedTablesCache,
authBackend,
surrogateKeysCache
);
this.dataviewLayergroupController = new DataviewLayergroupController(
dataviewBackend,
pgConnection,
mapStore,
userLimitsBackend,
layergroupAffectedTablesCache,
authBackend,
surrogateKeysCache
);
this.previewLayergroupController = new PreviewLayergroupController(
previewBackend,
pgConnection,
mapStore,
userLimitsBackend,
layergroupAffectedTablesCache,
authBackend,
surrogateKeysCache
);
this.tileLayergroupController = new TileLayergroupController(
tileBackend,
pgConnection,
mapStore,
userLimitsBackend,
layergroupAffectedTablesCache,
authBackend,
surrogateKeysCache
);
this.anonymousMapController = new AnonymousMapController(
pgConnection,
templateMaps,
mapBackend,
metadataBackend,
surrogateKeysCache,
userLimitsBackend,
layergroupAffectedTablesCache,
mapConfigAdapter,
statsBackend,
authBackend,
layergroupMetadata
);
this.previewTemplateController = new PreviewTemplateController(
namedMapProviderCache,
previewBackend,
surrogateKeysCache,
tablesExtentBackend,
metadataBackend,
pgConnection,
authBackend,
userLimitsBackend
);
this.analysesController = new AnalysesCatalogController(
pgConnection,
authBackend,
userLimitsBackend
);
}
register (apiRouter, mapPaths) {
const mapRouter = router({ mergeParams: true });
this.analysisLayergroupController.register(mapRouter);
this.attributesLayergroupController.register(mapRouter);
this.dataviewLayergroupController.register(mapRouter);
this.previewLayergroupController.register(mapRouter);
this.tileLayergroupController.register(mapRouter);
this.anonymousMapController.register(mapRouter);
this.previewTemplateController.register(mapRouter);
this.analysesController.register(mapRouter);
mapPaths.forEach(path => apiRouter.use(path, mapRouter));
}
};

109
lib/cartodb/controllers/layergroup/static.js → lib/cartodb/api/map/preview-layergroup-controller.js
View File

@ -1,94 +1,85 @@
const cors = require('../../middleware/cors');
const user = require('../../middleware/user');
const layergroupToken = require('../../middleware/layergroup-token');
const coordinates = require('../../middleware/coordinates');
const cleanUpQueryParams = require('../../middleware/clean-up-query-params');
const credentials = require('../../middleware/credentials');
const dbConnSetup = require('../../middleware/db-conn-setup');
const authorize = require('../../middleware/authorize');
const rateLimit = require('../../middleware/rate-limit');
const layergroupToken = require('../middlewares/layergroup-token');
const coordinates = require('../middlewares/coordinates');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const noop = require('../middlewares/noop');
const dbConnSetup = require('../middlewares/db-conn-setup');
const authorize = require('../middlewares/authorize');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider');
const cacheControlHeader = require('../../middleware/cache-control-header');
const cacheChannelHeader = require('../../middleware/cache-channel-header');
const surrogateKeyHeader = require('../../middleware/surrogate-key-header');
const lastModifiedHeader = require('../../middleware/last-modified-header');
const sendResponse = require('../../middleware/send-response');
const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider');
const cacheControlHeader = require('../middlewares/cache-control-header');
const cacheChannelHeader = require('../middlewares/cache-channel-header');
const surrogateKeyHeader = require('../middlewares/surrogate-key-header');
const lastModifiedHeader = require('../middlewares/last-modified-header');
const checkStaticImageFormat = require('../middlewares/check-static-image-format');
module.exports = class StaticController {
module.exports = class PreviewLayergroupController {
constructor (
previewBackend,
pgConnection,
mapStore,
userLimitsApi,
userLimitsBackend,
layergroupAffectedTablesCache,
authApi,
authBackend,
surrogateKeysCache
) {
this.previewBackend = previewBackend;
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.userLimitsApi = userLimitsApi;
this.userLimitsBackend = userLimitsBackend;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.authApi = authApi;
this.authBackend = authBackend;
this.surrogateKeysCache = surrogateKeysCache;
}
register (app) {
const { base_url_mapconfig: mapConfigBasePath } = app;
register (mapRouter) {
mapRouter.get('/static/center/:token/:z/:lat/:lng/:width/:height.:format', this.middlewares({
validateZoom: true,
previewType: 'centered'
}));
mapRouter.get('/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format', this.middlewares({
validateZoom: false,
previewType: 'bbox'
}));
}
middlewares ({ validateZoom, previewType }) {
const forcedFormat = 'png';
app.get(
`${mapConfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`,
cors(),
user(),
layergroupToken(),
coordinates({ z: true, x: false, y: false }),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
cleanUpQueryParams(['layer']),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache,
forcedFormat
),
getPreviewImageByCenter(this.previewBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
let getPreviewImage;
app.get(
`${mapConfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`,
cors(),
user(),
if (previewType === 'centered') {
getPreviewImage = getPreviewImageByCenter;
}
if (previewType === 'bbox') {
getPreviewImage = getPreviewImageByBoundingBox;
}
return [
layergroupToken(),
validateZoom ? coordinates({ z: true, x: false, y: false }) : noop(),
credentials(),
authorize(this.authApi),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
cleanUpQueryParams(['layer']),
checkStaticImageFormat(),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.userLimitsBackend,
this.pgConnection,
this.layergroupAffectedTablesCache,
forcedFormat
),
getPreviewImageByBoundingBox(this.previewBackend),
getPreviewImage(this.previewBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
lastModifiedHeader()
];
}
};
@ -120,6 +111,7 @@ function getPreviewImageByCenter (previewBackend) {
res.set('Content-Type', headers['Content-Type'] || `image/${format}`);
res.statusCode = 200;
res.body = image;
next();
@ -155,6 +147,7 @@ function getPreviewImageByBoundingBox (previewBackend) {
res.set('Content-Type', headers['Content-Type'] || `image/${format}`);
res.statusCode = 200;
res.body = image;
next();

166
lib/cartodb/controllers/named_maps.js → lib/cartodb/api/map/preview-template-controller.js
View File

@ -1,17 +1,14 @@
const cors = require('../middleware/cors');
const user = require('../middleware/user');
const cleanUpQueryParams = require('../middleware/clean-up-query-params');
const coordinates = require('../middleware/coordinates');
const credentials = require('../middleware/credentials');
const dbConnSetup = require('../middleware/db-conn-setup');
const authorize = require('../middleware/authorize');
const cacheControlHeader = require('../middleware/cache-control-header');
const cacheChannelHeader = require('../middleware/cache-channel-header');
const surrogateKeyHeader = require('../middleware/surrogate-key-header');
const lastModifiedHeader = require('../middleware/last-modified-header');
const sendResponse = require('../middleware/send-response');
const vectorError = require('../middleware/vector-error');
const rateLimit = require('../middleware/rate-limit');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
const authorize = require('../middlewares/authorize');
const namedMapProvider = require('../middlewares/named-map-provider');
const cacheControlHeader = require('../middlewares/cache-control-header');
const cacheChannelHeader = require('../middlewares/cache-channel-header');
const surrogateKeyHeader = require('../middlewares/surrogate-key-header');
const lastModifiedHeader = require('../middlewares/last-modified-header');
const checkStaticImageFormat = require('../middlewares/check-static-image-format');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const DEFAULT_ZOOM_CENTER = {
@ -26,70 +23,40 @@ function numMapper(n) {
return +n;
}
function NamedMapsController (
module.exports = class PreviewTemplateController {
constructor (
namedMapProviderCache,
tileBackend,
previewBackend,
surrogateKeysCache,
tablesExtentApi,
tablesExtentBackend,
metadataBackend,
pgConnection,
authApi,
userLimitsApi
) {
authBackend,
userLimitsBackend
) {
this.namedMapProviderCache = namedMapProviderCache;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.tablesExtentApi = tablesExtentApi;
this.tablesExtentBackend = tablesExtentBackend;
this.metadataBackend = metadataBackend;
this.pgConnection = pgConnection;
this.authApi = authApi;
this.userLimitsApi = userLimitsApi;
}
this.authBackend = authBackend;
this.userLimitsBackend = userLimitsBackend;
}
module.exports = NamedMapsController;
register (mapRouter) {
mapRouter.get('/static/named/:template_id/:width/:height.:format', this.middlewares());
}
NamedMapsController.prototype.register = function(app) {
const { base_url_mapconfig: mapconfigBasePath, base_url_templated: templateBasePath } = app;
app.get(
`${templateBasePath}/:template_id/:layer/:z/:x/:y.(:format)`,
cors(),
user(),
coordinates(),
middlewares () {
return [
credentials(),
authorize(this.authApi),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES),
cleanUpQueryParams(),
getNamedMapProvider({
namedMapProviderCache: this.namedMapProviderCache,
label: 'NAMED_MAP_TILE'
}),
getTile({
tileBackend: this.tileBackend,
label: 'NAMED_MAP_TILE'
}),
setContentTypeHeader(),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse(),
vectorError()
);
app.get(
`${mapconfigBasePath}/static/named/:template_id/:width/:height.:format`,
cors(),
user(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED),
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED),
cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
getNamedMapProvider({
checkStaticImageFormat(),
namedMapProvider({
namedMapProviderCache: this.namedMapProviderCache,
label: 'STATIC_VIZ_MAP', forcedFormat: 'png'
}),
@ -98,51 +65,18 @@ NamedMapsController.prototype.register = function(app) {
namedMapProviderCache: this.namedMapProviderCache,
label: 'STATIC_VIZ_MAP'
}),
getStaticImageOptions({ tablesExtentApi: this.tablesExtentApi }),
getStaticImageOptions({ tablesExtentBackend: this.tablesExtentBackend }),
getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }),
setContentTypeHeader(),
incrementMapViews({ metadataBackend: this.metadataBackend }),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
lastModifiedHeader()
];
}
};
function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = null }) {
return function getNamedMapProviderMiddleware (req, res, next) {
const { user, token, cache_buster, api_key } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const { template_id, layer: layerFromParams, z, x, y, format } = req.params;
const { layer: layerFromQuery } = req.query;
const params = {
user, token, cache_buster, api_key,
dbuser, dbname, dbpassword, dbhost, dbport,
template_id, layer: (layerFromQuery || layerFromParams), z, x, y, format
};
if (forcedFormat) {
params.format = forcedFormat;
params.layer = params.layer || 'all';
}
const { config, auth_token } = req.query;
namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => {
if (err) {
err.label = label;
return next(err);
}
res.locals.mapConfigProvider = namedMapProvider;
next();
});
};
}
function getTemplate ({ label }) {
return function getTemplateMiddleware (req, res, next) {
const { mapConfigProvider } = res.locals;
@ -209,33 +143,7 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label })
};
}
function getTile ({ tileBackend, label }) {
return function getTileMiddleware (req, res, next) {
const { mapConfigProvider } = res.locals;
const { layer, z, x, y, format } = req.params;
const params = { layer, z, x, y, format };
tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats) => {
req.profiler.add(stats);
req.profiler.done('render-' + format);
if (err) {
err.label = label;
return next(err);
}
if (headers) {
res.set(headers);
}
res.body = tile;
next();
});
};
}
function getStaticImageOptions ({ tablesExtentApi }) {
function getStaticImageOptions ({ tablesExtentBackend }) {
return function getStaticImageOptionsMiddleware(req, res, next) {
const { user, mapConfigProvider, template } = res.locals;
const { zoom, lon, lat, bbox } = req.query;
@ -261,7 +169,7 @@ function getStaticImageOptions ({ tablesExtentApi }) {
return next();
}
tablesExtentApi.getBounds(user, tables, (err, bounds) => {
tablesExtentBackend.getBounds(user, tables, (err, bounds) => {
if (err) {
return next();
}
@ -365,6 +273,7 @@ function getImage({ previewBackend, label }) {
res.set(headers);
}
res.statusCode = 200;
res.body = image;
next();
@ -384,6 +293,7 @@ function getImage({ previewBackend, label }) {
res.set(headers);
}
res.statusCode = 200;
res.body = image;
next();

170
lib/cartodb/api/map/tile-layergroup-controller.js Normal file
View File

@ -0,0 +1,170 @@
const layergroupToken = require('../middlewares/layergroup-token');
const coordinates = require('../middlewares/coordinates');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
const authorize = require('../middlewares/authorize');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const createMapStoreMapConfigProvider = require('../middlewares/map-store-map-config-provider');
const cacheControlHeader = require('../middlewares/cache-control-header');
const cacheChannelHeader = require('../middlewares/cache-channel-header');
const surrogateKeyHeader = require('../middlewares/surrogate-key-header');
const lastModifiedHeader = require('../middlewares/last-modified-header');
const vectorError = require('../middlewares/vector-error');
const SUPPORTED_FORMATS = {
grid_json: true,
json_torque: true,
torque_json: true,
png: true,
png32: true,
mvt: true
};
module.exports = class TileLayergroupController {
constructor (
tileBackend,
pgConnection,
mapStore,
userLimitsBackend,
layergroupAffectedTablesCache,
authBackend,
surrogateKeysCache
) {
this.tileBackend = tileBackend;
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.userLimitsBackend = userLimitsBackend;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.authBackend = authBackend;
this.surrogateKeysCache = surrogateKeysCache;
}
register (mapRouter) {
// REGEXP: doesn't match with `val`
const not = (val) => `(?!${val})([^\/]+?)`;
// Sadly the path that matches 1 also matches with 2 so we need to tell to express
// that performs only the middlewares of the first path that matches
// for that we use one array to group all paths.
mapRouter.get([
`/:token/:z/:x/:y@:scale_factor?x.:format`, // 1
`/:token/:z/:x/:y.:format`, // 2
`/:token${not('static')}/:layer/:z/:x/:y.(:format)`
], this.middlewares());
}
middlewares () {
return [
layergroupToken(),
coordinates(),
credentials(),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
cleanUpQueryParams(),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsBackend,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getTile(this.tileBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
incrementSuccessMetrics(global.statsClient),
incrementErrorMetrics(global.statsClient),
tileError(),
vectorError()
];
}
};
function parseFormat (format = '') {
const prettyFormat = format.replace('.', '_');
return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid';
}
function getStatusCode(tile, format){
return tile.length === 0 && format === 'mvt' ? 204 : 200;
}
function getTile (tileBackend) {
return function getTileMiddleware (req, res, next) {
req.profiler.start(`windshaft.${req.params.layer ? 'maplayer_tile' : 'map_tile'}`);
const { mapConfigProvider } = res.locals;
const { token } = res.locals;
const { layer, z, x, y, format } = req.params;
const params = { token, layer, z, x, y, format };
tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => {
req.profiler.add(stats);
if (err) {
return next(err);
}
if (headers) {
res.set(headers);
}
const formatStat = parseFormat(req.params.format);
res.statusCode = getStatusCode(tile, formatStat);
res.body = tile;
next();
});
};
}
function incrementSuccessMetrics (statsClient) {
return function incrementSuccessMetricsMiddleware (req, res, next) {
const formatStat = parseFormat(req.params.format);
statsClient.increment('windshaft.tiles.success');
statsClient.increment(`windshaft.tiles.${formatStat}.success`);
next();
};
}
function incrementErrorMetrics (statsClient) {
return function incrementErrorMetricsMiddleware (err, req, res, next) {
const formatStat = parseFormat(req.params.format);
statsClient.increment('windshaft.tiles.error');
statsClient.increment(`windshaft.tiles.${formatStat}.error`);
next(err);
};
}
function tileError () {
return function tileErrorMiddleware (err, req, res, next) {
if (err.message === 'Tile does not exist' && req.params.format === 'mvt') {
res.statusCode = 204;
return next();
}
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
let errMsg = err.message ? ( '' + err.message ) : ( '' + err );
// Rewrite mapnik parsing errors to start with layer number
const matches = errMsg.match("(.*) in style 'layer([0-9]+)'");
if (matches) {
errMsg = `style${matches[2]}: ${matches[1]}`;
}
err.message = errMsg;
err.label = 'TILE RENDER';
next(err);
};
}

14
lib/cartodb/api/middlewares/augment-layergroup-data.js Normal file
View File

@ -0,0 +1,14 @@
const _ = require('underscore');
module.exports = function augmentLayergroupData () {
return function augmentLayergroupDataMiddleware (req, res, next) {
const layergroup = res.body;
// include in layergroup response the variables in serverMedata
// those variables are useful to send to the client information
// about how to reach this server or information about it
_.extend(layergroup, global.environment.serverMetadata);
next();
};
};

4
lib/cartodb/middleware/authorize.js → lib/cartodb/api/middlewares/authorize.js
View File

@ -1,6 +1,6 @@
module.exports = function authorize (authApi) {
module.exports = function authorize (authBackend) {
return function authorizeMiddleware (req, res, next) {
authApi.authorize(req, res, (err, authorized) => {
authBackend.authorize(req, res, (err, authorized) => {
req.profiler.done('authorize');
if (err) {

0
lib/cartodb/middleware/cache-channel-header.js → lib/cartodb/api/middlewares/cache-channel-header.js
View File

0
lib/cartodb/middleware/cache-control-header.js → lib/cartodb/api/middlewares/cache-control-header.js
View File

11
lib/cartodb/api/middlewares/check-json-content-type.js Normal file
View File

@ -0,0 +1,11 @@
module.exports = function checkJsonContentType () {
return function checkJsonContentTypeMiddleware(req, res, next) {
if (req.method === 'POST' && !req.is('application/json')) {
return next(new Error('POST data must be of type application/json'));
}
req.profiler.done('checkJsonContentTypeMiddleware');
next();
};
};

11
lib/cartodb/api/middlewares/check-static-image-format.js Normal file
View File

@ -0,0 +1,11 @@
const VALID_IMAGE_FORMATS = ['png', 'jpg'];
module.exports = function checkStaticImageFormat () {
return function checkStaticImageFormatMiddleware (req, res, next) {
if(!VALID_IMAGE_FORMATS.includes(req.params.format)) {
return next(new Error(`Unsupported image format "${req.params.format}"`));
}
next();
};
};

0
lib/cartodb/middleware/clean-up-query-params.js → lib/cartodb/api/middlewares/clean-up-query-params.js
View File

0
lib/cartodb/middleware/coordinates.js → lib/cartodb/api/middlewares/coordinates.js
View File

18
lib/cartodb/api/middlewares/cors.js Normal file
View File

@ -0,0 +1,18 @@
module.exports = function cors () {
return function corsMiddleware (req, res, next) {
const headers = [
'X-Requested-With',
'X-Prototype-Version',
'X-CSRF-Token'
];
if (req.method === 'OPTIONS') {
headers.push('Content-Type');
}
res.set("Access-Control-Allow-Origin", "*");
res.set("Access-Control-Allow-Headers", headers.join(', '));
next();
};
};

0
lib/cartodb/middleware/credentials.js → lib/cartodb/api/middlewares/credentials.js
View File

0
lib/cartodb/middleware/db-conn-setup.js → lib/cartodb/api/middlewares/db-conn-setup.js
View File

0
lib/cartodb/middleware/error-middleware.js → lib/cartodb/api/middlewares/error-middleware.js
View File

16
lib/cartodb/api/middlewares/increment-map-view-count.js Normal file
View File

@ -0,0 +1,16 @@
module.exports = function incrementMapViewCount (metadataBackend) {
return function incrementMapViewCountMiddleware(req, res, next) {
const { mapConfig, user } = res.locals;
// Error won't blow up, just be logged.
metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => {
req.profiler.done('incMapviewCount');
if (err) {
global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`);
}
next();
});
};
};

9
lib/cartodb/api/middlewares/init-profiler.js Normal file
View File

@ -0,0 +1,9 @@
module.exports = function initProfiler (isTemplateInstantiation) {
const operation = isTemplateInstantiation ? 'instance_template' : 'createmap';
return function initProfilerMiddleware (req, res, next) {
req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`);
req.profiler.done(`${operation}.initProfilerMiddleware`);
next();
};
};

9
lib/cartodb/api/middlewares/initialize-status-code.js Normal file
View File

@ -0,0 +1,9 @@
module.exports = function initializeStatusCode () {
return function initializeStatusCodeMiddleware (req, res, next) {
if (req.method !== 'OPTIONS') {
res.statusCode = 404;
}
next();
};
};

0
lib/cartodb/middleware/last-modified-header.js → lib/cartodb/api/middlewares/last-modified-header.js
View File

39
lib/cartodb/api/middlewares/last-updated-time-layergroup.js Normal file
View File

@ -0,0 +1,39 @@
module.exports = function setLastUpdatedTimeToLayergroup () {
return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) {
const { mapConfigProvider, analysesResults } = res.locals;
const layergroup = res.body;
mapConfigProvider.createAffectedTables((err, affectedTables) => {
if (err) {
return next(err);
}
if (!affectedTables) {
return next();
}
var lastUpdateTime = affectedTables.getLastUpdatedAt();
lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime;
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
layergroup.last_updated = new Date(lastUpdateTime).toISOString();
next();
});
};
};
function getLastUpdatedTime(analysesResults, lastUpdateTime) {
if (!Array.isArray(analysesResults)) {
return lastUpdateTime;
}
return analysesResults.reduce(function(lastUpdateTime, analysis) {
return analysis.getNodes().reduce(function(lastNodeUpdatedAtTime, node) {
var nodeUpdatedAtDate = node.getUpdatedAt();
var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0;
return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime;
}, lastUpdateTime);
}, lastUpdateTime);
}

26
lib/cartodb/api/middlewares/layer-stats.js Normal file
View File

@ -0,0 +1,26 @@
module.exports = function setLayerStats (pgConnection, statsBackend) {
return function setLayerStatsMiddleware(req, res, next) {
const { user, mapConfig } = res.locals;
const layergroup = res.body;
pgConnection.getConnection(user, (err, connection) => {
if (err) {
return next(err);
}
statsBackend.getStats(mapConfig, connection, function(err, layersStats) {
if (err) {
return next(err);
}
if (layersStats.length > 0) {
layergroup.metadata.layers.forEach(function (layer, index) {
layer.meta.stats = layersStats[index];
});
}
next();
});
});
};
};

15
lib/cartodb/api/middlewares/layergroup-id-header.js Normal file
View File

@ -0,0 +1,15 @@
module.exports = function setLayergroupIdHeader (templateMaps, useTemplateHash) {
return function setLayergroupIdHeaderMiddleware (req, res, next) {
const { user, template } = res.locals;
const layergroup = res.body;
if (useTemplateHash) {
var templateHash = templateMaps.fingerPrint(template).substring(0, 8);
layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`;
}
res.set('X-Layergroup-Id', layergroup.layergroupid);
next();
};
};

14
lib/cartodb/api/middlewares/layergroup-metadata.js Normal file
View File

@ -0,0 +1,14 @@
module.exports = function setMetadataToLayergroup (layergroupMetadata, includeQuery) {
return function setMetadataToLayergroupMiddleware (req, res, next) {
const { user, mapConfig, analysesResults = [], context } = res.locals;
const layergroup = res.body;
layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj());
layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery);
layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context);
layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context);
layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig);
next();
};
};

2
lib/cartodb/middleware/layergroup-token.js → lib/cartodb/api/middlewares/layergroup-token.js
View File

@ -1,4 +1,4 @@
const LayergroupToken = require('../models/layergroup-token');
const LayergroupToken = require('../../models/layergroup-token');
const authErrorMessageTemplate = function (signer, user) {
return `Cannot use map signature of user "${signer}" on db of user "${user}"`;
};

22
lib/cartodb/api/middlewares/logger.js Normal file
View File

@ -0,0 +1,22 @@
module.exports = function logger (options) {
if (!global.log4js || !options.log_format) {
return function dummyLoggerMiddleware (req, res, next) {
next();
};
}
const opts = {
level: 'info',
// Allowing for unbuffered logging is mainly
// used to avoid hanging during unit testing.
// TODO: provide an explicit teardown function instead,
// releasing any event handler or timer set by
// this component.
buffer: !options.unbuffered_logging,
// optional log format
format: options.log_format
};
const logger = global.log4js.getLogger();
return global.log4js.connectLogger(logger, opts);
};

0
lib/cartodb/middleware/lzma.js → lib/cartodb/api/middlewares/lzma.js
View File

35
lib/cartodb/api/middlewares/map-error.js Normal file
View File

@ -0,0 +1,35 @@
module.exports = function mapError (options) {
const { addContext = false, label = 'MAPS CONTROLLER' } = options;
return function mapErrorMiddleware (err, req, res, next) {
req.profiler.done('error');
const { mapConfig } = res.locals;
if (addContext) {
err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err;
}
err.label = label;
next(err);
};
};
function populateError(err, mapConfig) {
var error = new Error(err.message);
error.http_status = err.http_status;
if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) {
error.http_status = 400;
}
error.type = 'layer';
error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined;
error.layer = {
id: mapConfig.getLayerId(err.layerIndex),
index: err.layerIndex,
type: mapConfig.layerType(err.layerIndex)
};
return error;
}

6
lib/cartodb/controllers/layergroup/middlewares/map-store-map-config-provider.js → lib/cartodb/api/middlewares/map-store-map-config-provider.js
View File

@ -1,8 +1,8 @@
const MapStoreMapConfigProvider = require('../../../models/mapconfig/provider/map-store-provider');
const MapStoreMapConfigProvider = require('../../models/mapconfig/provider/map-store-provider');
module.exports = function createMapStoreMapConfigProvider (
mapStore,
userLimitsApi,
userLimitsBackend,
pgConnection,
affectedTablesCache,
forcedFormat = null
@ -26,7 +26,7 @@ module.exports = function createMapStoreMapConfigProvider (
res.locals.mapConfigProvider = new MapStoreMapConfigProvider(
mapStore,
user,
userLimitsApi,
userLimitsBackend,
pgConnection,
affectedTablesCache,
params

32
lib/cartodb/api/middlewares/named-map-provider.js Normal file
View File

@ -0,0 +1,32 @@
module.exports = function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = null }) {
return function getNamedMapProviderMiddleware (req, res, next) {
const { user, token, cache_buster, api_key } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const { template_id, layer: layerFromParams, z, x, y, format } = req.params;
const { layer: layerFromQuery } = req.query;
const params = {
user, token, cache_buster, api_key,
dbuser, dbname, dbpassword, dbhost, dbport,
template_id, layer: (layerFromQuery || layerFromParams), z, x, y, format
};
if (forcedFormat) {
params.format = forcedFormat;
params.layer = params.layer || 'all';
}
const { config, auth_token } = req.query;
namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => {
if (err) {
err.label = label;
return next(err);
}
res.locals.mapConfigProvider = namedMapProvider;
next();
});
};
};

5
lib/cartodb/api/middlewares/noop.js Normal file
View File

@ -0,0 +1,5 @@
module.exports = function noop () {
return function noopMiddleware (req, res, next) {
next();
};
};

4
lib/cartodb/middleware/rate-limit.js → lib/cartodb/api/middlewares/rate-limit.js
View File

@ -19,13 +19,13 @@ const RATE_LIMIT_ENDPOINTS_GROUPS = {
NAMED_TILES: 'named_tiles'
};
function rateLimit(userLimitsApi, endpointGroup = null) {
function rateLimit(userLimitsBackend, endpointGroup = null) {
if (!isRateLimitEnabled(endpointGroup)) {
return function rateLimitDisabledMiddleware(req, res, next) { next(); };
}
return function rateLimitMiddleware(req, res, next) {
userLimitsApi.getRateLimit(res.locals.user, endpointGroup, function (err, userRateLimit) {
userLimitsBackend.getRateLimit(res.locals.user, endpointGroup, function (err, userRateLimit) {
if (err) {
return next(err);
}

2
lib/cartodb/middleware/send-response.js → lib/cartodb/api/middlewares/send-response.js
View File

@ -2,7 +2,7 @@ module.exports = function sendResponse () {
return function sendResponseMiddleware (req, res) {
req.profiler.done('res');
res.status(res.statusCode || 200);
res.status(res.statusCode);
if (Buffer.isBuffer(res.body)) {
return res.send(res.body);

11
lib/cartodb/api/middlewares/served-by-host-header.js Normal file
View File

@ -0,0 +1,11 @@
const os = require('os');
module.exports = function servedByHostHeader () {
const hostname = os.hostname().split('.')[0];
return function servedByHostHeaderMiddleware (req, res, next) {
res.set('X-Served-By-Host', hostname);
next();
};
};

2
lib/cartodb/middleware/stats.js → lib/cartodb/api/middlewares/stats.js
View File

@ -1,4 +1,4 @@
const Profiler = require('../stats/profiler_proxy');
const Profiler = require('../../stats/profiler_proxy');
const debug = require('debug')('windshaft:cartodb:stats');
const onHeaders = require('on-headers');

4
lib/cartodb/middleware/surrogate-key-header.js → lib/cartodb/api/middlewares/surrogate-key-header.js
View File

@ -1,5 +1,5 @@
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
const NamedMapsCacheEntry = require('../../cache/model/named_maps_entry');
const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider');
module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) {
return function setSurrogateKeyHeaderMiddleware(req, res, next) {

10
lib/cartodb/api/middlewares/syntax-error.js Normal file
View File

@ -0,0 +1,10 @@
module.exports = function syntaxError () {
return function syntaxErrorMiddleware (err, req, res, next) {
if (err.name === 'SyntaxError') {
err.http_status = 400;
err.message = `${err.name}: ${err.message}`;
}
next(err);
};
};

2
lib/cartodb/middleware/user.js → lib/cartodb/api/middlewares/user.js
View File

@ -1,4 +1,4 @@
const CdbRequest = require('../models/cdb_request');
const CdbRequest = require('../../models/cdb_request');
module.exports = function user () {
const cdbRequest = new CdbRequest();

2
lib/cartodb/middleware/vector-error.js → lib/cartodb/api/middlewares/vector-error.js
View File

@ -1,5 +1,5 @@
const fs = require('fs');
const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../assets/render-timeout-fallback.mvt');
const timeoutErrorVectorTile = fs.readFileSync(__dirname + '/../../../../assets/render-timeout-fallback.mvt');
module.exports = function vectorError() {
return function vectorErrorMiddleware(err, req, res, next) {

159
lib/cartodb/controllers/named_maps_admin.js → lib/cartodb/api/template/admin-template-controller.js
View File

@ -1,95 +1,92 @@
const { templateName } = require('../backends/template_maps');
const cors = require('../middleware/cors');
const user = require('../middleware/user');
const credentials = require('../middleware/credentials');
const rateLimit = require('../middleware/rate-limit');
const { templateName } = require('../../backends/template_maps');
const credentials = require('../middlewares/credentials');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const sendResponse = require('../middleware/send-response');
/**
* @param {AuthApi} authApi
module.exports = class AdminTemplateController {
/**
* @param {AuthBackend} authBackend
* @param {PgConnection} pgConnection
* @param {TemplateMaps} templateMaps
* @constructor
*/
function NamedMapsAdminController(authApi, templateMaps, userLimitsApi) {
this.authApi = authApi;
constructor (authBackend, templateMaps, userLimitsBackend) {
this.authBackend = authBackend;
this.templateMaps = templateMaps;
this.userLimitsApi = userLimitsApi;
}
this.userLimitsBackend = userLimitsBackend;
}
module.exports = NamedMapsAdminController;
register (templateRouter) {
templateRouter.options(`/:template_id`);
NamedMapsAdminController.prototype.register = function (app) {
const { base_url_templated: templateBasePath } = app;
templateRouter.post('/', this.middlewares({
action: 'create',
label: 'POST TEMPLATE',
rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE
}));
app.post(
`${templateBasePath}/`,
cors(),
user(),
templateRouter.put('/:template_id', this.middlewares({
action: 'update',
label: 'PUT TEMPLATE',
rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE
}));
templateRouter.get('/:template_id', this.middlewares({
action: 'get',
label: 'GET TEMPLATE',
rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET
}));
templateRouter.delete('/:template_id', this.middlewares({
action: 'delete',
label: 'DELETE TEMPLATE',
rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE
}));
templateRouter.get('/', this.middlewares({
action: 'list',
label: 'GET TEMPLATE LIST',
rateLimitGroup: RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST
}));
}
middlewares ({ action, label, rateLimitGroup }) {
let template;
if (action === 'create') {
template = createTemplate;
}
if (action === 'update') {
template = updateTemplate;
}
if (action === 'get') {
template = retrieveTemplate;
}
if (action === 'delete') {
template = destroyTemplate;
}
if (action === 'list') {
template = listTemplates;
}
return [
credentials(),
authorizedByAPIKey({ authBackend: this.authBackend, action, label }),
rateLimit(this.userLimitsBackend, rateLimitGroup),
checkContentType({ action: 'POST', label: 'POST TEMPLATE' }),
authorizedByAPIKey({ authApi: this.authApi, action: 'create', label: 'POST TEMPLATE' }),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE),
createTemplate({ templateMaps: this.templateMaps }),
sendResponse()
);
app.put(
`${templateBasePath}/:template_id`,
cors(),
user(),
credentials(),
checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }),
authorizedByAPIKey({ authApi: this.authApi, action: 'update', label: 'PUT TEMPLATE' }),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE),
updateTemplate({ templateMaps: this.templateMaps }),
sendResponse()
);
app.get(
`${templateBasePath}/:template_id`,
cors(),
user(),
credentials(),
authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET),
retrieveTemplate({ templateMaps: this.templateMaps }),
sendResponse()
);
app.delete(
`${templateBasePath}/:template_id`,
cors(),
user(),
credentials(),
authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE),
destroyTemplate({ templateMaps: this.templateMaps }),
sendResponse()
);
app.get(
`${templateBasePath}/`,
cors(),
user(),
credentials(),
authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST),
listTemplates({ templateMaps: this.templateMaps }),
sendResponse()
);
app.options(
`${templateBasePath}/:template_id`,
cors('Content-Type')
);
template({ templateMaps: this.templateMaps })
];
}
};
function checkContentType ({ action, label }) {
function checkContentType ({ label }) {
return function checkContentTypeMiddleware (req, res, next) {
if (!req.is('application/json')) {
const error = new Error(`template ${action} data must be of type application/json`);
if ((req.method === 'POST' || req.method === 'PUT') && !req.is('application/json')) {
const error = new Error(`${req.method} template data must be of type application/json`);
error.label = label;
return next(error);
}
@ -98,11 +95,11 @@ function checkContentType ({ action, label }) {
};
}
function authorizedByAPIKey ({ authApi, action, label }) {
function authorizedByAPIKey ({ authBackend, action, label }) {
return function authorizedByAPIKeyMiddleware (req, res, next) {
const { user } = res.locals;
authApi.authorizedByAPIKey(user, res, (err, authenticated, apikey) => {
authBackend.authorizedByAPIKey(user, res, (err, authenticated, apikey) => {
if (err) {
return next(err);
}
@ -138,6 +135,7 @@ function createTemplate ({ templateMaps }) {
return next(err);
}
res.statusCode = 200;
res.body = { template_id: templateId };
next();
@ -156,6 +154,7 @@ function updateTemplate ({ templateMaps }) {
return next(err);
}
res.statusCode = 200;
res.body = { template_id: templateId };
next();
@ -184,6 +183,7 @@ function retrieveTemplate ({ templateMaps }) {
// so we remove it before returning to the user
delete template.auth_id;
res.statusCode = 200;
res.body = { template };
next();
@ -222,6 +222,7 @@ function listTemplates ({ templateMaps }) {
return next(err);
}
res.statusCode = 200;
res.body = { template_ids: templateIds };
next();

213
lib/cartodb/api/template/named-template-controller.js Normal file
View File

@ -0,0 +1,213 @@
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
const authorize = require('../middlewares/authorize');
const initProfiler = require('../middlewares/init-profiler');
const checkJsonContentType = require('../middlewares/check-json-content-type');
const incrementMapViewCount = require('../middlewares/increment-map-view-count');
const augmentLayergroupData = require('../middlewares/augment-layergroup-data');
const cacheControlHeader = require('../middlewares/cache-control-header');
const cacheChannelHeader = require('../middlewares/cache-channel-header');
const surrogateKeyHeader = require('../middlewares/surrogate-key-header');
const lastModifiedHeader = require('../middlewares/last-modified-header');
const lastUpdatedTimeLayergroup = require('../middlewares/last-updated-time-layergroup');
const layerStats = require('../middlewares/layer-stats');
const layergroupIdHeader = require('../middlewares/layergroup-id-header');
const layergroupMetadata = require('../middlewares/layergroup-metadata');
const mapError = require('../middlewares/map-error');
const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named-map-provider');
const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
module.exports = class NamedMapController {
/**
* @param {PgConnection} pgConnection
* @param {TemplateMaps} templateMaps
* @param {MapBackend} mapBackend
* @param metadataBackend
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsBackend} userLimitsBackend
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @param {MapConfigAdapter} mapConfigAdapter
* @param {StatsBackend} statsBackend
* @param {AuthBackend} authBackend
* @param layergroupMetadata
* @constructor
*/
constructor (
pgConnection,
templateMaps,
mapBackend,
metadataBackend,
surrogateKeysCache,
userLimitsBackend,
layergroupAffectedTables,
mapConfigAdapter,
statsBackend,
authBackend,
layergroupMetadata
) {
this.pgConnection = pgConnection;
this.templateMaps = templateMaps;
this.mapBackend = mapBackend;
this.metadataBackend = metadataBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsBackend = userLimitsBackend;
this.layergroupAffectedTables = layergroupAffectedTables;
this.mapConfigAdapter = mapConfigAdapter;
this.statsBackend = statsBackend;
this.authBackend = authBackend;
this.layergroupMetadata = layergroupMetadata;
}
register (templateRouter) {
templateRouter.get('/:template_id/jsonp', this.middlewares());
templateRouter.post('/:template_id', this.middlewares());
}
middlewares () {
const isTemplateInstantiation = true;
const useTemplateHash = true;
const includeQuery = false;
const label = 'NAMED MAP LAYERGROUP';
const addContext = false;
return [
credentials(),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED),
cleanUpQueryParams(['aggregation']),
initProfiler(isTemplateInstantiation),
checkJsonContentType(),
checkInstantiteLayergroup(),
getTemplate(
this.templateMaps,
this.pgConnection,
this.metadataBackend,
this.userLimitsBackend,
this.mapConfigAdapter,
this.layergroupAffectedTables
),
instantiateLayergroup(
this.mapBackend,
this.userLimitsBackend,
this.pgConnection,
this.layergroupAffectedTables
),
incrementMapViewCount(this.metadataBackend),
augmentLayergroupData(),
cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader({ now: true }),
lastUpdatedTimeLayergroup(),
layerStats(this.pgConnection, this.statsBackend),
layergroupIdHeader(this.templateMaps ,useTemplateHash),
layergroupMetadata(this.layergroupMetadata, includeQuery),
mapError({ label, addContext })
];
}
};
function checkInstantiteLayergroup () {
return function checkInstantiteLayergroupMiddleware(req, res, next) {
if (req.method === 'GET') {
const { callback, config } = req.query;
if (callback === undefined || callback.length === 0) {
return next(new Error('callback parameter should be present and be a function name'));
}
if (config) {
try {
req.body = JSON.parse(config);
} catch(e) {
return next(new Error('Invalid config parameter, should be a valid JSON'));
}
}
}
req.profiler.done('checkInstantiteLayergroup');
return next();
};
}
function getTemplate (
templateMaps,
pgConnection,
metadataBackend,
userLimitsBackend,
mapConfigAdapter,
affectedTablesCache
) {
return function getTemplateMiddleware (req, res, next) {
const templateParams = req.body;
const { user, dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const { template_id } = req.params;
const { auth_token } = req.query;
const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
const mapConfigProvider = new NamedMapMapConfigProvider(
templateMaps,
pgConnection,
metadataBackend,
userLimitsBackend,
mapConfigAdapter,
affectedTablesCache,
user,
template_id,
templateParams,
auth_token,
params
);
mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => {
req.profiler.done('named.getMapConfig');
if (err) {
return next(err);
}
res.locals.mapConfig = mapConfig;
res.locals.rendererParams = rendererParams;
res.locals.mapConfigProvider = mapConfigProvider;
next();
});
};
}
function instantiateLayergroup (mapBackend, userLimitsBackend, pgConnection, affectedTablesCache) {
return function instantiateLayergroupMiddleware (req, res, next) {
const { user, mapConfig, rendererParams } = res.locals;
const mapConfigProvider = new CreateLayergroupMapConfigProvider(
mapConfig,
user,
userLimitsBackend,
pgConnection,
affectedTablesCache,
rendererParams
);
mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => {
req.profiler.done('createLayergroup');
if (err) {
return next(err);
}
res.statusCode = 200;
res.body = layergroup;
const { mapConfigProvider } = res.locals;
res.locals.analysesResults = mapConfigProvider.analysesResults;
res.locals.template = mapConfigProvider.template;
res.locals.context = mapConfigProvider.context;
next();
});
};
}

64
lib/cartodb/api/template/template-router.js Normal file
View File

@ -0,0 +1,64 @@
const { Router: router } = require('express');
const NamedMapController = require('./named-template-controller');
const AdminTemplateController = require('./admin-template-controller');
const TileTemplateController = require('./tile-template-controller');
module.exports = class TemplateRouter {
constructor ({ collaborators }) {
const {
pgConnection,
templateMaps,
mapBackend,
metadataBackend,
surrogateKeysCache,
userLimitsBackend,
layergroupAffectedTablesCache,
mapConfigAdapter,
statsBackend,
authBackend,
layergroupMetadata,
namedMapProviderCache,
tileBackend,
} = collaborators;
this.namedMapController = new NamedMapController(
pgConnection,
templateMaps,
mapBackend,
metadataBackend,
surrogateKeysCache,
userLimitsBackend,
layergroupAffectedTablesCache,
mapConfigAdapter,
statsBackend,
authBackend,
layergroupMetadata
);
this.tileTemplateController = new TileTemplateController(
namedMapProviderCache,
tileBackend,
surrogateKeysCache,
pgConnection,
authBackend,
userLimitsBackend
);
this.adminTemplateController = new AdminTemplateController(
authBackend,
templateMaps,
userLimitsBackend
);
}
register (apiRouter, templatePaths) {
const templateRouter = router({ mergeParams: true });
this.namedMapController.register(templateRouter);
this.tileTemplateController.register(templateRouter);
this.adminTemplateController.register(templateRouter);
templatePaths.forEach(path => apiRouter.use(path, templateRouter));
}
};

95
lib/cartodb/api/template/tile-template-controller.js Normal file
View File

@ -0,0 +1,95 @@
const coordinates = require('../middlewares/coordinates');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
const authorize = require('../middlewares/authorize');
const namedMapProvider = require('../middlewares/named-map-provider');
const cacheControlHeader = require('../middlewares/cache-control-header');
const cacheChannelHeader = require('../middlewares/cache-channel-header');
const surrogateKeyHeader = require('../middlewares/surrogate-key-header');
const lastModifiedHeader = require('../middlewares/last-modified-header');
const vectorError = require('../middlewares/vector-error');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
module.exports = class TileTemplateController {
constructor (
namedMapProviderCache,
tileBackend,
surrogateKeysCache,
pgConnection,
authBackend,
userLimitsBackend
) {
this.namedMapProviderCache = namedMapProviderCache;
this.tileBackend = tileBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.pgConnection = pgConnection;
this.authBackend = authBackend;
this.userLimitsBackend = userLimitsBackend;
}
register (templateRouter) {
templateRouter.get('/:template_id/:layer/:z/:x/:y.(:format)', this.middlewares());
}
middlewares () {
return [
coordinates(),
credentials(),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsBackend, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES),
cleanUpQueryParams(),
namedMapProvider({
namedMapProviderCache: this.namedMapProviderCache,
label: 'NAMED_MAP_TILE'
}),
getTile({
tileBackend: this.tileBackend,
label: 'NAMED_MAP_TILE'
}),
setContentTypeHeader(),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
vectorError()
];
}
};
function getTile ({ tileBackend, label }) {
return function getTileMiddleware (req, res, next) {
const { mapConfigProvider } = res.locals;
const { layer, z, x, y, format } = req.params;
const params = { layer, z, x, y, format };
tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats) => {
req.profiler.add(stats);
req.profiler.done('render-' + format);
if (err) {
err.label = label;
return next(err);
}
if (headers) {
res.set(headers);
}
res.statusCode = 200;
res.body = tile;
next();
});
};
}
function setContentTypeHeader () {
return function setContentTypeHeaderMiddleware(req, res, next) {
res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png');
next();
};
}

12
lib/cartodb/api/auth_api.js → lib/cartodb/backends/auth.js
View File

@ -5,16 +5,16 @@
* @param {MapStore} mapStore
* @param {TemplateMaps} templateMaps
* @constructor
* @type {AuthApi}
* @type {AuthBackend}
*/
function AuthApi(pgConnection, metadataBackend, mapStore, templateMaps) {
function AuthBackend(pgConnection, metadataBackend, mapStore, templateMaps) {
this.pgConnection = pgConnection;
this.metadataBackend = metadataBackend;
this.mapStore = mapStore;
this.templateMaps = templateMaps;
}
module.exports = AuthApi;
module.exports = AuthBackend;
// Check if the user is authorized by a signer
//
@ -23,7 +23,7 @@ module.exports = AuthApi;
// null if the request is not signed by anyone
// or will be a string cartodb username otherwise.
//
AuthApi.prototype.authorizedBySigner = function(req, res, callback) {
AuthBackend.prototype.authorizedBySigner = function(req, res, callback) {
if ( ! res.locals.token || ! res.locals.signer ) {
return callback(null, false); // no signer requested
}
@ -58,7 +58,7 @@ function isValidApiKey(apikey) {
// @param callback function(err, authorized)
// NOTE: authorized is expected to be 0 or 1 (integer)
//
AuthApi.prototype.authorizedByAPIKey = function(user, res, callback) {
AuthBackend.prototype.authorizedByAPIKey = function(user, res, callback) {
const apikeyToken = res.locals.api_key;
const basicAuthUsername = res.locals.basicAuthUsername;
@ -121,7 +121,7 @@ function usernameMatches (basicAuthUsername, requestUsername) {
* @param res - standard res object. Contains the auth parameters in locals
* @param callback function(err, allowed) is access allowed not?
*/
AuthApi.prototype.authorize = function(req, res, callback) {
AuthBackend.prototype.authorize = function(req, res, callback) {
var user = res.locals.user;
this.authorizedByAPIKey(user, res, (err, isAuthorizedByApikey) => {

6
lib/cartodb/api/filter_stats_api.js → lib/cartodb/backends/filter-stats.js
View File

@ -2,11 +2,11 @@ var _ = require('underscore');
var step = require('step');
var AnalysisFilter = require('../models/filter/analysis');
function FilterStatsApi(pgQueryRunner) {
function FilterStatsBackends(pgQueryRunner) {
this.pgQueryRunner = pgQueryRunner;
}
module.exports = FilterStatsApi;
module.exports = FilterStatsBackends;
function getEstimatedRows(pgQueryRunner, username, query, callback) {
pgQueryRunner.run(username, "EXPLAIN (FORMAT JSON)"+query, function(err, result_rows) {
@ -23,7 +23,7 @@ function getEstimatedRows(pgQueryRunner, username, query, callback) {
});
}
FilterStatsApi.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) {
FilterStatsBackends.prototype.getFilterStats = function (username, unfiltered_query, filters, callback) {
var stats = {};
var self = this;
step(

284
lib/cartodb/backends/layer-stats/mapnik-layer-stats.js
View File

@ -1,4 +1,27 @@
var queryUtils = require('../../utils/query-utils');
const AggregationMapConfig = require('../../models/aggregation/aggregation-mapconfig');
var SubstitutionTokens = require('../../utils/substitution-tokens');
// Instantiate a query with tokens for a given zoom level
function queryForZoom(sql, zoom, singleTile=false) {
const tileRes = 256;
const wmSize = 6378137.0*2*Math.PI;
const nTiles = Math.pow(2, zoom);
const tileSize = wmSize / nTiles;
const resolution = tileSize / tileRes;
const scaleDenominator = resolution / 0.00028;
const x0 = -wmSize/2, y0 = -wmSize/2;
let bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0+wmSize}, ${y0+wmSize})`;
if (singleTile) {
bbox = `ST_MakeEnvelope(${x0}, ${y0}, ${x0 + tileSize}, ${y0 + tileSize})`;
}
return SubstitutionTokens.replace(sql, {
bbox: bbox,
scale_denominator: scaleDenominator,
pixel_width: resolution,
pixel_height: resolution
});
}
function MapnikLayerStats () {
this._types = {
@ -11,17 +34,260 @@ MapnikLayerStats.prototype.is = function (type) {
return this._types[type] ? this._types[type] : false;
};
function queryPromise(dbConnection, query) {
return new Promise((resolve, reject) => {
dbConnection.query(query, (err, res) => err ? reject(err) : resolve(res));
});
}
function columnAggregations(field) {
if (field.type === 'number') {
return ['min', 'max', 'avg', 'sum'];
}
if (field.type === 'date') { // TODO other types too?
return ['min', 'max'];
}
return [];
}
function _getSQL(ctx, query, type='pre', zoom=0) {
let sql;
if (type === 'pre') {
sql = ctx.preQuery;
}
else {
sql = ctx.aggrQuery;
}
sql = queryForZoom(sql, zoom || 0);
return query(sql);
}
function _estimatedFeatureCount(ctx) {
return queryPromise(ctx.dbConnection, _getSQL(ctx, queryUtils.getQueryRowEstimation))
.then(res => ({ estimatedFeatureCount: res.rows[0].rows }))
.catch(() => ({ estimatedFeatureCount: -1 }));
}
function _featureCount(ctx) {
if (ctx.metaOptions.featureCount) {
// TODO: if ctx.metaOptions.columnStats we can combine this with column stats query
return queryPromise(ctx.dbConnection, _getSQL(ctx, queryUtils.getQueryActualRowCount))
.then(res => ({ featureCount: res.rows[0].rows }));
}
return Promise.resolve();
}
function _aggrFeatureCount(ctx) {
if (ctx.metaOptions.hasOwnProperty('aggrFeatureCount')) {
// We expect as zoom level as the value of aggrFeatureCount
// TODO: it'd be nice to admit an array of zoom levels to
// return metadata for multiple levels.
return queryPromise(
ctx.dbConnection,
_getSQL(ctx, queryUtils.getQueryActualRowCount, 'post', ctx.metaOptions.aggrFeatureCount)
).then(res => ({ aggrFeatureCount: res.rows[0].rows }));
}
return Promise.resolve();
}
function _geometryType(ctx) {
if (ctx.metaOptions.geometryType) {
const geometryColumn = AggregationMapConfig.getAggregationGeometryColumn();
return queryPromise(ctx.dbConnection, _getSQL(ctx, sql => queryUtils.getQueryGeometryType(sql, geometryColumn)))
.then(res => ({ geometryType: res.rows[0].geom_type }));
}
return Promise.resolve();
}
function _columns(ctx) {
if (ctx.metaOptions.columns || ctx.metaOptions.columnStats) {
// note: post-aggregation columns are in layer.options.columns when aggregation is present
return queryPromise(ctx.dbConnection, _getSQL(ctx, sql => queryUtils.getQueryLimited(sql, 0)))
.then(res => formatResultFields(ctx.dbConnection, res.fields));
}
return Promise.resolve();
}
// combine a list of results merging the properties of all the objects
// undefined results are admitted and ignored
function mergeResults(results) {
if (results) {
if (results.length === 0) {
return {};
}
return results.reduce((a, b) => {
if (a === undefined) {
return b;
}
if (b === undefined) {
return a;
}
return Object.assign({}, a, b);
});
}
}
// deeper (1 level) combination of a list of objects:
// mergeColumns([{ col1: { a: 1 }, col2: { a: 2 } }, { col1: { b: 3 } }]) => { col1: { a: 1, b: 3 }, col2: { a: 2 } }
function mergeColumns(results) {
if (results) {
if (results.length === 0) {
return {};
}
return results.reduce((a, b) => {
let c = Object.assign({}, b || {}, a || {});
Object.keys(c).forEach(key => {
if (b.hasOwnProperty(key)) {
c[key] = Object.assign(c[key], b[key]);
}
});
return c;
});
}
}
const SAMPLE_SEED = 0.5;
const DEFAULT_SAMPLE_ROWS = 100;
function _sample(ctx, numRows) {
if (ctx.metaOptions.sample) {
const sampleProb = Math.min(ctx.metaOptions.sample.num_rows / numRows, 1);
// We'll use a safety limit just in case numRows is a bad estimate
const requestedRows = ctx.metaOptions.sample.num_rows || DEFAULT_SAMPLE_ROWS;
const limit = Math.ceil(requestedRows * 1.5);
let columns = ctx.metaOptions.sample.include_columns;
return queryPromise(ctx.dbConnection, _getSQL(
ctx,
sql => queryUtils.getQuerySample(sql, sampleProb, limit, SAMPLE_SEED, columns)
)).then(res => ({ sample: res.rows }));
}
return Promise.resolve();
}
function _columnStats(ctx, columns) {
if (!columns) {
return Promise.resolve();
}
if (ctx.metaOptions.columnStats) {
let queries = [];
let aggr = [];
queries.push(new Promise(resolve => resolve(columns))); // add columns as first result
Object.keys(columns).forEach(name => {
aggr = aggr.concat(
columnAggregations(columns[name])
.map(fn => `${fn}(${name}) AS ${name}_${fn}`)
);
if (columns[name].type === 'string') {
const topN = ctx.metaOptions.columnStats.topCategories || 1024;
const includeNulls = ctx.metaOptions.columnStats.hasOwnProperty('includeNulls') ?
ctx.metaOptions.columnStats.includeNulls :
true;
// TODO: ctx.metaOptions.columnStats.maxCategories
// => use PG stats to dismiss columns with more distinct values
queries.push(
queryPromise(
ctx.dbConnection,
_getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN, includeNulls))
).then(res => ({ [name]: { categories: res.rows } }))
);
}
});
queries.push(
queryPromise(
ctx.dbConnection,
_getSQL(ctx, sql => `SELECT ${aggr.join(',')} FROM (${sql}) AS __cdb_query`)
).then(res => {
let stats = {};
Object.keys(columns).forEach(name => {
stats[name] = {};
columnAggregations(columns[name]).forEach(fn => {
stats[name][fn] = res.rows[0][`${name}_${fn}`];
});
});
return stats;
})
);
return Promise.all(queries).then(results => ({ columns: mergeColumns(results) }));
}
return Promise.resolve({ columns });
}
// This is adapted from SQL API:
function fieldType(cname) {
let tname;
switch (true) {
case /bool/.test(cname):
tname = 'boolean';
break;
case /int|float|numeric/.test(cname):
tname = 'number';
break;
case /text|char|unknown/.test(cname):
tname = 'string';
break;
case /date|time/.test(cname):
tname = 'date';
break;
default:
tname = cname;
}
if ( tname && cname.match(/^_/) ) {
tname += '[]';
}
return tname;
}
// columns are returned as an object { columnName1: { type1: ...}, ..}
// for consistency with SQL API
function formatResultFields(dbConnection, fields = []) {
let nfields = {};
for (let field of fields) {
const cname = dbConnection.typeName(field.dataTypeID);
let tname;
if ( ! cname ) {
tname = 'unknown(' + field.dataTypeID + ')';
} else {
tname = fieldType(cname);
}
nfields[field.name] = { type: tname };
}
return nfields;
}
MapnikLayerStats.prototype.getStats =
function (layer, dbConnection, callback) {
var queryRowCountSql = queryUtils.getQueryRowCount(layer.options.sql);
// This query would gather stats for postgresql table if not exists
dbConnection.query(queryRowCountSql, function (err, res) {
if (err) {
return callback(null, {estimatedFeatureCount: -1});
} else {
// We decided that the relation is 1 row == 1 feature
return callback(null, {estimatedFeatureCount: res.rows[0].rows});
}
let aggrQuery = layer.options.sql;
let preQuery = layer.options.sql_raw || aggrQuery;
let ctx = {
dbConnection,
preQuery,
aggrQuery,
metaOptions: layer.options.metadata || {}
};
// TODO: could save some queries if queryUtils.getAggregationMetadata() has been used and kept somewhere
// we would set queries.results.estimatedFeatureCount and queries.results.geometryType
// (if metaOptions.geometryType) from it.
// TODO: compute _sample with _featureCount when available
// TODO: add support for sample.exclude option by, in that case, forcing the columns query and
// passing the results to the sample query function.
Promise.all([
_estimatedFeatureCount(ctx).then(
({ estimatedFeatureCount }) => _sample(ctx, estimatedFeatureCount)
.then(sampleResults => mergeResults([sampleResults, { estimatedFeatureCount }]))
),
_featureCount(ctx),
_aggrFeatureCount(ctx),
_geometryType(ctx),
_columns(ctx).then(columns => _columnStats(ctx, columns))
]).then(results => {
callback(null, mergeResults(results));
}).catch(error => {
callback(error);
});
};

6
lib/cartodb/api/overviews_metadata_api.js → lib/cartodb/backends/overviews-metadata.js
View File

@ -1,10 +1,10 @@
var SubstitutionTokens = require('../utils/substitution-tokens');
function OverviewsMetadataApi(pgQueryRunner) {
function OverviewsMetadataBackend(pgQueryRunner) {
this.pgQueryRunner = pgQueryRunner;
}
module.exports = OverviewsMetadataApi;
module.exports = OverviewsMetadataBackend;
function prepareSql(sql) {
return sql && SubstitutionTokens.replace(sql, {
@ -15,7 +15,7 @@ function prepareSql(sql) {
});
}
OverviewsMetadataApi.prototype.getOverviewsMetadata = function (username, sql, callback) {
OverviewsMetadataBackend.prototype.getOverviewsMetadata = function (username, sql, callback) {
// FIXME: Currently using internal function _cdb_schema_name
// CDB_Overviews should provide the schema information directly.
var query = 'SELECT *, _cdb_schema_name(base_table)' +

6
lib/cartodb/api/tables_extent_api.js → lib/cartodb/backends/tables-extent.js
View File

@ -1,8 +1,8 @@
function TablesExtentApi(pgQueryRunner) {
function TablesExtentBackend(pgQueryRunner) {
this.pgQueryRunner = pgQueryRunner;
}
module.exports = TablesExtentApi;
module.exports = TablesExtentBackend;
/**
* Given a username and a list of tables it will return the estimated extent in SRID 4326 for all the tables based on
@ -13,7 +13,7 @@ module.exports = TablesExtentApi;
* `table_name` format as valid input
* @param {Function} callback function(err, result) {Object} result with `west`, `south`, `east`, `north`
*/
TablesExtentApi.prototype.getBounds = function (username, tables, callback) {
TablesExtentBackend.prototype.getBounds = function (username, tables, callback) {
var estimatedExtentSQLs = tables.map(function(table) {
return "ST_EstimatedExtent('" + table.schema_name + "', '" + table.table_name + "', 'the_geom_webmercator')";
});

14
lib/cartodb/api/user_limits_api.js → lib/cartodb/backends/user-limits.js
View File

@ -5,9 +5,9 @@ var step = require('step');
* @param metadataBackend
* @param options
* @constructor
* @type {UserLimitsApi}
* @type {UserLimitsBackend}
*/
function UserLimitsApi(metadataBackend, options) {
function UserLimitsBackend(metadataBackend, options) {
this.metadataBackend = metadataBackend;
this.options = options || {};
this.options.limits = this.options.limits || {};
@ -15,9 +15,9 @@ function UserLimitsApi(metadataBackend, options) {
this.preprareRateLimit();
}
module.exports = UserLimitsApi;
module.exports = UserLimitsBackend;
UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback) {
UserLimitsBackend.prototype.getRenderLimits = function (username, apiKey, callback) {
var self = this;
var limits = {
@ -40,7 +40,7 @@ UserLimitsApi.prototype.getRenderLimits = function (username, apiKey, callback)
});
};
UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) {
UserLimitsBackend.prototype.getTimeoutRenderLimit = function (username, apiKey, callback) {
var self = this;
step(
@ -80,12 +80,12 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call
);
};
UserLimitsApi.prototype.preprareRateLimit = function () {
UserLimitsBackend.prototype.preprareRateLimit = function () {
if (this.options.limits.rateLimitsEnabled) {
this.metadataBackend.loadRateLimitsScript();
}
};
UserLimitsApi.prototype.getRateLimit = function (user, endpointGroup, callback) {
UserLimitsBackend.prototype.getRateLimit = function (user, endpointGroup, callback) {
this.metadataBackend.getRateLimit(user, 'maps', endpointGroup, callback);
};

6
lib/cartodb/cache/named_map_provider_cache.js vendored
View File

@ -10,14 +10,14 @@ function NamedMapProviderCache(
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
userLimitsBackend,
mapConfigAdapter,
affectedTablesCache
) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.metadataBackend = metadataBackend;
this.userLimitsApi = userLimitsApi;
this.userLimitsBackend = userLimitsBackend;
this.mapConfigAdapter = mapConfigAdapter;
this.affectedTablesCache = affectedTablesCache;
@ -36,7 +36,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok
this.templateMaps,
this.pgConnection,
this.metadataBackend,
this.userLimitsApi,
this.userLimitsBackend,
this.mapConfigAdapter,
this.affectedTablesCache,
user,

8
lib/cartodb/controllers/index.js
View File

@ -1,8 +0,0 @@
module.exports = {
Analyses: require('./analyses'),
Layergroup: require('./layergroup'),
Map: require('./map'),
NamedMaps: require('./named_maps'),
NamedMapsAdmin: require('./named_maps_admin'),
ServerInfo: require('./server_info')
};

75
lib/cartodb/controllers/layergroup/analysis.js
View File

@ -1,75 +0,0 @@
const cors = require('../../middleware/cors');
const user = require('../../middleware/user');
const layergroupToken = require('../../middleware/layergroup-token');
const cleanUpQueryParams = require('../../middleware/clean-up-query-params');
const credentials = require('../../middleware/credentials');
const dbConnSetup = require('../../middleware/db-conn-setup');
const authorize = require('../../middleware/authorize');
const rateLimit = require('../../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const sendResponse = require('../../middleware/send-response');
const dbParamsFromResLocals = require('../../utils/database-params');
module.exports = class AnalysisController {
constructor (
analysisStatusBackend,
pgConnection,
mapStore,
userLimitsApi,
layergroupAffectedTablesCache,
authApi,
surrogateKeysCache
) {
this.analysisStatusBackend = analysisStatusBackend;
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.authApi = authApi;
this.surrogateKeysCache = surrogateKeysCache;
}
register (app) {
const { base_url_mapconfig: mapConfigBasePath } = app;
app.get(
`${mapConfigBasePath}/:token/analysis/node/:nodeId`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS),
cleanUpQueryParams(),
analysisNodeStatus(this.analysisStatusBackend),
sendResponse()
);
}
};
function analysisNodeStatus (analysisStatusBackend) {
return function analysisNodeStatusMiddleware(req, res, next) {
const { nodeId } = req.params;
const dbParams = dbParamsFromResLocals(res.locals);
analysisStatusBackend.getNodeStatus(nodeId, dbParams, (err, nodeStatus, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET NODE STATUS';
return next(err);
}
res.set({
'Cache-Control': 'public,max-age=5',
'Last-Modified': new Date().toUTCString()
});
res.body = nodeStatus;
next();
});
};
}

199
lib/cartodb/controllers/layergroup/dataview.js
View File

@ -1,199 +0,0 @@
const cors = require('../../middleware/cors');
const user = require('../../middleware/user');
const layergroupToken = require('../../middleware/layergroup-token');
const cleanUpQueryParams = require('../../middleware/clean-up-query-params');
const credentials = require('../../middleware/credentials');
const dbConnSetup = require('../../middleware/db-conn-setup');
const authorize = require('../../middleware/authorize');
const rateLimit = require('../../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider');
const cacheControlHeader = require('../../middleware/cache-control-header');
const cacheChannelHeader = require('../../middleware/cache-channel-header');
const surrogateKeyHeader = require('../../middleware/surrogate-key-header');
const lastModifiedHeader = require('../../middleware/last-modified-header');
const sendResponse = require('../../middleware/send-response');
const ALLOWED_DATAVIEW_QUERY_PARAMS = [
'filters', // json
'own_filter', // 0, 1
'no_filters', // 0, 1
'bbox', // w,s,e,n
'start', // number
'end', // number
'column_type', // string
'bins', // number
'aggregation', //string
'offset', // number
'q', // widgets search
'categories', // number
];
module.exports = class DataviewController {
constructor (
dataviewBackend,
pgConnection,
mapStore,
userLimitsApi,
layergroupAffectedTablesCache,
authApi,
surrogateKeysCache
) {
this.dataviewBackend = dataviewBackend;
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.authApi = authApi;
this.surrogateKeysCache = surrogateKeysCache;
}
register (app) {
const { base_url_mapconfig: mapConfigBasePath } = app;
// Undocumented/non-supported API endpoint methods.
// Use at your own peril.
app.get(
`${mapConfigBasePath}/:token/dataview/:dataviewName`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getDataview(this.dataviewBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
app.get(
`${mapConfigBasePath}/:token/:layer/widget/:dataviewName`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getDataview(this.dataviewBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
app.get(
`${mapConfigBasePath}/:token/dataview/:dataviewName/search`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
dataviewSearch(this.dataviewBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
app.get(
`${mapConfigBasePath}/:token/:layer/widget/:dataviewName/search`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
dataviewSearch(this.dataviewBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
}
};
function getDataview (dataviewBackend) {
return function getDataviewMiddleware (req, res, next) {
const { user, mapConfigProvider } = res.locals;
const { dataviewName } = req.params;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const params = Object.assign({ dataviewName, dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET DATAVIEW';
return next(err);
}
res.body = dataview;
next();
});
};
}
function dataviewSearch (dataviewBackend) {
return function dataviewSearchMiddleware (req, res, next) {
const { user, mapConfigProvider } = res.locals;
const { dataviewName } = req.params;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET DATAVIEW SEARCH';
return next(err);
}
res.body = searchResult;
next();
});
};
}

114
lib/cartodb/controllers/layergroup/index.js
View File

@ -1,114 +0,0 @@
const DataviewBackend = require('../../backends/dataview');
const AnalysisStatusBackend = require('../../backends/analysis-status');
const TileController = require('./tile');
const AttributesController = require('./attributes');
const StaticController = require('./static');
const DataviewController = require('./dataview');
const AnalysisController = require('./analysis');
/**
* @param {prepareContext} prepareContext
* @param {PgConnection} pgConnection
* @param {MapStore} mapStore
* @param {TileBackend} tileBackend
* @param {PreviewBackend} previewBackend
* @param {AttributesBackend} attributesBackend
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @param {AnalysisBackend} analysisBackend
* @constructor
*/
function LayergroupController(
pgConnection,
mapStore,
tileBackend,
previewBackend,
attributesBackend,
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTablesCache,
analysisBackend,
authApi
) {
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
this.attributesBackend = attributesBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.dataviewBackend = new DataviewBackend(analysisBackend);
this.analysisStatusBackend = new AnalysisStatusBackend();
this.authApi = authApi;
}
module.exports = LayergroupController;
LayergroupController.prototype.register = function(app) {
const tileController = new TileController(
this.tileBackend,
this.pgConnection,
this.mapStore,
this.userLimitsApi,
this.layergroupAffectedTablesCache,
this.authApi,
this.surrogateKeysCache
);
tileController.register(app);
const attributesController = new AttributesController(
this.attributesBackend,
this.pgConnection,
this.mapStore,
this.userLimitsApi,
this.layergroupAffectedTablesCache,
this.authApi,
this.surrogateKeysCache
);
attributesController.register(app);
const staticController = new StaticController(
this.previewBackend,
this.pgConnection,
this.mapStore,
this.userLimitsApi,
this.layergroupAffectedTablesCache,
this.authApi,
this.surrogateKeysCache
);
staticController.register(app);
const dataviewController = new DataviewController(
this.dataviewBackend,
this.pgConnection,
this.mapStore,
this.userLimitsApi,
this.layergroupAffectedTablesCache,
this.authApi,
this.surrogateKeysCache
);
dataviewController.register(app);
const analysisController = new AnalysisController(
this.analysisStatusBackend,
this.pgConnection,
this.mapStore,
this.userLimitsApi,
this.layergroupAffectedTablesCache,
this.authApi,
this.surrogateKeysCache
);
analysisController.register(app);
};

234
lib/cartodb/controllers/layergroup/tile.js
View File

@ -1,234 +0,0 @@
const cors = require('../../middleware/cors');
const user = require('../../middleware/user');
const layergroupToken = require('../../middleware/layergroup-token');
const coordinates = require('../../middleware/coordinates');
const cleanUpQueryParams = require('../../middleware/clean-up-query-params');
const credentials = require('../../middleware/credentials');
const dbConnSetup = require('../../middleware/db-conn-setup');
const authorize = require('../../middleware/authorize');
const rateLimit = require('../../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider');
const cacheControlHeader = require('../../middleware/cache-control-header');
const cacheChannelHeader = require('../../middleware/cache-channel-header');
const surrogateKeyHeader = require('../../middleware/surrogate-key-header');
const lastModifiedHeader = require('../../middleware/last-modified-header');
const sendResponse = require('../../middleware/send-response');
const vectorError = require('../../middleware/vector-error');
const SUPPORTED_FORMATS = {
grid_json: true,
json_torque: true,
torque_json: true,
png: true,
png32: true,
mvt: true
};
module.exports = class TileController {
constructor (
tileBackend,
pgConnection,
mapStore,
userLimitsApi,
layergroupAffectedTablesCache,
authApi,
surrogateKeysCache
) {
this.tileBackend = tileBackend;
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.authApi = authApi;
this.surrogateKeysCache = surrogateKeysCache;
}
register (app) {
const { base_url_mapconfig: mapConfigBasePath } = app;
app.get(
`${mapConfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`,
cors(),
user(),
layergroupToken(),
coordinates(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
cleanUpQueryParams(),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getTile(this.tileBackend, 'map_tile'),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
incrementSuccessMetrics(global.statsClient),
incrementErrorMetrics(global.statsClient),
tileError(),
vectorError(),
sendResponse()
);
app.get(
`${mapConfigBasePath}/:token/:z/:x/:y.:format`,
cors(),
user(),
layergroupToken(),
coordinates(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
cleanUpQueryParams(),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getTile(this.tileBackend, 'map_tile'),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
incrementSuccessMetrics(global.statsClient),
incrementErrorMetrics(global.statsClient),
tileError(),
vectorError(),
sendResponse()
);
app.get(
`${mapConfigBasePath}/:token/:layer/:z/:x/:y.(:format)`,
distinguishLayergroupFromStaticRoute(),
cors(),
user(),
layergroupToken(),
coordinates(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
cleanUpQueryParams(),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getTile(this.tileBackend, 'maplayer_tile'),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
incrementSuccessMetrics(global.statsClient),
incrementErrorMetrics(global.statsClient),
tileError(),
vectorError(),
sendResponse()
);
}
};
function distinguishLayergroupFromStaticRoute () {
return function distinguishLayergroupFromStaticRouteMiddleware(req, res, next) {
if (req.params.token === 'static') {
return next('route');
}
next();
};
}
function parseFormat (format = '') {
const prettyFormat = format.replace('.', '_');
return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid';
}
function getStatusCode(tile, format){
return tile.length === 0 && format === 'mvt' ? 204 : 200;
}
function getTile (tileBackend, profileLabel = 'tile') {
return function getTileMiddleware (req, res, next) {
req.profiler.start(`windshaft.${profileLabel}`);
const { mapConfigProvider } = res.locals;
const { token } = res.locals;
const { layer, z, x, y, format } = req.params;
const params = { token, layer, z, x, y, format };
tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => {
req.profiler.add(stats);
if (err) {
return next(err);
}
if (headers) {
res.set(headers);
}
const formatStat = parseFormat(req.params.format);
res.statusCode = getStatusCode(tile, formatStat);
res.body = tile;
next();
});
};
}
function incrementSuccessMetrics (statsClient) {
return function incrementSuccessMetricsMiddleware (req, res, next) {
const formatStat = parseFormat(req.params.format);
statsClient.increment('windshaft.tiles.success');
statsClient.increment(`windshaft.tiles.${formatStat}.success`);
next();
};
}
function incrementErrorMetrics (statsClient) {
return function incrementErrorMetricsMiddleware (err, req, res, next) {
const formatStat = parseFormat(req.params.format);
statsClient.increment('windshaft.tiles.error');
statsClient.increment(`windshaft.tiles.${formatStat}.error`);
next(err);
};
}
function tileError () {
return function tileErrorMiddleware (err, req, res, next) {
if (err.message === 'Tile does not exist' && req.params.format === 'mvt') {
res.statusCode = 204;
return next();
}
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
let errMsg = err.message ? ( '' + err.message ) : ( '' + err );
// Rewrite mapnik parsing errors to start with layer number
const matches = errMsg.match("(.*) in style 'layer([0-9]+)'");
if (matches) {
errMsg = `style${matches[2]}: ${matches[1]}`;
}
err.message = errMsg;
err.label = 'TILE RENDER';
next(err);
};
}

592
lib/cartodb/controllers/map.js
View File

@ -1,592 +0,0 @@
const _ = require('underscore');
const windshaft = require('windshaft');
const MapConfig = windshaft.model.MapConfig;
const Datasource = windshaft.model.Datasource;
const ResourceLocator = require('../models/resource-locator');
const cors = require('../middleware/cors');
const user = require('../middleware/user');
const cleanUpQueryParams = require('../middleware/clean-up-query-params');
const credentials = require('../middleware/credentials');
const dbConnSetup = require('../middleware/db-conn-setup');
const authorize = require('../middleware/authorize');
const cacheControlHeader = require('../middleware/cache-control-header');
const cacheChannelHeader = require('../middleware/cache-channel-header');
const surrogateKeyHeader = require('../middleware/surrogate-key-header');
const lastModifiedHeader = require('../middleware/last-modified-header');
const sendResponse = require('../middleware/send-response');
const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
const LayergroupMetadata = require('../utils/layergroup-metadata');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
/**
* @param {AuthApi} authApi
* @param {PgConnection} pgConnection
* @param {TemplateMaps} templateMaps
* @param {MapBackend} mapBackend
* @param metadataBackend
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @param {MapConfigAdapter} mapConfigAdapter
* @param {StatsBackend} statsBackend
* @constructor
*/
function MapController (
pgConnection,
templateMaps,
mapBackend,
metadataBackend,
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTables,
mapConfigAdapter,
statsBackend,
authApi
) {
this.pgConnection = pgConnection;
this.templateMaps = templateMaps;
this.mapBackend = mapBackend;
this.metadataBackend = metadataBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTables = layergroupAffectedTables;
this.mapConfigAdapter = mapConfigAdapter;
const resourceLocator = new ResourceLocator(global.environment);
this.layergroupMetadata = new LayergroupMetadata(resourceLocator);
this.statsBackend = statsBackend;
this.authApi = authApi;
}
module.exports = MapController;
MapController.prototype.register = function(app) {
const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = app;
app.get(
`${mapConfigBasePath}`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
app.post(
`${mapConfigBasePath}`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
const useTemplate = true;
app.get(
`${templateBasePath}/:template_id/jsonp`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate)
);
app.post(
`${templateBasePath}/:template_id`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate)
);
app.options(`${mapConfigBasePath}`, cors('Content-Type'));
};
MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) {
const isTemplateInstantiation = useTemplate;
const useTemplateHash = useTemplate;
const includeQuery = !useTemplate;
const label = useTemplate ? 'NAMED MAP LAYERGROUP' : 'ANONYMOUS LAYERGROUP';
const addContext = !useTemplate;
return [
cors(),
user(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, endpointGroup),
cleanUpQueryParams(['aggregation']),
initProfiler(isTemplateInstantiation),
checkJsonContentType(),
this.getCreateMapMiddlewares(useTemplate),
incrementMapViewCount(this.metadataBackend),
augmentLayergroupData(),
cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader({ now: true }),
setLastUpdatedTimeToLayergroup(),
setLayerStats(this.pgConnection, this.statsBackend),
setLayergroupIdHeader(this.templateMaps ,useTemplateHash),
setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata),
setAnalysesMetadataToLayergroup(this.layergroupMetadata, includeQuery),
setTurboCartoMetadataToLayergroup(this.layergroupMetadata),
setAggregationMetadataToLayergroup(this.layergroupMetadata),
setTilejsonMetadataToLayergroup(this.layergroupMetadata),
sendResponse(),
augmentError({ label, addContext })
];
};
MapController.prototype.getCreateMapMiddlewares = function (useTemplate) {
if (useTemplate) {
return [
checkInstantiteLayergroup(),
getTemplate(
this.templateMaps,
this.pgConnection,
this.metadataBackend,
this.userLimitsApi,
this.mapConfigAdapter,
this.layergroupAffectedTables
),
instantiateLayergroup(
this.mapBackend,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTables
)
];
}
return [
checkCreateLayergroup(),
prepareAdapterMapConfig(this.mapConfigAdapter),
createLayergroup (
this.mapBackend,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTables
)
];
};
function initProfiler (isTemplateInstantiation) {
const operation = isTemplateInstantiation ? 'instance_template' : 'createmap';
return function initProfilerMiddleware (req, res, next) {
req.profiler.start(`windshaft-cartodb.${operation}_${req.method.toLowerCase()}`);
req.profiler.done(`${operation}.initProfilerMiddleware`);
next();
};
}
function checkJsonContentType () {
return function checkJsonContentTypeMiddleware(req, res, next) {
if (req.method === 'POST' && !req.is('application/json')) {
return next(new Error('POST data must be of type application/json'));
}
req.profiler.done('checkJsonContentTypeMiddleware');
next();
};
}
function checkInstantiteLayergroup () {
return function checkInstantiteLayergroupMiddleware(req, res, next) {
if (req.method === 'GET') {
const { callback, config } = req.query;
if (callback === undefined || callback.length === 0) {
return next(new Error('callback parameter should be present and be a function name'));
}
if (config) {
try {
req.body = JSON.parse(config);
} catch(e) {
return next(new Error('Invalid config parameter, should be a valid JSON'));
}
}
}
req.profiler.done('checkInstantiteLayergroup');
return next();
};
}
function checkCreateLayergroup () {
return function checkCreateLayergroupMiddleware (req, res, next) {
if (req.method === 'GET') {
const { config } = req.query;
if (!config) {
return next(new Error('layergroup GET needs a "config" parameter'));
}
try {
req.body = JSON.parse(config);
} catch (err) {
return next(err);
}
}
req.profiler.done('checkCreateLayergroup');
return next();
};
}
function getTemplate (
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache
) {
return function getTemplateMiddleware (req, res, next) {
const templateParams = req.body;
const { user, dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const { template_id } = req.params;
const { auth_token } = req.query;
const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
const mapConfigProvider = new NamedMapMapConfigProvider(
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache,
user,
template_id,
templateParams,
auth_token,
params
);
mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => {
req.profiler.done('named.getMapConfig');
if (err) {
return next(err);
}
res.locals.mapConfig = mapConfig;
res.locals.rendererParams = rendererParams;
res.locals.mapConfigProvider = mapConfigProvider;
next();
});
};
}
function prepareAdapterMapConfig (mapConfigAdapter) {
return function prepareAdapterMapConfigMiddleware(req, res, next) {
const requestMapConfig = req.body;
const { user, api_key } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
const context = {
analysisConfiguration: {
user,
db: {
host: dbhost,
port: dbport,
dbname: dbname,
user: dbuser,
pass: dbpassword
},
batch: {
username: user,
apiKey: api_key
}
}
};
mapConfigAdapter.getMapConfig(user, requestMapConfig, params, context, (err, requestMapConfig) => {
req.profiler.done('anonymous.getMapConfig');
if (err) {
return next(err);
}
req.body = requestMapConfig;
res.locals.context = context;
next();
});
};
}
function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) {
return function createLayergroupMiddleware (req, res, next) {
const requestMapConfig = req.body;
const { context } = res.locals;
const { user, cache_buster, api_key } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const params = {
cache_buster, api_key,
dbuser, dbname, dbpassword, dbhost, dbport
};
const datasource = context.datasource || Datasource.EmptyDatasource();
const mapConfig = new MapConfig(requestMapConfig, datasource);
const mapConfigProvider = new CreateLayergroupMapConfigProvider(
mapConfig,
user,
userLimitsApi,
pgConnection,
affectedTablesCache,
params
);
res.locals.mapConfig = mapConfig;
res.locals.analysesResults = context.analysesResults;
const mapParams = { dbuser, dbname, dbpassword, dbhost, dbport };
mapBackend.createLayergroup(mapConfig, mapParams, mapConfigProvider, (err, layergroup) => {
req.profiler.done('createLayergroup');
if (err) {
return next(err);
}
res.body = layergroup;
res.locals.mapConfigProvider = mapConfigProvider;
next();
});
};
}
function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) {
return function instantiateLayergroupMiddleware (req, res, next) {
const { user, mapConfig, rendererParams } = res.locals;
const mapConfigProvider = new CreateLayergroupMapConfigProvider(
mapConfig,
user,
userLimitsApi,
pgConnection,
affectedTablesCache,
rendererParams
);
mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => {
req.profiler.done('createLayergroup');
if (err) {
return next(err);
}
res.body = layergroup;
const { mapConfigProvider } = res.locals;
res.locals.analysesResults = mapConfigProvider.analysesResults;
res.locals.template = mapConfigProvider.template;
res.locals.context = mapConfigProvider.context;
next();
});
};
}
function incrementMapViewCount (metadataBackend) {
return function incrementMapViewCountMiddleware(req, res, next) {
const { mapConfig, user } = res.locals;
// Error won't blow up, just be logged.
metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => {
req.profiler.done('incMapviewCount');
if (err) {
global.logger.log(`ERROR: failed to increment mapview count for user '${user}': ${err.message}`);
}
next();
});
};
}
function augmentLayergroupData () {
return function augmentLayergroupDataMiddleware (req, res, next) {
const layergroup = res.body;
// include in layergroup response the variables in serverMedata
// those variables are useful to send to the client information
// about how to reach this server or information about it
_.extend(layergroup, global.environment.serverMetadata);
next();
};
}
function setLastUpdatedTimeToLayergroup () {
return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) {
const { mapConfigProvider, analysesResults } = res.locals;
const layergroup = res.body;
mapConfigProvider.createAffectedTables((err, affectedTables) => {
if (err) {
return next(err);
}
if (!affectedTables) {
return next();
}
var lastUpdateTime = affectedTables.getLastUpdatedAt();
lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime;
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
layergroup.last_updated = new Date(lastUpdateTime).toISOString();
next();
});
};
}
function getLastUpdatedTime(analysesResults, lastUpdateTime) {
if (!Array.isArray(analysesResults)) {
return lastUpdateTime;
}
return analysesResults.reduce(function(lastUpdateTime, analysis) {
return analysis.getNodes().reduce(function(lastNodeUpdatedAtTime, node) {
var nodeUpdatedAtDate = node.getUpdatedAt();
var nodeUpdatedTimeAt = (nodeUpdatedAtDate && nodeUpdatedAtDate.getTime()) || 0;
return nodeUpdatedTimeAt > lastNodeUpdatedAtTime ? nodeUpdatedTimeAt : lastNodeUpdatedAtTime;
}, lastUpdateTime);
}, lastUpdateTime);
}
function setLayerStats (pgConnection, statsBackend) {
return function setLayerStatsMiddleware(req, res, next) {
const { user, mapConfig } = res.locals;
const layergroup = res.body;
pgConnection.getConnection(user, (err, connection) => {
if (err) {
return next(err);
}
statsBackend.getStats(mapConfig, connection, function(err, layersStats) {
if (err) {
return next(err);
}
if (layersStats.length > 0) {
layergroup.metadata.layers.forEach(function (layer, index) {
layer.meta.stats = layersStats[index];
});
}
next();
});
});
};
}
function setLayergroupIdHeader (templateMaps, useTemplateHash) {
return function setLayergroupIdHeaderMiddleware (req, res, next) {
const { user, template } = res.locals;
const layergroup = res.body;
if (useTemplateHash) {
var templateHash = templateMaps.fingerPrint(template).substring(0, 8);
layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`;
}
res.set('X-Layergroup-Id', layergroup.layergroupid);
next();
};
}
function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) {
return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) {
const { user, mapConfig } = res.locals;
const layergroup = res.body;
layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj());
next();
};
}
function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) {
return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) {
const { user, analysesResults = [] } = res.locals;
const layergroup = res.body;
layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery);
next();
};
}
function setTurboCartoMetadataToLayergroup (layergroupMetadata) {
return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) {
const { mapConfig, context } = res.locals;
const layergroup = res.body;
layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context);
next();
};
}
function setAggregationMetadataToLayergroup (layergroupMetadata) {
return function setAggregationMetadataToLayergroupMiddleware (req, res, next) {
const { mapConfig, context } = res.locals;
const layergroup = res.body;
layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context);
next();
};
}
function setTilejsonMetadataToLayergroup (layergroupMetadata) {
return function augmentLayergroupTilejsonMiddleware (req, res, next) {
const { user, mapConfig } = res.locals;
const layergroup = res.body;
layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig);
next();
};
}
function augmentError (options) {
const { addContext = false, label = 'MAPS CONTROLLER' } = options;
return function augmentErrorMiddleware (err, req, res, next) {
req.profiler.done('error');
const { mapConfig } = res.locals;
if (addContext) {
err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err;
}
err.label = label;
next(err);
};
}
function populateError(err, mapConfig) {
var error = new Error(err.message);
error.http_status = err.http_status;
if (!err.http_status && err.message.indexOf('column "the_geom_webmercator" does not exist') >= 0) {
error.http_status = 400;
}
error.type = 'layer';
error.subtype = err.message.indexOf('Postgis Plugin') >= 0 ? 'query' : undefined;
error.layer = {
id: mapConfig.getLayerId(err.layerIndex),
index: err.layerIndex,
type: mapConfig.layerType(err.layerIndex)
};
return error;
}

14
lib/cartodb/middleware/cors.js
View File

@ -1,14 +0,0 @@
module.exports = function cors (extraHeaders) {
return function corsMiddleware (req, res, next) {
let baseHeaders = "X-Requested-With, X-Prototype-Version, X-CSRF-Token";
if(extraHeaders) {
baseHeaders += ", " + extraHeaders;
}
res.set("Access-Control-Allow-Origin", "*");
res.set("Access-Control-Allow-Headers", baseHeaders);
next();
};
};

5
lib/cartodb/models/dataview/base.js
View File

@ -52,6 +52,11 @@ module.exports = class BaseDataview {
if (err) {
return callback(err);
}
if (!result || !result.rows || !result.rows.length) {
return callback(new Error('The column type could not be determined'));
}
const pgType = result.rows[0].pg_typeof;
callback(null, getPGTypeName(pgType));
}, readOnlyTransaction);

7
lib/cartodb/models/mapconfig/adapter/aggregation-mapconfig-adapter.js
View File

@ -101,6 +101,12 @@ module.exports = class AggregationMapConfigAdapter {
aggregationSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, aggregationSql);
}
if (!layer.options.sql_raw) {
// if sql_wrap is present, the original query should already be
// in sql_raw (with sql being the wrapped query);
// otherwise we keep the now the original query in sql_raw
layer.options.sql_raw = layer.options.sql;
}
layer.options.sql = aggregationSql;
mapConfig.getLayerAggregationColumns(index, (err, columns) => {
@ -159,6 +165,7 @@ module.exports = class AggregationMapConfigAdapter {
}
_getAggregationMetadata (mapConfig, layer, adapted) {
// also: pre-aggr query, columns, ...
if (!adapted) {
return { png: false, mvt: false };
}

10
lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter.js
View File

@ -2,9 +2,9 @@ var step = require('step');
var queue = require('queue-async');
var _ = require('underscore');
function MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi) {
this.overviewsMetadataApi = overviewsMetadataApi;
this.filterStatsApi = filterStatsApi;
function MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend) {
this.overviewsMetadataBackend = overviewsMetadataBackend;
this.filterStatsBackend = filterStatsBackend;
}
module.exports = MapConfigOverviewsAdapter;
@ -25,7 +25,7 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConf
if ( layer.type !== 'mapnik' && layer.type !== 'cartodb' ) {
return done(null, layer);
}
self.overviewsMetadataApi.getOverviewsMetadata(user, layer.options.sql, function(err, metadata){
self.overviewsMetadataBackend.getOverviewsMetadata(user, layer.options.sql, function(err, metadata){
if (err) {
done(err, layer);
} else {
@ -53,7 +53,7 @@ MapConfigOverviewsAdapter.prototype.getMapConfig = function(user, requestMapConf
function collectStatsData(err, filters, unfiltered_query) {
var next_step = this;
if ( filters ) {
self.filterStatsApi.getFilterStats(
self.filterStatsBackend.getFilterStats(
user,
unfiltered_query, filters,
function(err, stats) {

4
lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js
View File

@ -117,7 +117,9 @@ TurboCartoAdapter.prototype._parseCartoCss = function (username, params, layer,
var layerSql = layer.options.sql;
var layerRawSql = layer.options.sql_raw;
if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql) {
if (SubstitutionTokens.hasTokens(layerSql) && layerRawSql && layer.options.sql_wrap) {
// For wrapped queries we'll derive the tokens from the data extent
// instead of the whole Earth/root tile.
var self = this;
var tokensQuery = tokensQueryTpl({_sql: layerRawSql});
return pg.query(tokensQuery, function(err, resultSet) {

15
lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js
View File

@ -7,16 +7,23 @@ const QueryTables = require('cartodb-query-tables');
/**
* @param {MapConfig} mapConfig
* @param {String} user
* @param {UserLimitsApi} userLimitsApi
* @param {UserLimitsBackend} userLimitsBackend
* @param {Object} params
* @constructor
* @type {CreateLayergroupMapConfigProvider}
*/
function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, pgConnection, affectedTablesCache, params) {
function CreateLayergroupMapConfigProvider(
mapConfig,
user,
userLimitsBackend,
pgConnection,
affectedTablesCache,
params
) {
this.mapConfig = mapConfig;
this.user = user;
this.userLimitsApi = userLimitsApi;
this.userLimitsBackend = userLimitsBackend;
this.pgConnection = pgConnection;
this.affectedTablesCache = affectedTablesCache;
this.params = params;
@ -36,7 +43,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) {
step(
function prepareContextLimits() {
self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
self.userLimitsBackend.getRenderLimits(self.user, self.params.api_key, this);
},
function handleRenderLimits(err, renderLimits) {
assert.ifError(err);

8
lib/cartodb/models/mapconfig/provider/map-store-provider.js
View File

@ -7,15 +7,15 @@ const QueryTables = require('cartodb-query-tables');
/**
* @param {MapStore} mapStore
* @param {String} user
* @param {UserLimitsApi} userLimitsApi
* @param {UserLimitsBackend} userLimitsBackend
* @param {Object} params
* @constructor
* @type {MapStoreMapConfigProvider}
*/
function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, pgConnection, affectedTablesCache, params) {
function MapStoreMapConfigProvider(mapStore, user, userLimitsBackend, pgConnection, affectedTablesCache, params) {
this.mapStore = mapStore;
this.user = user;
this.userLimitsApi = userLimitsApi;
this.userLimitsBackend = userLimitsBackend;
this.pgConnection = pgConnection;
this.affectedTablesCache = affectedTablesCache;
this.token = params.token;
@ -38,7 +38,7 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
step(
function prepareContextLimits() {
self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
self.userLimitsBackend.getRenderLimits(self.user, self.params.api_key, this);
},
function handleRenderLimits(err, renderLimits) {
assert.ifError(err);

6
lib/cartodb/models/mapconfig/provider/named-map-provider.js
View File

@ -15,7 +15,7 @@ function NamedMapMapConfigProvider(
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
userLimitsBackend,
mapConfigAdapter,
affectedTablesCache,
owner,
@ -27,7 +27,7 @@ function NamedMapMapConfigProvider(
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.metadataBackend = metadataBackend;
this.userLimitsApi = userLimitsApi;
this.userLimitsBackend = userLimitsBackend;
this.mapConfigAdapter = mapConfigAdapter;
this.owner = owner;
@ -125,7 +125,7 @@ NamedMapMapConfigProvider.prototype.getMapConfig = function(callback) {
function prepareContextLimits(err, _mapConfig) {
assert.ifError(err);
mapConfig = _mapConfig;
self.userLimitsApi.getRenderLimits(self.owner, self.params.api_key, this);
self.userLimitsBackend.getRenderLimits(self.owner, self.params.api_key, this);
},
function cacheAndReturnMapConfig(err, renderLimits) {
self.err = err;

10
lib/cartodb/controllers/server_info.js → lib/cartodb/server-info-controller.js
View File

@ -1,4 +1,4 @@
var HealthCheck = require('../monitoring/health_check');
var HealthCheck = require('./monitoring/health_check');
var WELCOME_MSG = "This is the CartoDB Maps API, " +
"see the documentation at http://docs.cartodb.com/cartodb-platform/maps-api.html";
@ -12,10 +12,10 @@ function ServerInfoController(versions) {
module.exports = ServerInfoController;
ServerInfoController.prototype.register = function(app) {
app.get('/health', this.health.bind(this));
app.get('/', this.welcome.bind(this));
app.get('/version', this.version.bind(this));
ServerInfoController.prototype.register = function(monitorRouter) {
monitorRouter.get('/health', this.health.bind(this));
monitorRouter.get('/', this.welcome.bind(this));
monitorRouter.get('/version', this.version.bind(this));
};
ServerInfoController.prototype.welcome = function(req, res) {

411
lib/cartodb/server.js
View File

@ -1,273 +1,39 @@
var express = require('express');
var bodyParser = require('body-parser');
var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var _ = require('underscore');
const _ = require('underscore');
const express = require('express');
const windshaft = require('windshaft');
const { mapnik } = windshaft;
var controller = require('./controllers');
const jsonReplacer = require('./utils/json-replacer');
var SurrogateKeysCache = require('./cache/surrogate_keys_cache');
var NamedMapsCacheEntry = require('./cache/model/named_maps_entry');
var VarnishHttpCacheBackend = require('./cache/backend/varnish_http');
var FastlyCacheBackend = require('./cache/backend/fastly');
const ApiRouter = require('./api/api-router');
const ServerInfoController = require('./server-info-controller');
var StatsClient = require('./stats/client');
const stats = require('./middleware/stats');
const StatsClient = require('./stats/client');
var RendererStatsReporter = require('./stats/reporter/renderer');
module.exports = function createServer (serverOptions) {
validateOptions(serverOptions);
var windshaft = require('windshaft');
var mapnik = windshaft.mapnik;
var TemplateMaps = require('./backends/template_maps.js');
var OverviewsMetadataApi = require('./api/overviews_metadata_api');
var FilterStatsApi = require('./api/filter_stats_api');
var UserLimitsApi = require('./api/user_limits_api');
var AuthApi = require('./api/auth_api');
var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
var NamedMapProviderCache = require('./cache/named_map_provider_cache');
var PgQueryRunner = require('./backends/pg_query_runner');
var PgConnection = require('./backends/pg_connection');
var AnalysisBackend = require('./backends/analysis');
var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter');
var MapConfigBufferSizeAdapter = require('./models/mapconfig/adapter/mapconfig-buffer-size-adapter');
var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter');
var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter');
var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter');
var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter');
var AggregationMapConfigAdapter = require('./models/mapconfig/adapter/aggregation-mapconfig-adapter');
var MapConfigAdapter = require('./models/mapconfig/adapter');
var StatsBackend = require('./backends/stats');
const lzmaMiddleware = require('./middleware/lzma');
const errorMiddleware = require('./middleware/error-middleware');
module.exports = function(serverOptions) {
// Make stats client globally accessible
global.statsClient = StatsClient.getInstance(serverOptions.statsd);
var redisPool = new RedisPool(_.defaults(global.environment.redis, {
name: 'windshaft-server',
unwatchOnRelease: false,
noReadyCheck: true
}));
redisPool.on('status', function(status) {
var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.';
global.statsClient.gauge(keyPrefix + 'count', status.count);
global.statsClient.gauge(keyPrefix + 'unused', status.unused);
global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
});
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
var filterStatsApi = new FilterStatsApi(pgQueryRunner);
var userLimitsApi = new UserLimitsApi(metadataBackend, {
limits: {
cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
render: serverOptions.renderer.mapnik.limits.render || 0,
rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled
}
});
var templateMaps = new TemplateMaps(redisPool, {
max_user_templates: global.environment.maxUserTemplates
});
var surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends(serverOptions));
function invalidateNamedMap (owner, templateName) {
var startTime = Date.now();
surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) {
var logMessage = JSON.stringify({
username: owner,
type: 'named_map_invalidation',
elapsed: Date.now() - startTime,
error: !!err ? JSON.stringify(err.message) : undefined
});
if (err) {
global.logger.warn(logMessage);
} else {
global.logger.info(logMessage);
}
});
}
['update', 'delete'].forEach(function(eventType) {
templateMaps.on(eventType, invalidateNamedMap);
});
serverOptions.grainstore.mapnik_version = mapnikVersion(serverOptions);
validateOptions(serverOptions);
bootstrapFonts(serverOptions);
// initialize express server
var app = bootstrap(serverOptions);
// Extend windshaft with all the elements of the options object
_.extend(app, serverOptions);
const app = express();
var mapStore = new windshaft.storage.MapStore({
pool: redisPool,
expire_time: serverOptions.grainstore.default_layergroup_ttl
});
app.enable('jsonp callback');
app.disable('x-powered-by');
app.disable('etag');
app.set('json replacer', jsonReplacer());
var onTileErrorStrategy;
if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) {
const apiRouter = new ApiRouter({ serverOptions, environmentOptions: global.environment });
apiRouter.register(app);
function isRenderTimeoutError (err) {
return err.message === 'Render timed out';
}
const versions = getAndValidateVersions(serverOptions);
function isDatasourceTimeoutError (err) {
return err.message && err.message.match(/canceling statement due to statement timeout/i);
}
function isTimeoutError (err) {
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
}
function isRasterFormat (format) {
return format === 'png' || format === 'jpg';
}
if (isTimeoutError(err) && isRasterFormat(format)) {
return callback(null, timeoutErrorTile, {
'Content-Type': 'image/png',
}, {});
} else {
return callback(err, tile, headers, stats);
}
};
}
var rendererFactory = new windshaft.renderer.Factory({
onTileErrorStrategy: onTileErrorStrategy,
mapnik: {
redisPool: redisPool,
grainstore: serverOptions.grainstore,
mapnik: serverOptions.renderer.mapnik
},
http: serverOptions.renderer.http,
mvt: serverOptions.renderer.mvt
});
// initialize render cache
var rendererCacheOpts = _.defaults(serverOptions.renderCache || {}, {
ttl: 60000, // 60 seconds TTL by default
statsInterval: 60000 // reports stats every milliseconds defined here
});
var rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts);
var rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval);
rendererStatsReporter.start();
var attributesBackend = new windshaft.backend.Attributes();
var previewBackend = new windshaft.backend.Preview(rendererCache);
var tileBackend = new windshaft.backend.Tile(rendererCache);
var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
var statsBackend = new StatsBackend();
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
var mapConfigAdapter = new MapConfigAdapter(
new MapConfigNamedLayersAdapter(templateMaps, pgConnection),
new MapConfigBufferSizeAdapter(),
new SqlWrapMapConfigAdapter(),
new DataviewsWidgetsAdapter(),
new AnalysisMapConfigAdapter(analysisBackend),
new AggregationMapConfigAdapter(pgConnection),
new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi),
new TurboCartoAdapter()
);
var namedMapProviderCache = new NamedMapProviderCache(
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
layergroupAffectedTablesCache
);
['update', 'delete'].forEach(function(eventType) {
templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
});
var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps);
var TablesExtentApi = require('./api/tables_extent_api');
var tablesExtentApi = new TablesExtentApi(pgQueryRunner);
var versions = getAndValidateVersions(serverOptions);
/*******************************************************************************************************************
* Routing
******************************************************************************************************************/
new controller.Layergroup(
pgConnection,
mapStore,
tileBackend,
previewBackend,
attributesBackend,
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTablesCache,
analysisBackend,
authApi
).register(app);
new controller.Map(
pgConnection,
templateMaps,
mapBackend,
metadataBackend,
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTablesCache,
mapConfigAdapter,
statsBackend,
authApi
).register(app);
new controller.NamedMaps(
namedMapProviderCache,
tileBackend,
previewBackend,
surrogateKeysCache,
tablesExtentApi,
metadataBackend,
pgConnection,
authApi,
userLimitsApi
).register(app);
new controller.NamedMapsAdmin(authApi, templateMaps, userLimitsApi).register(app);
new controller.Analyses(pgConnection, authApi, userLimitsApi).register(app);
new controller.ServerInfo(versions).register(app);
/*******************************************************************************************************************
* END Routing
******************************************************************************************************************/
app.use(errorMiddleware());
const serverInfoController = new ServerInfoController(versions);
serverInfoController.register(app);
return app;
};
@ -278,6 +44,22 @@ function validateOptions(opts) {
}
}
function bootstrapFonts(opts) {
// Set carto renderer configuration for MMLStore
opts.grainstore.carto_env = opts.grainstore.carto_env || {};
var cenv = opts.grainstore.carto_env;
cenv.validation_data = cenv.validation_data || {};
if ( ! cenv.validation_data.fonts ) {
mapnik.register_system_fonts();
mapnik.register_default_fonts();
cenv.validation_data.fonts = _.keys(mapnik.fontFiles());
}
}
function mapnikVersion(opts) {
return opts.grainstore.mapnik_version || mapnik.versions.mapnik;
}
function getAndValidateVersions(options) {
// jshint undef:false
var warn = console.warn.bind(console);
@ -309,127 +91,10 @@ function getAndValidateVersions(options) {
});
// Be nice and warn if configured mapnik version is != installed mapnik version
if (mapnik.versions.mapnik !== options.grainstore.mapnik_version) {
warn('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' +
if (windshaft.mapnik.versions.mapnik !== options.grainstore.mapnik_version) {
warn('WARNING: detected mapnik version (' + windshaft.mapnik.versions.mapnik + ')' +
' != configured mapnik version (' + options.grainstore.mapnik_version + ')');
}
return installedDependenciesVersions;
}
function bootstrapFonts(opts) {
// Set carto renderer configuration for MMLStore
opts.grainstore.carto_env = opts.grainstore.carto_env || {};
var cenv = opts.grainstore.carto_env;
cenv.validation_data = cenv.validation_data || {};
if ( ! cenv.validation_data.fonts ) {
mapnik.register_system_fonts();
mapnik.register_default_fonts();
cenv.validation_data.fonts = _.keys(mapnik.fontFiles());
}
}
function bootstrap(opts) {
var app;
if (_.isObject(opts.https)) {
// use https if possible
app = express.createServer(opts.https);
} else {
// fall back to http by default
app = express();
}
app.enable('jsonp callback');
app.disable('x-powered-by');
app.disable('etag');
// Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705
// See: http://expressjs.com/en/4x/api.html#app.set
app.set('json replacer', function (key, value) {
if (value !== value) {
return 'NaN';
}
if (value === Infinity) {
return 'Infinity';
}
if (value === -Infinity) {
return '-Infinity';
}
return value;
});
app.use(bodyParser.json());
app.use(function bootstrap$prepareRequestResponse(req, res, next) {
if (global.environment && global.environment.api_hostname) {
res.set('X-Served-By-Host', global.environment.api_hostname);
}
next();
});
app.use(stats({
enabled: opts.useProfiler,
statsClient: global.statsClient
}));
app.use(lzmaMiddleware());
// temporary measure until we upgrade to newer version expressjs so we can check err.status
app.use(function(err, req, res, next) {
if (err) {
if (err.name === 'SyntaxError') {
res.status(400).json({ errors: [err.name + ': ' + err.message] });
} else {
next(err);
}
} else {
next();
}
});
setupLogger(app, opts);
return app;
}
function setupLogger(app, opts) {
if (global.log4js && opts.log_format) {
var loggerOpts = {
// Allowing for unbuffered logging is mainly
// used to avoid hanging during unit testing.
// TODO: provide an explicit teardown function instead,
// releasing any event handler or timer set by
// this component.
buffer: !opts.unbuffered_logging,
// optional log format
format: opts.log_format
};
app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level: 'info'})));
}
}
function surrogateKeysCacheBackends(serverOptions) {
var cacheBackends = [];
if (serverOptions.varnish_purge_enabled) {
cacheBackends.push(
new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port)
);
}
if (serverOptions.fastly &&
!!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) {
cacheBackends.push(
new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId)
);
}
return cacheBackends;
}
function mapnikVersion(opts) {
return opts.grainstore.mapnik_version || mapnik.versions.mapnik;
}

75
lib/cartodb/server_options.js
View File

@ -16,6 +16,15 @@ var rendererConfig = _.defaults(global.environment.renderer || {}, {
snapToGrid: false,
clipByBox2d: false,
metrics: false,
postgis: {
simplify_geometries: false,
extent: '-20037508.3,-20037508.3,20037508.3,20037508.3',
row_limit: 65535,
persist_connection: false,
use_overviews: true,
max_size: 500,
twkb_encoding: true
},
limits: {}
},
http: {}
@ -49,12 +58,14 @@ module.exports = {
port: global.environment.port,
host: global.environment.host
},
// FIXME: Remove it. This is no longer needed, paths are defined in routers
// This is for inline maps and table maps
base_url: global.environment.base_url_legacy || '/tiles/:table',
/// @deprecated with Windshaft-0.17.0
///base_url_notable: '/tiles',
// FIXME: Remove it. This is no longer needed, paths are defined in routers
// This is for Detached maps
//
// "maps" is the official, while
@ -62,14 +73,61 @@ module.exports = {
//
base_url_mapconfig: global.environment.base_url_detached || '(?:/maps|/tiles/layergroup)',
// FIXME: Remove it. This is no longer needed, paths are defined in routers
base_url_templated: global.environment.base_url_templated || '(?:/maps/named|/tiles/template)',
// Base URLs for the APIs
//
// See http://github.com/CartoDB/Windshaft-cartodb/wiki/Unified-Map-API
routes: global.environment.routes || {
v1: {
paths: [
'/api/v1',
'/user/:user/api/v1',
],
// Base url for the Detached Maps API
// "/api/v1/map" is the new API,
map: {
paths: [
'/map',
]
},
// Base url for the Templated Maps API
// "/api/v1/map/named" is the new API,
template: {
paths: [
'/map/named'
]
}
},
// For compatibility with versions up to 1.6.x
v0: {
paths: [
'/tiles'
],
// Base url for the Detached Maps API
// "/tiles/layergroup" is for compatibility with versions up to 1.6.x
map: {
paths: [
'/layergroup'
]
},
// Base url for the Templated Maps API
// "/tiles/template" is for compatibility with versions up to 1.6.x
template: {
paths: [
'/template'
]
}
}
},
grainstore: {
map: {
// TODO: allow to specify in configuration
srid: 3857
},
datasource: global.environment.postgres,
datasource: rendererConfig.mapnik.postgis || global.environment.postgres || {},
cachedir: global.environment.millstone.cache_basedir,
use_workers: rendererConfig.mapnik.useCartocssWorkers || false,
mapnik_version: global.environment.mapnik_version,
@ -82,18 +140,9 @@ module.exports = {
statsInterval: rendererConfig.statsInterval
},
renderer: {
mvt: rendererConfig.mvt,
mapnik: _.defaults(rendererConfig.mapnik, {
geojson: {
dbPoolParams: {
size: 16,
idleTimeout: 3000,
reapInterval: 1000
},
clipByBox2d: false
}
}),
torque: rendererConfig.torque,
mvt: Object.assign({ dbPoolParams: global.environment.postgres.pool }, rendererConfig.mvt),
mapnik: rendererConfig.mapnik,
torque: Object.assign({ dbPoolParams: global.environment.postgres.pool }, rendererConfig.torque),
http: rendererConfig.http
},

19
lib/cartodb/utils/json-replacer.js Normal file
View File

@ -0,0 +1,19 @@
module.exports = function jsonReplacerFactory () {
// Fix: https://github.com/CartoDB/Windshaft-cartodb/issues/705
// See: http://expressjs.com/en/4x/api.html#app.set
return function jsonReplacer (key, value) {
if (value !== value) {
return 'NaN';
}
if (value === Infinity) {
return 'Infinity';
}
if (value === -Infinity) {
return '-Infinity';
}
return value;
};
};

100
lib/cartodb/utils/query-utils.js
View File

@ -21,10 +21,15 @@ module.exports.extractTableNames = function extractTableNames(query) {
].join('');
};
module.exports.getQueryActualRowCount = function (query) {
return `select COUNT(*) AS rows FROM (${query}) AS __cdb_query`;
};
function getQueryRowEstimation(query) {
return 'select CDB_EstimateRowCount($windshaft$' + query + '$windshaft$) as rows';
}
module.exports.getQueryRowCount = getQueryRowEstimation;
module.exports.getQueryRowEstimation = getQueryRowEstimation;
module.exports.getAggregationMetadata = ctx => `
WITH
@ -71,3 +76,96 @@ module.exports.countNaNs = function countNaNs(ctx) {
`sum(CASE WHEN (${ctx.column} = 'NaN'::float) THEN 1 ELSE 0 END)`
}`;
};
module.exports.getQueryTopCategories = function(query, column, topN, includeNulls=false) {
const where = includeNulls ? '' : `WHERE ${column} IS NOT NULL`;
return `
SELECT ${column} AS category, COUNT(*) AS frequency
FROM (${query}) AS __cdb_query
${where}
GROUP BY ${column} ORDER BY 2 DESC
LIMIT ${topN}
`;
};
function columnSelector(columns) {
if (!columns) {
return '*';
}
if (typeof columns === 'string') {
return columns;
}
if (Array.isArray(columns)) {
return columns.map(name => `"${name}"`).join(', ');
}
throw new TypeError(`Bad argument type for columns: ${typeof columns}`);
}
module.exports.getQuerySample = function(query, sampleProb, limit = null, randomSeed = 0.5, columns = null) {
const singleTable = simpleQueryTable(query);
if (singleTable) {
return getTableSample(singleTable.table, columns || singleTable.columns, sampleProb, limit, randomSeed);
}
const limitClause = limit ? `LIMIT ${limit}` : '';
return `
WITH __cdb_rndseed AS (
SELECT setseed(${randomSeed})
)
SELECT ${columnSelector(columns)}
FROM (${query}) AS __cdb_query
WHERE random() < ${sampleProb}
${limitClause}
`;
};
function getTableSample(table, columns, sampleProb, limit = null, randomSeed = 0.5) {
const limitClause = limit ? `LIMIT ${limit}` : '';
sampleProb *= 100;
randomSeed *= Math.pow(2, 31) -1;
return `
SELECT ${columnSelector(columns)}
FROM ${table}
TABLESAMPLE BERNOULLI (${sampleProb}) REPEATABLE (${randomSeed}) ${limitClause}
`;
}
function simpleQueryTable(sql) {
const basicQuery =
/\s*SELECT\s+([\*a-z0-9_,\s]+?)\s+FROM\s+((\"[^"]+\"|[a-z0-9_]+)\.)?(\"[^"]+\"|[a-z0-9_]+)\s*;?\s*/i;
const unwrappedQuery = new RegExp("^"+basicQuery.source+"$", 'i');
// queries for named maps are wrapped like this:
var wrappedQuery = new RegExp(
"^\\s*SELECT\\s+\\*\\s+FROM\\s+\\(" +
basicQuery.source +
"\\)\\s+AS\\s+wrapped_query\\s+WHERE\\s+\\d+=1\\s*$",
'i'
);
let match = sql.match(unwrappedQuery);
if (!match) {
match = sql.match(wrappedQuery);
}
if (match) {
const columns = match[1];
const schema = match[3];
const table = match[4];
return { table: schema ? `${schema}.${table}` : table, columns };
}
return false;
}
module.exports.getQueryGeometryType = function(query, geometryColumn) {
return `
SELECT ST_GeometryType(${geometryColumn}) AS geom_type
FROM (${query}) AS __cdb_query
WHERE ${geometryColumn} IS NOT NULL
LIMIT 1
`;
};
module.exports.getQueryLimited = function(query, limit=0) {
return `
SELECT *
FROM (${query}) AS __cdb_query
LIMIT ${limit}
`;
};

54
package.json
View File

@ -24,41 +24,41 @@
"Simon Martin <simon@carto.com>"
],
"dependencies": {
"basic-auth": "^2.0.0",
"body-parser": "^1.18.2",
"camshaft": "0.61.8",
"cartodb-psql": "0.10.2",
"basic-auth": "2.0.0",
"body-parser": "1.18.3",
"camshaft": "0.61.10",
"cartodb-psql": "0.11.0",
"cartodb-query-tables": "0.3.0",
"cartodb-redis": "git://github.com/CartoDB/node-cartodb-redis.git#remove-auth-fallback",
"debug": "^3.1.0",
"dot": "~1.0.2",
"express": "~4.16.0",
"fastly-purge": "~1.0.1",
"glob": "^7.1.2",
"debug": "3.1.0",
"dot": "1.1.2",
"express": "4.16.3",
"fastly-purge": "1.0.1",
"glob": "7.1.2",
"log4js": "cartodb/log4js-node#cdb",
"lru-cache": "2.6.5",
"lzma": "~2.3.2",
"node-statsd": "~0.0.7",
"on-headers": "^1.0.1",
"queue-async": "~1.0.7",
"lru-cache": "4.1.3",
"lzma": "2.3.2",
"node-statsd": "0.1.1",
"on-headers": "1.0.1",
"queue-async": "1.1.0",
"redis-mpool": "0.5.0",
"request": "2.85.0",
"semver": "~5.3.0",
"step": "~0.0.6",
"step-profiler": "~0.3.0",
"request": "2.87.0",
"semver": "5.5.0",
"step": "1.0.0",
"step-profiler": "0.3.0",
"turbo-carto": "0.20.2",
"underscore": "~1.6.0",
"windshaft": "4.7.1",
"yargs": "~5.0.0"
"underscore": "1.6.0",
"windshaft": "4.8.1",
"yargs": "11.1.0"
},
"devDependencies": {
"istanbul": "~0.4.3",
"jshint": "~2.9.4",
"mocha": "~3.4.1",
"moment": "~2.18.1",
"nock": "~2.11.0",
"istanbul": "0.4.5",
"jshint": "2.9.5",
"mocha": "3.5.3",
"moment": "2.22.1",
"nock": "9.2.6",
"redis": "2.8.0",
"strftime": "~0.8.2"
"strftime": "0.10.0"
},
"scripts": {
"lint": "jshint lib test",

176
test/acceptance/aggregation.js
View File

@ -100,6 +100,18 @@ describe('aggregation', function () {
from generate_series(-3, 3) x
`;
const POINTS_SQL_PAIRS = `
-- Generate pairs of near points
select
x + 4 as cartodb_id,
st_setsrid(st_makepoint(Floor(x/2)*10 + x/1000.0, Floor(x/2)*10 + x/1000.0), 4326) as the_geom,
st_transform(
st_setsrid(st_makepoint(Floor(x/2)*10 + x/1000.0, Floor(x/2)*10 + x/1000.0),4326),
3857) as the_geom_webmercator,
x as value
from generate_series(-6, 6) x
`;
function createVectorMapConfig (layers = [
{
type: 'cartodb',
@ -130,6 +142,7 @@ describe('aggregation', function () {
before(function () {
serverOptions.renderer.mvt.usePostGIS = usePostGIS;
this.layerStatsConfig = global.environment.enabledFeatures.layerStats;
});
after(function (){
@ -138,6 +151,7 @@ describe('aggregation', function () {
afterEach(function (done) {
this.testClient.drain(done);
global.environment.enabledFeatures.layerStats = this.layerStatsConfig;
});
it('should return a layergroup indicating the mapconfig was aggregated', function (done) {
@ -2173,6 +2187,168 @@ describe('aggregation', function () {
});
});
['default', 'centroid', 'point-sample', 'point-grid'].forEach(placement => {
it(`default pre-aggregation stats are available with ${placement} aggregation`, function (done) {
global.environment.enabledFeatures.layerStats = true;
this.mapConfig = {
version: '1.6.0',
buffersize: { 'mvt': 0 },
layers: [
{
type: 'cartodb',
options: {
sql: POINTS_SQL_PAIRS,
resolution: 1,
aggregation: {
threshold: 1
}
}
}
]
};
if (placement !== 'default') {
this.mapConfig.layers[0].options.aggregation.placement = placement;
}
this.testClient = new TestClient(this.mapConfig);
this.testClient.getLayergroup((err, body) => {
if (err) {
return done(err);
}
assert.equal(typeof body.metadata, 'object');
assert.ok(Array.isArray(body.metadata.layers));
assert.ok(body.metadata.layers[0].meta.aggregation.mvt);
assert.ok(body.metadata.layers[0].meta.stats.estimatedFeatureCount > 0);
done();
});
});
it(`on demand post-aggregation stats are available with ${placement} aggregation`, function (done) {
global.environment.enabledFeatures.layerStats = true;
this.mapConfig = {
version: '1.6.0',
buffersize: { 'mvt': 0 },
layers: [
{
type: 'cartodb',
options: {
sql: POINTS_SQL_PAIRS,
resolution: 1,
aggregation: {
threshold: 1
},
metadata: {
aggrFeatureCount: 10
}
}
}
]
};
if (placement !== 'default') {
this.mapConfig.layers[0].options.aggregation.placement = placement;
}
this.testClient = new TestClient(this.mapConfig);
this.testClient.getLayergroup((err, body) => {
if (err) {
return done(err);
}
assert.equal(typeof body.metadata, 'object');
assert.ok(Array.isArray(body.metadata.layers));
assert.ok(body.metadata.layers[0].meta.aggregation.mvt);
assert.equal(body.metadata.layers[0].meta.stats.aggrFeatureCount, 13);
done();
});
});
it(`post-aggregation count adapts to zoom level with ${placement} aggregation`, function (done) {
global.environment.enabledFeatures.layerStats = true;
this.mapConfig = {
version: '1.6.0',
buffersize: { 'mvt': 0 },
layers: [
{
type: 'cartodb',
options: {
sql: POINTS_SQL_PAIRS,
resolution: 1,
aggregation: {
threshold: 1
},
metadata: {
aggrFeatureCount: 0
}
}
}
]
};
if (placement !== 'default') {
this.mapConfig.layers[0].options.aggregation.placement = placement;
}
this.testClient = new TestClient(this.mapConfig);
this.testClient.getLayergroup((err, body) => {
if (err) {
return done(err);
}
assert.equal(typeof body.metadata, 'object');
assert.ok(Array.isArray(body.metadata.layers));
assert.ok(body.metadata.layers[0].meta.aggregation.mvt);
assert.equal(body.metadata.layers[0].meta.stats.aggrFeatureCount, 9);
done();
});
});
it(`on-demand pre-aggregation stats are available with ${placement} aggregation`, function (done) {
global.environment.enabledFeatures.layerStats = true;
this.mapConfig = {
version: '1.6.0',
buffersize: { 'mvt': 0 },
layers: [
{
type: 'cartodb',
options: {
sql: POINTS_SQL_PAIRS,
resolution: 1,
aggregation: {
threshold: 1
},
metadata: {
featureCount: true
}
}
}
]
};
if (placement !== 'default') {
this.mapConfig.layers[0].options.aggregation.placement = placement;
}
this.testClient = new TestClient(this.mapConfig);
this.testClient.getLayergroup((err, body) => {
if (err) {
return done(err);
}
assert.equal(typeof body.metadata, 'object');
assert.ok(Array.isArray(body.metadata.layers));
assert.ok(body.metadata.layers[0].meta.aggregation.mvt);
assert.equal(body.metadata.layers[0].meta.stats.featureCount, 13);
done();
});
});
});
});
});

35
test/acceptance/analysis/named-maps.js
View File

@ -261,6 +261,41 @@ describe('named-maps analysis', function() {
);
});
it('should return and an error requesting unsupported image format', function(done) {
assert.response(
server,
{
url: '/api/v1/map/static/center/' + layergroupid + '/4/42/-3/320/240.gif',
method: 'GET',
encoding: 'binary',
headers: {
host: username
}
},
{
status: 400,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function(res, err) {
assert.ifError(err);
assert.deepEqual(
JSON.parse(res.body),
{
errors:['Unsupported image format \"gif\"'],
errors_with_context:[{
type: 'unknown',
message: 'Unsupported image format \"gif\"'
}]
}
);
done();
}
);
});
});
describe('auto-instantiation', function() {

128
test/acceptance/dataviews/error-cases.js
View File

@ -3,7 +3,9 @@ require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
describe('histogram-dataview', function() {
describe('dataview error cases', function() {
describe('generic errors', function() {
afterEach(function(done) {
if (this.testClient) {
@ -84,4 +86,128 @@ describe('histogram-dataview', function() {
done();
});
});
});
describe('pg_typeof', function() {
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
function createMapConfig(query) {
query = query || 'select * from populated_places_simple_reduced';
return {
version: '1.5.0',
layers: [
{
type: 'mapnik',
options: {
sql: query,
cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
cartocss_version: '2.0.1',
source: { id: "a0" }
}
}
],
analyses: [{
id: "a0",
type: "source",
params: {
query: query
}
}],
dataviews: {
aggregation_count_dataview: {
type: "aggregation",
source: { id: "a0" },
options: {
column: "adm0name",
aggregation: "count",
aggregationColumn: "adm0name"
}
}
}
};
}
it('should work without filters', function(done) {
this.testClient = new TestClient(createMapConfig());
this.testClient.getDataview('aggregation_count_dataview', { own_filter: 0 }, function(err) {
assert.ifError(err);
done();
});
});
it('should work with filters', function(done) {
var params = {
filters: {
dataviews: {aggregation_count_dataview: {accept: ['Canada']}}
}
};
this.testClient = new TestClient(createMapConfig());
this.testClient.getDataview('aggregation_count_dataview', params, function(err) {
assert.ifError(err);
done();
});
});
it('should return an error if the column used by dataview does not exist', function(done) {
const query = 'select cartodb_id, the_geom, the_geom_webmercator from populated_places_simple_reduced';
const params = {
response: {
status: 404,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
const expectedResponseBody = {
errors:['column "adm0name" does not exist'],
errors_with_context:[{
type:'unknown',
message:'column "adm0name" does not exist'
}]
};
this.testClient = new TestClient(createMapConfig(query));
this.testClient.getDataview('aggregation_count_dataview', params, function(err, result) {
assert.ifError(err);
assert.deepEqual(result, expectedResponseBody);
done();
});
});
it('should not fail if query does not return rows', function(done) {
const query = 'select * from populated_places_simple_reduced limit 0';
const expectedResponseBody = {
aggregation: 'count',
count: 0,
nulls: 0,
nans: 0,
infinities: 0,
min: 0,
max: 0,
categoriesCount: 0,
categories: [],
type: 'aggregation'
};
this.testClient = new TestClient(createMapConfig(query));
this.testClient.getDataview('aggregation_count_dataview', { own_filter: 0 }, function(err, result) {
assert.ifError(err);
assert.deepEqual(result, expectedResponseBody);
done();
});
});
});
});

3
test/acceptance/multilayer_server.js
View File

@ -234,7 +234,7 @@ describe('tests from old api translated to multilayer', function() {
var parsed = JSON.parse(res.body);
assert.ok(parsed.errors);
assert.equal(parsed.errors.length, 1);
assert.ok(parsed.errors[0].match(/^Unexpected token W/));
assert.ok(parsed.errors[0].match(/Unexpected token W/));
done();
}
@ -373,6 +373,7 @@ describe('tests from old api translated to multilayer', function() {
};
// reset internal cacheChannel cache
// FIXME: we need a better way to reset cache while running tests
server.layergroupAffectedTablesCache.cache.reset();
assert.response(server,

53
test/acceptance/named_maps_static_view.js
View File

@ -282,4 +282,57 @@ describe('named maps static view', function() {
});
});
it('should return an error requesting unsupported image format', function (done) {
var view = {
zoom: 4,
center: {
lng: 40,
lat: 20
}
};
templateMaps.addTemplate(username, createTemplate(view), function (err) {
if (err) {
return done(err);
}
var url = `/api/v1/map/static/named/${templateName}/640/480.gif`;
var requestOptions = {
url: url,
method: 'GET',
headers: {
host: username
},
encoding: 'binary'
};
var expectedResponse = {
status: 400,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
// this could be removed once named maps are invalidated, otherwise you hits the cache
var server = new CartodbWindshaft(serverOptions);
assert.response(server, requestOptions, expectedResponse, function (res, err) {
assert.ifError(err);
assert.deepEqual(
JSON.parse(res.body),
{
errors:['Unsupported image format \"gif\"'],
errors_with_context:[{
type: 'unknown',
message: 'Unsupported image format \"gif\"'
}]
}
);
done();
});
});
});
});

10
test/acceptance/ported/server_gettile.js
View File

@ -51,19 +51,19 @@ describe('server_gettile', function() {
var mapConfig = testClient.defaultTableMapConfig('test_table');
testClient.withLayergroup(mapConfig, function (err, requestTile, finish) {
requestTile(tileUrl, function (err, res) {
var xwc = res.headers['x-windshaft-cache'];
var xwc = parseInt(res.headers['x-windshaft-cache'], 10);
assert.ok(xwc);
assert.ok(xwc > 0);
lastXwc = xwc;
requestTile(tileUrl, function (err, res) {
var xwc = res.headers['x-windshaft-cache'];
var xwc = parseInt(res.headers['x-windshaft-cache'], 10);
assert.ok(xwc);
assert.ok(xwc > 0);
assert.ok(xwc >= lastXwc);
requestTile(tileUrl, { cache_buster: 'wadus' }, function (err, res) {
var xwc = res.headers['x-windshaft-cache'];
var xwc = parseInt(res.headers['x-windshaft-cache'], 10);
assert.ok(!xwc);
finish(done);
@ -103,7 +103,7 @@ describe('server_gettile', function() {
testClient.withLayergroup(mapConfig, validateLayergroup, function(err, requestTile, finish) {
requestTile(tileUrl, function(err, res) {
var xwc = res.headers['x-windshaft-cache'];
var xwc = parseInt(res.headers['x-windshaft-cache'], 10);
assert.ok(!xwc);
requestTile(tileUrl, function (err, res) {
@ -111,7 +111,7 @@ describe('server_gettile', function() {
res.headers.hasOwnProperty('x-windshaft-cache'),
"Did not hit renderer cache on second time"
);
assert.ok(res.headers['x-windshaft-cache'] >= 0);
assert.ok(parseInt(res.headers['x-windshaft-cache'], 10) >= 0);
assert.imageBufferIsSimilarToFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err) {

27
test/acceptance/ported/support/ported_server_options.js
View File

@ -7,8 +7,35 @@ var overviewsQueryRewriter = new OverviewsQueryRewriter({
});
module.exports = _.extend({}, serverOptions, {
// FIXME: Remove it. This is no longer needed, paths are defined in routers
base_url: '/database/:dbname/table/:table',
// FIXME: Remove it. This is no longer needed, paths are defined in routers
base_url_mapconfig: '/database/:dbname/layergroup',
routes: {
v0: {
paths: [
'/tiles',
'/database/:dbname'
],
// Base url for the Detached Maps API
// "/tiles/layergroup" is for compatibility with versions up to 1.6.x
map: {
paths: [
'/layergroup'
]
},
// Base url for the Templated Maps API
// "/tiles/template" is for compatibility with versions up to 1.6.x
template: {
paths: [
'/template'
]
}
}
},
grainstore: {
datasource: {
geometry_field: 'the_geom',

6
test/acceptance/rate-limit.test.js
View File

@ -5,8 +5,8 @@ const redis = require('redis');
const RedisPool = require('redis-mpool');
const cartodbRedis = require('cartodb-redis');
const TestClient = require('../support/test-client');
const UserLimitsApi = require('../../lib/cartodb/api/user_limits_api');
const rateLimitMiddleware = require('../../lib/cartodb/middleware/rate-limit');
const UserLimitsBackend = require('../../lib/cartodb/backends/user-limits');
const rateLimitMiddleware = require('../../lib/cartodb/api/middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware;
let userLimitsApi;
@ -218,7 +218,7 @@ describe('rate limit middleware', function () {
const redisPool = new RedisPool(global.environment.redis);
const metadataBackend = cartodbRedis({ pool: redisPool });
userLimitsApi = new UserLimitsApi(metadataBackend, {
userLimitsApi = new UserLimitsBackend(metadataBackend, {
limits: {
rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled
}

46
test/acceptance/regressions.js
View File

@ -1,7 +1,10 @@
require('../support/test_helper');
var assert = require('../support/assert');
const helper = require('../support/test_helper');
var TestClient = require('../support/test-client');
const LayergroupToken = require('../../lib/cartodb/models/layergroup-token');
const CartodbWindshaft = require(__dirname + '/../../lib/cartodb/server');
const serverOptions = require(__dirname + '/../../lib/cartodb/server_options');
describe('regressions', function() {
@ -38,6 +41,49 @@ describe('regressions', function() {
});
});
// See: https://github.com/CartoDB/Windshaft-cartodb/pull/956
it('"/user/localhost/api/v1/map" should create an anonymous map', function (done) {
const server = new CartodbWindshaft(serverOptions);
const layergroup = {
version: '1.7.0',
layers: [
{
type: 'mapnik',
options: {
sql: TestClient.SQL.ONE_POINT,
cartocss: TestClient.CARTOCSS.POINTS,
cartocss_version: '2.3.0'
}
}
]
};
const keysToDelete = {};
assert.response(server,
{
url: '/user/localhost/api/v1/map',
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
},
function(res, err) {
if (err) {
return done(err);
}
const body = JSON.parse(res.body);
assert.ok(body.layergroupid);
keysToDelete['map_cfg|' + LayergroupToken.parse(body.layergroupid).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
helper.deleteRedisKeys(keysToDelete, done);
}
);
});
describe('map instantiation', function () {
const apikeyToken = 'regular1';
const mapConfig = {

323
test/acceptance/stats/mapnik_stats_layergroup.js
View File

@ -49,7 +49,7 @@ describe('Create mapnik layergroup', function() {
sql: [
'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address',
' from test_table t1, test_table_2 t2',
' where t1.cartodb_id = t2.cartodb_id;'
' where t1.cartodb_id = t2.cartodb_id'
].join(''),
cartocss_version: cartocssVersion,
cartocss: cartocss
@ -74,6 +74,26 @@ describe('Create mapnik layergroup', function() {
}
};
var mapnikLayerNullCats = {
type: 'mapnik',
options: {
sql: `
WITH geom AS (
SELECT
'SRID=4326;POINT(0 0)'::geometry AS the_geom,
'SRID=3857;POINT(0 0)'::geometry AS the_geom_webmercator
)
SELECT 1 AS cartodb_id, 'A' As cat, geom.* FROM geom
UNION
SELECT 2 AS cartodb_id, 'B' As cat, geom.* FROM geom
UNION
SELECT 2 AS cartodb_id, NULL::text As cat, geom.* FROM geom
`,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
function mapnikBasicLayerId(index) {
return 'layer' + index;
}
@ -90,7 +110,7 @@ describe('Create mapnik layergroup', function() {
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
testClient.drain(done);
@ -107,7 +127,7 @@ describe('Create mapnik layergroup', function() {
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
@ -127,7 +147,7 @@ describe('Create mapnik layergroup', function() {
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
@ -147,7 +167,7 @@ describe('Create mapnik layergroup', function() {
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
testClient.drain(done);
@ -164,7 +184,7 @@ describe('Create mapnik layergroup', function() {
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
@ -183,7 +203,7 @@ describe('Create mapnik layergroup', function() {
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3);
assert.ok(!layergroup.metadata.layers[0].meta.stats[1]);
@ -204,7 +224,7 @@ describe('Create mapnik layergroup', function() {
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].type, 'mapnik');
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
@ -224,7 +244,7 @@ describe('Create mapnik layergroup', function() {
});
testClient.getLayergroup(function (err, layergroup) {
assert.ok(!err);
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[0].type, 'http');
assert.ok(!layergroup.metadata.layers[0].meta.cartocss);
@ -245,7 +265,7 @@ describe('Create mapnik layergroup', function() {
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
// we don't care about stats here as is an aliased column
assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount'));
@ -265,7 +285,7 @@ describe('Create mapnik layergroup', function() {
});
testClient.getLayergroup(function(err, layergroup) {
assert.ok(!err);
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[0].type, 'http');
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
@ -273,7 +293,288 @@ describe('Create mapnik layergroup', function() {
assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats'));
assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1));
assert.equal(layergroup.metadata.layers[2].type, 'http');
global.environment.enabledFeatures.layerStats = true;
testClient.drain(done);
});
});
function layerWithMetadata(layer, metadata) {
return Object.assign(layer, {
options: Object.assign(layer.options, { metadata })
});
}
it('should provide columns as optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
columns: true
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
const expectedColumns = {
cartodb_id: { type: 'number' },
the_geom: { type: 'geometry' },
the_geom_webmercator: { type: 'geometry' },
address: { type: 'string' }
};
assert.deepEqual(layergroup.metadata.layers[0].meta.stats.columns, expectedColumns);
testClient.drain(done);
});
});
// metadata categories are ordered only partially by descending frequency;
// this orders them completely to avoid ambiguities when comparing
function withSortedCategories(columns) {
function catOrder(a, b) {
if (a.frequency !== b.frequency) {
return b.frequency - a.frequency;
}
if (a.category < b.category) {
return -1;
}
if (a.category > b.category) {
return +1;
}
return 0;
}
let sorted = {};
Object.keys(columns).forEach(name => {
let data = columns[name];
if (data.hasOwnProperty('categories')) {
data = Object.assign(data, { categories: data.categories.sort(catOrder)});
}
sorted[name] = data;
});
return sorted;
}
it('should provide column stats as optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
columnStats: true
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
const expectedColumns = {
cartodb_id: {
type: 'number',
avg: 3,
max: 5,
min: 1,
sum: 15
},
the_geom: { type: 'geometry' },
the_geom_webmercator: { type: 'geometry' },
address: {
type: 'string',
categories: [
{
category: "Calle de la Palma 72, Madrid, Spain",
frequency: 1
},
{
category: "Calle de Pérez Galdós 9, Madrid, Spain",
frequency: 1
},
{
category: "Calle Divino Pastor 12, Madrid, Spain",
frequency: 1
},
{
category: "Manuel Fernández y González 8, Madrid, Spain",
frequency: 1
},
{
category: "Plaza Conde de Toreno 2, Madrid, Spain",
frequency: 1
}
]
}
};
assert.deepEqual(
withSortedCategories(layergroup.metadata.layers[0].meta.stats.columns),
withSortedCategories(expectedColumns)
);
testClient.drain(done);
});
});
it('should limit the number of categories as requested', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
columnStats: { topCategories: 2 }
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns;
assert.equal(columnsMetadata.address.categories.length, 2);
testClient.drain(done);
});
});
it('should include null categories if requested', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayerNullCats, {
columnStats: { includeNulls: true }
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns;
assert.equal(columnsMetadata.cat.categories.length, 3);
testClient.drain(done);
});
});
it('should not include null categories if not requested', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayerNullCats, {
columnStats: { includeNulls: false }
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns;
assert.equal(columnsMetadata.cat.categories.length, 2);
testClient.drain(done);
});
});
it('should provide row count as optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
featureCount: true
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5);
testClient.drain(done);
});
});
it('should provide geometry type as optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
geometryType: true
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point');
testClient.drain(done);
});
});
it('should provide a sample as optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
sample: { num_rows: 3 }
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0);
const expectedCols = [ 'cartodb_id', 'address', 'the_geom', 'the_geom_webmercator' ].sort();
assert.deepEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols);
testClient.drain(done);
});
});
it('can specify sample columns', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
sample: {
num_rows: 3,
include_columns: [ 'cartodb_id', 'address', 'the_geom' ]
}
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0);
const expectedCols = [ 'cartodb_id', 'address', 'the_geom' ].sort();
assert.deepEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols);
testClient.drain(done);
});
});
it('should only provide requested optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
geometryType: true,
featureCount: true
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point');
assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5);
assert.equal(layergroup.metadata.layers[0].meta.stats.sample, undefined);
assert.equal(layergroup.metadata.layers[0].meta.stats.columns, undefined);
testClient.drain(done);
});
});
});

2
test/acceptance/templates.js
View File

@ -29,6 +29,7 @@ describe('template_api', function() {
before(function () {
server = new CartodbWindshaft(serverOptions);
server.setMaxListeners(0);
// FIXME: we need a better way to reset cache while running tests
server.layergroupAffectedTablesCache.cache.reset();
});
@ -1060,6 +1061,7 @@ describe('template_api', function() {
assert.ok(cc);
assert.equal(cc, expectedCC);
// hack simulating restart...
// FIXME: we need a better way to reset cache while running tests
server.layergroupAffectedTablesCache.cache.reset(); // need to clean channel cache
var get_request = {
url: '/api/v1/map/' + layergroupid + ':cb1/0/0/0/1.json.torque?auth_token=valid1',

72
test/acceptance/user-database-timeout-limit.js
View File

@ -2,6 +2,7 @@ require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
const serverOptions = require('../../lib/cartodb/server_options');
const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`;
@ -438,6 +439,77 @@ describe('user database timeout limit', function () {
});
});
if (process.env.POSTGIS_VERSION === '2.4') {
describe('fetching vector tiles via PostGIS renderer', function() {
const usePostGIS = true;
const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
beforeEach(function (done) {
serverOptions.renderer.mvt.usePostGIS = usePostGIS;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup({ response: expectedResponse }, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function () {
serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('"mvt" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'mvt',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/x-protobuf'
}
},
cacheBuster: true
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
var tileJSON = tile.toJSON();
assert.equal(Array.isArray(tileJSON), true);
assert.equal(tileJSON.length, 2);
assert.equal(tileJSON[0].name, 'errorTileSquareLayer');
assert.equal(tileJSON[1].name, 'errorTileStripesLayer');
done();
});
});
});
});
}
});
});

33
test/acceptance/user-render-timeout-limit.js
View File

@ -139,7 +139,7 @@ describe('user render timeout limit', function () {
});
it('layergroup creation works but tile request fails due to render timeout', function (done) {
this.testClient.getTile(0, 0, 0, {}, (err, res, tile) => {
this.testClient.getTile(0, 0, 0, { cacheBuster: true }, (err, res, tile) => {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
@ -180,7 +180,8 @@ describe('user render timeout limit', function () {
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
},
cacheBuster: true
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
@ -201,24 +202,20 @@ describe('user render timeout limit', function () {
});
});
if (process.env.POSTGIS_VERSION === '2.4') {
describe('vector (PostGIS)', vector(true));
}
describe('vector (mapnik)', vector(false));
function vector(usePostGIS) {
describe('vector tile via mapnik renderer', function () {
const usePostGIS = false;
const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
return function () {
beforeEach(function (done) {
serverOptions.renderer.mvt.usePostGIS = usePostGIS;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(50, done);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
@ -234,7 +231,8 @@ describe('user render timeout limit', function () {
headers: {
'Content-Type': 'application/x-protobuf'
}
}
},
cacheBuster: true
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
@ -249,8 +247,7 @@ describe('user render timeout limit', function () {
done();
});
});
};
}
});
describe('interativity', function () {
beforeEach(function (done) {
@ -277,7 +274,8 @@ describe('user render timeout limit', function () {
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
},
cacheBuster: true
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
@ -365,7 +363,7 @@ describe('user render timeout limit', function () {
});
});
it('layergroup creation works and render tile fails', function (done) {
it('layergroup creation works and render static center tile fails', function (done) {
const params = {
zoom: 0,
lat: 0,
@ -378,7 +376,8 @@ describe('user render timeout limit', function () {
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
},
cacheBuster: true
};
this.testClient.getStaticCenter(params, function (err, res, timeoutError) {

10
test/integration/mapconfig_overviews_adapter.js
View File

@ -5,18 +5,18 @@ var RedisPool = require('redis-mpool');
var cartodbRedis = require('cartodb-redis');
var PgConnection = require(__dirname + '/../../lib/cartodb/backends/pg_connection');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
var FilterStatsApi = require('../../lib/cartodb/api/filter_stats_api');
var OverviewsMetadataBackend = require('../../lib/cartodb/backends/overviews-metadata');
var FilterStatsBackend = require('../../lib/cartodb/backends/filter-stats');
var MapConfigOverviewsAdapter = require('../../lib/cartodb/models/mapconfig/adapter/mapconfig-overviews-adapter');
var redisPool = new RedisPool(global.environment.redis);
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
var filterStatsApi = new FilterStatsApi(pgQueryRunner);
var overviewsMetadataBackend = new OverviewsMetadataBackend(pgQueryRunner);
var filterStatsBackend = new FilterStatsBackend(pgQueryRunner);
var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi);
var mapConfigOverviewsAdapter = new MapConfigOverviewsAdapter(overviewsMetadataBackend, filterStatsBackend);
describe('MapConfigOverviewsAdapter', function() {

12
test/integration/overviews-metadata-api.js
View File

@ -7,24 +7,24 @@ var cartodbRedis = require('cartodb-redis');
var PgConnection = require('../../lib/cartodb/backends/pg_connection');
var PgQueryRunner = require('../../lib/cartodb/backends/pg_query_runner');
var OverviewsMetadataApi = require('../../lib/cartodb/api/overviews_metadata_api');
var OverviewsMetadataBackend = require('../../lib/cartodb/backends/overviews-metadata');
describe('OverviewsMetadataApi', function() {
describe('OverviewsMetadataBackend', function() {
var overviewsMetadataApi;
var overviewsMetadataBackend;
before(function() {
var redisPool = new RedisPool(global.environment.redis);
var metadataBackend = cartodbRedis({pool: redisPool});
var pgConnection = new PgConnection(metadataBackend);
var pgQueryRunner = new PgQueryRunner(pgConnection);
overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
overviewsMetadataBackend = new OverviewsMetadataBackend(pgQueryRunner);
});
it('should return an empty relation for tables that have no overviews', function(done) {
var query = 'select * from test_table';
overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) {
overviewsMetadataBackend.getOverviewsMetadata('localhost', query, function(err, result) {
assert.ok(!err, err);
assert.deepEqual(result, {});
@ -35,7 +35,7 @@ describe('OverviewsMetadataApi', function() {
it('should return overviews metadata', function(done) {
var query = 'select * from test_table_overviews';
overviewsMetadataApi.getOverviewsMetadata('localhost', query, function(err, result) {
overviewsMetadataBackend.getOverviewsMetadata('localhost', query, function(err, result) {
assert.ok(!err, err);
assert.deepEqual(result, {

15
test/support/test-client.js
View File

@ -125,6 +125,14 @@ function resErr2errRes(callback) {
};
}
function layergroupidTemplate (layergroupId, params) {
const { token, signer, cacheBuster } = LayergroupToken.parse(layergroupId);
// {user}@{token}:{cache_buster}
// {token}:{cache_buster}
return `${signer ? signer + '@' : ''}${token}:${params.cacheBuster ? Date.now() : cacheBuster }`;
}
TestClient.prototype.getWidget = function(widgetName, params, callback) {
var self = this;
@ -726,7 +734,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
url = '/api/v1/map/' + layergroupId + '/';
url = `/api/v1/map/${layergroupidTemplate(layergroupId, params)}/`;
var layers = params.layers;
@ -769,7 +777,6 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
}
}, params.response);
var isPng = format.match(/png$/);
if (isPng) {
@ -954,7 +961,9 @@ TestClient.prototype.getStaticCenter = function (params, callback) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
url = `/api/v1/map/static/center/${layergroupId}/${zoom}/${lat}/${lng}/${width}/${height}.${format}`;
const layergroupid = layergroupidTemplate(layergroupId, params);
url = `/api/v1/map/static/center/${layergroupid}/${zoom}/${lat}/${lng}/${width}/${height}.${format}`;
if (self.apiKey) {
url += '?' + qs.stringify({api_key: self.apiKey});

2
test/unit/cartodb/error-middleware.test.js
View File

@ -1,7 +1,7 @@
require('../../support/test_helper.js');
var assert = require('assert');
var errorMiddleware = require('../../../lib/cartodb/middleware/error-middleware');
var errorMiddleware = require('../../../lib/cartodb/api/middlewares/error-middleware');
describe('error-middleware', function() {

2
test/unit/cartodb/error_messages.test.js
View File

@ -2,7 +2,7 @@ require('../../support/test_helper');
var assert = require('assert');
var errorMiddleware = require('../../../lib/cartodb/middleware/error-middleware');
var errorMiddleware = require('../../../lib/cartodb/api/middlewares/error-middleware');
describe('error messages clean up', function() {

Some files were not shown because too many files have changed in this diff Show More