cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into cartogl

Browse Source
This commit is contained in:
Javier Goizueta 2018-04-04 12:21:43 +02:00
commit fefb0b23af
62 changed files with 2186 additions and 1737 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -11,4 +11,3 @@ redis.pid
*.log
coverage/
.DS_Store
libredis_cell.so

4
NEWS.md
View File

@ -1,5 +1,8 @@
# Changelog
## 6.0.1
Released 2018-mm-dd
## 6.0.0
Released 2018-03-19
Backward incompatible changes:
@ -9,6 +12,7 @@ New features:
- Upgrades camshaft to 0.61.8
- Upgrades cartodb-redis to 1.0.0
- Rate limit feature (disabled by default)
- Fixes for tests with PG11
## 5.4.0
Released 2018-03-15

6
lib/cartodb/api/auth_api.js
View File

@ -25,7 +25,7 @@ module.exports = AuthApi;
// null if the request is not signed by anyone
// or will be a string cartodb username otherwise.
//
AuthApi.prototype.authorizedBySigner = function(res, callback) {
AuthApi.prototype.authorizedBySigner = function(req, res, callback) {
if ( ! res.locals.token || ! res.locals.signer ) {
return callback(null, false); // no signer requested
}
@ -33,7 +33,7 @@ AuthApi.prototype.authorizedBySigner = function(res, callback) {
var self = this;
var layergroup_id = res.locals.token;
var auth_token = res.locals.auth_token;
var auth_token = req.query.auth_token;
this.mapStore.load(layergroup_id, function(err, mapConfig) {
if (err) {
@ -180,7 +180,7 @@ AuthApi.prototype.authorize = function(req, res, callback) {
});
}
this.authorizedBySigner(res, (err, isAuthorizedBySigner) => {
this.authorizedBySigner(req, res, (err, isAuthorizedBySigner) => {
if (err) {
return callback(err);
}

28
lib/cartodb/backends/analysis-status.js
View File

@ -5,16 +5,14 @@ function AnalysisStatusBackend() {
module.exports = AnalysisStatusBackend;
AnalysisStatusBackend.prototype.getNodeStatus = function (params, callback) {
var nodeId = params.nodeId;
AnalysisStatusBackend.prototype.getNodeStatus = function (nodeId, dbParams, callback) {
var statusQuery = [
'SELECT node_id, status, updated_at, last_error_message as error_message',
'FROM cdb_analysis_catalog where node_id = \'' + nodeId + '\''
].join(' ');
var pg = new PSQL(dbParamsFromReqParams(params));
var pg = new PSQL(dbParams);
pg.query(statusQuery, function(err, result) {
if (err) {
return callback(err, result);
@ -36,23 +34,3 @@ AnalysisStatusBackend.prototype.getNodeStatus = function (params, callback) {
return callback(null, statusResponse);
}, true); // use read-only transaction
};
function dbParamsFromReqParams(params) {
var dbParams = {};
if ( params.dbuser ) {
dbParams.user = params.dbuser;
}
if ( params.dbpassword ) {
dbParams.pass = params.dbpassword;
}
if ( params.dbhost ) {
dbParams.host = params.dbhost;
}
if ( params.dbport ) {
dbParams.port = params.dbport;
}
if ( params.dbname ) {
dbParams.dbname = params.dbname;
}
return dbParams;
}

26
lib/cartodb/backends/dataview.js
View File

@ -1,13 +1,11 @@
var assert = require('assert');
var _ = require('underscore');
var PSQL = require('cartodb-psql');
var step = require('step');
var BBoxFilter = require('../models/filter/bbox');
var DataviewFactory = require('../models/dataview/factory');
var DataviewFactoryWithOverviews = require('../models/dataview/overviews/factory');
const dbParamsFromReqParams = require('../utils/database-params');
var OverviewsQueryRewriter = require('../utils/overviews_query_rewriter');
var overviewsQueryRewriter = new OverviewsQueryRewriter({
zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
@ -48,7 +46,7 @@ DataviewBackend.prototype.getDataview = function (mapConfigProvider, user, param
}
var pg = new PSQL(dbParamsFromReqParams(params));
var query = getDataviewQuery(dataviewDefinition, ownFilter, noFilters);
if (params.bbox) {
var bboxFilter = new BBoxFilter({column: 'the_geom_webmercator', srid: 3857}, {bbox: params.bbox});
@ -170,23 +168,3 @@ function getDataviewDefinition(mapConfig, dataviewName) {
var dataviews = mapConfig.dataviews || {};
return dataviews[dataviewName];
}
function dbParamsFromReqParams(params) {
var dbParams = {};
if ( params.dbuser ) {
dbParams.user = params.dbuser;
}
if ( params.dbpassword ) {
dbParams.pass = params.dbpassword;
}
if ( params.dbhost ) {
dbParams.host = params.dbhost;
}
if ( params.dbport ) {
dbParams.port = params.dbport;
}
if ( params.dbname ) {
dbParams.dbname = params.dbname;
}
return dbParams;
}

11
lib/cartodb/cache/named_map_provider_cache.js vendored
View File

@ -6,12 +6,20 @@ var queue = require('queue-async');
var LruCache = require("lru-cache");
function NamedMapProviderCache(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) {
function NamedMapProviderCache(
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache
) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.metadataBackend = metadataBackend;
this.userLimitsApi = userLimitsApi;
this.mapConfigAdapter = mapConfigAdapter;
this.affectedTablesCache = affectedTablesCache;
this.providerCache = new LruCache({ max: 2000 });
}
@ -30,6 +38,7 @@ NamedMapProviderCache.prototype.get = function(user, templateId, config, authTok
this.metadataBackend,
this.userLimitsApi,
this.mapConfigAdapter,
this.affectedTablesCache,
user,
templateId,
config,

75
lib/cartodb/controllers/analyses.js
View File

@ -1,28 +1,41 @@
var PSQL = require('cartodb-psql');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
const PSQL = require('cartodb-psql');
const cors = require('../middleware/cors');
const user = require('../middleware/user');
const cleanUpQueryParams = require('../middleware/clean-up-query-params');
const credentials = require('../middleware/credentials');
const authorize = require('../middleware/authorize');
const dbConnSetup = require('../middleware/db-conn-setup');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const cacheControlHeader = require('../middleware/cache-control-header');
const sendResponse = require('../middleware/send-response');
const dbParamsFromResLocals = require('../utils/database-params');
function AnalysesController(prepareContext, userLimitsApi) {
this.prepareContext = prepareContext;
function AnalysesController(pgConnection, authApi, userLimitsApi) {
this.pgConnection = pgConnection;
this.authApi = authApi;
this.userLimitsApi = userLimitsApi;
}
module.exports = AnalysesController;
AnalysesController.prototype.register = function (app) {
const { base_url_mapconfig: mapconfigBasePath } = app;
app.get(
`${app.base_url_mapconfig}/analyses/catalog`,
`${mapconfigBasePath}/analyses/catalog`,
cors(),
userMiddleware(),
user(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG),
this.prepareContext,
cleanUpQueryParams(),
createPGClient(),
getDataFromQuery({ queryTemplate: catalogQueryTpl, key: 'catalog' }),
getDataFromQuery({ queryTemplate: tablesQueryTpl, key: 'tables' }),
prepareResponse(),
setCacheControlHeader(),
cacheControlHeader({ ttl: 10, revalidate: true }),
sendResponse(),
unauthorizedError()
);
@ -30,7 +43,10 @@ AnalysesController.prototype.register = function (app) {
function createPGClient () {
return function createPGClientMiddleware (req, res, next) {
res.locals.pg = new PSQL(dbParamsFromReqParams(res.locals));
const dbParams = dbParamsFromResLocals(res.locals);
res.locals.pg = new PSQL(dbParams);
next();
};
}
@ -97,25 +113,6 @@ function prepareResponse () {
};
}
function setCacheControlHeader () {
return function setCacheControlHeaderMiddleware (req, res, next) {
res.set('Cache-Control', 'public,max-age=10,must-revalidate');
next();
};
}
function sendResponse () {
return function sendResponseMiddleware (req, res) {
res.status(200);
if (req.query && req.query.callback) {
res.jsonp(res.body);
} else {
res.json(res.body);
}
};
}
function unauthorizedError () {
return function unathorizedErrorMiddleware(err, req, res, next) {
if (err.message.match(/permission\sdenied/)) {
@ -149,23 +146,3 @@ var tablesQueryTpl = ctx => `
FROM analysis_tables
ORDER BY size DESC
`;
function dbParamsFromReqParams(params) {
var dbParams = {};
if ( params.dbuser ) {
dbParams.user = params.dbuser;
}
if ( params.dbpassword ) {
dbParams.pass = params.dbpassword;
}
if ( params.dbhost ) {
dbParams.host = params.dbhost;
}
if ( params.dbport ) {
dbParams.port = params.dbport;
}
if ( params.dbname ) {
dbParams.dbname = params.dbname;
}
return dbParams;
}

663
lib/cartodb/controllers/layergroup.js
View File

@ -1,663 +0,0 @@
const cors = require('../middleware/cors');
const userMiddleware = require('../middleware/user');
const allowQueryParams = require('../middleware/allow-query-params');
const vectorError = require('../middleware/vector-error');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const DataviewBackend = require('../backends/dataview');
const AnalysisStatusBackend = require('../backends/analysis-status');
const MapStoreMapConfigProvider = require('../models/mapconfig/provider/map-store-provider');
const QueryTables = require('cartodb-query-tables');
const SUPPORTED_FORMATS = {
grid_json: true,
json_torque: true,
torque_json: true,
png: true,
png32: true,
mvt: true
};
/**
* @param {prepareContext} prepareContext
* @param {PgConnection} pgConnection
* @param {MapStore} mapStore
* @param {TileBackend} tileBackend
* @param {PreviewBackend} previewBackend
* @param {AttributesBackend} attributesBackend
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @param {AnalysisBackend} analysisBackend
* @constructor
*/
function LayergroupController(
prepareContext,
pgConnection,
mapStore,
tileBackend,
previewBackend,
attributesBackend,
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTables,
analysisBackend
) {
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
this.attributesBackend = attributesBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTables = layergroupAffectedTables;
this.dataviewBackend = new DataviewBackend(analysisBackend);
this.analysisStatusBackend = new AnalysisStatusBackend();
this.prepareContext = prepareContext;
}
module.exports = LayergroupController;
LayergroupController.prototype.register = function(app) {
const { base_url_mapconfig: basePath } = app;
app.get(
`${basePath}/:token/:z/:x/:y@:scale_factor?x.:format`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getTile(this.tileBackend, 'map_tile'),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
incrementSuccessMetrics(global.statsClient),
sendResponse(),
incrementErrorMetrics(global.statsClient),
tileError(),
vectorError()
);
app.get(
`${basePath}/:token/:z/:x/:y.:format`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getTile(this.tileBackend, 'map_tile'),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
incrementSuccessMetrics(global.statsClient),
sendResponse(),
incrementErrorMetrics(global.statsClient),
tileError(),
vectorError()
);
app.get(
`${basePath}/:token/:layer/:z/:x/:y.(:format)`,
distinguishLayergroupFromStaticRoute(),
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getTile(this.tileBackend, 'maplayer_tile'),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
incrementSuccessMetrics(global.statsClient),
sendResponse(),
incrementErrorMetrics(global.statsClient),
tileError(),
vectorError()
);
app.get(
`${basePath}/:token/:layer/attributes/:fid`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getFeatureAttributes(this.attributesBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
const forcedFormat = 'png';
app.get(
`${basePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
allowQueryParams(['layer']),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat),
getPreviewImageByCenter(this.previewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
app.get(
`${basePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
allowQueryParams(['layer']),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi, forcedFormat),
getPreviewImageByBoundingBox(this.previewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
// Undocumented/non-supported API endpoint methods.
// Use at your own peril.
const allowedDataviewQueryParams = [
'filters', // json
'own_filter', // 0, 1
'no_filters', // 0, 1
'bbox', // w,s,e,n
'start', // number
'end', // number
'column_type', // string
'bins', // number
'aggregation', //string
'offset', // number
'q', // widgets search
'categories', // number
];
app.get(
`${basePath}/:token/dataview/:dataviewName`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
allowQueryParams(allowedDataviewQueryParams),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getDataview(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
app.get(
`${basePath}/:token/:layer/widget/:dataviewName`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
allowQueryParams(allowedDataviewQueryParams),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
getDataview(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
app.get(
`${basePath}/:token/dataview/:dataviewName/search`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
allowQueryParams(allowedDataviewQueryParams),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
dataviewSearch(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
app.get(
`${basePath}/:token/:layer/widget/:dataviewName/search`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
allowQueryParams(allowedDataviewQueryParams),
this.prepareContext,
createMapStoreMapConfigProvider(this.mapStore, this.userLimitsApi),
dataviewSearch(this.dataviewBackend),
setCacheControlHeader(),
setLastModifiedHeader(),
getAffectedTables(this.layergroupAffectedTables, this.pgConnection, this.mapStore),
setCacheChannelHeader(),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse()
);
app.get(
`${basePath}/:token/analysis/node/:nodeId`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS),
this.prepareContext,
analysisNodeStatus(this.analysisStatusBackend),
sendResponse()
);
};
function distinguishLayergroupFromStaticRoute () {
return function distinguishLayergroupFromStaticRouteMiddleware(req, res, next) {
if (req.params.token === 'static') {
return next('route');
}
next();
};
}
function analysisNodeStatus (analysisStatusBackend) {
return function analysisNodeStatusMiddleware(req, res, next) {
analysisStatusBackend.getNodeStatus(res.locals, (err, nodeStatus, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET NODE STATUS';
return next(err);
}
res.set({
'Cache-Control': 'public,max-age=5',
'Last-Modified': new Date().toUTCString()
});
res.body = nodeStatus;
next();
});
};
}
function getRequestParams(locals) {
const params = Object.assign({}, locals);
delete params.mapConfigProvider;
delete params.allowedQueryParams;
return params;
}
function createMapStoreMapConfigProvider (mapStore, userLimitsApi, forcedFormat = null) {
return function createMapStoreMapConfigProviderMiddleware (req, res, next) {
const { user } = res.locals;
const params = getRequestParams(res.locals);
if (forcedFormat) {
params.format = forcedFormat;
params.layer = params.layer || 'all';
}
res.locals.mapConfigProvider = new MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params);
next();
};
}
function getDataview (dataviewBackend) {
return function getDataviewMiddleware (req, res, next) {
const { user, mapConfigProvider } = res.locals;
const params = getRequestParams(res.locals);
dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET DATAVIEW';
return next(err);
}
res.body = dataview;
next();
});
};
}
function dataviewSearch (dataviewBackend) {
return function dataviewSearchMiddleware (req, res, next) {
const { user, dataviewName, mapConfigProvider } = res.locals;
const params = getRequestParams(res.locals);
dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET DATAVIEW SEARCH';
return next(err);
}
res.body = searchResult;
next();
});
};
}
function getFeatureAttributes (attributesBackend) {
return function getFeatureAttributesMiddleware (req, res, next) {
req.profiler.start('windshaft.maplayer_attribute');
const { mapConfigProvider } = res.locals;
const params = getRequestParams(res.locals);
attributesBackend.getFeatureAttributes(mapConfigProvider, params, false, (err, tile, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET ATTRIBUTES';
return next(err);
}
res.body = tile;
next();
});
};
}
function getStatusCode(tile, format){
return tile.length === 0 && format === 'mvt'? 204 : 200;
}
function parseFormat (format = '') {
const prettyFormat = format.replace('.', '_');
return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid';
}
function getTile (tileBackend, profileLabel = 'tile') {
return function getTileMiddleware (req, res, next) {
req.profiler.start(`windshaft.${profileLabel}`);
const { mapConfigProvider } = res.locals;
const params = getRequestParams(res.locals);
tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => {
req.profiler.add(stats);
if (err) {
return next(err);
}
if (headers) {
res.set(headers);
}
const formatStat = parseFormat(req.params.format);
res.statusCode = getStatusCode(tile, formatStat);
res.body = tile;
next();
});
};
}
function getPreviewImageByCenter (previewBackend) {
return function getPreviewImageByCenterMiddleware (req, res, next) {
const width = +req.params.width;
const height = +req.params.height;
const zoom = +req.params.z;
const center = {
lng: +req.params.lng,
lat: +req.params.lat
};
const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
const { mapConfigProvider: provider } = res.locals;
previewBackend.getImage(provider, format, width, height, zoom, center, (err, image, headers, stats = {}) => {
req.profiler.done(`render-${format}`);
req.profiler.add(stats);
if (err) {
err.label = 'STATIC_MAP';
return next(err);
}
if (headers) {
res.set(headers);
}
res.set('Content-Type', headers['Content-Type'] || `image/${format}`);
res.body = image;
next();
});
};
}
function getPreviewImageByBoundingBox (previewBackend) {
return function getPreviewImageByBoundingBoxMiddleware (req, res, next) {
const width = +req.params.width;
const height = +req.params.height;
const bounds = {
west: +req.params.west,
north: +req.params.north,
east: +req.params.east,
south: +req.params.south
};
const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
const { mapConfigProvider: provider } = res.locals;
previewBackend.getImage(provider, format, width, height, bounds, (err, image, headers, stats = {}) => {
req.profiler.done(`render-${format}`);
req.profiler.add(stats);
if (err) {
err.label = 'STATIC_MAP';
return next(err);
}
if (headers) {
res.set(headers);
}
res.set('Content-Type', headers['Content-Type'] || `image/${format}`);
res.body = image;
next();
});
};
}
function setLastModifiedHeader () {
return function setLastModifiedHeaderMiddleware (req, res, next) {
let { cache_buster: cacheBuster } = res.locals;
cacheBuster = parseInt(cacheBuster, 10);
const lastUpdated = res.locals.cache_buster ? new Date(cacheBuster) : new Date();
res.set('Last-Modified', lastUpdated.toUTCString());
next();
};
}
function setCacheControlHeader () {
return function setCacheControlHeaderMiddleware (req, res, next) {
res.set('Cache-Control', 'public,max-age=31536000');
next();
};
}
function getAffectedTables (layergroupAffectedTables, pgConnection, mapStore) {
return function getAffectedTablesMiddleware (req, res, next) {
const { user, dbname, token } = res.locals;
if (layergroupAffectedTables.hasAffectedTables(dbname, token)) {
res.locals.affectedTables = layergroupAffectedTables.get(dbname, token);
return next();
}
mapStore.load(token, (err, mapconfig) => {
if (err) {
global.logger.warn('ERROR generating cache channel:', err);
return next();
}
const queries = [];
mapconfig.getLayers().forEach(function(layer) {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(function(table) {
queries.push(`SELECT * FROM ${table} LIMIT 0`);
});
}
});
const sql = queries.length ? queries.join(';') : null;
if (!sql) {
global.logger.warn('ERROR generating cache channel:' +
' this request doesn\'t need an X-Cache-Channel generated');
return next();
}
pgConnection.getConnection(user, (err, connection) => {
if (err) {
global.logger.warn('ERROR generating cache channel:', err);
return next();
}
QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => {
req.profiler.done('getAffectedTablesFromQuery');
if (err) {
global.logger.warn('ERROR generating cache channel: ', err);
return next();
}
// feed affected tables cache so it can be reused from, for instance, map controller
layergroupAffectedTables.set(dbname, token, affectedTables);
res.locals.affectedTables = affectedTables;
next();
});
});
});
};
}
function setCacheChannelHeader () {
return function setCacheChannelHeaderMiddleware (req, res, next) {
const { affectedTables } = res.locals;
if (affectedTables) {
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
}
next();
};
}
function setSurrogateKeyHeader (surrogateKeysCache) {
return function setSurrogateKeyHeaderMiddleware (req, res, next) {
const { affectedTables } = res.locals;
if (affectedTables) {
surrogateKeysCache.tag(res, affectedTables);
}
next();
};
}
function incrementSuccessMetrics (statsClient) {
return function incrementSuccessMetricsMiddleware (req, res, next) {
const formatStat = parseFormat(req.params.format);
statsClient.increment('windshaft.tiles.success');
statsClient.increment(`windshaft.tiles.${formatStat}.success`);
next();
};
}
function sendResponse () {
return function sendResponseMiddleware (req, res) {
req.profiler.done('res');
res.status(res.statusCode || 200);
if (!Buffer.isBuffer(res.body) && typeof res.body === 'object') {
if (req.query && req.query.callback) {
res.jsonp(res.body);
} else {
res.json(res.body);
}
} else {
res.send(res.body);
}
};
}
function incrementErrorMetrics (statsClient) {
return function incrementErrorMetricsMiddleware (err, req, res, next) {
const formatStat = parseFormat(req.params.format);
statsClient.increment('windshaft.tiles.error');
statsClient.increment(`windshaft.tiles.${formatStat}.error`);
next(err);
};
}
function tileError () {
return function tileErrorMiddleware (err, req, res, next) {
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
let errMsg = err.message ? ( '' + err.message ) : ( '' + err );
// Rewrite mapnik parsing errors to start with layer number
const matches = errMsg.match("(.*) in style 'layer([0-9]+)'");
if (matches) {
errMsg = `style${matches[2]}: ${matches[1]}`;
}
err.message = errMsg;
err.label = 'TILE RENDER';
next(err);
};
}

75
lib/cartodb/controllers/layergroup/analysis.js Normal file
View File

@ -0,0 +1,75 @@
const cors = require('../../middleware/cors');
const user = require('../../middleware/user');
const layergroupToken = require('../../middleware/layergroup-token');
const cleanUpQueryParams = require('../../middleware/clean-up-query-params');
const credentials = require('../../middleware/credentials');
const dbConnSetup = require('../../middleware/db-conn-setup');
const authorize = require('../../middleware/authorize');
const rateLimit = require('../../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const sendResponse = require('../../middleware/send-response');
const dbParamsFromResLocals = require('../../utils/database-params');
module.exports = class AnalysisController {
constructor (
analysisStatusBackend,
pgConnection,
mapStore,
userLimitsApi,
layergroupAffectedTablesCache,
authApi,
surrogateKeysCache
) {
this.analysisStatusBackend = analysisStatusBackend;
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.authApi = authApi;
this.surrogateKeysCache = surrogateKeysCache;
}
register (app) {
const { base_url_mapconfig: mapConfigBasePath } = app;
app.get(
`${mapConfigBasePath}/:token/analysis/node/:nodeId`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS),
cleanUpQueryParams(),
analysisNodeStatus(this.analysisStatusBackend),
sendResponse()
);
}
};
function analysisNodeStatus (analysisStatusBackend) {
return function analysisNodeStatusMiddleware(req, res, next) {
const { nodeId } = req.params;
const dbParams = dbParamsFromResLocals(res.locals);
analysisStatusBackend.getNodeStatus(nodeId, dbParams, (err, nodeStatus, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET NODE STATUS';
return next(err);
}
res.set({
'Cache-Control': 'public,max-age=5',
'Last-Modified': new Date().toUTCString()
});
res.body = nodeStatus;
next();
});
};
}

93
lib/cartodb/controllers/layergroup/attributes.js Normal file
View File

@ -0,0 +1,93 @@
const cors = require('../../middleware/cors');
const user = require('../../middleware/user');
const layergroupToken = require('../../middleware/layergroup-token');
const cleanUpQueryParams = require('../../middleware/clean-up-query-params');
const credentials = require('../../middleware/credentials');
const dbConnSetup = require('../../middleware/db-conn-setup');
const authorize = require('../../middleware/authorize');
const rateLimit = require('../../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider');
const cacheControlHeader = require('../../middleware/cache-control-header');
const cacheChannelHeader = require('../../middleware/cache-channel-header');
const surrogateKeyHeader = require('../../middleware/surrogate-key-header');
const lastModifiedHeader = require('../../middleware/last-modified-header');
const sendResponse = require('../../middleware/send-response');
module.exports = class AttribitesController {
constructor (
attributesBackend,
pgConnection,
mapStore,
userLimitsApi,
layergroupAffectedTablesCache,
authApi,
surrogateKeysCache
) {
this.attributesBackend = attributesBackend;
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.authApi = authApi;
this.surrogateKeysCache = surrogateKeysCache;
}
register (app) {
const { base_url_mapconfig: mapConfigBasePath } = app;
app.get(
`${mapConfigBasePath}/:token/:layer/attributes/:fid`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES),
cleanUpQueryParams(),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getFeatureAttributes(this.attributesBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
}
};
function getFeatureAttributes (attributesBackend) {
return function getFeatureAttributesMiddleware (req, res, next) {
req.profiler.start('windshaft.maplayer_attribute');
const { mapConfigProvider } = res.locals;
const { token } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const { layer, fid } = req.params;
const params = {
token,
dbuser, dbname, dbpassword, dbhost, dbport,
layer, fid
};
attributesBackend.getFeatureAttributes(mapConfigProvider, params, false, (err, tile, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET ATTRIBUTES';
return next(err);
}
res.body = tile;
next();
});
};
}

199
lib/cartodb/controllers/layergroup/dataview.js Normal file
View File

@ -0,0 +1,199 @@
const cors = require('../../middleware/cors');
const user = require('../../middleware/user');
const layergroupToken = require('../../middleware/layergroup-token');
const cleanUpQueryParams = require('../../middleware/clean-up-query-params');
const credentials = require('../../middleware/credentials');
const dbConnSetup = require('../../middleware/db-conn-setup');
const authorize = require('../../middleware/authorize');
const rateLimit = require('../../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider');
const cacheControlHeader = require('../../middleware/cache-control-header');
const cacheChannelHeader = require('../../middleware/cache-channel-header');
const surrogateKeyHeader = require('../../middleware/surrogate-key-header');
const lastModifiedHeader = require('../../middleware/last-modified-header');
const sendResponse = require('../../middleware/send-response');
const ALLOWED_DATAVIEW_QUERY_PARAMS = [
'filters', // json
'own_filter', // 0, 1
'no_filters', // 0, 1
'bbox', // w,s,e,n
'start', // number
'end', // number
'column_type', // string
'bins', // number
'aggregation', //string
'offset', // number
'q', // widgets search
'categories', // number
];
module.exports = class DataviewController {
constructor (
dataviewBackend,
pgConnection,
mapStore,
userLimitsApi,
layergroupAffectedTablesCache,
authApi,
surrogateKeysCache
) {
this.dataviewBackend = dataviewBackend;
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.authApi = authApi;
this.surrogateKeysCache = surrogateKeysCache;
}
register (app) {
const { base_url_mapconfig: mapConfigBasePath } = app;
// Undocumented/non-supported API endpoint methods.
// Use at your own peril.
app.get(
`${mapConfigBasePath}/:token/dataview/:dataviewName`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getDataview(this.dataviewBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
app.get(
`${mapConfigBasePath}/:token/:layer/widget/:dataviewName`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getDataview(this.dataviewBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
app.get(
`${mapConfigBasePath}/:token/dataview/:dataviewName/search`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
dataviewSearch(this.dataviewBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
app.get(
`${mapConfigBasePath}/:token/:layer/widget/:dataviewName/search`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
cleanUpQueryParams(ALLOWED_DATAVIEW_QUERY_PARAMS),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
dataviewSearch(this.dataviewBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
}
};
function getDataview (dataviewBackend) {
return function getDataviewMiddleware (req, res, next) {
const { user, mapConfigProvider } = res.locals;
const { dataviewName } = req.params;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const params = Object.assign({ dataviewName, dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
dataviewBackend.getDataview(mapConfigProvider, user, params, (err, dataview, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET DATAVIEW';
return next(err);
}
res.body = dataview;
next();
});
};
}
function dataviewSearch (dataviewBackend) {
return function dataviewSearchMiddleware (req, res, next) {
const { user, mapConfigProvider } = res.locals;
const { dataviewName } = req.params;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
dataviewBackend.search(mapConfigProvider, user, dataviewName, params, (err, searchResult, stats = {}) => {
req.profiler.add(stats);
if (err) {
err.label = 'GET DATAVIEW SEARCH';
return next(err);
}
res.body = searchResult;
next();
});
};
}

114
lib/cartodb/controllers/layergroup/index.js Normal file
View File

@ -0,0 +1,114 @@
const DataviewBackend = require('../../backends/dataview');
const AnalysisStatusBackend = require('../../backends/analysis-status');
const TileController = require('./tile');
const AttributesController = require('./attributes');
const StaticController = require('./static');
const DataviewController = require('./dataview');
const AnalysisController = require('./analysis');
/**
* @param {prepareContext} prepareContext
* @param {PgConnection} pgConnection
* @param {MapStore} mapStore
* @param {TileBackend} tileBackend
* @param {PreviewBackend} previewBackend
* @param {AttributesBackend} attributesBackend
* @param {SurrogateKeysCache} surrogateKeysCache
* @param {UserLimitsApi} userLimitsApi
* @param {LayergroupAffectedTables} layergroupAffectedTables
* @param {AnalysisBackend} analysisBackend
* @constructor
*/
function LayergroupController(
pgConnection,
mapStore,
tileBackend,
previewBackend,
attributesBackend,
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTablesCache,
analysisBackend,
authApi
) {
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
this.attributesBackend = attributesBackend;
this.surrogateKeysCache = surrogateKeysCache;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.dataviewBackend = new DataviewBackend(analysisBackend);
this.analysisStatusBackend = new AnalysisStatusBackend();
this.authApi = authApi;
}
module.exports = LayergroupController;
LayergroupController.prototype.register = function(app) {
const tileController = new TileController(
this.tileBackend,
this.pgConnection,
this.mapStore,
this.userLimitsApi,
this.layergroupAffectedTablesCache,
this.authApi,
this.surrogateKeysCache
);
tileController.register(app);
const attributesController = new AttributesController(
this.attributesBackend,
this.pgConnection,
this.mapStore,
this.userLimitsApi,
this.layergroupAffectedTablesCache,
this.authApi,
this.surrogateKeysCache
);
attributesController.register(app);
const staticController = new StaticController(
this.previewBackend,
this.pgConnection,
this.mapStore,
this.userLimitsApi,
this.layergroupAffectedTablesCache,
this.authApi,
this.surrogateKeysCache
);
staticController.register(app);
const dataviewController = new DataviewController(
this.dataviewBackend,
this.pgConnection,
this.mapStore,
this.userLimitsApi,
this.layergroupAffectedTablesCache,
this.authApi,
this.surrogateKeysCache
);
dataviewController.register(app);
const analysisController = new AnalysisController(
this.analysisStatusBackend,
this.pgConnection,
this.mapStore,
this.userLimitsApi,
this.layergroupAffectedTablesCache,
this.authApi,
this.surrogateKeysCache
);
analysisController.register(app);
};

37
lib/cartodb/controllers/layergroup/middlewares/map-store-map-config-provider.js Normal file
View File

@ -0,0 +1,37 @@
const MapStoreMapConfigProvider = require('../../../models/mapconfig/provider/map-store-provider');
module.exports = function createMapStoreMapConfigProvider (
mapStore,
userLimitsApi,
pgConnection,
affectedTablesCache,
forcedFormat = null
) {
return function createMapStoreMapConfigProviderMiddleware (req, res, next) {
const { user, token, cache_buster, api_key } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const { layer, z, x, y, scale_factor, format } = req.params;
const params = {
user, token, cache_buster, api_key,
dbuser, dbname, dbpassword, dbhost, dbport,
layer, z, x, y, scale_factor, format
};
if (forcedFormat) {
params.format = forcedFormat;
params.layer = params.layer || 'all';
}
res.locals.mapConfigProvider = new MapStoreMapConfigProvider(
mapStore,
user,
userLimitsApi,
pgConnection,
affectedTablesCache,
params
);
next();
};
};

161
lib/cartodb/controllers/layergroup/static.js Normal file
View File

@ -0,0 +1,161 @@
const cors = require('../../middleware/cors');
const user = require('../../middleware/user');
const layergroupToken = require('../../middleware/layergroup-token');
const cleanUpQueryParams = require('../../middleware/clean-up-query-params');
const credentials = require('../../middleware/credentials');
const dbConnSetup = require('../../middleware/db-conn-setup');
const authorize = require('../../middleware/authorize');
const rateLimit = require('../../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider');
const cacheControlHeader = require('../../middleware/cache-control-header');
const cacheChannelHeader = require('../../middleware/cache-channel-header');
const surrogateKeyHeader = require('../../middleware/surrogate-key-header');
const lastModifiedHeader = require('../../middleware/last-modified-header');
const sendResponse = require('../../middleware/send-response');
module.exports = class StaticController {
constructor (
previewBackend,
pgConnection,
mapStore,
userLimitsApi,
layergroupAffectedTablesCache,
authApi,
surrogateKeysCache
) {
this.previewBackend = previewBackend;
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.authApi = authApi;
this.surrogateKeysCache = surrogateKeysCache;
}
register (app) {
const { base_url_mapconfig: mapConfigBasePath } = app;
const forcedFormat = 'png';
app.get(
`${mapConfigBasePath}/static/center/:token/:z/:lat/:lng/:width/:height.:format`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
cleanUpQueryParams(['layer']),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache,
forcedFormat
),
getPreviewImageByCenter(this.previewBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
app.get(
`${mapConfigBasePath}/static/bbox/:token/:west,:south,:east,:north/:width/:height.:format`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
cleanUpQueryParams(['layer']),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache,
forcedFormat
),
getPreviewImageByBoundingBox(this.previewBackend),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
}
};
function getPreviewImageByCenter (previewBackend) {
return function getPreviewImageByCenterMiddleware (req, res, next) {
const width = +req.params.width;
const height = +req.params.height;
const zoom = +req.params.z;
const center = {
lng: +req.params.lng,
lat: +req.params.lat
};
const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
const { mapConfigProvider: provider } = res.locals;
previewBackend.getImage(provider, format, width, height, zoom, center, (err, image, headers, stats = {}) => {
req.profiler.done(`render-${format}`);
req.profiler.add(stats);
if (err) {
err.label = 'STATIC_MAP';
return next(err);
}
if (headers) {
res.set(headers);
}
res.set('Content-Type', headers['Content-Type'] || `image/${format}`);
res.body = image;
next();
});
};
}
function getPreviewImageByBoundingBox (previewBackend) {
return function getPreviewImageByBoundingBoxMiddleware (req, res, next) {
const width = +req.params.width;
const height = +req.params.height;
const bounds = {
west: +req.params.west,
north: +req.params.north,
east: +req.params.east,
south: +req.params.south
};
const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
const { mapConfigProvider: provider } = res.locals;
previewBackend.getImage(provider, format, width, height, bounds, (err, image, headers, stats = {}) => {
req.profiler.done(`render-${format}`);
req.profiler.add(stats);
if (err) {
err.label = 'STATIC_MAP';
return next(err);
}
if (headers) {
res.set(headers);
}
res.set('Content-Type', headers['Content-Type'] || `image/${format}`);
res.body = image;
next();
});
};
}

230
lib/cartodb/controllers/layergroup/tile.js Normal file
View File

@ -0,0 +1,230 @@
const cors = require('../../middleware/cors');
const user = require('../../middleware/user');
const layergroupToken = require('../../middleware/layergroup-token');
const cleanUpQueryParams = require('../../middleware/clean-up-query-params');
const credentials = require('../../middleware/credentials');
const dbConnSetup = require('../../middleware/db-conn-setup');
const authorize = require('../../middleware/authorize');
const rateLimit = require('../../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const createMapStoreMapConfigProvider = require('./middlewares/map-store-map-config-provider');
const cacheControlHeader = require('../../middleware/cache-control-header');
const cacheChannelHeader = require('../../middleware/cache-channel-header');
const surrogateKeyHeader = require('../../middleware/surrogate-key-header');
const lastModifiedHeader = require('../../middleware/last-modified-header');
const sendResponse = require('../../middleware/send-response');
const vectorError = require('../../middleware/vector-error');
const SUPPORTED_FORMATS = {
grid_json: true,
json_torque: true,
torque_json: true,
png: true,
png32: true,
mvt: true
};
module.exports = class TileController {
constructor (
tileBackend,
pgConnection,
mapStore,
userLimitsApi,
layergroupAffectedTablesCache,
authApi,
surrogateKeysCache
) {
this.tileBackend = tileBackend;
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.userLimitsApi = userLimitsApi;
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
this.authApi = authApi;
this.surrogateKeysCache = surrogateKeysCache;
}
register (app) {
const { base_url_mapconfig: mapConfigBasePath } = app;
app.get(
`${mapConfigBasePath}/:token/:z/:x/:y@:scale_factor?x.:format`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
cleanUpQueryParams(),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getTile(this.tileBackend, 'map_tile'),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
incrementSuccessMetrics(global.statsClient),
incrementErrorMetrics(global.statsClient),
tileError(),
vectorError(),
sendResponse()
);
app.get(
`${mapConfigBasePath}/:token/:z/:x/:y.:format`,
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
cleanUpQueryParams(),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getTile(this.tileBackend, 'map_tile'),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
incrementSuccessMetrics(global.statsClient),
incrementErrorMetrics(global.statsClient),
tileError(),
vectorError(),
sendResponse()
);
app.get(
`${mapConfigBasePath}/:token/:layer/:z/:x/:y.(:format)`,
distinguishLayergroupFromStaticRoute(),
cors(),
user(),
layergroupToken(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
cleanUpQueryParams(),
createMapStoreMapConfigProvider(
this.mapStore,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTablesCache
),
getTile(this.tileBackend, 'maplayer_tile'),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
incrementSuccessMetrics(global.statsClient),
incrementErrorMetrics(global.statsClient),
tileError(),
vectorError(),
sendResponse()
);
}
};
function distinguishLayergroupFromStaticRoute () {
return function distinguishLayergroupFromStaticRouteMiddleware(req, res, next) {
if (req.params.token === 'static') {
return next('route');
}
next();
};
}
function parseFormat (format = '') {
const prettyFormat = format.replace('.', '_');
return SUPPORTED_FORMATS[prettyFormat] ? prettyFormat : 'invalid';
}
function getStatusCode(tile, format){
return tile.length === 0 && format === 'mvt' ? 204 : 200;
}
function getTile (tileBackend, profileLabel = 'tile') {
return function getTileMiddleware (req, res, next) {
req.profiler.start(`windshaft.${profileLabel}`);
const { mapConfigProvider } = res.locals;
const { token } = res.locals;
const { layer, z, x, y, format } = req.params;
const params = { token, layer, z, x, y, format };
tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats = {}) => {
req.profiler.add(stats);
if (err) {
return next(err);
}
if (headers) {
res.set(headers);
}
const formatStat = parseFormat(req.params.format);
res.statusCode = getStatusCode(tile, formatStat);
res.body = tile;
next();
});
};
}
function incrementSuccessMetrics (statsClient) {
return function incrementSuccessMetricsMiddleware (req, res, next) {
const formatStat = parseFormat(req.params.format);
statsClient.increment('windshaft.tiles.success');
statsClient.increment(`windshaft.tiles.${formatStat}.success`);
next();
};
}
function incrementErrorMetrics (statsClient) {
return function incrementErrorMetricsMiddleware (err, req, res, next) {
const formatStat = parseFormat(req.params.format);
statsClient.increment('windshaft.tiles.error');
statsClient.increment(`windshaft.tiles.${formatStat}.error`);
next(err);
};
}
function tileError () {
return function tileErrorMiddleware (err, req, res, next) {
if (err.message === 'Tile does not exist' && req.params.format === 'mvt') {
res.statusCode = 204;
return next();
}
// See https://github.com/Vizzuality/Windshaft-cartodb/issues/68
let errMsg = err.message ? ( '' + err.message ) : ( '' + err );
// Rewrite mapnik parsing errors to start with layer number
const matches = errMsg.match("(.*) in style 'layer([0-9]+)'");
if (matches) {
errMsg = `style${matches[2]}: ${matches[1]}`;
}
err.message = errMsg;
err.label = 'TILE RENDER';
next(err);
};
}

358
lib/cartodb/controllers/map.js
View File

@ -2,12 +2,18 @@ const _ = require('underscore');
const windshaft = require('windshaft');
const MapConfig = windshaft.model.MapConfig;
const Datasource = windshaft.model.Datasource;
const QueryTables = require('cartodb-query-tables');
const ResourceLocator = require('../models/resource-locator');
const cors = require('../middleware/cors');
const userMiddleware = require('../middleware/user');
const allowQueryParams = require('../middleware/allow-query-params');
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const user = require('../middleware/user');
const cleanUpQueryParams = require('../middleware/clean-up-query-params');
const credentials = require('../middleware/credentials');
const dbConnSetup = require('../middleware/db-conn-setup');
const authorize = require('../middleware/authorize');
const cacheControlHeader = require('../middleware/cache-control-header');
const cacheChannelHeader = require('../middleware/cache-channel-header');
const surrogateKeyHeader = require('../middleware/surrogate-key-header');
const lastModifiedHeader = require('../middleware/last-modified-header');
const sendResponse = require('../middleware/send-response');
const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
const LayergroupMetadata = require('../utils/layergroup-metadata');
@ -27,9 +33,18 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
* @param {StatsBackend} statsBackend
* @constructor
*/
function MapController(prepareContext, pgConnection, templateMaps, mapBackend, metadataBackend,
surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter,
statsBackend) {
function MapController (
pgConnection,
templateMaps,
mapBackend,
metadataBackend,
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTables,
mapConfigAdapter,
statsBackend,
authApi
) {
this.pgConnection = pgConnection;
this.templateMaps = templateMaps;
this.mapBackend = mapBackend;
@ -43,35 +58,37 @@ function MapController(prepareContext, pgConnection, templateMaps, mapBackend, m
this.layergroupMetadata = new LayergroupMetadata(resourceLocator);
this.statsBackend = statsBackend;
this.prepareContext = prepareContext;
this.authApi = authApi;
}
module.exports = MapController;
MapController.prototype.register = function(app) {
const { base_url_mapconfig, base_url_templated } = app;
const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = app;
app.get(
`${mapConfigBasePath}`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
app.post(
`${mapConfigBasePath}`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
const useTemplate = true;
app.get(
base_url_mapconfig,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
app.post(
base_url_mapconfig,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
app.get(
`${base_url_templated}/:template_id/jsonp`,
`${templateBasePath}/:template_id/jsonp`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate)
);
app.post(
`${base_url_templated}/:template_id`,
`${templateBasePath}/:template_id`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate)
);
app.options(
app.base_url_mapconfig,
cors('Content-Type')
);
app.options(`${mapConfigBasePath}`, cors('Content-Type'));
};
MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) {
@ -83,20 +100,22 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us
return [
cors(),
userMiddleware(),
user(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, endpointGroup),
allowQueryParams(['aggregation']),
this.prepareContext,
cleanUpQueryParams(['aggregation']),
initProfiler(isTemplateInstantiation),
checkJsonContentType(),
this.getCreateMapMiddlewares(useTemplate),
incrementMapViewCount(this.metadataBackend),
augmentLayergroupData(),
getAffectedTables(this.pgConnection, this.layergroupAffectedTables),
setCacheChannel(),
setLastModified(),
cacheControlHeader({ ttl: global.environment.varnish.layergroupTtl || 86400, revalidate: true }),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader({ now: true }),
setLastUpdatedTimeToLayergroup(),
setCacheControl(),
setLayerStats(this.pgConnection, this.statsBackend),
setLayergroupIdHeader(this.templateMaps ,useTemplateHash),
setDataviewsAndWidgetsUrlsToLayergroupMetadata(this.layergroupMetadata),
@ -104,7 +123,6 @@ MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, us
setTurboCartoMetadataToLayergroup(this.layergroupMetadata),
setAggregationMetadataToLayergroup(this.layergroupMetadata),
setTilejsonMetadataToLayergroup(this.layergroupMetadata),
setSurrogateKeyHeader(this.surrogateKeysCache),
sendResponse(),
augmentError({ label, addContext })
];
@ -119,16 +137,27 @@ MapController.prototype.getCreateMapMiddlewares = function (useTemplate) {
this.pgConnection,
this.metadataBackend,
this.userLimitsApi,
this.mapConfigAdapter
this.mapConfigAdapter,
this.layergroupAffectedTables
),
instantiateLayergroup(this.mapBackend, this.userLimitsApi)
instantiateLayergroup(
this.mapBackend,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTables
)
];
}
return [
checkCreateLayergroup(),
prepareAdapterMapConfig(this.mapConfigAdapter),
createLayergroup (this.mapBackend, this.userLimitsApi)
createLayergroup (
this.mapBackend,
this.userLimitsApi,
this.pgConnection,
this.layergroupAffectedTables
)
];
};
@ -181,7 +210,7 @@ function checkInstantiteLayergroup () {
function checkCreateLayergroup () {
return function checkCreateLayergroupMiddleware (req, res, next) {
if (req.method === 'GET') {
const { config } = res.locals;
const { config } = req.query;
if (!config) {
return next(new Error('layergroup GET needs a "config" parameter'));
@ -199,33 +228,45 @@ function checkCreateLayergroup () {
};
}
function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter) {
function getTemplate (
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache
) {
return function getTemplateMiddleware (req, res, next) {
const templateParams = req.body;
const { user } = res.locals;
const { user, dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const { template_id } = req.params;
const { auth_token } = req.query;
const mapconfigProvider = new NamedMapMapConfigProvider(
const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
const mapConfigProvider = new NamedMapMapConfigProvider(
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache,
user,
req.params.template_id,
template_id,
templateParams,
res.locals.auth_token,
res.locals
auth_token,
params
);
mapconfigProvider.getMapConfig((err, mapconfig, rendererParams) => {
mapConfigProvider.getMapConfig((err, mapConfig, rendererParams) => {
req.profiler.done('named.getMapConfig');
if (err) {
return next(err);
}
res.locals.mapconfig = mapconfig;
res.locals.mapConfig = mapConfig;
res.locals.rendererParams = rendererParams;
res.locals.mapconfigProvider = mapconfigProvider;
res.locals.mapConfigProvider = mapConfigProvider;
next();
});
@ -235,7 +276,10 @@ function getTemplate (templateMaps, pgConnection, metadataBackend, userLimitsApi
function prepareAdapterMapConfig (mapConfigAdapter) {
return function prepareAdapterMapConfigMiddleware(req, res, next) {
const requestMapConfig = req.body;
const { user, dbhost, dbport, dbname, dbuser, dbpassword, api_key } = res.locals;
const { user, api_key } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const params = Object.assign({ dbuser, dbname, dbpassword, dbhost, dbport }, req.query);
const context = {
analysisConfiguration: {
@ -254,7 +298,7 @@ function prepareAdapterMapConfig (mapConfigAdapter) {
}
};
mapConfigAdapter.getMapConfig(user, requestMapConfig, res.locals, context, (err, requestMapConfig) => {
mapConfigAdapter.getMapConfig(user, requestMapConfig, params, context, (err, requestMapConfig) => {
req.profiler.done('anonymous.getMapConfig');
if (err) {
return next(err);
@ -268,51 +312,75 @@ function prepareAdapterMapConfig (mapConfigAdapter) {
};
}
function createLayergroup (mapBackend, userLimitsApi) {
function createLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) {
return function createLayergroupMiddleware (req, res, next) {
const requestMapConfig = req.body;
const { context, user } = res.locals;
const datasource = context.datasource || Datasource.EmptyDatasource();
const mapconfig = new MapConfig(requestMapConfig, datasource);
const mapconfigProvider =
new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, res.locals);
res.locals.mapconfig = mapconfig;
const { context } = res.locals;
const { user, cache_buster, api_key } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const params = {
cache_buster, api_key,
dbuser, dbname, dbpassword, dbhost, dbport
};
const datasource = context.datasource || Datasource.EmptyDatasource();
const mapConfig = new MapConfig(requestMapConfig, datasource);
const mapConfigProvider = new CreateLayergroupMapConfigProvider(
mapConfig,
user,
userLimitsApi,
pgConnection,
affectedTablesCache,
params
);
res.locals.mapConfig = mapConfig;
res.locals.analysesResults = context.analysesResults;
mapBackend.createLayergroup(mapconfig, res.locals, mapconfigProvider, (err, layergroup) => {
const mapParams = { dbuser, dbname, dbpassword, dbhost, dbport };
mapBackend.createLayergroup(mapConfig, mapParams, mapConfigProvider, (err, layergroup) => {
req.profiler.done('createLayergroup');
if (err) {
return next(err);
}
res.locals.layergroup = layergroup;
res.body = layergroup;
res.locals.mapConfigProvider = mapConfigProvider;
next();
});
};
}
function instantiateLayergroup (mapBackend, userLimitsApi) {
function instantiateLayergroup (mapBackend, userLimitsApi, pgConnection, affectedTablesCache) {
return function instantiateLayergroupMiddleware (req, res, next) {
const { user, mapconfig, rendererParams } = res.locals;
const mapconfigProvider =
new CreateLayergroupMapConfigProvider(mapconfig, user, userLimitsApi, rendererParams);
const { user, mapConfig, rendererParams } = res.locals;
const mapConfigProvider = new CreateLayergroupMapConfigProvider(
mapConfig,
user,
userLimitsApi,
pgConnection,
affectedTablesCache,
rendererParams
);
mapBackend.createLayergroup(mapconfig, rendererParams, mapconfigProvider, (err, layergroup) => {
mapBackend.createLayergroup(mapConfig, rendererParams, mapConfigProvider, (err, layergroup) => {
req.profiler.done('createLayergroup');
if (err) {
return next(err);
}
res.locals.layergroup = layergroup;
res.body = layergroup;
const { mapconfigProvider } = res.locals;
const { mapConfigProvider } = res.locals;
res.locals.analysesResults = mapconfigProvider.analysesResults;
res.locals.template = mapconfigProvider.template;
res.locals.templateName = mapconfigProvider.getTemplateName();
res.locals.context = mapconfigProvider.context;
res.locals.analysesResults = mapConfigProvider.analysesResults;
res.locals.template = mapConfigProvider.template;
res.locals.context = mapConfigProvider.context;
next();
});
@ -321,10 +389,10 @@ function instantiateLayergroup (mapBackend, userLimitsApi) {
function incrementMapViewCount (metadataBackend) {
return function incrementMapViewCountMiddleware(req, res, next) {
const { mapconfig, user } = res.locals;
const { mapConfig, user } = res.locals;
// Error won't blow up, just be logged.
metadataBackend.incMapviewCount(user, mapconfig.obj().stat_tag, (err) => {
metadataBackend.incMapviewCount(user, mapConfig.obj().stat_tag, (err) => {
req.profiler.done('incMapviewCount');
if (err) {
@ -338,7 +406,7 @@ function incrementMapViewCount (metadataBackend) {
function augmentLayergroupData () {
return function augmentLayergroupDataMiddleware (req, res, next) {
const { layergroup } = res.locals;
const layergroup = res.body;
// include in layergroup response the variables in serverMedata
// those variables are useful to send to the client information
@ -349,80 +417,33 @@ function augmentLayergroupData () {
};
}
function getAffectedTables (pgConnection, layergroupAffectedTables) {
return function getAffectedTablesMiddleware (req, res, next) {
const { dbname, layergroup, user, mapconfig } = res.locals;
function setLastUpdatedTimeToLayergroup () {
return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) {
const { mapConfigProvider, analysesResults } = res.locals;
const layergroup = res.body;
pgConnection.getConnection(user, (err, connection) => {
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
return next(err);
}
const sql = [];
mapconfig.getLayers().forEach(function(layer) {
sql.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(function(table) {
sql.push('SELECT * FROM ' + table + ' LIMIT 0');
});
}
});
if (!affectedTables) {
return next();
}
QueryTables.getAffectedTablesFromQuery(connection, sql.join(';'), (err, affectedTables) => {
req.profiler.done('getAffectedTablesFromQuery');
if (err) {
return next(err);
}
var lastUpdateTime = affectedTables.getLastUpdatedAt();
// feed affected tables cache so it can be reused from, for instance, layergroup controller
layergroupAffectedTables.set(dbname, layergroup.layergroupId, affectedTables);
lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime;
res.locals.affectedTables = affectedTables;
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
layergroup.last_updated = new Date(lastUpdateTime).toISOString();
next();
});
next();
});
};
}
function setCacheChannel () {
return function setCacheChannelMiddleware (req, res, next) {
const { affectedTables } = res.locals;
if (req.method === 'GET') {
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
}
next();
};
}
function setLastModified () {
return function setLastModifiedMiddleware (req, res, next) {
if (req.method === 'GET') {
res.set('Last-Modified', (new Date()).toUTCString());
}
next();
};
}
function setLastUpdatedTimeToLayergroup () {
return function setLastUpdatedTimeToLayergroupMiddleware (req, res, next) {
const { affectedTables, layergroup, analysesResults } = res.locals;
var lastUpdateTime = affectedTables.getLastUpdatedAt();
lastUpdateTime = getLastUpdatedTime(analysesResults, lastUpdateTime) || lastUpdateTime;
// last update for layergroup cache buster
layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
layergroup.last_updated = new Date(lastUpdateTime).toISOString();
next();
};
}
function getLastUpdatedTime(analysesResults, lastUpdateTime) {
if (!Array.isArray(analysesResults)) {
return lastUpdateTime;
@ -436,27 +457,17 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) {
}, lastUpdateTime);
}
function setCacheControl () {
return function setCacheControlMiddleware (req, res, next) {
if (req.method === 'GET') {
var ttl = global.environment.varnish.layergroupTtl || 86400;
res.set('Cache-Control', 'public,max-age='+ttl+',must-revalidate');
}
next();
};
}
function setLayerStats (pgConnection, statsBackend) {
return function setLayerStatsMiddleware(req, res, next) {
const { user, mapconfig, layergroup } = res.locals;
const { user, mapConfig } = res.locals;
const layergroup = res.body;
pgConnection.getConnection(user, (err, connection) => {
if (err) {
return next(err);
}
statsBackend.getStats(mapconfig, connection, function(err, layersStats) {
statsBackend.getStats(mapConfig, connection, function(err, layersStats) {
if (err) {
return next(err);
}
@ -475,7 +486,8 @@ function setLayerStats (pgConnection, statsBackend) {
function setLayergroupIdHeader (templateMaps, useTemplateHash) {
return function setLayergroupIdHeaderMiddleware (req, res, next) {
const { layergroup, user, template } = res.locals;
const { user, template } = res.locals;
const layergroup = res.body;
if (useTemplateHash) {
var templateHash = templateMaps.fingerPrint(template).substring(0, 8);
@ -490,9 +502,10 @@ function setLayergroupIdHeader (templateMaps, useTemplateHash) {
function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) {
return function setDataviewsAndWidgetsUrlsToLayergroupMetadataMiddleware (req, res, next) {
const { layergroup, user, mapconfig } = res.locals;
const { user, mapConfig } = res.locals;
const layergroup = res.body;
layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapconfig.obj());
layergroupMetadata.addDataviewsAndWidgetsUrls(user, layergroup, mapConfig.obj());
next();
};
@ -500,7 +513,8 @@ function setDataviewsAndWidgetsUrlsToLayergroupMetadata (layergroupMetadata) {
function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) {
return function setAnalysesMetadataToLayergroupMiddleware (req, res, next) {
const { layergroup, user, analysesResults = [] } = res.locals;
const { user, analysesResults = [] } = res.locals;
const layergroup = res.body;
layergroupMetadata.addAnalysesMetadata(user, layergroup, analysesResults, includeQuery);
@ -510,9 +524,10 @@ function setAnalysesMetadataToLayergroup (layergroupMetadata, includeQuery) {
function setTurboCartoMetadataToLayergroup (layergroupMetadata) {
return function setTurboCartoMetadataToLayergroupMiddleware (req, res, next) {
const { layergroup, mapconfig, context } = res.locals;
const { mapConfig, context } = res.locals;
const layergroup = res.body;
layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapconfig.obj(), context);
layergroupMetadata.addTurboCartoContextMetadata(layergroup, mapConfig.obj(), context);
next();
};
@ -520,9 +535,10 @@ function setTurboCartoMetadataToLayergroup (layergroupMetadata) {
function setAggregationMetadataToLayergroup (layergroupMetadata) {
return function setAggregationMetadataToLayergroupMiddleware (req, res, next) {
const { layergroup, mapconfig, context } = res.locals;
const { mapConfig, context } = res.locals;
const layergroup = res.body;
layergroupMetadata.addAggregationContextMetadata(layergroup, mapconfig.obj(), context);
layergroupMetadata.addAggregationContextMetadata(layergroup, mapConfig.obj(), context);
next();
};
@ -530,54 +546,24 @@ function setAggregationMetadataToLayergroup (layergroupMetadata) {
function setTilejsonMetadataToLayergroup (layergroupMetadata) {
return function augmentLayergroupTilejsonMiddleware (req, res, next) {
const { layergroup, user, mapconfig } = res.locals;
const { user, mapConfig } = res.locals;
const layergroup = res.body;
layergroupMetadata.addTileJsonMetadata(layergroup, user, mapconfig);
layergroupMetadata.addTileJsonMetadata(layergroup, user, mapConfig);
next();
};
}
function setSurrogateKeyHeader (surrogateKeysCache) {
return function setSurrogateKeyHeaderMiddleware(req, res, next) {
const { affectedTables, user, templateName } = res.locals;
if (req.method === 'GET' && affectedTables.tables && affectedTables.tables.length > 0) {
surrogateKeysCache.tag(res, affectedTables);
}
if (templateName) {
surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, templateName));
}
next();
};
}
function sendResponse () {
return function sendResponseMiddleware (req, res) {
req.profiler.done('res');
const { layergroup } = res.locals;
res.status(200);
if (req.query && req.query.callback) {
res.jsonp(layergroup);
} else {
res.json(layergroup);
}
};
}
function augmentError (options) {
const { addContext = false, label = 'MAPS CONTROLLER' } = options;
return function augmentErrorMiddleware (err, req, res, next) {
req.profiler.done('error');
const { mapconfig } = res.locals;
const { mapConfig } = res.locals;
if (addContext) {
err = Number.isFinite(err.layerIndex) ? populateError(err, mapconfig) : err;
err = Number.isFinite(err.layerIndex) ? populateError(err, mapConfig) : err;
}
err.label = label;

292
lib/cartodb/controllers/named_maps.js
View File

@ -1,7 +1,14 @@
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const cors = require('../middleware/cors');
const userMiddleware = require('../middleware/user');
const allowQueryParams = require('../middleware/allow-query-params');
const user = require('../middleware/user');
const cleanUpQueryParams = require('../middleware/clean-up-query-params');
const credentials = require('../middleware/credentials');
const dbConnSetup = require('../middleware/db-conn-setup');
const authorize = require('../middleware/authorize');
const cacheControlHeader = require('../middleware/cache-control-header');
const cacheChannelHeader = require('../middleware/cache-channel-header');
const surrogateKeyHeader = require('../middleware/surrogate-key-header');
const lastModifiedHeader = require('../middleware/last-modified-header');
const sendResponse = require('../middleware/send-response');
const vectorError = require('../middleware/vector-error');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
@ -18,25 +25,15 @@ function numMapper(n) {
return +n;
}
function getRequestParams(locals) {
const params = Object.assign({}, locals);
delete params.template;
delete params.affectedTablesAndLastUpdate;
delete params.namedMapProvider;
delete params.allowedQueryParams;
return params;
}
function NamedMapsController(
prepareContext,
namedMapProviderCache,
tileBackend,
function NamedMapsController (
namedMapProviderCache,
tileBackend,
previewBackend,
surrogateKeysCache,
tablesExtentApi,
metadataBackend,
surrogateKeysCache,
tablesExtentApi,
metadataBackend,
pgConnection,
authApi,
userLimitsApi
) {
this.namedMapProviderCache = namedMapProviderCache;
@ -45,51 +42,55 @@ function NamedMapsController(
this.surrogateKeysCache = surrogateKeysCache;
this.tablesExtentApi = tablesExtentApi;
this.metadataBackend = metadataBackend;
this.pgConnection = pgConnection;
this.authApi = authApi;
this.userLimitsApi = userLimitsApi;
this.prepareContext = prepareContext;
}
module.exports = NamedMapsController;
NamedMapsController.prototype.register = function(app) {
const { base_url_mapconfig, base_url_templated } = app;
const { base_url_mapconfig: mapconfigBasePath, base_url_templated: templateBasePath } = app;
app.get(
`${base_url_templated}/:template_id/:layer/:z/:x/:y.(:format)`,
`${templateBasePath}/:template_id/:layer/:z/:x/:y.(:format)`,
cors(),
userMiddleware(),
user(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES),
this.prepareContext,
cleanUpQueryParams(),
getNamedMapProvider({
namedMapProviderCache: this.namedMapProviderCache,
label: 'NAMED_MAP_TILE'
}),
getAffectedTables(),
getTile({
tileBackend: this.tileBackend,
label: 'NAMED_MAP_TILE'
}),
setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
setCacheChannelHeader(),
setLastModifiedHeader(),
setCacheControlHeader(),
setContentTypeHeader(),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse(),
vectorError()
);
app.get(
`${base_url_mapconfig}/static/named/:template_id/:width/:height.:format`,
`${mapconfigBasePath}/static/named/:template_id/:width/:height.:format`,
cors(),
userMiddleware(),
user(),
credentials(),
authorize(this.authApi),
dbConnSetup(this.pgConnection),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED),
allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
this.prepareContext,
cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
getNamedMapProvider({
namedMapProviderCache: this.namedMapProviderCache,
label: 'STATIC_VIZ_MAP', forcedFormat: 'png'
}),
getAffectedTables(),
getTemplate({ label: 'STATIC_VIZ_MAP' }),
prepareLayerFilterFromPreviewLayers({
namedMapProviderCache: this.namedMapProviderCache,
@ -97,28 +98,35 @@ NamedMapsController.prototype.register = function(app) {
}),
getStaticImageOptions({ tablesExtentApi: this.tablesExtentApi }),
getImage({ previewBackend: this.previewBackend, label: 'STATIC_VIZ_MAP' }),
incrementMapViews({ metadataBackend: this.metadataBackend }),
setSurrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
setCacheChannelHeader(),
setLastModifiedHeader(),
setCacheControlHeader(),
setContentTypeHeader(),
incrementMapViews({ metadataBackend: this.metadataBackend }),
cacheControlHeader(),
cacheChannelHeader(),
surrogateKeyHeader({ surrogateKeysCache: this.surrogateKeysCache }),
lastModifiedHeader(),
sendResponse()
);
};
function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = null }) {
return function getNamedMapProviderMiddleware (req, res, next) {
const { user } = res.locals;
const { config, auth_token } = req.query;
const { template_id } = req.params;
const { user, token, cache_buster, api_key } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const { template_id, layer: layerFromParams, z, x, y, format } = req.params;
const { layer: layerFromQuery } = req.query;
const params = {
user, token, cache_buster, api_key,
dbuser, dbname, dbpassword, dbhost, dbport,
template_id, layer: (layerFromQuery || layerFromParams), z, x, y, format
};
if (forcedFormat) {
res.locals.format = forcedFormat;
res.locals.layer = res.locals.layer || 'all';
params.format = forcedFormat;
params.layer = params.layer || 'all';
}
const params = getRequestParams(res.locals);
const { config, auth_token } = req.query;
namedMapProviderCache.get(user, template_id, config, auth_token, params, (err, namedMapProvider) => {
if (err) {
@ -126,25 +134,7 @@ function getNamedMapProvider ({ namedMapProviderCache, label, forcedFormat = nul
return next(err);
}
res.locals.namedMapProvider = namedMapProvider;
next();
});
};
}
function getAffectedTables () {
return function getAffectedTables (req, res, next) {
const { namedMapProvider } = res.locals;
namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
req.profiler.done('affectedTables');
if (err) {
return next(err);
}
res.locals.affectedTablesAndLastUpdate = affectedTablesAndLastUpdate;
res.locals.mapConfigProvider = namedMapProvider;
next();
});
@ -153,9 +143,9 @@ function getAffectedTables () {
function getTemplate ({ label }) {
return function getTemplateMiddleware (req, res, next) {
const { namedMapProvider } = res.locals;
const { mapConfigProvider } = res.locals;
namedMapProvider.getTemplate((err, template) => {
mapConfigProvider.getTemplate((err, template) => {
if (err) {
err.label = label;
return next(err);
@ -170,8 +160,7 @@ function getTemplate ({ label }) {
function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label }) {
return function prepareLayerFilterFromPreviewLayersMiddleware (req, res, next) {
const { user, template } = res.locals;
const { template_id } = req.params;
const { template } = res.locals;
const { config, auth_token } = req.query;
if (!template || !template.view || !template.view.preview_layers) {
@ -191,7 +180,15 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label })
return next();
}
const params = getRequestParams(res.locals);
const { user, token, cache_buster, api_key } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const { template_id, format } = req.params;
const params = {
user, token, cache_buster, api_key,
dbuser, dbname, dbpassword, dbhost, dbport,
template_id, format
};
// overwrites 'all' default filter
params.layer = layerVisibilityFilter.join(',');
@ -203,7 +200,7 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label })
return next(err);
}
res.locals.namedMapProvider = provider;
res.locals.mapConfigProvider = provider;
next();
});
@ -212,19 +209,24 @@ function prepareLayerFilterFromPreviewLayers ({ namedMapProviderCache, label })
function getTile ({ tileBackend, label }) {
return function getTileMiddleware (req, res, next) {
const { namedMapProvider } = res.locals;
const { mapConfigProvider } = res.locals;
const { layer, z, x, y, format } = req.params;
const params = { layer, z, x, y, format };
tileBackend.getTile(namedMapProvider, req.params, (err, tile, headers, stats) => {
tileBackend.getTile(mapConfigProvider, params, (err, tile, headers, stats) => {
req.profiler.add(stats);
req.profiler.done('render-' + format);
if (err) {
err.label = label;
return next(err);
}
res.locals.body = tile;
res.locals.headers = headers;
res.locals.stats = stats;
if (headers) {
res.set(headers);
}
res.body = tile;
next();
});
@ -233,9 +235,11 @@ function getTile ({ tileBackend, label }) {
function getStaticImageOptions ({ tablesExtentApi }) {
return function getStaticImageOptionsMiddleware(req, res, next) {
const { user, namedMapProvider, template } = res.locals;
const { user, mapConfigProvider, template } = res.locals;
const { zoom, lon, lat, bbox } = req.query;
const params = { zoom, lon, lat, bbox };
const imageOpts = getImageOptions(res.locals, template);
const imageOpts = getImageOptions(params, template);
if (imageOpts) {
res.locals.imageOpts = imageOpts;
@ -244,18 +248,18 @@ function getStaticImageOptions ({ tablesExtentApi }) {
res.locals.imageOpts = DEFAULT_ZOOM_CENTER;
namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
return next();
}
var affectedTables = affectedTablesAndLastUpdate.tables || [];
var tables = affectedTables.tables || [];
if (affectedTables.length === 0) {
if (tables.length === 0) {
return next();
}
tablesExtentApi.getBounds(user, affectedTables, (err, bounds) => {
tablesExtentApi.getBounds(user, tables, (err, bounds) => {
if (err) {
return next();
}
@ -335,7 +339,7 @@ function getImageOptionsFromBoundingBox (bbox = '') {
function getImage({ previewBackend, label }) {
return function getImageMiddleware (req, res, next) {
const { imageOpts, namedMapProvider } = res.locals;
const { imageOpts, mapConfigProvider } = res.locals;
const { zoom, center, bounds } = imageOpts;
let { width, height } = req.params;
@ -346,45 +350,62 @@ function getImage({ previewBackend, label }) {
const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
if (zoom !== undefined && center) {
return previewBackend.getImage(namedMapProvider, format, width, height, zoom, center,
return previewBackend.getImage(mapConfigProvider, format, width, height, zoom, center,
(err, image, headers, stats) => {
req.profiler.add(stats);
if (err) {
err.label = label;
return next(err);
}
res.locals.body = image;
res.locals.headers = headers;
res.locals.stats = stats;
if (headers) {
res.set(headers);
}
res.body = image;
next();
});
}
previewBackend.getImage(namedMapProvider, format, width, height, bounds, (err, image, headers, stats) => {
previewBackend.getImage(mapConfigProvider, format, width, height, bounds, (err, image, headers, stats) => {
req.profiler.add(stats);
req.profiler.done('render-' + format);
if (err) {
err.label = label;
return next(err);
}
res.locals.body = image;
res.locals.headers = headers;
res.locals.stats = stats;
if (headers) {
res.set(headers);
}
res.body = image;
next();
});
};
}
function setContentTypeHeader () {
return function setContentTypeHeaderMiddleware(req, res, next) {
res.set('Content-Type', res.get('content-type') || res.get('Content-Type') || 'image/png');
next();
};
}
function incrementMapViewsError (ctx) {
return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`;
}
function incrementMapViews ({ metadataBackend }) {
return function incrementMapViewsMiddleware(req, res, next) {
const { user, namedMapProvider } = res.locals;
const { user, mapConfigProvider } = res.locals;
namedMapProvider.getMapConfig((err, mapConfig) => {
mapConfigProvider.getMapConfig((err, mapConfig) => {
if (err) {
global.logger.log(incrementMapViewsError({ user, err }));
return next();
@ -432,86 +453,3 @@ function templateBounds(view) {
}
return false;
}
function setSurrogateKeyHeader ({ surrogateKeysCache }) {
return function setSurrogateKeyHeaderMiddleware(req, res, next) {
const { user, namedMapProvider, affectedTablesAndLastUpdate } = res.locals;
surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, namedMapProvider.getTemplateName()));
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
if (affectedTablesAndLastUpdate.tables.length > 0) {
surrogateKeysCache.tag(res, affectedTablesAndLastUpdate);
}
}
next();
};
}
function setCacheChannelHeader () {
return function setCacheChannelHeaderMiddleware (req, res, next) {
const { affectedTablesAndLastUpdate } = res.locals;
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
res.set('X-Cache-Channel', affectedTablesAndLastUpdate.getCacheChannel());
}
next();
};
}
function setLastModifiedHeader () {
return function setLastModifiedHeaderMiddleware(req, res, next) {
const { affectedTablesAndLastUpdate } = res.locals;
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
var lastModifiedDate;
if (Number.isFinite(affectedTablesAndLastUpdate.lastUpdatedTime)) {
lastModifiedDate = new Date(affectedTablesAndLastUpdate.getLastUpdatedAt());
} else {
lastModifiedDate = new Date();
}
res.set('Last-Modified', lastModifiedDate.toUTCString());
}
next();
};
}
function setCacheControlHeader () {
return function setCacheControlHeaderMiddleware(req, res, next) {
const { affectedTablesAndLastUpdate } = res.locals;
res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
// we increase cache control as we can invalidate it
res.set('Cache-Control', 'public,max-age=31536000');
}
next();
};
}
function setContentTypeHeader () {
return function setContentTypeHeaderMiddleware(req, res, next) {
const { headers = {} } = res.locals;
res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
next();
};
}
function sendResponse () {
return function sendResponseMiddleware (req, res) {
const { body, stats = {}, format } = res.locals;
req.profiler.done('render-' + format);
req.profiler.add(stats);
res.status(200);
res.send(body);
};
}

64
lib/cartodb/controllers/named_maps_admin.js
View File

@ -1,10 +1,10 @@
const { templateName } = require('../backends/template_maps');
const cors = require('../middleware/cors');
const userMiddleware = require('../middleware/user');
const user = require('../middleware/user');
const credentials = require('../middleware/credentials');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const localsMiddleware = require('../middleware/context/locals');
const credentialsMiddleware = require('../middleware/context/credentials');
const sendResponse = require('../middleware/send-response');
/**
* @param {AuthApi} authApi
@ -21,72 +21,67 @@ function NamedMapsAdminController(authApi, templateMaps, userLimitsApi) {
module.exports = NamedMapsAdminController;
NamedMapsAdminController.prototype.register = function (app) {
const { base_url_templated } = app;
const { base_url_templated: templateBasePath } = app;
app.post(
`${base_url_templated}/`,
`${templateBasePath}/`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE),
localsMiddleware(),
credentialsMiddleware(),
user(),
credentials(),
checkContentType({ action: 'POST', label: 'POST TEMPLATE' }),
authorizedByAPIKey({ authApi: this.authApi, action: 'create', label: 'POST TEMPLATE' }),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE),
createTemplate({ templateMaps: this.templateMaps }),
sendResponse()
);
app.put(
`${base_url_templated}/:template_id`,
`${templateBasePath}/:template_id`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE),
localsMiddleware(),
credentialsMiddleware(),
user(),
credentials(),
checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }),
authorizedByAPIKey({ authApi: this.authApi, action: 'update', label: 'PUT TEMPLATE' }),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE),
updateTemplate({ templateMaps: this.templateMaps }),
sendResponse()
);
app.get(
`${base_url_templated}/:template_id`,
`${templateBasePath}/:template_id`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET),
localsMiddleware(),
credentialsMiddleware(),
user(),
credentials(),
authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET),
retrieveTemplate({ templateMaps: this.templateMaps }),
sendResponse()
);
app.delete(
`${base_url_templated}/:template_id`,
`${templateBasePath}/:template_id`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE),
localsMiddleware(),
credentialsMiddleware(),
user(),
credentials(),
authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE),
destroyTemplate({ templateMaps: this.templateMaps }),
sendResponse()
);
app.get(
`${base_url_templated}/`,
`${templateBasePath}/`,
cors(),
userMiddleware(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST),
localsMiddleware(),
credentialsMiddleware(),
user(),
credentials(),
authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST),
listTemplates({ templateMaps: this.templateMaps }),
sendResponse()
);
app.options(
`${base_url_templated}/:template_id`,
`${templateBasePath}/:template_id`,
cors('Content-Type')
);
};
@ -224,12 +219,3 @@ function listTemplates ({ templateMaps }) {
});
};
}
function sendResponse () {
return function sendResponseMiddleware (req, res) {
res.status(res.statusCode || 200);
const method = req.query.callback ? 'jsonp' : 'json';
res[method](res.body);
};
}

10
lib/cartodb/middleware/allow-query-params.js
View File

@ -1,10 +0,0 @@
module.exports = function allowQueryParams (params) {
if (!Array.isArray(params)) {
throw new Error('allowQueryParams must receive an Array of params');
}
return function allowQueryParamsMiddleware (req, res, next) {
res.locals.allowedQueryParams = params;
next();
};
};

0
lib/cartodb/middleware/context/authorize.js → lib/cartodb/middleware/authorize.js
View File

24
lib/cartodb/middleware/cache-channel-header.js Normal file
View File

@ -0,0 +1,24 @@
module.exports = function setCacheChannelHeader () {
return function setCacheChannelHeaderMiddleware (req, res, next) {
if (req.method !== 'GET') {
return next();
}
const { mapConfigProvider } = res.locals;
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
global.logger.warn('ERROR generating Cache Channel Header:', err);
return next();
}
if (!affectedTables) {
return next();
}
res.set('X-Cache-Channel', affectedTables.getCacheChannel());
next();
});
};
};

19
lib/cartodb/middleware/cache-control-header.js Normal file
View File

@ -0,0 +1,19 @@
const ONE_YEAR_IN_SECONDS = 60 * 60 * 24 * 365;
module.exports = function setCacheControlHeader ({ ttl = ONE_YEAR_IN_SECONDS, revalidate = false } = {}) {
return function setCacheControlHeaderMiddleware (req, res, next) {
if (req.method !== 'GET') {
return next();
}
const directives = [ 'public', `max-age=${ttl}` ];
if (revalidate) {
directives.push('must-revalidate');
}
res.set('Cache-Control', directives.join(','));
next();
};
};

15
lib/cartodb/middleware/context/clean-up-query-params.js → lib/cartodb/middleware/clean-up-query-params.js
View File

@ -14,19 +14,16 @@ const REQUEST_QUERY_PARAMS_WHITELIST = [
'filters' // json
];
module.exports = function cleanUpQueryParamsMiddleware () {
return function cleanUpQueryParams (req, res, next) {
var allowedQueryParams = REQUEST_QUERY_PARAMS_WHITELIST;
module.exports = function cleanUpQueryParamsMiddleware (customQueryParams = []) {
if (!Array.isArray(customQueryParams)) {
throw new Error('customQueryParams must receive an Array of params');
}
if (Array.isArray(res.locals.allowedQueryParams)) {
allowedQueryParams = allowedQueryParams.concat(res.locals.allowedQueryParams);
}
return function cleanUpQueryParams (req, res, next) {
const allowedQueryParams = [...REQUEST_QUERY_PARAMS_WHITELIST, ...customQueryParams];
req.query = _.pick(req.query, allowedQueryParams);
// bring all query values onto res.locals object
_.extend(res.locals, req.query);
next();
};
};

17
lib/cartodb/middleware/context/index.js
View File

@ -1,17 +0,0 @@
const locals = require('./locals');
const cleanUpQueryParams = require('./clean-up-query-params');
const layergroupToken = require('./layergroup-token');
const credentials = require('./credentials');
const authorize = require('./authorize');
const dbConnSetup = require('./db-conn-setup');
module.exports = function prepareContextMiddleware(authApi, pgConnection) {
return [
locals(),
cleanUpQueryParams(),
layergroupToken(),
credentials(),
authorize(authApi),
dbConnSetup(pgConnection)
];
};

7
lib/cartodb/middleware/context/locals.js
View File

@ -1,7 +0,0 @@
module.exports = function locals () {
return function localsMiddleware (req, res, next) {
res.locals = Object.assign(req.params || {}, res.locals);
next();
};
};

0
lib/cartodb/middleware/context/credentials.js → lib/cartodb/middleware/credentials.js
View File

0
lib/cartodb/middleware/context/db-conn-setup.js → lib/cartodb/middleware/db-conn-setup.js
View File

12
lib/cartodb/middleware/error-middleware.js
View File

@ -15,10 +15,6 @@ module.exports = function errorMiddleware (/* options */) {
var statusCode = findStatusCode(err);
if (err.message === 'Tile does not exist' && res.locals.format === 'mvt') {
statusCode = 204;
}
setErrorHeader(allErrors, statusCode, res);
debug('[%s ERROR] -- %d: %s, %s', label, statusCode, err, err.stack);
@ -186,15 +182,15 @@ function setErrorHeader(errors, statusCode, res) {
subtype: error.subtype
};
});
res.set('X-Tiler-Errors', stringifyForLogs(errorsLog));
}
/**
* Remove problematic nested characters
* Remove problematic nested characters
* from object for logs RegEx
*
* @param {Object} object
*
* @param {Object} object
*/
function stringifyForLogs(object) {
Object.keys(object).map(key => {

45
lib/cartodb/middleware/last-modified-header.js Normal file
View File

@ -0,0 +1,45 @@
module.exports = function setLastModifiedHeader ({ now = false } = {}) {
return function setLastModifiedHeaderMiddleware(req, res, next) {
if (req.method !== 'GET') {
return next();
}
const { mapConfigProvider, cache_buster } = res.locals;
if (cache_buster) {
const cacheBuster = parseInt(cache_buster, 10);
const lastModifiedDate = Number.isFinite(cacheBuster) ? new Date(cacheBuster) : new Date();
res.set('Last-Modified', lastModifiedDate.toUTCString());
return next();
}
// REVIEW: to keep 100% compatibility with maps controller
if (now) {
res.set('Last-Modified', new Date().toUTCString());
return next();
}
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
global.logger.warn('ERROR generating Last Modified Header:', err);
return next();
}
if (!affectedTables) {
res.set('Last-Modified', new Date().toUTCString());
return next();
}
const lastUpdatedAt = affectedTables.getLastUpdatedAt();
const lastModifiedDate = Number.isFinite(lastUpdatedAt) ? new Date(lastUpdatedAt) : new Date();
res.set('Last-Modified', lastModifiedDate.toUTCString());
next();
});
};
};

9
lib/cartodb/middleware/context/layergroup-token.js → lib/cartodb/middleware/layergroup-token.js
View File

@ -1,17 +1,12 @@
const LayergroupToken = require('../../models/layergroup-token');
const LayergroupToken = require('../models/layergroup-token');
const authErrorMessageTemplate = function (signer, user) {
return `Cannot use map signature of user "${signer}" on db of user "${user}"`;
};
module.exports = function layergroupToken () {
return function layergroupTokenMiddleware (req, res, next) {
if (!res.locals.token) {
return next();
}
const user = res.locals.user;
const layergroupToken = LayergroupToken.parse(res.locals.token);
const layergroupToken = LayergroupToken.parse(req.params.token);
res.locals.token = layergroupToken.token;
res.locals.cache_buster = layergroupToken.cacheBuster;

8
lib/cartodb/middleware/rate-limit.js
View File

@ -39,12 +39,16 @@ function rateLimit(userLimitsApi, endpointGroup = null) {
res.set({
'Carto-Rate-Limit-Limit': limit,
'Carto-Rate-Limit-Remaining': remaining,
'Retry-After': retry,
'Carto-Rate-Limit-Reset': reset
});
if (isBlocked) {
const rateLimitError = new Error('You are over the limits.');
// retry is floor rounded in seconds by redis-cell
res.set('Retry-After', retry + 1);
let rateLimitError = new Error(
'You are over platform\'s limits. Please contact us to know more details'
);
rateLimitError.http_status = 429;
rateLimitError.type = 'limit';
rateLimitError.subtype = 'rate-limit';

17
lib/cartodb/middleware/send-response.js Normal file
View File

@ -0,0 +1,17 @@
module.exports = function sendResponse () {
return function sendResponseMiddleware (req, res) {
req.profiler.done('res');
res.status(res.statusCode || 200);
if (Buffer.isBuffer(res.body)) {
return res.send(res.body);
}
if (req.query.callback) {
return res.jsonp(res.body);
}
res.json(res.body);
};
};

31
lib/cartodb/middleware/surrogate-key-header.js Normal file
View File

@ -0,0 +1,31 @@
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
module.exports = function setSurrogateKeyHeader ({ surrogateKeysCache }) {
return function setSurrogateKeyHeaderMiddleware(req, res, next) {
const { user, mapConfigProvider } = res.locals;
if (mapConfigProvider instanceof NamedMapMapConfigProvider) {
surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, mapConfigProvider.getTemplateName()));
}
if (req.method !== 'GET') {
return next();
}
mapConfigProvider.getAffectedTables((err, affectedTables) => {
if (err) {
global.logger.warn('ERROR generating Surrogate Key Header:', err);
return next();
}
if (!affectedTables || !affectedTables.tables || affectedTables.tables.length === 0) {
return next();
}
surrogateKeysCache.tag(res, affectedTables);
next();
});
};
};

6
lib/cartodb/middleware/vector-error.js
View File

@ -5,7 +5,7 @@ module.exports = function vectorError() {
return function vectorErrorMiddleware(err, req, res, next) {
if(req.params.format === 'mvt') {
if (isTimeoutError(err)) {
if (isTimeoutError(err) || isRateLimitError(err)) {
res.set('Content-Type', 'application/x-protobuf');
return res.status(429).send(timeoutErrorVectorTile);
}
@ -27,3 +27,7 @@ function isDatasourceTimeoutError (err) {
function isTimeoutError (err) {
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
}
function isRateLimitError (err) {
return err.type === 'limit' && err.subtype === 'rate-limit';
}

22
lib/cartodb/models/mapconfig/adapter/turbo-carto-adapter.js
View File

@ -11,6 +11,8 @@ var PostgresDatasource = require('../../../backends/turbo-carto-postgres-datasou
var MapConfig = require('windshaft').model.MapConfig;
const dbParamsFromReqParams = require('../../../utils/database-params');
function TurboCartoAdapter() {
}
@ -158,23 +160,3 @@ TurboCartoAdapter.prototype.process = function (psql, cartocss, sql, callback) {
function shouldParseLayerCartocss(layer) {
return layer && layer.options && layer.options.cartocss && layer.options.sql;
}
function dbParamsFromReqParams(params) {
var dbParams = {};
if ( params.dbuser ) {
dbParams.user = params.dbuser;
}
if ( params.dbpassword ) {
dbParams.pass = params.dbpassword;
}
if ( params.dbhost ) {
dbParams.host = params.dbhost;
}
if ( params.dbport ) {
dbParams.port = params.dbport;
}
if ( params.dbname ) {
dbParams.dbname = params.dbname;
}
return dbParams;
}

62
lib/cartodb/models/mapconfig/provider/create-layergroup-provider.js
View File

@ -2,6 +2,7 @@ var assert = require('assert');
var step = require('step');
var MapStoreMapConfigProvider = require('./map-store-provider');
const QueryTables = require('cartodb-query-tables');
/**
* @param {MapConfig} mapConfig
@ -11,10 +12,13 @@ var MapStoreMapConfigProvider = require('./map-store-provider');
* @constructor
* @type {CreateLayergroupMapConfigProvider}
*/
function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, params) {
function CreateLayergroupMapConfigProvider(mapConfig, user, userLimitsApi, pgConnection, affectedTablesCache, params) {
this.mapConfig = mapConfig;
this.user = user;
this.userLimitsApi = userLimitsApi;
this.pgConnection = pgConnection;
this.affectedTablesCache = affectedTablesCache;
this.params = params;
this.cacheBuster = params.cache_buster || 0;
}
@ -23,7 +27,13 @@ module.exports = CreateLayergroupMapConfigProvider;
CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) {
var self = this;
if (this.mapConfig && this.params && this.context) {
return callback(null, this.mapConfig, this.params, this.context);
}
var context = {};
step(
function prepareContextLimits() {
self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
@ -31,6 +41,7 @@ CreateLayergroupMapConfigProvider.prototype.getMapConfig = function(callback) {
function handleRenderLimits(err, renderLimits) {
assert.ifError(err);
context.limits = renderLimits;
self.context = context;
return null;
},
function finish(err) {
@ -46,3 +57,52 @@ CreateLayergroupMapConfigProvider.prototype.getCacheBuster = MapStoreMapConfigPr
CreateLayergroupMapConfigProvider.prototype.filter = MapStoreMapConfigProvider.prototype.filter;
CreateLayergroupMapConfigProvider.prototype.createKey = MapStoreMapConfigProvider.prototype.createKey;
CreateLayergroupMapConfigProvider.prototype.getAffectedTables = function (callback) {
this.getMapConfig((err, mapConfig) => {
if (err) {
return callback(err);
}
const { dbname } = this.params;
const token = mapConfig.id();
if (this.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = this.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
const queries = [];
this.mapConfig.getLayers().forEach(layer => {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(table => {
queries.push(`SELECT * FROM ${table} LIMIT 0`);
});
}
});
const sql = queries.length ? queries.join(';') : null;
if (!sql) {
return callback();
}
this.pgConnection.getConnection(this.user, (err, connection) => {
if (err) {
return callback(err);
}
QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => {
if (err) {
return callback(err);
}
this.affectedTablesCache.set(dbname, token, affectedTables);
callback(null, affectedTables);
});
});
});
};

69
lib/cartodb/models/mapconfig/provider/map-store-provider.js
View File

@ -2,6 +2,7 @@ var _ = require('underscore');
var assert = require('assert');
var dot = require('dot');
var step = require('step');
const QueryTables = require('cartodb-query-tables');
/**
* @param {MapStore} mapStore
@ -11,20 +12,30 @@ var step = require('step');
* @constructor
* @type {MapStoreMapConfigProvider}
*/
function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, params) {
function MapStoreMapConfigProvider(mapStore, user, userLimitsApi, pgConnection, affectedTablesCache, params) {
this.mapStore = mapStore;
this.user = user;
this.userLimitsApi = userLimitsApi;
this.params = params;
this.pgConnection = pgConnection;
this.affectedTablesCache = affectedTablesCache;
this.token = params.token;
this.cacheBuster = params.cache_buster || 0;
this.mapConfig = null;
this.params = params;
this.context = null;
}
module.exports = MapStoreMapConfigProvider;
MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
var self = this;
if (this.mapConfig !== null) {
return callback(null, this.mapConfig, this.params, this.context);
}
var context = {};
step(
function prepareContextLimits() {
self.userLimitsApi.getRenderLimits(self.user, self.params.api_key, this);
@ -39,6 +50,8 @@ MapStoreMapConfigProvider.prototype.getMapConfig = function(callback) {
self.mapStore.load(self.token, this);
},
function finish(err, mapConfig) {
self.mapConfig = mapConfig;
self.context = context;
return callback(err, mapConfig, self.params, context);
}
);
@ -74,4 +87,54 @@ MapStoreMapConfigProvider.prototype.createKey = function(base) {
scale_factor: 1
});
return (base) ? baseKeyTpl(tplValues) : rendererKeyTpl(tplValues);
};
};
MapStoreMapConfigProvider.prototype.getAffectedTables = function(callback) {
this.getMapConfig((err, mapConfig) => {
if (err) {
return callback(err);
}
const { dbname } = this.params;
const token = mapConfig.id();
if (this.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = this.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
const queries = [];
mapConfig.getLayers().forEach(layer => {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(table => {
queries.push(`SELECT * FROM ${table} LIMIT 0`);
});
}
});
const sql = queries.length ? queries.join(';') : null;
if (!sql) {
return callback();
}
this.pgConnection.getConnection(this.user, (err, connection) => {
if (err) {
return callback(err);
}
QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => {
if (err) {
return callback(err);
}
this.affectedTablesCache.set(dbname, token, affectedTables);
callback(err, affectedTables);
});
});
});
};

99
lib/cartodb/models/mapconfig/provider/named-map-provider.js
View File

@ -11,8 +11,19 @@ var QueryTables = require('cartodb-query-tables');
* @constructor
* @type {NamedMapMapConfigProvider}
*/
function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend, userLimitsApi, mapConfigAdapter,
owner, templateId, config, authToken, params) {
function NamedMapMapConfigProvider(
templateMaps,
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter,
affectedTablesCache,
owner,
templateId,
config,
authToken,
params
) {
this.templateMaps = templateMaps;
this.pgConnection = pgConnection;
this.metadataBackend = metadataBackend;
@ -30,7 +41,7 @@ function NamedMapMapConfigProvider(templateMaps, pgConnection, metadataBackend,
// use template after call to mapConfig
this.template = null;
this.affectedTablesAndLastUpdate = null;
this.affectedTablesCache = affectedTablesCache;
// providing
this.err = null;
@ -189,7 +200,7 @@ NamedMapMapConfigProvider.prototype.getCacheBuster = function() {
NamedMapMapConfigProvider.prototype.reset = function() {
this.template = null;
this.affectedTablesAndLastUpdate = null;
this.affectedTables = null;
this.err = null;
this.mapConfig = null;
@ -251,39 +262,51 @@ NamedMapMapConfigProvider.prototype.getTemplateName = function() {
return this.templateName;
};
NamedMapMapConfigProvider.prototype.getAffectedTablesAndLastUpdatedTime = function(callback) {
var self = this;
if (this.affectedTablesAndLastUpdate !== null) {
return callback(null, this.affectedTablesAndLastUpdate);
}
step(
function getMapConfig() {
self.getMapConfig(this);
},
function getSql(err, mapConfig) {
assert.ifError(err);
return mapConfig.getLayers().map(function(layer) {
return layer.options.sql;
}).join(';');
},
function getAffectedTables(err, sql) {
assert.ifError(err);
step(
function getConnection() {
self.pgConnection.getConnection(self.owner, this);
},
function getAffectedTables(err, connection) {
assert.ifError(err);
QueryTables.getAffectedTablesFromQuery(connection, sql, this);
},
this
);
},
function finish(err, result) {
self.affectedTablesAndLastUpdate = result;
return callback(err, result);
NamedMapMapConfigProvider.prototype.getAffectedTables = function(callback) {
this.getMapConfig((err, mapConfig) => {
if (err) {
return callback(err);
}
);
const { dbname } = this.rendererParams;
const token = mapConfig.id();
if (this.affectedTablesCache.hasAffectedTables(dbname, token)) {
const affectedTables = this.affectedTablesCache.get(dbname, token);
return callback(null, affectedTables);
}
const queries = [];
mapConfig.getLayers().forEach(layer => {
queries.push(layer.options.sql);
if (layer.options.affected_tables) {
layer.options.affected_tables.map(table => {
queries.push(`SELECT * FROM ${table} LIMIT 0`);
});
}
});
const sql = queries.length ? queries.join(';') : null;
if (!sql) {
return callback();
}
this.pgConnection.getConnection(this.owner, (err, connection) => {
if (err) {
return callback(err);
}
QueryTables.getAffectedTablesFromQuery(connection, sql, (err, affectedTables) => {
if (err) {
return callback(err);
}
this.affectedTablesCache.set(dbname, token, affectedTables);
callback(err, affectedTables);
});
});
});
};

22
lib/cartodb/server.js
View File

@ -49,8 +49,6 @@ var StatsBackend = require('./backends/stats');
const lzmaMiddleware = require('./middleware/lzma');
const errorMiddleware = require('./middleware/error-middleware');
const prepareContextMiddleware = require('./middleware/context');
module.exports = function(serverOptions) {
// Make stats client globally accessible
global.statsClient = StatsClient.getInstance(serverOptions.statsd);
@ -202,7 +200,8 @@ module.exports = function(serverOptions) {
pgConnection,
metadataBackend,
userLimitsApi,
mapConfigAdapter
mapConfigAdapter,
layergroupAffectedTablesCache
);
['update', 'delete'].forEach(function(eventType) {
@ -216,16 +215,11 @@ module.exports = function(serverOptions) {
var versions = getAndValidateVersions(serverOptions);
const prepareContext = typeof serverOptions.req2params === 'function' ?
serverOptions.req2params :
prepareContextMiddleware(authApi, pgConnection);
/*******************************************************************************************************************
* Routing
******************************************************************************************************************/
new controller.Layergroup(
prepareContext,
pgConnection,
mapStore,
tileBackend,
@ -234,11 +228,11 @@ module.exports = function(serverOptions) {
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTablesCache,
analysisBackend
analysisBackend,
authApi
).register(app);
new controller.Map(
prepareContext,
pgConnection,
templateMaps,
mapBackend,
@ -247,23 +241,25 @@ module.exports = function(serverOptions) {
userLimitsApi,
layergroupAffectedTablesCache,
mapConfigAdapter,
statsBackend
statsBackend,
authApi
).register(app);
new controller.NamedMaps(
prepareContext,
namedMapProviderCache,
tileBackend,
previewBackend,
surrogateKeysCache,
tablesExtentApi,
metadataBackend,
pgConnection,
authApi,
userLimitsApi
).register(app);
new controller.NamedMapsAdmin(authApi, templateMaps, userLimitsApi).register(app);
new controller.Analyses(prepareContext, userLimitsApi).register(app);
new controller.Analyses(pgConnection, authApi, userLimitsApi).register(app);
new controller.ServerInfo(versions).register(app);

25
lib/cartodb/utils/database-params.js Normal file
View File

@ -0,0 +1,25 @@
module.exports = function getDatabaseConnectionParams (params) {
const dbParams = {};
if (params.dbuser) {
dbParams.user = params.dbuser;
}
if (params.dbpassword) {
dbParams.pass = params.dbpassword;
}
if (params.dbhost) {
dbParams.host = params.dbhost;
}
if (params.dbport) {
dbParams.port = params.dbport;
}
if (params.dbname) {
dbParams.dbname = params.dbname;
}
return dbParams;
};

2
package.json
View File

@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "6.0.0",
"version": "6.0.1",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"

19
run_tests.sh
View File

@ -50,17 +50,6 @@ die() {
exit 1
}
get_redis_cell() {
if test x"$OPT_REDIS_CELL" = xyes; then
echo "Downloading redis-cell"
curl -L https://github.com/brandur/redis-cell/releases/download/v0.2.2/redis-cell-v0.2.2-x86_64-unknown-linux-gnu.tar.gz --output redis-cell.tar.gz > /dev/null 2>&1
tar xvzf redis-cell.tar.gz > /dev/null 2>&1
mv libredis_cell.so ${BASEDIR}/test/support/libredis_cell.so
rm redis-cell.tar.gz
rm libredis_cell.d
fi
}
trap 'cleanup_and_exit' 1 2 3 5 9 13
while [ -n "$1" ]; do
@ -122,9 +111,13 @@ fi
TESTS=$@
if test x"$OPT_CREATE_REDIS" = xyes; then
get_redis_cell
echo "Starting redis on port ${REDIS_PORT}"
echo "port ${REDIS_PORT}" | redis-server - --loadmodule ${BASEDIR}/test/support/libredis_cell.so > ${BASEDIR}/test.log &
REDIS_CELL_PATH="${BASEDIR}/test/support/libredis_cell.so"
if [[ "$OSTYPE" == "darwin"* ]]; then
REDIS_CELL_PATH="${BASEDIR}/test/support/libredis_cell.dylib"
fi
echo "port ${REDIS_PORT}" | redis-server - --loadmodule ${REDIS_CELL_PATH} > ${BASEDIR}/test.log &
PID_REDIS=$!
echo ${PID_REDIS} > ${BASEDIR}/redis.pid
fi

6
test/acceptance/multilayer.js
View File

@ -1272,6 +1272,8 @@ describe(suiteName, function() {
it("cache control for layergroup default value", function(done) {
global.environment.varnish.layergroupTtl = null;
var server = new CartodbWindshaft(serverOptions);
assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation,
function(res) {
assert.equal(res.headers['cache-control'], 'public,max-age=86400,must-revalidate');
@ -1287,6 +1289,8 @@ describe(suiteName, function() {
var layergroupTtl = 300;
global.environment.varnish.layergroupTtl = layergroupTtl;
var server = new CartodbWindshaft(serverOptions);
assert.response(server, layergroupTtlRequest, layergroupTtlResponseExpectation,
function(res) {
assert.equal(res.headers['cache-control'], 'public,max-age=' + layergroupTtl + ',must-revalidate');
@ -1336,7 +1340,7 @@ describe(suiteName, function() {
status: 403
},
function(res) {
assert.ok(res.body.match(/permission denied for relation test_table_private_1/));
assert.ok(res.body.match(/permission denied for .+?test_table_private_1/));
done();
}
);

2
test/acceptance/named_layers.js
View File

@ -659,7 +659,7 @@ describe('named_layers', function() {
}
var parsedBody = JSON.parse(response.body);
assert.ok(parsedBody.errors[0].match(/permission denied for relation test_table_private_1/));
assert.ok(parsedBody.errors[0].match(/permission denied for .+?test_table_private_1/));
return null;
},

280
test/acceptance/ported/attributes.js
View File

@ -1,10 +1,9 @@
var testHelper = require('../../support/test_helper');
var assert = require('../../support/assert');
var step = require('step');
var assert = require('../../support/assert');
var step = require('step');
var cartodbServer = require('../../../lib/cartodb/server');
var PortedServerOptions = require('./support/ported_server_options');
var LayergroupToken = require('../../../lib/cartodb/models/layergroup-token');
describe('attributes', function() {
@ -47,132 +46,147 @@ describe('attributes', function() {
});
it("can only be fetched from layer having an attributes spec", function(done) {
var expected_token;
step(
function do_post()
{
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
data: JSON.stringify(test_mapconfig_1)
}, {}, function(res, err) { next(err, res); });
},
function checkPost(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
// CORS headers should be sent with response
// from layergroup creation via POST
checkCORSHeaders(res);
var parsedBody = JSON.parse(res.body);
if ( expected_token ) {
assert.deepEqual(parsedBody, {layergroupid: expected_token, layercount: 2});
} else {
expected_token = parsedBody.layergroupid;
}
return null;
},
function do_get_attr_0(err)
{
assert.ifError(err);
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/attributes/1',
method: 'GET'
}, {}, function(res, err) { next(err, res); });
},
function check_error_0(err, res) {
assert.ifError(err);
assert.equal(
res.statusCode,
400,
res.statusCode + ( res.statusCode !== 200 ? (': ' + res.body) : '' )
);
var parsed = JSON.parse(res.body);
assert.equal(parsed.errors[0], "Layer 0 has no exposed attributes");
return null;
},
function do_get_attr_1(err)
{
assert.ifError(err);
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/1',
method: 'GET'
}, {}, function(res, err) { next(err, res); });
},
function check_attr_1(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var parsed = JSON.parse(res.body);
assert.deepEqual(parsed, {"n":6});
return null;
},
function do_get_attr_1_404(err)
{
assert.ifError(err);
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/-666',
method: 'GET'
}, {}, function(res, err) { next(err, res); });
},
function check_attr_1_404(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 404, res.statusCode + ': ' + res.body);
var parsed = JSON.parse(res.body);
assert.ok(parsed.errors);
var msg = parsed.errors[0];
assert.equal(msg, "Multiple features (0) identified by 'i' = -666 in layer 1");
return null;
},
function finish(err) {
keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
done(err);
var expected_token;
step(
function do_post()
{
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(test_mapconfig_1)
}, {}, function(res, err) { next(err, res); });
},
function checkPost(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
// CORS headers should be sent with response
// from layergroup creation via POST
checkCORSHeaders(res);
var parsedBody = JSON.parse(res.body);
if ( expected_token ) {
assert.deepEqual(parsedBody, {layergroupid: expected_token, layercount: 2});
} else {
expected_token = parsedBody.layergroupid;
}
);
});
return null;
},
function do_get_attr_0(err)
{
assert.ifError(err);
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/attributes/1',
method: 'GET',
headers: {
host: 'localhost'
},
}, {}, function(res, err) { next(err, res); });
},
function check_error_0(err, res) {
assert.ifError(err);
assert.equal(
res.statusCode,
400,
res.statusCode + ( res.statusCode !== 200 ? (': ' + res.body) : '' )
);
var parsed = JSON.parse(res.body);
assert.equal(parsed.errors[0], "Layer 0 has no exposed attributes");
return null;
},
function do_get_attr_1(err)
{
assert.ifError(err);
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/1',
method: 'GET',
headers: {
host: 'localhost'
}
}, {}, function(res, err) { next(err, res); });
},
function check_attr_1(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var parsed = JSON.parse(res.body);
assert.deepEqual(parsed, {"n":6});
return null;
},
function do_get_attr_1_404(err)
{
assert.ifError(err);
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/-666',
method: 'GET',
headers: {
host: 'localhost'
}
}, {}, function(res, err) { next(err, res); });
},
function check_attr_1_404(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 404, res.statusCode + ': ' + res.body);
var parsed = JSON.parse(res.body);
assert.ok(parsed.errors);
var msg = parsed.errors[0];
assert.equal(msg, "Multiple features (0) identified by 'i' = -666 in layer 1");
return null;
},
function finish(err) {
keysToDelete['map_cfg|' + LayergroupToken.parse(expected_token).token] = 0;
keysToDelete['user:localhost:mapviews:global'] = 5;
done(err);
}
);
});
// See https://github.com/CartoDB/Windshaft/issues/131
it("are checked at map creation time", function(done) {
// clone the mapconfig test
var mapconfig = JSON.parse(JSON.stringify(test_mapconfig_1));
// append unexistant attribute name
mapconfig.layers[1].options.sql = 'SELECT * FROM test_table';
mapconfig.layers[1].options.attributes.id = 'unexistant';
mapconfig.layers[1].options.attributes.columns = ['cartodb_id'];
// clone the mapconfig test
var mapconfig = JSON.parse(JSON.stringify(test_mapconfig_1));
// append unexistant attribute name
mapconfig.layers[1].options.sql = 'SELECT * FROM test_table';
mapconfig.layers[1].options.attributes.id = 'unexistant';
mapconfig.layers[1].options.attributes.columns = ['cartodb_id'];
step(
function do_post()
{
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
data: JSON.stringify(mapconfig)
}, {}, function(res, err) { next(err, res); });
},
function checkPost(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 404, res.statusCode + ': ' + (res.statusCode===200?'...':res.body));
var parsed = JSON.parse(res.body);
assert.ok(parsed.errors);
assert.equal(parsed.errors.length, 1);
var msg = parsed.errors[0];
assert.equal(msg, 'column "unexistant" does not exist');
return null;
},
function finish(err) {
done(err);
}
);
});
step(
function do_post()
{
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(mapconfig)
}, {}, function(res, err) { next(err, res); });
},
function checkPost(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 404, res.statusCode + ': ' + (res.statusCode===200?'...':res.body));
var parsed = JSON.parse(res.body);
assert.ok(parsed.errors);
assert.equal(parsed.errors.length, 1);
var msg = parsed.errors[0];
assert.equal(msg, 'column "unexistant" does not exist');
return null;
},
function finish(err) {
done(err);
}
);
});
it("can be used with jsonp", function(done) {
@ -184,7 +198,10 @@ describe('attributes', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(test_mapconfig_1)
}, {}, function(res, err) { next(err, res); });
},
@ -209,7 +226,10 @@ describe('attributes', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token +
'/0/attributes/1?callback=test',
method: 'GET'
method: 'GET',
headers: {
host: 'localhost'
}
}, {}, function(res, err) { next(err, res); });
},
function check_error_0(err, res) {
@ -232,7 +252,10 @@ describe('attributes', function() {
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/1',
method: 'GET'
method: 'GET',
headers: {
host: 'localhost'
}
}, {}, function(res, err) { next(err, res); });
},
function check_attr_1(err, res) {
@ -270,7 +293,10 @@ describe('attributes', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(mapconfig)
}, {}, function(res, err) { next(err, res); });
},

160
test/acceptance/ported/multilayer.js
View File

@ -44,7 +44,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json; charset=utf-8' },
headers: { host: 'localhost', 'Content-Type': 'application/json; charset=utf-8' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
@ -85,7 +85,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
@ -101,7 +101,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png',
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
@ -154,7 +155,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
@ -177,7 +178,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png',
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
@ -194,7 +196,8 @@ describe('multilayer', function() {
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
@ -212,7 +215,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token +
'/1/0/0/0.grid.json?interactivity=cartodb_id',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
@ -266,7 +270,7 @@ describe('multilayer', function() {
config: JSON.stringify(layergroup)
}),
method: 'GET',
headers: {'Content-Type': 'application/json' }
headers: { host: 'localhost', 'Content-Type': 'application/json' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
// CORS headers should be sent with response
@ -289,7 +293,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png',
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
@ -307,7 +312,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token +
'/0/0/0/0.grid.json?interactivity=cartodb_id',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
@ -325,7 +331,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token +
'/1/0/0/0.grid.json?interactivity=cartodb_id',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
@ -380,7 +387,7 @@ describe('multilayer', function() {
callback: 'jsonp_test'
}),
method: 'GET',
headers: {'Content-Type': 'application/json' }
headers: { host: 'localhost', 'Content-Type': 'application/json' }
}, {}, function(res, err) { next(err, res); });
},
function do_check_token(err, res) {
@ -413,7 +420,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png',
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
@ -431,7 +439,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token +
'/0/0/0/0.grid.json?interactivity=cartodb_id',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
@ -449,7 +458,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token +
'/1/0/0/0.grid.json?interactivity=cartodb_id',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
@ -510,7 +520,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res, err) {
next(err, res);
@ -530,7 +540,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png',
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
@ -548,7 +559,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token +
'/0/0/0/0.grid.json?interactivity=cartodb_id',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
@ -566,7 +578,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token +
'/1/0/0/0.grid.json?interactivity=cartodb_id',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
@ -583,7 +596,8 @@ describe('multilayer', function() {
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/1/attributes/4',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res, err) {
next(err, res);
});
@ -602,7 +616,8 @@ describe('multilayer', function() {
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/2/0/0/0.json.torque',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res, err) { next(err, res); });
},
function do_check_torque2(err, res) {
@ -624,7 +639,8 @@ describe('multilayer', function() {
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/1/0/0/0.json.torque',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res, err) { next(err, res); });
},
function do_check_torque1(err, res) {
@ -684,7 +700,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup1)
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
@ -700,7 +716,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup2)
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
@ -717,7 +733,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + token1 + '/0/0/0.png',
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
@ -734,7 +751,8 @@ describe('multilayer', function() {
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + token1 + '/0/0/0/0.grid.json?interactivity=cartodb_id',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
@ -752,7 +770,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + token2 + '/0/0/0.png',
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
@ -769,7 +788,8 @@ describe('multilayer', function() {
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + token2 + '/0/0/0/0.grid.json?interactivity=cartodb_id',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "application/json; charset=utf-8");
@ -826,7 +846,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
try {
@ -848,7 +868,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png',
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");
@ -893,7 +914,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
@ -922,7 +943,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
@ -957,7 +978,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: _.template(tpl, {font:'bogus'})
}, function(res) { next(null, res); });
},
@ -977,7 +998,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {host: 'localhost', 'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: _.template(tpl, {font:available_system_fonts[0]})
}, function(res) { next(null, res); });
},
@ -1040,7 +1061,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
try {
@ -1061,7 +1082,8 @@ describe('multilayer', function() {
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json',
method: 'GET'
method: 'GET',
headers: { host: 'localhost' }
}, {}, function(res) {
next(null, res);
});
@ -1093,63 +1115,8 @@ describe('multilayer', function() {
);
});
// See http://github.com/CartoDB/Windshaft/issues/157
it("req2params is called only once for a multilayer post", function(done) {
var layergroup = {
version: '1.0.1',
layers: [
{ options: {
sql: 'select cartodb_id, ST_Translate(the_geom, 50, 0) as the_geom from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
cartocss_version: '2.0.1',
interactivity: [ 'cartodb_id' ],
geom_column: 'the_geom'
} },
{ options: {
sql: 'select cartodb_id, ST_Translate(the_geom, -50, 0) as the_geom from test_table limit 2 offset 2',
cartocss: '#layer { marker-fill:blue; marker-allow-overlap:true; }',
cartocss_version: '2.0.2',
interactivity: [ 'cartodb_id' ],
geom_column: 'the_geom'
} }
]
};
var expected_token;
step(
function do_post()
{
global.req2params_calls = 0;
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res, err) { next(err,res); });
},
function check_post(err, res) {
assert.ifError(err);
assert.equal(res.statusCode, 200, res.statusCode + ': ' + res.body);
var parsedBody = JSON.parse(res.body);
expected_token = LayergroupToken.parse(parsedBody.layergroupid).token;
assert.equal(global.req2params_calls, 1);
return null;
},
function finish(err) {
if (err) {
return done(err);
}
var keysToDelete = {'user:localhost:mapviews:global': 5};
keysToDelete['map_cfg|' + expected_token] = 0;
testHelper.deleteRedisKeys(keysToDelete, done);
}
);
});
// See https://github.com/CartoDB/Windshaft/issues/163
it("has different token for different database", function(done) {
it.skip("has different token for different database", function(done) {
var layergroup = {
version: '1.0.1',
layers: [
@ -1170,7 +1137,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res, err) { next(err,res); });
},
@ -1187,7 +1154,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test2/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'cartodb250user', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res, err) { next(err,res); });
},
@ -1233,7 +1200,7 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res, err) { next(err,res); });
},
@ -1251,7 +1218,8 @@ describe('multilayer', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + token1 + '/0/0/0.png',
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: { host: 'localhost' }
}, {}, function(res) {
assert.equal(res.statusCode, 200, res.body);
assert.equal(res.headers['content-type'], "image/png");

90
test/acceptance/ported/multilayer_error_cases.js
View File

@ -24,7 +24,10 @@ describe('multilayer error cases', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/x-www-form-urlencoded' }
headers: {
host: 'localhost',
'Content-Type': 'application/x-www-form-urlencoded'
}
}, {}, function(res) {
assert.equal(res.statusCode, 400, res.body);
var parsedBody = JSON.parse(res.body);
@ -37,7 +40,10 @@ describe('multilayer error cases', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' }
headers: {
host: 'localhost',
'Content-Type': 'application/json'
}
}, {}, function(res) {
assert.equal(res.statusCode, 400, res.body);
var parsedBody = JSON.parse(res.body);
@ -50,7 +56,10 @@ describe('multilayer error cases', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup?callback=test',
method: 'POST',
headers: {'Content-Type': 'application/json' }
headers: {
host: 'localhost',
'Content-Type': 'application/json'
}
}, {}, function(res) {
assert.equal(res.statusCode, 200);
assert.equal(
@ -65,27 +74,30 @@ describe('multilayer error cases', function() {
});
it("layergroup with no cartocss_version", function(done) {
var layergroup = {
version: '1.0.0',
layers: [
{ options: {
sql: 'select cartodb_id, ST_Translate(the_geom, 50, 0) as the_geom from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
geom_column: 'the_geom'
} }
]
};
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 400, res.body);
var parsedBody = JSON.parse(res.body);
assert.deepEqual(parsedBody.errors, ["Missing cartocss_version for layer 0 options"]);
done();
});
var layergroup = {
version: '1.0.0',
layers: [
{ options: {
sql: 'select cartodb_id, ST_Translate(the_geom, 50, 0) as the_geom from test_table limit 2',
cartocss: '#layer { marker-fill:red; marker-width:32; marker-allow-overlap:true; }',
geom_column: 'the_geom'
} }
]
};
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 400, res.body);
var parsedBody = JSON.parse(res.body);
assert.deepEqual(parsedBody.errors, ["Missing cartocss_version for layer 0 options"]);
done();
});
});
it("sql/cartocss combination errors", function(done) {
@ -98,17 +110,18 @@ describe('multilayer error cases', function() {
geom_column: 'the_geom'
}}]
};
ServerOptions.afterLayergroupCreateCalls = 0;
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
}, {}, function(res) {
try {
assert.equal(res.statusCode, 400, res.statusCode + ': ' + res.body);
// See http://github.com/CartoDB/Windshaft/issues/159
assert.equal(ServerOptions.afterLayergroupCreateCalls, 0);
var parsed = JSON.parse(res.body);
assert.ok(parsed);
assert.equal(parsed.errors.length, 1);
@ -149,12 +162,9 @@ describe('multilayer error cases', function() {
}}
]
};
ServerOptions.afterLayergroupCreateCalls = 0;
this.client = new TestClient(layergroup);
this.client.getLayergroup({ response: { status: 400 } }, function(err, parsed) {
assert.ok(!err, err);
// See http://github.com/CartoDB/Windshaft/issues/159
assert.equal(ServerOptions.afterLayergroupCreateCalls, 0);
assert.ok(parsed);
assert.equal(parsed.errors.length, 1);
var error = parsed.errors[0];
@ -186,7 +196,10 @@ describe('multilayer error cases', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
}, {}, function(res) {
try {
@ -222,7 +235,10 @@ describe('multilayer error cases', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
}, {}, function(res) {
try {
@ -264,7 +280,10 @@ describe('multilayer error cases', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
}, {}, function(res) {
assert.equal(res.statusCode, 400, res.body);
@ -367,7 +386,10 @@ describe('multilayer error cases', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/deadbeef/0/0/0/0.grid.json',
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: {
host: 'localhost'
}
}, {}, function(res, err) { next(err, res); });
},
function checkResponse(err, res) {

5
test/acceptance/ported/multilayer_interactivity.js
View File

@ -23,13 +23,12 @@ describe('multilayer interactivity and layers order', function() {
layers: testScenario.layers
};
PortedServerOptions.afterLayergroupCreateCalls = 0;
assert.response(server,
{
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
@ -49,8 +48,6 @@ describe('multilayer interactivity and layers order', function() {
'\n\tLayer types: ' + layergroup.layers.map(layerType).join(', ')
);
// assert.equal(PortedServerOptions.afterLayergroupCreateCalls, 1);
var layergroupResponse = JSON.parse(response.body);
assert.ok(layergroupResponse);

6
test/acceptance/ported/raster.js
View File

@ -41,7 +41,7 @@ describe('raster', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(mapconfig)
}, {}, function(res, err) { next(err, res); });
},
@ -66,7 +66,8 @@ describe('raster', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png',
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: { host: 'localhost' }
}, {}, function(res, err) { next(err, res); });
},
function check_response(err, res) {
@ -124,6 +125,7 @@ describe('raster', function() {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(mapconfig)

15
test/acceptance/ported/regressions.js
View File

@ -8,21 +8,6 @@ describe('regressions', function() {
testHelper.rmdirRecursiveSync(global.environment.millstone.cache_basedir);
});
// See https://github.com/Vizzuality/Windshaft/issues/65
it("#65 catching non-Error exception doesn't kill the backend", function(done) {
var mapConfig = testClient.defaultTableMapConfig('test_table');
testClient.withLayergroup(mapConfig, function(err, requestTile, finish) {
var options = {
statusCode: 400,
contentType: 'application/json; charset=utf-8'
};
requestTile('/0/0/0.png?testUnexpectedError=1', options, function(err, res) {
assert.deepEqual(JSON.parse(res.body).errors, ["test unexpected error"]);
finish(done);
});
});
});
// Test that you cannot write to the database from a tile request
//
// See http://github.com/CartoDB/Windshaft/issues/130

6
test/acceptance/ported/retina.js
View File

@ -38,6 +38,7 @@ describe('retina support', function() {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(retinaSampleMapConfig)
@ -67,7 +68,10 @@ describe('retina support', function() {
{
url: '/database/windshaft_test/layergroup/' + layergroupId + '/0/0/0' + scaleFactor + '.png',
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: {
host: 'localhost'
}
},
responseHead,
assertFn

29
test/acceptance/ported/server_gettile.js
View File

@ -59,7 +59,7 @@ describe('server_gettile', function() {
assert.ok(xwc > 0);
assert.ok(xwc >= lastXwc);
requestTile(tileUrl + '?cache_buster=wadus', function (err, res) {
requestTile(tileUrl, { cache_buster: 'wadus' }, function (err, res) {
var xwc = res.headers['x-windshaft-cache'];
assert.ok(!xwc);
@ -99,18 +99,25 @@ describe('server_gettile', function() {
}
testClient.withLayergroup(mapConfig, validateLayergroup, function(err, requestTile, finish) {
requestTile(tileUrl, function(err, res) {
assert.ok(res.headers.hasOwnProperty('x-windshaft-cache'), "Did not hit renderer cache on second time");
assert.ok(res.headers['x-windshaft-cache'] >= 0);
var xwc = res.headers['x-windshaft-cache'];
assert.ok(!xwc);
assert.imageBufferIsSimilarToFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err) {
finish(function(finishErr) {
done(err || finishErr);
});
}
);
requestTile(tileUrl, function (err, res) {
assert.ok(
res.headers.hasOwnProperty('x-windshaft-cache'),
"Did not hit renderer cache on second time"
);
assert.ok(res.headers['x-windshaft-cache'] >= 0);
assert.imageBufferIsSimilarToFile(res.body, imageFixture, IMAGE_EQUALS_TOLERANCE_PER_MIL,
function(err) {
finish(function(finishErr) {
done(err || finishErr);
});
}
);
});
});
});
});

7
test/acceptance/ported/server_png8_format.js
View File

@ -62,6 +62,7 @@ describe('server_png8_format', function() {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroup)
@ -81,7 +82,10 @@ describe('server_png8_format', function() {
var requestPayload = {
url: '/database/windshaft_test/layergroup/' + layergroupId + tilePartialUrl,
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: {
host: 'localhost'
}
};
var requestHeaders = {
@ -179,4 +183,3 @@ describe('server_png8_format', function() {
});
});
});

46
test/acceptance/ported/support/ported_server_options.js
View File

@ -1,7 +1,6 @@
var _ = require('underscore');
var serverOptions = require('../../../../lib/cartodb/server_options');
var mapnik = require('windshaft').mapnik;
var LayergroupToken = require('../../../../lib/cartodb/models/layergroup-token');
var OverviewsQueryRewriter = require('../../../../lib/cartodb/utils/overviews_query_rewriter');
var overviewsQueryRewriter = new OverviewsQueryRewriter({
zoom_level: 'CDB_ZoomFromScale(!scale_denominator!)'
@ -47,48 +46,5 @@ module.exports = _.extend({}, serverOptions, {
enable_cors: global.environment.enable_cors,
unbuffered_logging: true, // for smoother teardown from tests
log_format: null, // do not log anything
afterLayergroupCreateCalls: 0,
useProfiler: true,
req2params: function(req, res, callback){
if ( req.query.testUnexpectedError ) {
return callback('test unexpected error');
}
// this is in case you want to test sql parameters eg ...png?sql=select * from my_table limit 10
req.params = _.extend({}, req.params);
if (req.params.token) {
req.params.token = LayergroupToken.parse(req.params.token).token;
}
_.extend(req.params, req.query);
req.params.user = 'localhost';
res.locals.user = 'localhost';
req.params.dbhost = global.environment.postgres.host;
req.params.dbport = req.params.dbport || global.environment.postgres.port;
req.params.dbuser = 'test_windshaft_publicuser';
if (req.params.dbname !== 'windshaft_test2') {
req.params.dbuser = 'test_windshaft_cartodb_user_1';
}
req.params.dbname = 'test_windshaft_cartodb_user_1_db';
// add all params to res.locals
res.locals = _.extend({}, req.params);
// increment number of calls counter
global.req2params_calls = global.req2params_calls ? global.req2params_calls + 1 : 1;
// send the finished req object on
callback(null,req);
},
afterLayergroupCreate: function(req, cfg, res, callback) {
res.layercount = cfg.layers.length;
// config.afterLayergroupCreateCalls++;
callback(null);
}
useProfiler: true
});

31
test/acceptance/ported/support/test_client.js
View File

@ -72,7 +72,7 @@ function createLayergroup(layergroupConfig, options, callback) {
});
},
function validateLayergroup(err, res) {
assert.ok(!err, 'Failed to request layergroup');
assert.ifError(err);
var parsedBody;
var layergroupid;
@ -129,6 +129,7 @@ function layergroupRequest(layergroupConfig, method, callbackName, extraParams)
var request = {
url: '/database/windshaft_test/layergroup',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
}
};
@ -337,6 +338,7 @@ function getGeneric(layergroupConfig, url, expectedResponse, callback) {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {
host: 'localhost',
'Content-Type': 'application/json'
},
data: JSON.stringify(layergroupConfig)
@ -372,7 +374,10 @@ function getGeneric(layergroupConfig, url, expectedResponse, callback) {
var request = {
url: finalUrl,
method: 'GET'
method: 'GET',
headers: {
host: 'localhost'
}
};
if (contentType === pngContentType) {
@ -449,12 +454,28 @@ function withLayergroup(layergroupConfig, options, callback) {
};
}
var baseUrlTpl = '/database/windshaft_test/layergroup/<%= layergroupid %>';
var finalUrl = _.template(baseUrlTpl, { layergroupid: layergroupid }) + layergroupUrl;
const signerTpl = function ({ signer }) {
return `${signer ? `:${signer}@` : ''}`;
};
const cacheTpl = function ({ cache_buster, cacheBuster }) {
return `${cache_buster ? `:${cache_buster}` : `:${cacheBuster}`}`;
};
const urlTpl = function ({layergroupid, cache_buster = null, tile }) {
const { signer, token , cacheBuster } = LayergroupToken.parse(layergroupid);
const base = '/database/windshaft_test/layergroup/';
return `${base}${signerTpl({signer})}${token}${cacheTpl({cache_buster, cacheBuster})}${tile}`;
};
const finalUrl = urlTpl({ layergroupid, cache_buster: options.cache_buster, tile: layergroupUrl });
var request = {
url: finalUrl,
method: 'GET'
method: 'GET',
headers: {
host: 'localhost'
}
};
if (options.contentType === pngContentType) {

46
test/acceptance/ported/torque.js
View File

@ -48,7 +48,7 @@ describe('torque', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) { next(null, res); });
},
@ -71,7 +71,7 @@ describe('torque', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) { next(null, res); });
},
@ -94,7 +94,7 @@ describe('torque', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) { next(null, res); });
},
@ -136,7 +136,7 @@ describe('torque', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) { next(null, res); });
},
@ -179,7 +179,7 @@ describe('torque', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(mapconfig)
}, {}, function(res, err) { next(err, res); });
},
@ -218,7 +218,10 @@ describe('torque', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0.png',
method: 'GET',
encoding: 'binary'
encoding: 'binary',
headers: {
host: 'localhost'
}
}, {}, function(res, err) { next(err, res); });
},
function check_mapnik_error_1(err, res) {
@ -235,7 +238,10 @@ describe('torque', function() {
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.grid.json',
method: 'GET'
method: 'GET',
headers: {
host: 'localhost'
}
}, {}, function(res, err) { next(err, res); });
},
function check_mapnik_error_2(err, res) {
@ -252,7 +258,10 @@ describe('torque', function() {
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.json.torque',
method: 'GET'
method: 'GET',
headers: {
host: 'localhost'
}
}, {}, function(res, err) { next(err, res); });
},
function check_torque0_response(err, res) {
@ -270,7 +279,10 @@ describe('torque', function() {
var next = this;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/0/0/0.torque.json',
method: 'GET'
method: 'GET',
headers: {
host: 'localhost'
}
}, {}, function(res, err) { next(err, res); });
},
function check_torque0_response_1(err, res) {
@ -315,7 +327,7 @@ describe('torque', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(mapconfig)
}, {}, function(res, err) { next(err, res); });
},
@ -354,19 +366,25 @@ describe('torque', function() {
]
};
let defautlPort = global.environment.postgres.port;
step(
function do_post()
{
var next = this;
global.environment.postgres.port = 54777;
assert.response(server, {
url: '/database/windshaft_test/layergroup?dbport=54777',
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(mapconfig)
}, {}, function(res, err) { next(err, res); });
},
function checkPost(err, res) {
assert.ifError(err);
global.environment.postgres.port = defautlPort;
assert.equal(res.statusCode, 500, res.statusCode + ': ' + res.body);
var parsed = JSON.parse(res.body);
assert.ok(parsed.errors, parsed);
@ -407,9 +425,9 @@ describe('torque', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(layergroup)
}, {}, function(res) { next(null, res); });
}, {}, function (res) { next(null, res); });
},
function checkResponse(err, res) {
assert.ifError(err);

21
test/acceptance/ported/torque_boundaries.js
View File

@ -237,7 +237,7 @@ describe('torque boundary points', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(boundaryPointsMapConfig)
}, {}, function (res, err) {
@ -250,7 +250,10 @@ describe('torque boundary points', function() {
var partialUrl = tileRequest.z + '/' + tileRequest.x + '/' + tileRequest.y;
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + expected_token + '/0/' + partialUrl + '.json.torque',
method: 'GET'
method: 'GET',
headers: {
host: 'localhost'
}
}, {}, function (res, err) {
assert.ok(!err, 'Failed to get json');
@ -351,7 +354,7 @@ describe('torque boundary points', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(londonPointMapConfig)
}, {}, function (res, err) {
assert.ok(!err, 'Failed to create layergroup');
@ -363,7 +366,10 @@ describe('torque boundary points', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + layergroupId + '/0/2/1/1.json.torque',
method: 'GET'
method: 'GET',
headers: {
host: 'localhost'
}
}, {}, function (res, err) {
assert.ok(!err, 'Failed to request torque.json');
@ -414,7 +420,7 @@ describe('torque boundary points', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup',
method: 'POST',
headers: {'Content-Type': 'application/json' },
headers: { host: 'localhost', 'Content-Type': 'application/json' },
data: JSON.stringify(londonPointMapConfig)
}, {}, function (res, err) {
assert.ok(!err, 'Failed to create layergroup');
@ -426,7 +432,10 @@ describe('torque boundary points', function() {
assert.response(server, {
url: '/database/windshaft_test/layergroup/' + layergroupId + '/0/13/4255/2765.json.torque',
method: 'GET'
method: 'GET',
headers: {
host: 'localhost'
}
}, {}, function (res, err) {
assert.ok(!err, 'Failed to request torque.json');

166
test/acceptance/rate-limit.test.js
View File

@ -15,6 +15,7 @@ let redisClient;
let testClient;
let keysToDelete = ['user:localhost:mapviews:global'];
const user = 'localhost';
let layergroupid;
const query = `
SELECT
@ -68,13 +69,13 @@ const createMapConfig = ({
});
function setLimit(count, period, burst) {
function setLimit(count, period, burst, endpoint = RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS) {
redisClient.SELECT(8, err => {
if (err) {
return;
}
const key = `limits:rate:store:${user}:maps:${RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS}`;
const key = `limits:rate:store:${user}:maps:${endpoint}`;
redisClient.rpush(key, burst);
redisClient.rpush(key, count);
redisClient.rpush(key, period);
@ -87,8 +88,12 @@ function getReqAndRes() {
req: {},
res: {
headers: {},
set(headers) {
this.headers = headers;
set(headers, value) {
if(typeof headers === 'object') {
this.headers = headers;
} else {
this.headers[headers] = value;
}
},
locals: {
user: 'localhost'
@ -97,18 +102,21 @@ function getReqAndRes() {
};
}
function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done = null) {
const response = {
function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done) {
let response = {
status,
headers: {
'Content-Type': 'application/json; charset=utf-8',
'Carto-Rate-Limit-Limit': limit,
'Carto-Rate-Limit-Remaining': remaining,
'Carto-Rate-Limit-Reset': reset,
'Retry-After': retry
'Carto-Rate-Limit-Reset': reset
}
};
if(retry) {
response.headers['Retry-After'] = retry;
}
testClient.getLayergroup({ response }, err => {
assert.ifError(err);
if (done) {
@ -117,21 +125,26 @@ function assertGetLayergroupRequest (status, limit, remaining, reset, retry, don
});
}
function assertRateLimitRequest (status, limit, remaining, reset, retry, done = null) {
function assertRateLimitRequest (status, limit, remaining, reset, retry, done) {
const { req, res } = getReqAndRes();
rateLimit(req, res, function (err) {
assert.deepEqual(res.headers, {
let expectedHeaders = {
"Carto-Rate-Limit-Limit": limit,
"Carto-Rate-Limit-Remaining": remaining,
"Carto-Rate-Limit-Reset": reset,
"Retry-After": retry
});
"Carto-Rate-Limit-Reset": reset
};
if(retry) {
expectedHeaders['Retry-After'] = retry;
}
assert.deepEqual(res.headers, expectedHeaders);
if(status === 200) {
assert.ifError(err);
} else {
assert.ok(err);
assert.equal(err.message, 'You are over the limits.');
assert.equal(err.message, 'You are over platform\'s limits. Please contact us to know more details');
assert.equal(err.http_status, 429);
assert.equal(err.type, 'limit');
assert.equal(err.subtype, 'rate-limit');
@ -178,7 +191,7 @@ describe('rate limit', function() {
const burst = 1;
setLimit(count, period, burst);
assertGetLayergroupRequest(200, '2', '1', '1', '-1', done);
assertGetLayergroupRequest(200, '2', '1', '1', null, done);
});
it("1 req/sec: 2 req/seg should be limited", function(done) {
@ -187,12 +200,12 @@ describe('rate limit', function() {
const burst = 1;
setLimit(count, period, burst);
assertGetLayergroupRequest(200, '2', '1', '1', '-1');
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1'), 250);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 500);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 750);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 950);
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1', done), 1050);
assertGetLayergroupRequest(200, '2', '1', '1');
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1'), 250);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 500);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 750);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '1'), 950);
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', null, done), 1050);
});
});
@ -234,12 +247,12 @@ describe('rate limit middleware', function () {
});
it("1 req/sec: 2 req/seg should be limited", function (done) {
assertRateLimitRequest(200, 1, 0, 1, -1);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 250);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950);
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050);
assertRateLimitRequest(200, 1, 0, 1);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 250);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 750);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 950);
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, null, done), 1050);
});
it("1 req/sec: 2 req/seg should be limited, removing SHA script from Redis", function (done) {
@ -248,13 +261,100 @@ describe('rate limit middleware', function () {
'SCRIPT',
['FLUSH'],
function () {
assertRateLimitRequest(200, 1, 0, 1, -1);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950);
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050);
assertRateLimitRequest(200, 1, 0, 1);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 750);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 1), 950);
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, null, done), 1050);
}
);
});
});
describe('rate limit and vector tiles', function () {
before(function(done) {
global.environment.enabledFeatures.rateLimitsEnabled = true;
global.environment.enabledFeatures.rateLimitsByEndpoint.tile = true;
redisClient = redis.createClient(global.environment.redis.port);
const count = 1;
const period = 1;
const burst = 0;
setLimit(count, period, burst, RATE_LIMIT_ENDPOINTS_GROUPS.TILE);
testClient = new TestClient(createMapConfig(), 1234);
testClient.getLayergroup({status: 200}, (err, res) => {
assert.ifError(err);
layergroupid = res.layergroupid;
done();
});
});
after(function() {
global.environment.enabledFeatures.rateLimitsEnabled = false;
global.environment.enabledFeatures.rateLimitsByEndpoint.tile = false;
});
afterEach(function(done) {
keysToDelete.forEach( key => {
redisClient.del(key);
});
redisClient.SELECT(0, () => {
redisClient.del('user:localhost:mapviews:global');
redisClient.SELECT(5, () => {
redisClient.del('user:localhost:mapviews:global');
done();
});
});
});
it('mvt rate limited', function (done) {
const tileParams = (status, limit, remaining, reset, retry, contentType) => {
let headers = {
"Content-Type": contentType,
"Carto-Rate-Limit-Limit": limit,
"Carto-Rate-Limit-Remaining": remaining,
"Carto-Rate-Limit-Reset": reset
};
if (retry) {
headers['Retry-After'] = retry;
}
return {
layergroupid: layergroupid,
format: 'mvt',
response: {status, headers}
};
};
testClient.getTile(0, 0, 0, tileParams(204, '1', '0', '1'), (err) => {
assert.ifError(err);
testClient.getTile(
0,
0,
0,
tileParams(429, '1', '0', '0', '1', 'application/x-protobuf'),
(err, res, tile) => {
assert.ifError(err);
var tileJSON = tile.toJSON();
assert.equal(Array.isArray(tileJSON), true);
assert.equal(tileJSON.length, 2);
assert.equal(tileJSON[0].name, 'errorTileSquareLayer');
assert.equal(tileJSON[1].name, 'errorTileStripesLayer');
done();
}
);
});
});
});

2
test/acceptance/turbo-carto/regressions.js
View File

@ -88,7 +88,7 @@ describe('turbo-carto regressions', function() {
var turboCartoError = layergroup.errors_with_context[0];
assert.ok(turboCartoError);
assert.equal(turboCartoError.type, 'layer');
assert.ok(turboCartoError.message.match(/permission\sdenied\sfor\srelation\stest_table_private_1/));
assert.ok(turboCartoError.message.match(/permission\sdenied\sfor\s.+?test_table_private_1/));
done();
});

BIN
test/support/libredis_cell.dylib Executable file
View File

Binary file not shown.

BIN
test/support/libredis_cell.so Executable file
View File

Binary file not shown.

28
test/unit/cartodb/prepare-context.test.js
View File

@ -7,11 +7,10 @@ var PgConnection = require('../../../lib/cartodb/backends/pg_connection');
var AuthApi = require('../../../lib/cartodb/api/auth_api');
var TemplateMaps = require('../../../lib/cartodb/backends/template_maps');
const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/middleware/context/clean-up-query-params');
const authorizeMiddleware = require('../../../lib/cartodb/middleware/context/authorize');
const dbConnSetupMiddleware = require('../../../lib/cartodb/middleware/context/db-conn-setup');
const credentialsMiddleware = require('../../../lib/cartodb/middleware/context/credentials');
const localsMiddleware = require('../../../lib/cartodb/middleware/context/locals');
const cleanUpQueryParamsMiddleware = require('../../../lib/cartodb/middleware/clean-up-query-params');
const authorizeMiddleware = require('../../../lib/cartodb/middleware/authorize');
const dbConnSetupMiddleware = require('../../../lib/cartodb/middleware/db-conn-setup');
const credentialsMiddleware = require('../../../lib/cartodb/middleware/credentials');
var windshaft = require('windshaft');
@ -66,18 +65,6 @@ describe('prepare-context', function() {
return res;
}
it('res.locals are created', function(done) {
const locals = localsMiddleware();
let req = {};
let res = {};
locals(prepareRequest(req), prepareResponse(res), function(err) {
if ( err ) { done(err); return; }
assert.ok(res.hasOwnProperty('locals'), 'response has locals');
done();
});
});
it('cleans up request', function(done){
var req = {headers: { host:'localhost' }, query: {dbuser:'hacker',dbname:'secret'}};
var res = {};
@ -178,10 +165,9 @@ describe('prepare-context', function() {
return done(err);
}
var query = res.locals;
assert.deepEqual(config, query.config);
assert.equal('test', query.api_key);
assert.equal(undefined, query.non_included);
assert.deepEqual(config, req.query.config);
assert.equal('test', req.query.api_key);
assert.equal(undefined, req.query.non_included);
done();
});
});