cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'spread-prepare-context-middleware' into unify-middlewares

Browse Source
This commit is contained in:
Daniel García Aubert 2018-03-20 11:06:31 +01:00
commit df999e040c
20 changed files with 596 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -11,3 +11,4 @@ redis.pid
*.log
coverage/
.DS_Store
libredis_cell.so

21
NEWS.md
View File

@ -1,13 +1,32 @@
# Changelog
## 6.0.0
Released 2018-03-19
Backward incompatible changes:
- Needs Redis v4
New features:
- Upgrades camshaft to 0.61.8
- Upgrades cartodb-redis to 1.0.0
- Rate limit feature (disabled by default)
## 5.4.0
Released yyyy-mm-dd
Released 2018-03-15
- Upgrades Windshaft to 4.5.7 ([Mapnik top metrics](https://github.com/CartoDB/Windshaft/pull/597), [AttributesBackend allows multiple features if all the attributes are the same](https://github.com/CartoDB/Windshaft/pull/602))
- Implemented middleware to authorize users via new Api Key system
- Keep the old authorization system as fallback
- Aggregation widget: Remove NULL categories in 'count' aggregations too
- Update request to 2.85.0
- Update camshaft to 0.61.4 (Fixes for AOI and Merge analyses)
- Update windshaft to 4.6.0, which in turn updates @carto/mapnik to 3.6.2-carto.4 and related dependencies. It brings in a cache for rasterized symbols. See https://github.com/CartoDB/node-mapnik/blob/v3.6.2-carto/CHANGELOG.carto.md#362-carto4
- PostGIS: Variables in postgis SQL queries must now additionally be wrapped in `!` (refs [#29](https://github.com/CartoDB/mapnik/issues/29), [mapnik/#3618](https://github.com/mapnik/mapnik/pull/3618)):
```sql
-- Before
SELECT ... WHERE trait = @variable
-- Now
SELECT ... WHERE trait = !@variable!
```
## 5.3.1
Released 2018-02-13

23
config/environments/development.js.example
View File

@ -343,7 +343,28 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerStats: true
layerStats: true,
// whether it should rate limit endpoints (global configuration)
rateLimitsEnabled: false,
// whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
rateLimitsByEndpoint: {
anonymous: false,
static: false,
static_named: false,
dataview: false,
dataview_search: false,
analysis: false,
analysis_catalog: false,
tile: false,
attributes: false,
named_list: false,
named_create: false,
named_get: false,
named: false,
named_update: false,
named_delete: false,
named_tiles: false
}
}
};

23
config/environments/production.js.example
View File

@ -345,7 +345,28 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerStats: false
layerStats: false,
// whether it should rate limit endpoints (global configuration)
rateLimitsEnabled: false,
// whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
rateLimitsByEndpoint: {
anonymous: false,
static: false,
static_named: false,
dataview: false,
dataview_search: false,
analysis: false,
analysis_catalog: false,
tile: false,
attributes: false,
named_list: false,
named_create: false,
named_get: false,
named: false,
named_update: false,
named_delete: false,
named_tiles: false
}
}
};

23
config/environments/staging.js.example
View File

@ -345,7 +345,28 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerStats: true
layerStats: true,
// whether it should rate limit endpoints (global configuration)
rateLimitsEnabled: false,
// whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
rateLimitsByEndpoint: {
anonymous: false,
static: false,
static_named: false,
dataview: false,
dataview_search: false,
analysis: false,
analysis_catalog: false,
tile: false,
attributes: false,
named_list: false,
named_create: false,
named_get: false,
named: false,
named_update: false,
named_delete: false,
named_tiles: false
}
}
};

23
config/environments/test.js.example
View File

@ -339,7 +339,28 @@ var config = {
// whether the affected tables for a given SQL must query directly postgresql or use the SQL API
cdbQueryTablesFromPostgres: true,
// whether in mapconfig is available stats & metadata for each layer
layerStats: true
layerStats: true,
// whether it should rate limit endpoints (global configuration)
rateLimitsEnabled: false,
// whether it should rate limit one or more endpoints (only if rateLimitsEnabled = true)
rateLimitsByEndpoint: {
anonymous: false,
static: false,
static_named: false,
dataview: false,
dataview_search: false,
analysis: false,
analysis_catalog: false,
tile: false,
attributes: false,
named_list: false,
named_create: false,
named_get: false,
named: false,
named_update: false,
named_delete: false,
named_tiles: false
}
}
};

12
lib/cartodb/api/user_limits_api.js
View File

@ -11,6 +11,8 @@ function UserLimitsApi(metadataBackend, options) {
this.metadataBackend = metadataBackend;
this.options = options || {};
this.options.limits = this.options.limits || {};
this.preprareRateLimit();
}
module.exports = UserLimitsApi;
@ -77,3 +79,13 @@ UserLimitsApi.prototype.getTimeoutRenderLimit = function (username, apiKey, call
callback
);
};
UserLimitsApi.prototype.preprareRateLimit = function () {
if (this.options.limits.rateLimitsEnabled) {
this.metadataBackend.loadRateLimitsScript();
}
};
UserLimitsApi.prototype.getRateLimit = function (user, endpointGroup, callback) {
this.metadataBackend.getRateLimit(user, 'maps', endpointGroup, callback);
};

6
lib/cartodb/controllers/analyses.js
View File

@ -7,11 +7,14 @@ const layergroupToken = require('../middleware/layergroup-token');
const credentials = require('../middleware/credentials');
const authorize = require('../middleware/authorize');
const dbConnSetup = require('../middleware/db-conn-setup');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const sendResponse = require('../middleware/send-response');
function AnalysesController(pgConnection, authApi) {
function AnalysesController(pgConnection, authApi, userLimitsApi) {
this.pgConnection = pgConnection;
this.authApi = authApi;
this.userLimitsApi = userLimitsApi;
}
module.exports = AnalysesController;
@ -25,6 +28,7 @@ AnalysesController.prototype.register = function (app) {
cleanUpQueryParams(),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS_CATALOG),
layergroupToken(),
credentials(),
authorize(this.authApi),

27
lib/cartodb/controllers/layergroup.js
View File

@ -7,6 +7,8 @@ const layergroupToken = require('../middleware/layergroup-token');
const credentials = require('../middleware/credentials');
const dbConnSetup = require('../middleware/db-conn-setup');
const authorize = require('../middleware/authorize');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const sendResponse = require('../middleware/send-response');
const DataviewBackend = require('../backends/dataview');
const AnalysisStatusBackend = require('../backends/analysis-status');
@ -34,8 +36,18 @@ const SUPPORTED_FORMATS = {
* @param {AnalysisBackend} analysisBackend
* @constructor
*/
function LayergroupController(pgConnection, mapStore, tileBackend, previewBackend, attributesBackend,
surrogateKeysCache, userLimitsApi, layergroupAffectedTables, analysisBackend, authApi) {
function LayergroupController(
pgConnection,
mapStore,
tileBackend,
previewBackend,
attributesBackend,
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTables,
analysisBackend,
authApi
) {
this.pgConnection = pgConnection;
this.mapStore = mapStore;
this.tileBackend = tileBackend;
@ -61,6 +73,7 @@ LayergroupController.prototype.register = function(app) {
cleanUpQueryParams(),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
layergroupToken(),
credentials(),
authorize(this.authApi),
@ -85,6 +98,7 @@ LayergroupController.prototype.register = function(app) {
cleanUpQueryParams(),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
layergroupToken(),
credentials(),
authorize(this.authApi),
@ -110,6 +124,7 @@ LayergroupController.prototype.register = function(app) {
cleanUpQueryParams(),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.TILE),
layergroupToken(),
credentials(),
authorize(this.authApi),
@ -134,6 +149,7 @@ LayergroupController.prototype.register = function(app) {
cleanUpQueryParams(),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ATTRIBUTES),
layergroupToken(),
credentials(),
authorize(this.authApi),
@ -156,6 +172,7 @@ LayergroupController.prototype.register = function(app) {
cleanUpQueryParams(['layer']),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
layergroupToken(),
credentials(),
authorize(this.authApi),
@ -176,6 +193,7 @@ LayergroupController.prototype.register = function(app) {
cleanUpQueryParams(['layer']),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC),
layergroupToken(),
credentials(),
authorize(this.authApi),
@ -214,6 +232,7 @@ LayergroupController.prototype.register = function(app) {
cleanUpQueryParams(allowedDataviewQueryParams),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
layergroupToken(),
credentials(),
authorize(this.authApi),
@ -234,6 +253,7 @@ LayergroupController.prototype.register = function(app) {
cleanUpQueryParams(allowedDataviewQueryParams),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW),
layergroupToken(),
credentials(),
authorize(this.authApi),
@ -254,6 +274,7 @@ LayergroupController.prototype.register = function(app) {
cleanUpQueryParams(allowedDataviewQueryParams),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
layergroupToken(),
credentials(),
authorize(this.authApi),
@ -274,6 +295,7 @@ LayergroupController.prototype.register = function(app) {
cleanUpQueryParams(allowedDataviewQueryParams),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.DATAVIEW_SEARCH),
layergroupToken(),
credentials(),
authorize(this.authApi),
@ -294,6 +316,7 @@ LayergroupController.prototype.register = function(app) {
cleanUpQueryParams(),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANALYSIS),
layergroupToken(),
credentials(),
authorize(this.authApi),

42
lib/cartodb/controllers/map.js
View File

@ -17,6 +17,8 @@ const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const NamedMapMapConfigProvider = require('../models/mapconfig/provider/named-map-provider');
const CreateLayergroupMapConfigProvider = require('../models/mapconfig/provider/create-layergroup-provider');
const LayergroupMetadata = require('../utils/layergroup-metadata');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
/**
* @param {AuthApi} authApi
@ -31,9 +33,18 @@ const LayergroupMetadata = require('../utils/layergroup-metadata');
* @param {StatsBackend} statsBackend
* @constructor
*/
function MapController(pgConnection, templateMaps, mapBackend, metadataBackend,
surrogateKeysCache, userLimitsApi, layergroupAffectedTables, mapConfigAdapter,
statsBackend, authApi) {
function MapController (
pgConnection,
templateMaps,
mapBackend,
metadataBackend,
surrogateKeysCache,
userLimitsApi,
layergroupAffectedTables,
mapConfigAdapter,
statsBackend,
authApi
) {
this.pgConnection = pgConnection;
this.templateMaps = templateMaps;
this.mapBackend = mapBackend;
@ -55,18 +66,32 @@ module.exports = MapController;
MapController.prototype.register = function(app) {
const { base_url_mapconfig: mapconfigBasePath, base_url_templated: templateBasePath } = app;
app.get(`${mapconfigBasePath}`, this.composeCreateMapMiddleware());
app.post(`${mapconfigBasePath}`, this.composeCreateMapMiddleware());
app.get(
`${mapconfigBasePath}`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
app.post(
`${mapconfigBasePath}`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS)
);
const useTemplate = true;
app.get(`${templateBasePath}/:template_id/jsonp`, this.composeCreateMapMiddleware(useTemplate));
app.post(`${templateBasePath}/:template_id`, this.composeCreateMapMiddleware(useTemplate));
app.get(
`${templateBasePath}/:template_id/jsonp`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate)
);
app.post(
`${templateBasePath}/:template_id`,
this.composeCreateMapMiddleware(RATE_LIMIT_ENDPOINTS_GROUPS.NAMED, useTemplate)
);
app.options(app.base_url_mapconfig, cors('Content-Type'));
};
MapController.prototype.composeCreateMapMiddleware = function (useTemplate = false) {
MapController.prototype.composeCreateMapMiddleware = function (endpointGroup, useTemplate = false) {
const isTemplateInstantiation = useTemplate;
const useTemplateHash = useTemplate;
const includeQuery = !useTemplate;
@ -78,6 +103,7 @@ MapController.prototype.composeCreateMapMiddleware = function (useTemplate = fal
cleanUpQueryParams(['aggregation']),
locals(),
user(),
rateLimit(this.userLimitsApi, endpointGroup),
layergroupToken(),
credentials(),
authorize(this.authApi),

18
lib/cartodb/controllers/named_maps.js
View File

@ -9,6 +9,8 @@ const dbConnSetup = require('../middleware/db-conn-setup');
const authorize = require('../middleware/authorize');
const sendResponse = require('../middleware/send-response');
const vectorError = require('../middleware/vector-error');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const DEFAULT_ZOOM_CENTER = {
zoom: 1,
@ -33,8 +35,17 @@ function getRequestParams(locals) {
return params;
}
function NamedMapsController(namedMapProviderCache, tileBackend, previewBackend,
surrogateKeysCache, tablesExtentApi, metadataBackend, pgConnection, authApi) {
function NamedMapsController (
namedMapProviderCache,
tileBackend,
previewBackend,
surrogateKeysCache,
tablesExtentApi,
metadataBackend,
pgConnection,
authApi,
userLimitsApi
) {
this.namedMapProviderCache = namedMapProviderCache;
this.tileBackend = tileBackend;
this.previewBackend = previewBackend;
@ -43,6 +54,7 @@ function NamedMapsController(namedMapProviderCache, tileBackend, previewBackend,
this.metadataBackend = metadataBackend;
this.pgConnection = pgConnection;
this.authApi = authApi;
this.userLimitsApi = userLimitsApi;
}
module.exports = NamedMapsController;
@ -56,6 +68,7 @@ NamedMapsController.prototype.register = function(app) {
cleanUpQueryParams(),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_TILES),
layergroupToken(),
credentials(),
authorize(this.authApi),
@ -84,6 +97,7 @@ NamedMapsController.prototype.register = function(app) {
cleanUpQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.STATIC_NAMED),
layergroupToken(),
credentials(),
authorize(this.authApi),

10
lib/cartodb/controllers/named_maps_admin.js
View File

@ -3,6 +3,8 @@ const cors = require('../middleware/cors');
const user = require('../middleware/user');
const locals = require('../middleware/locals');
const credentials = require('../middleware/credentials');
const rateLimit = require('../middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const sendResponse = require('../middleware/send-response');
/**
@ -11,9 +13,10 @@ const sendResponse = require('../middleware/send-response');
* @param {TemplateMaps} templateMaps
* @constructor
*/
function NamedMapsAdminController(authApi, templateMaps) {
function NamedMapsAdminController(authApi, templateMaps, userLimitsApi) {
this.authApi = authApi;
this.templateMaps = templateMaps;
this.userLimitsApi = userLimitsApi;
}
module.exports = NamedMapsAdminController;
@ -26,6 +29,7 @@ NamedMapsAdminController.prototype.register = function (app) {
cors(),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_CREATE),
credentials(),
checkContentType({ action: 'POST', label: 'POST TEMPLATE' }),
authorizedByAPIKey({ authApi: this.authApi, action: 'create', label: 'POST TEMPLATE' }),
@ -38,6 +42,7 @@ NamedMapsAdminController.prototype.register = function (app) {
cors(),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_UPDATE),
credentials(),
checkContentType({ action: 'PUT', label: 'PUT TEMPLATE' }),
authorizedByAPIKey({ authApi: this.authApi, action: 'update', label: 'PUT TEMPLATE' }),
@ -50,6 +55,7 @@ NamedMapsAdminController.prototype.register = function (app) {
cors(),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_GET),
credentials(),
authorizedByAPIKey({ authApi: this.authApi, action: 'get', label: 'GET TEMPLATE' }),
retrieveTemplate({ templateMaps: this.templateMaps }),
@ -61,6 +67,7 @@ NamedMapsAdminController.prototype.register = function (app) {
cors(),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_DELETE),
credentials(),
authorizedByAPIKey({ authApi: this.authApi, action: 'delete', label: 'DELETE TEMPLATE' }),
destroyTemplate({ templateMaps: this.templateMaps }),
@ -72,6 +79,7 @@ NamedMapsAdminController.prototype.register = function (app) {
cors(),
locals(),
user(),
rateLimit(this.userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.NAMED_LIST),
credentials(),
authorizedByAPIKey({ authApi: this.authApi, action: 'list', label: 'GET TEMPLATE LIST' }),
listTemplates({ templateMaps: this.templateMaps }),

67
lib/cartodb/middleware/rate-limit.js Normal file
View File

@ -0,0 +1,67 @@
'use strict';
const RATE_LIMIT_ENDPOINTS_GROUPS = {
ANONYMOUS: 'anonymous',
STATIC: 'static',
STATIC_NAMED: 'static_named',
DATAVIEW: 'dataview',
DATAVIEW_SEARCH: 'dataview_search',
ANALYSIS: 'analysis',
ANALYSIS_CATALOG: 'analysis_catalog',
TILE: 'tile',
ATTRIBUTES: 'attributes',
NAMED_LIST: 'named_list',
NAMED_CREATE: 'named_create',
NAMED_GET: 'named_get',
NAMED: 'named',
NAMED_UPDATE: 'named_update',
NAMED_DELETE: 'named_delete',
NAMED_TILES: 'named_tiles'
};
function rateLimit(userLimitsApi, endpointGroup = null) {
if (!isRateLimitEnabled(endpointGroup)) {
return function rateLimitDisabledMiddleware(req, res, next) { next(); };
}
return function rateLimitMiddleware(req, res, next) {
userLimitsApi.getRateLimit(res.locals.user, endpointGroup, function (err, userRateLimit) {
if (err) {
return next(err);
}
if (!userRateLimit) {
return next();
}
const [isBlocked, limit, remaining, retry, reset] = userRateLimit;
res.set({
'Carto-Rate-Limit-Limit': limit,
'Carto-Rate-Limit-Remaining': remaining,
'Retry-After': retry,
'Carto-Rate-Limit-Reset': reset
});
if (isBlocked) {
const rateLimitError = new Error('You are over the limits.');
rateLimitError.http_status = 429;
rateLimitError.type = 'limit';
rateLimitError.subtype = 'rate-limit';
return next(rateLimitError);
}
return next();
});
};
}
function isRateLimitEnabled(endpointGroup) {
return global.environment.enabledFeatures.rateLimitsEnabled &&
endpointGroup &&
global.environment.enabledFeatures.rateLimitsByEndpoint[endpointGroup];
}
module.exports = rateLimit;
module.exports.RATE_LIMIT_ENDPOINTS_GROUPS = RATE_LIMIT_ENDPOINTS_GROUPS;

10
lib/cartodb/server.js
View File

@ -74,7 +74,8 @@ module.exports = function(serverOptions) {
var userLimitsApi = new UserLimitsApi(metadataBackend, {
limits: {
cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
render: serverOptions.renderer.mapnik.limits.render || 0
render: serverOptions.renderer.mapnik.limits.render || 0,
rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled
}
});
@ -251,12 +252,13 @@ module.exports = function(serverOptions) {
tablesExtentApi,
metadataBackend,
pgConnection,
authApi
authApi,
userLimitsApi
).register(app);
new controller.NamedMapsAdmin(authApi, templateMaps).register(app);
new controller.NamedMapsAdmin(authApi, templateMaps, userLimitsApi).register(app);
new controller.Analyses(pgConnection, authApi).register(app);
new controller.Analyses(pgConnection, authApi, userLimitsApi).register(app);
new controller.ServerInfo(versions).register(app);

8
package.json
View File

@ -1,7 +1,7 @@
{
"private": true,
"name": "windshaft-cartodb",
"version": "5.3.2",
"version": "6.0.0",
"description": "A map tile server for CartoDB",
"keywords": [
"cartodb"
@ -26,10 +26,10 @@
"dependencies": {
"basic-auth": "^2.0.0",
"body-parser": "^1.18.2",
"camshaft": "0.61.4",
"camshaft": "0.61.8",
"cartodb-psql": "0.10.2",
"cartodb-query-tables": "0.3.0",
"cartodb-redis": "0.16.0",
"cartodb-redis": "1.0.0",
"debug": "^3.1.0",
"dot": "~1.0.2",
"express": "~4.16.0",
@ -48,7 +48,7 @@
"step-profiler": "~0.3.0",
"turbo-carto": "0.20.2",
"underscore": "~1.6.0",
"windshaft": "4.5.7",
"windshaft": "4.6.0",
"yargs": "~5.0.0"
},
"devDependencies": {

20
run_tests.sh
View File

@ -6,6 +6,7 @@ OPT_DROP_REDIS=yes # drop the redis test environment
OPT_DROP_PGSQL=yes # drop the PostgreSQL test environment
OPT_COVERAGE=no # run tests with coverage
OPT_DOWNLOAD_SQL=yes # download a fresh copy of sql files
OPT_REDIS_CELL=yes # download redis cell
export PGAPPNAME=cartodb_tiler_tester
@ -49,6 +50,17 @@ die() {
exit 1
}
get_redis_cell() {
if test x"$OPT_REDIS_CELL" = xyes; then
echo "Downloading redis-cell"
curl -L https://github.com/brandur/redis-cell/releases/download/v0.2.2/redis-cell-v0.2.2-x86_64-unknown-linux-gnu.tar.gz --output redis-cell.tar.gz > /dev/null 2>&1
tar xvzf redis-cell.tar.gz > /dev/null 2>&1
mv libredis_cell.so ${BASEDIR}/test/support/libredis_cell.so
rm redis-cell.tar.gz
rm libredis_cell.d
fi
}
trap 'cleanup_and_exit' 1 2 3 5 9 13
while [ -n "$1" ]; do
@ -88,6 +100,10 @@ while [ -n "$1" ]; do
OPT_CREATE_PGSQL=no
shift
continue
elif test "$1" = "--norediscell"; then
OPT_REDIS_CELL=no
shift
continue
else
break
fi
@ -99,14 +115,16 @@ if [ -z "$1" ]; then
echo " --nocreate do not create the test environment on start" >&2
echo " --nodrop do not drop the test environment on exit" >&2
echo " --with-coverage use istanbul to determine code coverage" >&2
echo " --norediscell do not download redis-cell" >&2
exit 1
fi
TESTS=$@
if test x"$OPT_CREATE_REDIS" = xyes; then
get_redis_cell
echo "Starting redis on port ${REDIS_PORT}"
echo "port ${REDIS_PORT}" | redis-server - > ${BASEDIR}/test.log &
echo "port ${REDIS_PORT}" | redis-server - --loadmodule ${BASEDIR}/test/support/libredis_cell.so > ${BASEDIR}/test.log &
PID_REDIS=$!
echo ${PID_REDIS} > ${BASEDIR}/redis.pid
fi

4
test/acceptance/named_layers.js
View File

@ -108,7 +108,7 @@ describe('named_layers', function() {
});
beforeEach(function(done) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: true};
global.environment.enabledFeatures.cdbQueryTablesFromPostgres = true;
templateMaps.addTemplate(username, nestedNamedMapTemplate, function(err) {
if (err) {
return done(err);
@ -125,7 +125,7 @@ describe('named_layers', function() {
});
afterEach(function(done) {
global.environment.enabledFeatures = {cdbQueryTablesFromPostgres: false};
global.environment.enabledFeatures.cdbQueryTablesFromPostgres = false;
templateMaps.delTemplate(username, nestedNamedMapTemplateName, function(err) {
if (err) {
return done(err);

260
test/acceptance/rate-limit.test.js Normal file
View File

@ -0,0 +1,260 @@
require('../support/test_helper');
const assert = require('../support/assert');
const redis = require('redis');
const RedisPool = require('redis-mpool');
const cartodbRedis = require('cartodb-redis');
const TestClient = require('../support/test-client');
const UserLimitsApi = require('../../lib/cartodb/api/user_limits_api');
const rateLimitMiddleware = require('../../lib/cartodb/middleware/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware;
let userLimitsApi;
let rateLimit;
let redisClient;
let testClient;
let keysToDelete = ['user:localhost:mapviews:global'];
const user = 'localhost';
const query = `
SELECT
ST_Transform('SRID=4326;POINT(-180 85.05112877)'::geometry, 3857) the_geom_webmercator,
1 cartodb_id,
2 val
`;
const createMapConfig = ({
version = '1.6.0',
type = 'cartodb',
sql = query,
cartocss = TestClient.CARTOCSS.POINTS,
cartocss_version = '2.3.0',
interactivity = 'cartodb_id',
countBy = 'cartodb_id'
} = {}) => ({
version,
layers: [{
type,
options: {
source: {
id: 'a0'
},
cartocss,
cartocss_version,
interactivity
}
}],
analyses: [
{
id: 'a0',
type: 'source',
params: {
query: sql
}
}
],
dataviews: {
count: {
source: {
id: 'a0'
},
type: 'formula',
options: {
column: countBy,
operation: 'count'
}
}
}
});
function setLimit(count, period, burst) {
redisClient.SELECT(8, err => {
if (err) {
return;
}
const key = `limits:rate:store:${user}:maps:${RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS}`;
redisClient.rpush(key, burst);
redisClient.rpush(key, count);
redisClient.rpush(key, period);
keysToDelete.push(key);
});
}
function getReqAndRes() {
return {
req: {},
res: {
headers: {},
set(headers) {
this.headers = headers;
},
locals: {
user: 'localhost'
}
}
};
}
function assertGetLayergroupRequest (status, limit, remaining, reset, retry, done = null) {
const response = {
status,
headers: {
'Content-Type': 'application/json; charset=utf-8',
'Carto-Rate-Limit-Limit': limit,
'Carto-Rate-Limit-Remaining': remaining,
'Carto-Rate-Limit-Reset': reset,
'Retry-After': retry
}
};
testClient.getLayergroup({ response }, err => {
assert.ifError(err);
if (done) {
setTimeout(done, 1000);
}
});
}
function assertRateLimitRequest (status, limit, remaining, reset, retry, done = null) {
const { req, res } = getReqAndRes();
rateLimit(req, res, function (err) {
assert.deepEqual(res.headers, {
"Carto-Rate-Limit-Limit": limit,
"Carto-Rate-Limit-Remaining": remaining,
"Carto-Rate-Limit-Reset": reset,
"Retry-After": retry
});
if(status === 200) {
assert.ifError(err);
} else {
assert.ok(err);
assert.equal(err.message, 'You are over the limits.');
assert.equal(err.http_status, 429);
assert.equal(err.type, 'limit');
assert.equal(err.subtype, 'rate-limit');
}
if (done) {
setTimeout(done, 1000);
}
});
}
describe('rate limit', function() {
before(function() {
global.environment.enabledFeatures.rateLimitsEnabled = true;
global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true;
redisClient = redis.createClient(global.environment.redis.port);
testClient = new TestClient(createMapConfig(), 1234);
});
after(function() {
global.environment.enabledFeatures.rateLimitsEnabled = false;
global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = false;
});
afterEach(function(done) {
keysToDelete.forEach( key => {
redisClient.del(key);
});
redisClient.SELECT(0, () => {
redisClient.del('user:localhost:mapviews:global');
redisClient.SELECT(5, () => {
redisClient.del('user:localhost:mapviews:global');
done();
});
});
});
it('should not be rate limited', function (done) {
const count = 1;
const period = 1;
const burst = 1;
setLimit(count, period, burst);
assertGetLayergroupRequest(200, '2', '1', '1', '-1', done);
});
it("1 req/sec: 2 req/seg should be limited", function(done) {
const count = 1;
const period = 1;
const burst = 1;
setLimit(count, period, burst);
assertGetLayergroupRequest(200, '2', '1', '1', '-1');
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1'), 250);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 500);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 750);
setTimeout( () => assertGetLayergroupRequest(429, '2', '0', '1', '0'), 950);
setTimeout( () => assertGetLayergroupRequest(200, '2', '0', '1', '-1', done), 1050);
});
});
describe('rate limit middleware', function () {
before(function (done) {
global.environment.enabledFeatures.rateLimitsEnabled = true;
global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = true;
const redisPool = new RedisPool(global.environment.redis);
const metadataBackend = cartodbRedis({ pool: redisPool });
userLimitsApi = new UserLimitsApi(metadataBackend, {
limits: {
rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled
}
});
rateLimit = rateLimitMiddleware(userLimitsApi, RATE_LIMIT_ENDPOINTS_GROUPS.ANONYMOUS);
redisClient = redis.createClient(global.environment.redis.port);
testClient = new TestClient(createMapConfig(), 1234);
const count = 1;
const period = 1;
const burst = 0;
setLimit(count, period, burst);
setTimeout(done, 1000);
});
after(function () {
global.environment.enabledFeatures.rateLimitsEnabled = false;
global.environment.enabledFeatures.rateLimitsByEndpoint.anonymous = false;
keysToDelete.forEach(key => {
redisClient.del(key);
});
});
it("1 req/sec: 2 req/seg should be limited", function (done) {
assertRateLimitRequest(200, 1, 0, 1, -1);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 250);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950);
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050);
});
it("1 req/sec: 2 req/seg should be limited, removing SHA script from Redis", function (done) {
userLimitsApi.metadataBackend.redisCmd(
8,
'SCRIPT',
['FLUSH'],
function () {
assertRateLimitRequest(200, 1, 0, 1, -1);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 500);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 750);
setTimeout( () => assertRateLimitRequest(429, 1, 0, 0, 0), 950);
setTimeout( () => assertRateLimitRequest(200, 1, 0, 1, -1, done), 1050);
}
);
});
});

BIN
test/fixtures/analysis/basic-source-id-mapnik-layer.png vendored
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 28 KiB

After

Width:  |  Height:  |  Size: 27 KiB

56
yarn.lock
View File

@ -2,20 +2,20 @@
# yarn lockfile v1
"@carto/mapnik@3.6.2-carto.2", "@carto/mapnik@~3.6.2-carto.0":
version "3.6.2-carto.2"
resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.2.tgz#45a055fd2d39530a873ef9ce5a325baacc81c196"
"@carto/mapnik@3.6.2-carto.4":
version "3.6.2-carto.4"
resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.4.tgz#54042a5dbea293c54e1bd286b32277694c5dc2d2"
dependencies:
mapnik-vector-tile "1.5.0"
nan "~2.7.0"
node-pre-gyp "~0.6.30"
protozero "1.5.1"
"@carto/tilelive-bridge@cartodb/tilelive-bridge#2.5.1-cdb3":
version "2.5.1-cdb3"
resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/e61c7752c033595a273dcd1d4b267252b174bd28"
"@carto/tilelive-bridge@cartodb/tilelive-bridge#2.5.1-cdb4":
version "2.5.1-cdb4"
resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/3eb554e5109199f50f457cec72ee288cffa5d6b3"
dependencies:
"@carto/mapnik" "3.6.2-carto.2"
"@carto/mapnik" "3.6.2-carto.4"
"@mapbox/sphericalmercator" "~1.0.1"
mapnik-pool "~0.1.3"
@ -23,11 +23,11 @@
version "1.0.5"
resolved "https://registry.yarnpkg.com/@mapbox/sphericalmercator/-/sphericalmercator-1.0.5.tgz#70237b9774095ed1cfdbcea7a8fd1fc82b2691f2"
abaculus@cartodb/abaculus#2.0.3-cdb2:
version "2.0.3-cdb2"
resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/6468e0e3fddb2b23f60b9a3156117cff0307f6dc"
abaculus@cartodb/abaculus#2.0.3-cdb5:
version "2.0.3-cdb5"
resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/b899cbea04b3e6093aa3ef32331920acd5f839a1"
dependencies:
"@carto/mapnik" "~3.6.2-carto.0"
"@carto/mapnik" "3.6.2-carto.4"
d3-queue "^2.0.2"
sphericalmercator "1.0.x"
@ -238,9 +238,9 @@ camelcase@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-3.0.0.tgz#32fc4b9fcdaf845fcdf7e73bb97cac2261f0ab0a"
camshaft@0.61.4:
version "0.61.4"
resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.4.tgz#eeab7e69a07d6fbc39ebc01ad19acf979de8fb2a"
camshaft@0.61.8:
version "0.61.8"
resolved "https://registry.yarnpkg.com/camshaft/-/camshaft-0.61.8.tgz#75669c6c14791a93433e79a8892298e88cb0fce2"
dependencies:
async "^1.5.2"
bunyan "1.8.1"
@ -301,9 +301,9 @@ cartodb-query-tables@0.3.0:
version "0.3.0"
resolved "https://registry.yarnpkg.com/cartodb-query-tables/-/cartodb-query-tables-0.3.0.tgz#56e18d869666eb2e8e2cb57d0baf3acc923f8756"
cartodb-redis@0.16.0:
version "0.16.0"
resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-0.16.0.tgz#969312fd329b24a76bf6e5a4dd961445f2eda734"
cartodb-redis@1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/cartodb-redis/-/cartodb-redis-1.0.0.tgz#83b4888ba7abb5d5895c8958b7e15cf4882602aa"
dependencies:
dot "~1.0.2"
redis-mpool "^0.5.0"
@ -2239,11 +2239,11 @@ through@2:
version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb7:
version "0.6.18-cdb7"
resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/488d2acd65c89cc5382d996eabe8dc1f5051ce0f"
tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb8:
version "0.6.18-cdb8"
resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/9cb4546c8fdd34ced0a41dbf70e143475b4e2067"
dependencies:
"@carto/mapnik" "3.6.2-carto.2"
"@carto/mapnik" "3.6.2-carto.4"
generic-pool "~2.4.0"
mime "~1.6.0"
sphericalmercator "~1.0.4"
@ -2400,13 +2400,13 @@ window-size@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
windshaft@4.5.7:
version "4.5.7"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.5.7.tgz#1cb2950e529d0a7f8bb55d56f4f568152f05c27e"
windshaft@4.6.0:
version "4.6.0"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.6.0.tgz#d9394aff73c0aa761207ad2b0f12d1c23ac41244"
dependencies:
"@carto/mapnik" "3.6.2-carto.2"
"@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb3
abaculus cartodb/abaculus#2.0.3-cdb2
"@carto/mapnik" "3.6.2-carto.4"
"@carto/tilelive-bridge" cartodb/tilelive-bridge#2.5.1-cdb4
abaculus cartodb/abaculus#2.0.3-cdb5
canvas cartodb/node-canvas#1.6.2-cdb2
carto cartodb/carto#0.15.1-cdb3
cartodb-psql "^0.10.1"
@ -2420,7 +2420,7 @@ windshaft@4.5.7:
sphericalmercator "1.0.4"
step "~0.0.6"
tilelive "5.12.2"
tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb7
tilelive-mapnik cartodb/tilelive-mapnik#0.6.18-cdb8
torque.js "~2.11.0"
underscore "~1.6.0"