Merge branch 'master' into 984-ICU-dat-not-loading

This commit is contained in:
Eneko Lakasta 2018-01-10 11:19:29 +01:00
commit 1b85951e06
2 changed files with 364 additions and 249 deletions

View File

@ -1,12 +1,20 @@
var step = require('step');
var assert = require('assert');
var _ = require('underscore');
var NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
const cors = require('../middleware/cors');
const userMiddleware = require('../middleware/user');
const allowQueryParams = require('../middleware/allow-query-params');
const vectorError = require('../middleware/vector-error');
var cors = require('../middleware/cors');
var userMiddleware = require('../middleware/user');
var allowQueryParams = require('../middleware/allow-query-params');
var vectorError = require('../middleware/vector-error');
const DEFAULT_ZOOM_CENTER = {
zoom: 1,
center: {
lng: 0,
lat: 0
}
};
function numMapper(n) {
return +n;
}
function NamedMapsController(prepareContext, namedMapProviderCache, tileBackend, previewBackend,
surrogateKeysCache, tablesExtentApi, metadataBackend) {
@ -27,7 +35,15 @@ NamedMapsController.prototype.register = function(app) {
cors(),
userMiddleware,
this.prepareContext,
this.tile.bind(this),
this.getNamedMapProvider(),
this.getAffectedTables(),
this.getTile(),
this.setSurrogateKey(),
this.setCacheChannelHeader(),
this.setLastModifiedHeader(),
this.setCacheControlHeader(),
this.setContentTypeHeader(),
this.respond(),
vectorError()
);
@ -37,298 +53,315 @@ NamedMapsController.prototype.register = function(app) {
userMiddleware,
allowQueryParams(['layer', 'zoom', 'lon', 'lat', 'bbox']),
this.prepareContext,
this.staticMap.bind(this)
this.getNamedMapProvider('STATIC_VIZ_MAP'),
this.getAffectedTables(),
this.getTemplate('STATIC_VIZ_MAP'),
this.prepareLayerFilterFromPreviewLayers('STATIC_VIZ_MAP'),
this.getStaticImageOptions(),
this.getImage('STATIC_VIZ_MAP'),
this.incrementMapViews(),
this.setSurrogateKey(),
this.setCacheChannelHeader(),
this.setLastModifiedHeader(),
this.setCacheControlHeader(),
this.setContentTypeHeader(),
this.respond()
);
};
NamedMapsController.prototype.sendResponse = function(req, res, body, headers, namedMapProvider) {
this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(res.locals.user, namedMapProvider.getTemplateName()));
res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
NamedMapsController.prototype.getNamedMapProvider = function (label) {
return function getNamedMapProviderMiddleware (req, res, next) {
const { user } = res.locals;
const { config, auth_token } = req.query;
const { template_id } = req.params;
var self = this;
this.namedMapProviderCache.get(user, template_id, config, auth_token, res.locals, (err, namedMapProvider) => {
if (err) {
err.label = label;
return next(err);
}
step(
function getAffectedTablesAndLastUpdatedTime() {
namedMapProvider.getAffectedTablesAndLastUpdatedTime(this);
},
function sendResponse(err, result) {
res.locals.namedMapProvider = namedMapProvider;
next();
});
}.bind(this);
};
NamedMapsController.prototype.getAffectedTables = function () {
return function getAffectedTables (req, res, next) {
const { namedMapProvider } = res.locals;
namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
req.profiler.done('affectedTables');
if (err) {
global.logger.log('ERROR generating cache channel: ' + err);
}
if (!result || !!result.tables) {
// we increase cache control as we can invalidate it
res.set('Cache-Control', 'public,max-age=31536000');
var lastModifiedDate;
if (Number.isFinite(result.lastUpdatedTime)) {
lastModifiedDate = new Date(result.getLastUpdatedAt());
} else {
lastModifiedDate = new Date();
}
res.set('Last-Modified', lastModifiedDate.toUTCString());
res.set('X-Cache-Channel', result.getCacheChannel());
if (result.tables.length > 0) {
self.surrogateKeysCache.tag(res, result);
}
}
res.status(200);
res.send(body);
}
);
};
NamedMapsController.prototype.tile = function(req, res, next) {
var self = this;
var cdbUser = res.locals.user;
var namedMapProvider;
step(
function getNamedMapProvider() {
self.namedMapProviderCache.get(
cdbUser,
req.params.template_id,
req.query.config,
req.query.auth_token,
res.locals,
this
);
},
function getTile(err, _namedMapProvider) {
assert.ifError(err);
namedMapProvider = _namedMapProvider;
self.tileBackend.getTile(namedMapProvider, req.params, this);
},
function handleImage(err, tile, headers, stats) {
req.profiler.add(stats);
if (err) {
err.label = 'NAMED_MAP_TILE';
next(err);
} else {
self.sendResponse(req, res, tile, headers, namedMapProvider);
return next(err);
}
}
);
res.locals.affectedTablesAndLastUpdate = affectedTablesAndLastUpdate;
next();
});
}.bind(this);
};
NamedMapsController.prototype.staticMap = function(req, res, next) {
var self = this;
var cdbUser = res.locals.user;
var format = req.params.format === 'jpg' ? 'jpeg' : 'png';
// We force always the tile to be generated using PNG because
// is the only format we support by now
res.locals.format = 'png';
res.locals.layer = res.locals.layer || 'all';
var namedMapProvider;
step(
function getNamedMapProvider() {
self.namedMapProviderCache.get(
cdbUser,
req.params.template_id,
req.query.config,
req.query.auth_token,
res.locals,
this
);
},
function prepareLayerVisibility(err, _namedMapProvider) {
assert.ifError(err);
namedMapProvider = _namedMapProvider;
self.prepareLayerFilterFromPreviewLayers(cdbUser, req, res.locals, namedMapProvider, this);
},
function prepareImageOptions(err) {
assert.ifError(err);
self.getStaticImageOptions(cdbUser, res.locals, namedMapProvider, this);
},
function getImage(err, imageOpts) {
assert.ifError(err);
var width = +req.params.width;
var height = +req.params.height;
if (!_.isUndefined(imageOpts.zoom) && imageOpts.center) {
self.previewBackend.getImage(
namedMapProvider, format, width, height, imageOpts.zoom, imageOpts.center, this);
} else {
self.previewBackend.getImage(
namedMapProvider, format, width, height, imageOpts.bounds, this);
}
},
function incrementMapViews(err, image, headers, stats) {
assert.ifError(err);
var next = this;
namedMapProvider.getMapConfig(function(mapConfigErr, mapConfig) {
self.metadataBackend.incMapviewCount(cdbUser, mapConfig.obj().stat_tag, function(sErr) {
if (err) {
global.logger.log("ERROR: failed to increment mapview count for user '%s': %s", cdbUser, sErr);
}
next(err, image, headers, stats);
});
});
},
function handleImage(err, image, headers, stats) {
req.profiler.done('render-' + format);
req.profiler.add(stats || {});
NamedMapsController.prototype.getTemplate = function (label) {
return function getTemplateMiddleware (req, res, next) {
const { namedMapProvider } = res.locals;
namedMapProvider.getTemplate((err, template) => {
if (err) {
err.label = 'STATIC_VIZ_MAP';
next(err);
} else {
self.sendResponse(req, res, image, headers, namedMapProvider);
err.label = label;
return next(err);
}
}
);
res.locals.template = template;
next();
});
};
};
NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (
user,
req,
params,
namedMapProvider,
callback
) {
var self = this;
namedMapProvider.getTemplate(function (err, template) {
if (err) {
return callback(err);
}
NamedMapsController.prototype.prepareLayerFilterFromPreviewLayers = function (label) {
return function prepareLayerFilterFromPreviewLayersMiddleware (req, res, next) {
const { user, template } = res.locals;
const { template_id } = req.params;
const { config, auth_token } = req.query;
if (!template || !template.view || !template.view.preview_layers) {
return callback();
return next();
}
var previewLayers = template.view.preview_layers;
var layerVisibilityFilter = [];
template.layergroup.layers.forEach(function (layer, index) {
template.layergroup.layers.forEach((layer, index) => {
if (previewLayers[''+index] !== false && previewLayers[layer.id] !== false) {
layerVisibilityFilter.push(''+index);
}
});
if (!layerVisibilityFilter.length) {
return callback();
return next();
}
// overwrites 'all' default filter
params.layer = layerVisibilityFilter.join(',');
res.locals.layer = layerVisibilityFilter.join(',');
// recreates the provider
self.namedMapProviderCache.get(
user,
req.params.template_id,
req.query.config,
req.query.auth_token,
params,
callback
);
});
this.namedMapProviderCache.get(user, template_id, config, auth_token, res.locals, (err, provider) => {
if (err) {
err.label = label;
return next(err);
}
res.locals.namedMapProvider = provider;
next();
});
}.bind(this);
};
var DEFAULT_ZOOM_CENTER = {
zoom: 1,
center: {
lng: 0,
lat: 0
NamedMapsController.prototype.getTile = function () {
return function getTileMiddleware (req, res, next) {
const { namedMapProvider } = res.locals;
this.tileBackend.getTile(namedMapProvider, req.params, (err, tile, headers, stats) => {
req.profiler.add(stats);
if (err) {
err.label = 'NAMED_MAP_TILE';
return next(err);
}
res.locals.body = tile;
res.locals.headers = headers;
res.locals.stats = stats;
next();
});
}.bind(this);
};
NamedMapsController.prototype.getStaticImageOptions = function () {
return function getStaticImageOptionsMiddleware(req, res, next) {
const { user, namedMapProvider, template } = res.locals;
const imageOpts = getImageOptions(res.locals, template);
if (imageOpts) {
res.locals.imageOpts = imageOpts;
return next();
}
res.locals.imageOpts = DEFAULT_ZOOM_CENTER;
namedMapProvider.getAffectedTablesAndLastUpdatedTime((err, affectedTablesAndLastUpdate) => {
if (err) {
return next();
}
var affectedTables = affectedTablesAndLastUpdate.tables || [];
if (affectedTables.length === 0) {
return next();
}
this.tablesExtentApi.getBounds(user, affectedTables, (err, bounds) => {
if (err) {
return next();
}
res.locals.imageOpts = bounds;
return next();
});
});
}.bind(this);
};
function getImageOptions (params, template) {
const { zoom, lon, lat, bbox } = params;
let imageOpts = getImageOptionsFromCoordinates(zoom, lon, lat);
if (imageOpts) {
return imageOpts;
}
};
function numMapper(n) {
return +n;
imageOpts = getImageOptionsFromBoundingBox(bbox);
if (imageOpts) {
return imageOpts;
}
imageOpts = getImageOptionsFromTemplate(template, zoom);
if (imageOpts) {
return imageOpts;
}
}
NamedMapsController.prototype.getStaticImageOptions = function(cdbUser, params, namedMapProvider, callback) {
var self = this;
if ([params.zoom, params.lon, params.lat].map(numMapper).every(Number.isFinite)) {
return callback(null, {
zoom: params.zoom,
function getImageOptionsFromCoordinates (zoom, lon, lat) {
if ([zoom, lon, lat].map(numMapper).every(Number.isFinite)) {
return {
zoom: zoom,
center: {
lng: params.lon,
lat: params.lat
lng: lon,
lat: lat
}
});
};
}
}
if (params.bbox) {
var bbox = params.bbox.split(',').map(numMapper);
if (bbox.length === 4 && bbox.every(Number.isFinite)) {
return callback(null, {
bounds: {
west: bbox[0],
south: bbox[1],
east: bbox[2],
north: bbox[3]
}
});
function getImageOptionsFromTemplate (template, zoom) {
if (template.view) {
var zoomCenter = templateZoomCenter(template.view);
if (zoomCenter) {
if (Number.isFinite(+zoom)) {
zoomCenter.zoom = +zoom;
}
return zoomCenter;
}
var bounds = templateBounds(template.view);
if (bounds) {
return bounds;
}
}
}
step(
function getTemplate() {
namedMapProvider.getTemplate(this);
},
function handleTemplateView(err, template) {
assert.ifError(err);
function getImageOptionsFromBoundingBox (bbox = '') {
var _bbox = bbox.split(',').map(numMapper);
if (template.view) {
var zoomCenter = templateZoomCenter(template.view);
if (zoomCenter) {
if (Number.isFinite(+params.zoom)) {
zoomCenter.zoom = +params.zoom;
}
return zoomCenter;
}
var bounds = templateBounds(template.view);
if (bounds) {
return bounds;
}
if (_bbox.length === 4 && _bbox.every(Number.isFinite)) {
return {
bounds: {
west: _bbox[0],
south: _bbox[1],
east: _bbox[2],
north: _bbox[3]
}
};
}
}
return false;
},
function estimateBoundsIfNoImageOpts(err, imageOpts) {
if (imageOpts) {
return imageOpts;
}
NamedMapsController.prototype.getImage = function (label) {
return function getImageMiddleware (req, res, next) {
const { imageOpts, namedMapProvider } = res.locals;
const { zoom, center, bounds } = imageOpts;
var next = this;
namedMapProvider.getAffectedTablesAndLastUpdatedTime(function(err, affectedTablesAndLastUpdate) {
let { width, height } = req.params;
width = +width;
height = +height;
const format = req.params.format === 'jpg' ? 'jpeg' : 'png';
// We force always the tile to be generated using PNG because
// is the only format we support by now
res.locals.format = 'png';
res.locals.layer = res.locals.layer || 'all';
if (zoom !== undefined && center) {
return this.previewBackend.getImage(namedMapProvider, format, width, height, zoom, center,
(err, image, headers, stats) => {
if (err) {
return next(null);
err.label = label;
return next(err);
}
var affectedTables = affectedTablesAndLastUpdate.tables || [];
res.locals.body = image;
res.locals.headers = headers;
res.locals.stats = stats;
if (affectedTables.length === 0) {
return next(null);
}
self.tablesExtentApi.getBounds(cdbUser, affectedTables, function(err, result) {
return next(null, result);
});
next();
});
},
function returnCallback(err, imageOpts) {
return callback(err, imageOpts || DEFAULT_ZOOM_CENTER);
}
);
this.previewBackend.getImage(namedMapProvider, format, width, height, bounds, (err, image, headers, stats) => {
if (err) {
err.label = label;
return next(err);
}
res.locals.body = image;
res.locals.headers = headers;
res.locals.stats = stats;
next();
});
}.bind(this);
};
function incrementMapViewsError (ctx) {
return `ERROR: failed to increment mapview count for user '${ctx.user}': ${ctx.err}`;
}
NamedMapsController.prototype.incrementMapViews = function () {
return function incrementMapViewsMiddleware(req, res, next) {
const { user, namedMapProvider } = res.locals;
namedMapProvider.getMapConfig((err, mapConfig) => {
if (err) {
global.logger.log(incrementMapViewsError({ user, err }));
return next();
}
const statTag = mapConfig.obj().stat_tag;
this.metadataBackend.incMapviewCount(user, statTag, (err) => {
if (err) {
global.logger.log(incrementMapViewsError({ user, err }));
}
next();
});
});
}.bind(this);
};
function templateZoomCenter(view) {
if (!_.isUndefined(view.zoom) && view.center) {
if (view.zoom !== undefined && view.center) {
return {
zoom: view.zoom,
center: view.center
@ -339,9 +372,8 @@ function templateZoomCenter(view) {
function templateBounds(view) {
if (view.bounds) {
var hasAllBounds = _.every(['west', 'south', 'east', 'north'], function(prop) {
return Number.isFinite(view.bounds[prop]);
});
var hasAllBounds = ['west', 'south', 'east', 'north'].every(prop => Number.isFinite(view.bounds[prop]));
if (hasAllBounds) {
return {
bounds: {
@ -357,3 +389,86 @@ function templateBounds(view) {
}
return false;
}
NamedMapsController.prototype.setCacheChannelHeader = function () {
return function setCacheChannelHeaderMiddleware (req, res, next) {
const { affectedTablesAndLastUpdate } = res.locals;
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
res.set('X-Cache-Channel', affectedTablesAndLastUpdate.getCacheChannel());
}
next();
};
};
NamedMapsController.prototype.setSurrogateKey = function () {
return function setSurrogateKeyMiddleware(req, res, next) {
const { user, namedMapProvider, affectedTablesAndLastUpdate } = res.locals;
this.surrogateKeysCache.tag(res, new NamedMapsCacheEntry(user, namedMapProvider.getTemplateName()));
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
if (affectedTablesAndLastUpdate.tables.length > 0) {
this.surrogateKeysCache.tag(res, affectedTablesAndLastUpdate);
}
}
next();
}.bind(this);
};
NamedMapsController.prototype.setLastModifiedHeader = function () {
return function setLastModifiedHeaderMiddleware(req, res, next) {
const { affectedTablesAndLastUpdate } = res.locals;
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
var lastModifiedDate;
if (Number.isFinite(affectedTablesAndLastUpdate.lastUpdatedTime)) {
lastModifiedDate = new Date(affectedTablesAndLastUpdate.getLastUpdatedAt());
} else {
lastModifiedDate = new Date();
}
res.set('Last-Modified', lastModifiedDate.toUTCString());
}
next();
};
};
NamedMapsController.prototype.setCacheControlHeader = function () {
return function setCacheControlHeaderMiddleware(req, res, next) {
const { affectedTablesAndLastUpdate } = res.locals;
res.set('Cache-Control', 'public,max-age=7200,must-revalidate');
if (!affectedTablesAndLastUpdate || !!affectedTablesAndLastUpdate.tables) {
// we increase cache control as we can invalidate it
res.set('Cache-Control', 'public,max-age=31536000');
}
next();
};
};
NamedMapsController.prototype.setContentTypeHeader = function () {
return function setContentTypeHeaderMiddleware(req, res, next) {
const { headers = {} } = res.locals;
res.set('Content-Type', headers['content-type'] || headers['Content-Type'] || 'image/png');
next();
};
};
NamedMapsController.prototype.respond = function () {
return function respondMiddleware (req, res) {
const { body, stats = {}, format } = res.locals;
req.profiler.done('render-' + format);
req.profiler.add(stats);
res.status(200);
res.send(body);
};
};

View File

@ -2,7 +2,7 @@
# yarn lockfile v1
"abaculus@github:cartodb/abaculus#2.0.3-cdb1":
abaculus@cartodb/abaculus#2.0.3-cdb1:
version "2.0.3-cdb1"
resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf"
dependencies:
@ -226,7 +226,7 @@ camshaft@0.60.0:
dot "^1.0.3"
request "^2.69.0"
"canvas@github:cartodb/node-canvas#1.6.2-cdb2":
canvas@cartodb/node-canvas#1.6.2-cdb2:
version "1.6.2-cdb2"
resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766"
dependencies:
@ -252,7 +252,7 @@ carto@0.16.3:
optimist "~0.6.0"
underscore "~1.6.0"
"carto@github:cartodb/carto#0.15.1-cdb3":
carto@cartodb/carto#0.15.1-cdb3:
version "0.15.1-cdb3"
resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7"
dependencies:
@ -2223,7 +2223,7 @@ through@2:
version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4":
tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4:
version "2.3.1-cdb4"
resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6"
dependencies:
@ -2231,7 +2231,7 @@ through@2:
mapnik-pool "~0.1.3"
sphericalmercator "1.0.x"
"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3":
tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3:
version "0.6.18-cdb3"
resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01"
dependencies: