From 49f9904d00a7b502f1c03d5b711baede53294ed0 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 22 Nov 2016 16:41:31 +0100 Subject: [PATCH 1/6] Allow to set resource URL templates with substitution tokens --- config/environments/development.js.example | 15 ++++ config/environments/production.js.example | 15 ++++ config/environments/staging.js.example | 15 ++++ config/environments/test.js.example | 15 ++++ lib/cartodb/controllers/map.js | 92 ++++++++++++++++------ 5 files changed, 128 insertions(+), 24 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index be5b2350..491d9411 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -23,6 +23,21 @@ var config = { // "tiles/layergroup" is for compatibility with versions up to 1.6.x ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. + // + // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be + // configured to accept request with the {user} in the header host or in the request path. + // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`. + // + // This template allows to make the endpoints generation more flexible, the template exposes the following params: + // 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists. + // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. + // 3. {{=it.port}}: will use the `port` from this very same configuration file. + ,resources_url_templates: { + http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map', + https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map' + } + // Maximum number of connections for one process // 128 is a good value with a limit of 1024 open file descriptors ,maxConnections:128 diff --git a/config/environments/production.js.example b/config/environments/production.js.example index bcf2f8f2..2cb9a1f8 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -23,6 +23,21 @@ var config = { // "tiles/layergroup" is for compatibility with versions up to 1.6.x ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. + // + // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be + // configured to accept request with the {user} in the header host or in the request path. + // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`. + // + // This template allows to make the endpoints generation more flexible, the template exposes the following params: + // 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists. + // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. + // 3. {{=it.port}}: will use the `port` from this very same configuration file. + ,resources_url_templates: { + http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map', + https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map' + } + // Maximum number of connections for one process // 128 is a good value with a limit of 1024 open file descriptors ,maxConnections:128 diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index e3681a68..4449c847 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -23,6 +23,21 @@ var config = { // "/tiles/layergroup" is for compatibility with versions up to 1.6.x ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. + // + // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be + // configured to accept request with the {user} in the header host or in the request path. + // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`. + // + // This template allows to make the endpoints generation more flexible, the template exposes the following params: + // 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists. + // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. + // 3. {{=it.port}}: will use the `port` from this very same configuration file. + ,resources_url_templates: { + http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map', + https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map' + } + // Maximum number of connections for one process // 128 is a good value with a limit of 1024 open file descriptors ,maxConnections:128 diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 49e59217..f0fcafc0 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -23,6 +23,21 @@ var config = { // "tiles/layergroup" is for compatibility with versions up to 1.6.x ,base_url_detached: '(?:/api/v1/map|/user/:user/api/v1/map|/tiles/layergroup)' + // Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status. + // + // This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be + // configured to accept request with the {user} in the header host or in the request path. + // It also might depend on the configured cdn_url via `serverMetadata.cdn_url`. + // + // This template allows to make the endpoints generation more flexible, the template exposes the following params: + // 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists. + // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. + // 3. {{=it.port}}: will use the `port` from this very same configuration file. + ,resources_url_templates: { + http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map', + https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map' + } + // Maximum number of connections for one process // 128 is a good value with a limit of 1024 open file descriptors ,maxConnections:128 diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index 4b66972f..f3be4589 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -1,4 +1,6 @@ var _ = require('underscore'); +var dot = require('dot'); +dot.templateSettings.strip = false; var assert = require('assert'); var step = require('step'); var windshaft = require('windshaft'); @@ -44,6 +46,20 @@ function MapController(authApi, pgConnection, templateMaps, mapBackend, metadata this.layergroupAffectedTables = layergroupAffectedTables; this.mapConfigAdapter = mapConfigAdapter; + + this.resourcesUrlTemplates = null; + if (global.environment.resources_url_templates) { + var templates = global.environment.resources_url_templates; + + if (templates.http) { + this.resourcesUrlTemplates = this.resourcesUrlTemplates || {}; + this.resourcesUrlTemplates.http = dot.template(templates.http + '/{{=it.resource}}'); + } + if (templates.https) { + this.resourcesUrlTemplates = this.resourcesUrlTemplates || {}; + this.resourcesUrlTemplates.https = dot.template(templates.https + '/{{=it.resource}}'); + } + } } util.inherits(MapController, BaseController); @@ -187,8 +203,8 @@ MapController.prototype.create = function(req, res, prepareConfigFn) { self.sendError(req, res, err, 'ANONYMOUS LAYERGROUP'); } else { var analysesResults = context.analysesResults || []; - addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj()); - addAnalysesMetadata(req.context.user, layergroup, analysesResults, true); + self.addDataviewsAndWidgetsUrls(req.context.user, layergroup, mapConfig.obj()); + self.addAnalysesMetadata(req.context.user, layergroup, analysesResults, true); addContextMetadata(layergroup, mapConfig.obj(), context); res.set('X-Layergroup-Id', layergroup.layergroupid); self.send(req, res, layergroup, 200); @@ -260,8 +276,8 @@ MapController.prototype.instantiateTemplate = function(req, res, prepareParamsFn layergroup.layergroupid = cdbuser + '@' + templateHash + '@' + layergroup.layergroupid; var _mapConfig = mapConfig.obj(); - addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig); - addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults); + self.addDataviewsAndWidgetsUrls(cdbuser, layergroup, _mapConfig); + self.addAnalysesMetadata(cdbuser, layergroup, mapConfigProvider.analysesResults); addContextMetadata(layergroup, _mapConfig, mapConfigProvider.context); res.set('X-Layergroup-Id', layergroup.layergroupid); @@ -370,7 +386,7 @@ function getLastUpdatedTime(analysesResults, lastUpdateTime) { }, lastUpdateTime); } -function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery) { +MapController.prototype.addAnalysesMetadata = function(username, layergroup, analysesResults, includeQuery) { includeQuery = includeQuery || false; analysesResults = analysesResults || []; layergroup.metadata.analyses = []; @@ -383,7 +399,7 @@ function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery var nodeResource = layergroup.layergroupid + '/analysis/node/' + node.id(); var nodeRepr = { status: node.getStatus(), - url: getUrls(username, nodeResource) + url: this.getUrls(username, nodeResource) }; if (includeQuery) { nodeRepr.query = node.getQuery(); @@ -395,30 +411,30 @@ function addAnalysesMetadata(username, layergroup, analysesResults, includeQuery } return nodesIdMap; - }, {}) + }.bind(this), {}) }); - }); -} + }.bind(this)); +}; // TODO this should take into account several URL patterns -function addDataviewsAndWidgetsUrls(username, layergroup, mapConfig) { - addDataviewsUrls(username, layergroup, mapConfig); - addWidgetsUrl(username, layergroup, mapConfig); -} +MapController.prototype.addDataviewsAndWidgetsUrls = function(username, layergroup, mapConfig) { + this.addDataviewsUrls(username, layergroup, mapConfig); + this.addWidgetsUrl(username, layergroup, mapConfig); +}; -function addDataviewsUrls(username, layergroup, mapConfig) { +MapController.prototype.addDataviewsUrls = function(username, layergroup, mapConfig) { layergroup.metadata.dataviews = layergroup.metadata.dataviews || {}; var dataviews = mapConfig.dataviews || {}; Object.keys(dataviews).forEach(function(dataviewName) { var resource = layergroup.layergroupid + '/dataview/' + dataviewName; layergroup.metadata.dataviews[dataviewName] = { - url: getUrls(username, resource) + url: this.getUrls(username, resource) }; - }); -} + }.bind(this)); +}; -function addWidgetsUrl(username, layergroup, mapConfig) { +MapController.prototype.addWidgetsUrl = function(username, layergroup, mapConfig) { if (layergroup.metadata && Array.isArray(layergroup.metadata.layers) && Array.isArray(mapConfig.layers)) { layergroup.metadata.layers = layergroup.metadata.layers.map(function(layer, layerIndex) { var mapConfigLayer = mapConfig.layers[layerIndex]; @@ -428,16 +444,19 @@ function addWidgetsUrl(username, layergroup, mapConfig) { var resource = layergroup.layergroupid + '/' + layerIndex + '/widget/' + widgetName; layer.widgets[widgetName] = { type: mapConfigLayer.options.widgets[widgetName].type, - url: getUrls(username, resource) + url: this.getUrls(username, resource) }; - }); + }.bind(this)); } return layer; - }); + }.bind(this)); } -} +}; -function getUrls(username, resource) { +MapController.prototype.getUrls = function(username, resource) { + if (this.resourcesUrlTemplates) { + return this.getUrlsFromTemplate(username, resource); + } var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url; if (cdnUrl) { return { @@ -450,4 +469,29 @@ function getUrls(username, resource) { http: 'http://' + username + '.' + 'localhost.lan:' + port + '/api/v1/map/' + resource }; } -} +}; + +MapController.prototype.getUrlsFromTemplate = function(username, resource) { + var urls = {}; + var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url; + + if (this.resourcesUrlTemplates.http) { + urls.http = this.resourcesUrlTemplates.http({ + cdn_url: cdnUrl, + user: username, + port: global.environment.port, + resource: resource + }); + } + + if (this.resourcesUrlTemplates.https) { + urls.https = this.resourcesUrlTemplates.https({ + cdn_url: cdnUrl, + user: username, + port: global.environment.port, + resource: resource + }); + } + + return urls; +}; From 6c69ba54dbe95fc65aca9bf0ea6ebebdf351a3a6 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 22 Nov 2016 16:44:06 +0100 Subject: [PATCH 2/6] Use actual CDN url for HTTP and HTTPS --- lib/cartodb/controllers/map.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index f3be4589..a84fcf75 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -477,7 +477,7 @@ MapController.prototype.getUrlsFromTemplate = function(username, resource) { if (this.resourcesUrlTemplates.http) { urls.http = this.resourcesUrlTemplates.http({ - cdn_url: cdnUrl, + cdn_url: cdnUrl.http, user: username, port: global.environment.port, resource: resource @@ -486,7 +486,7 @@ MapController.prototype.getUrlsFromTemplate = function(username, resource) { if (this.resourcesUrlTemplates.https) { urls.https = this.resourcesUrlTemplates.https({ - cdn_url: cdnUrl, + cdn_url: cdnUrl.https, user: username, port: global.environment.port, resource: resource From 1f975e15c16d78ee832c183ceadc8c74a491f677 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Tue, 22 Nov 2016 17:01:34 +0100 Subject: [PATCH 3/6] Default to empty object for cdn URLs --- lib/cartodb/controllers/map.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cartodb/controllers/map.js b/lib/cartodb/controllers/map.js index a84fcf75..9aa3b34f 100644 --- a/lib/cartodb/controllers/map.js +++ b/lib/cartodb/controllers/map.js @@ -473,7 +473,7 @@ MapController.prototype.getUrls = function(username, resource) { MapController.prototype.getUrlsFromTemplate = function(username, resource) { var urls = {}; - var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url; + var cdnUrl = global.environment.serverMetadata && global.environment.serverMetadata.cdn_url || {}; if (this.resourcesUrlTemplates.http) { urls.http = this.resourcesUrlTemplates.http({ From 87a01a5cfd69ee0d2ac485b73f2fed02eded54d8 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 24 Nov 2016 12:56:02 +0100 Subject: [PATCH 4/6] Example configuration with just HTTP --- config/environments/test.js.example | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/environments/test.js.example b/config/environments/test.js.example index f0fcafc0..8961019d 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -34,8 +34,7 @@ var config = { // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. // 3. {{=it.port}}: will use the `port` from this very same configuration file. ,resources_url_templates: { - http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map', - https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map' + http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map' } // Maximum number of connections for one process From fe750f23bc5508d135c8b5bed4f22566ef063a16 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 24 Nov 2016 12:56:38 +0100 Subject: [PATCH 5/6] Closer to reality production example configuration --- config/environments/production.js.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 2cb9a1f8..d50eb1e4 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -34,7 +34,7 @@ var config = { // 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`. // 3. {{=it.port}}: will use the `port` from this very same configuration file. ,resources_url_templates: { - http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map', + http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map', https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map' } From 58c407aabb35f3651a2d8142070dd15e90dbfcf9 Mon Sep 17 00:00:00 2001 From: Raul Ochoa Date: Thu, 24 Nov 2016 12:57:45 +0100 Subject: [PATCH 6/6] Alternative development example configuration for user in path instead of host --- config/environments/development.js.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 491d9411..b66ee4d3 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -35,7 +35,7 @@ var config = { // 3. {{=it.port}}: will use the `port` from this very same configuration file. ,resources_url_templates: { http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map', - https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map' + https: 'http://localhost.lan:{{=it.port}}/user/{{=it.user}}/api/v1/map' } // Maximum number of connections for one process