cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into remove-auth-fallback

Browse Source
This commit is contained in:
Eneko Lakasta 2018-06-11 12:02:58 +02:00
commit 7f7eb78d8c
13 changed files with 94 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS.md
View File

@ -39,6 +39,7 @@ Bug Fixes:
- Static maps fails for unsupported formats
- Handling errors extracting the column type on dataviews
- Fix `meta.stats.estimatedFeatureCount` for aggregations and queries with tokens
- Static maps filters correctly if `layer` option is passed in the url.
## 6.1.0
Released 2018-04-16

21
config/environments/development.js.example
View File

@ -61,13 +61,13 @@ var config = {
// Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
//
// This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
// This URLs depend on how `routes` and `user_from_host` are configured: the application can be
// configured to accept request with the {user} in the header host or in the request path.
// It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
//
// This template allows to make the endpoints generation more flexible, the template exposes the following params:
// 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
// 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
// 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`.
// 3. {{=it.port}}: will use the `port` from this very same configuration file.
,resources_url_templates: {
http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
@ -87,12 +87,12 @@ var config = {
// idle socket timeout, in milliseconds
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: false
,cache_enabled: true
,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
// If log_filename is given logs will be written
// there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
,log_filename: undefined
,log_filename: 'logs/node-windshaft.log'
// Templated database username for authorized user
// Supported labels: 'user_id' (read from redis)
,postgres_auth_user: 'development_cartodb_user_<%= user_id %>'
@ -118,7 +118,7 @@ var config = {
,statsd: {
host: 'localhost',
port: 8125,
prefix: 'dev.',
prefix: 'dev.', // could be hostname, better not containing dots
cacheDns: true
// support all allowed node-statsd options
}
@ -204,7 +204,8 @@ var config = {
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
max_size: 500,
twkb_encoding: true
},
limits: {
@ -267,7 +268,7 @@ var config = {
// If filename is given logs comming from analysis client will be written
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
filename: '/tmp/analysis.log'
filename: 'logs/node-windshaft-analysis.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
@ -337,6 +338,12 @@ var config = {
// X-Tiler-Profile header containing elapsed timing for various
// steps taken for producing the response.
,useProfiler:true
,serverMetadata: {
cdn_url: {
http: undefined,
https: undefined
}
}
// Settings for the health check available at /health
,health: {
enabled: false,

19
config/environments/production.js.example
View File

@ -61,13 +61,13 @@ var config = {
// Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
//
// This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
// This URLs depend on how `routes` and `user_from_host` are configured: the application can be
// configured to accept request with the {user} in the header host or in the request path.
// It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
//
// This template allows to make the endpoints generation more flexible, the template exposes the following params:
// 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
// 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
// 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`.
// 3. {{=it.port}}: will use the `port` from this very same configuration file.
,resources_url_templates: {
http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map',
@ -130,15 +130,7 @@ var config = {
//If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
//PostGIS 2.4 is required for this to work
//If disabled it will use Mapnik MVT generation
usePostGIS: false,
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
usePostGIS: false
},
mapnik: {
// The size of the pool of internal mapnik backend
@ -212,7 +204,8 @@ var config = {
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
max_size: 500,
twkb_encoding: true
},
limits: {
@ -275,7 +268,7 @@ var config = {
// If filename is given logs comming from analysis client will be written
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
filename: 'logs/analysis.log'
filename: 'logs/node-windshaft-analysis.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.

17
config/environments/staging.js.example
View File

@ -61,16 +61,16 @@ var config = {
// Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
//
// This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
// This URLs depend on how `routes` and `user_from_host` are configured: the application can be
// configured to accept request with the {user} in the header host or in the request path.
// It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
//
// This template allows to make the endpoints generation more flexible, the template exposes the following params:
// 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
// 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
// 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`.
// 3. {{=it.port}}: will use the `port` from this very same configuration file.
,resources_url_templates: {
http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
http: 'http://{{=it.cdn_url}}/{{=it.user}}/api/v1/map',
https: 'https://{{=it.cdn_url}}/{{=it.user}}/api/v1/map'
}
@ -88,7 +88,7 @@ var config = {
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: true
,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms (:res[X-Tiler-Profiler]) -> :res[Content-Type] (:res[X-Tiler-Errors])'
,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
// If log_filename is given logs will be written
// there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
@ -118,7 +118,7 @@ var config = {
,statsd: {
host: 'localhost',
port: 8125,
prefix: 'stage.:host.',
prefix: 'stage.:host.', // could be hostname, better not containing dots
cacheDns: true
// support all allowed node-statsd options
}
@ -204,7 +204,8 @@ var config = {
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
max_size: 500,
twkb_encoding: true
},
limits: {
@ -267,7 +268,7 @@ var config = {
// If filename is given logs comming from analysis client will be written
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
filename: 'logs/analysis.log'
filename: 'logs/node-windshaft-analysis.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
@ -345,7 +346,7 @@ var config = {
}
// Settings for the health check available at /health
,health: {
enabled: false,
enabled: true,
username: 'localhost',
z: 0,
x: 0,

47
config/environments/test.js.example
View File

@ -61,16 +61,17 @@ var config = {
// Resource URLs expose endpoints to request/retrieve metadata associated to Maps: dataviews, analysis node status.
//
// This URLs depend on how `base_url_detached` and `user_from_host` are configured: the application can be
// This URLs depend on how `routes` and `user_from_host` are configured: the application can be
// configured to accept request with the {user} in the header host or in the request path.
// It also might depend on the configured cdn_url via `serverMetadata.cdn_url`.
//
// This template allows to make the endpoints generation more flexible, the template exposes the following params:
// 1. {{=it.cdn_url}}: will be used when `serverMetadata.cdn_url` exists.
// 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `base_url_detached`.
// 2. {{=it.user}}: will use the username as extraced from `user_from_host` or `routes`.
// 3. {{=it.port}}: will use the `port` from this very same configuration file.
,resources_url_templates: {
http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map'
http: 'http://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map',
https: 'https://{{=it.user}}.localhost.lan:{{=it.port}}/api/v1/map'
}
// Maximum number of connections for one process
@ -87,11 +88,11 @@ var config = {
,socket_timeout: 600000
,enable_cors: true
,cache_enabled: false
,log_format: '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
,log_format: ':req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-Tiler-Profiler]) (:res[X-Tiler-Errors])'
// If log_filename is given logs will be written
// there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
//,log_filename: 'logs/node-windshaft.log'
,log_filename: '/tmp/node-windshaft.log'
// Templated database username for authorized user
// Supported labels: 'user_id' (read from redis)
,postgres_auth_user: 'test_windshaft_cartodb_user_<%= user_id %>'
@ -99,8 +100,6 @@ var config = {
// Supported labels: 'user_id', 'user_password' (both read from redis)
,postgres_auth_pass: 'test_windshaft_cartodb_user_<%= user_id %>_pass'
,postgres: {
// Parameters to pass to datasource plugin of mapnik
// See http://github.com/mapnik/mapnik/wiki/PostGIS
user: "test_windshaft_publicuser",
password: "public",
host: '127.0.0.1',
@ -114,12 +113,12 @@ var config = {
reapInterval: 1000
}
}
,mapnik_version: ''
,mapnik_version: undefined
,mapnik_tile_format: 'png8:m=h'
,statsd: {
host: 'localhost',
port: 8125,
prefix: 'test.:host.',
prefix: 'test.:host.', // could be hostname, better not containing dots
cacheDns: true
// support all allowed node-statsd options
}
@ -131,15 +130,7 @@ var config = {
//If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
//PostGIS 2.4 is required for this to work
//If disabled it will use Mapnik MVT generation
usePostGIS: false,
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
usePostGIS: false
},
mapnik: {
// The size of the pool of internal mapnik backend
@ -213,7 +204,8 @@ var config = {
persist_connection: false,
simplify_geometries: true,
use_overviews: true, // use overviews to retrieve raster
max_size: 500
max_size: 500,
twkb_encoding: false
},
limits: {
@ -231,10 +223,15 @@ var config = {
// If enabled Mapnik will reuse the features retrieved from the database
// instead of requesting them once per style inside a layer
'cache-features': true,
'cache-features': false,
// Require metrics to the renderer
metrics: false
metrics: false,
// Options for markers attributes, ellipses and images caches
markers_symbolizer_caches: {
disabled: false
}
},
http: {
timeout: 2000, // the timeout in ms for a http tile request
@ -273,7 +270,7 @@ var config = {
// If filename is given logs comming from analysis client will be written
// there, in append mode. Otherwise 'log_filename' is used. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
filename: 'node-windshaft.log'
filename: '/tmp/node-windshaft-analysis.log'
},
// Define max execution time in ms for analyses or tags
// If analysis or tag are not found in redis this values will be used as default.
@ -343,6 +340,12 @@ var config = {
// X-Tiler-Profile header containing elapsed timing for various
// steps taken for producing the response.
,useProfiler:true
,serverMetadata: {
cdn_url: {
http: undefined,
https: undefined
}
}
// Settings for the health check available at /health
,health: {
enabled: false,

5
docs/static_maps_api.md
View File

@ -11,7 +11,7 @@ Begin by instantiating either a Named or Anonymous Map using the `layergroupid t
#### Definition
```bash
GET /api/v1/map/static/center/{token}/{z}/{lat}/{lng}/{width}/{height}.{format}
GET /api/v1/map/static/center/{token}/{z}/{lat}/{lng}/{width}/{height}.{format}{{?}extra_options}
```
#### Params
@ -58,6 +58,9 @@ Note: you can see this endpoint as
GET /api/v1/map/static/bbox/{token}/{west},{south},{east},{north}/{width}/{height}.{format}`
```
#### Extra options
* Layer: List of layers to be shown in the image (by default `all`), for example `?layer=0,1`.
### Named Map
#### Definition

5
lib/cartodb/api/middlewares/map-store-map-config-provider.js
View File

@ -10,12 +10,13 @@ module.exports = function createMapStoreMapConfigProvider (
return function createMapStoreMapConfigProviderMiddleware (req, res, next) {
const { user, token, cache_buster, api_key } = res.locals;
const { dbuser, dbname, dbpassword, dbhost, dbport } = res.locals;
const { layer, z, x, y, scale_factor, format } = req.params;
const { layer: layerFromParams, z, x, y, scale_factor, format } = req.params;
const { layer: layerFromQuery } = req.query;
const params = {
user, token, cache_buster, api_key,
dbuser, dbname, dbpassword, dbhost, dbport,
layer, z, x, y, scale_factor, format
layer: (layerFromQuery || layerFromParams), z, x, y, scale_factor, format
};
if (forcedFormat) {

5
lib/cartodb/server_options.js
View File

@ -1,4 +1,4 @@
var os = require('os');
const fqdn = require('@carto/fqdn-sync');
var _ = require('underscore');
var OverviewsQueryRewriter = require('./utils/overviews_query_rewriter');
@ -36,8 +36,7 @@ rendererConfig.mapnik.queryRewriter = overviewsQueryRewriter;
// See https://github.com/CartoDB/Windshaft-cartodb/issues/153
if (global.environment.statsd) {
if (global.environment.statsd.prefix) {
var host_token = os.hostname().split('.').reverse().join('.');
global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, host_token);
global.environment.statsd.prefix = global.environment.statsd.prefix.replace(/:host/, fqdn.reverse());
}
}

1
package.json
View File

@ -24,6 +24,7 @@
"Simon Martin <simon@carto.com>"
],
"dependencies": {
"@carto/fqdn-sync": "0.2.2",
"basic-auth": "2.0.0",
"body-parser": "1.18.3",
"camshaft": "0.61.10",

21
test/acceptance/analysis/named-maps.js
View File

@ -261,6 +261,27 @@ describe('named-maps analysis', function() {
);
});
it('should fail to retrieve static map preview via layergroup ' +
'when filtering by invalid layers', function(done) {
assert.response(
server,
{
url: '/api/v1/map/static/center/' + layergroupid + '/4/42/-3/320/240.png?layer=1',
method: 'GET',
encoding: 'binary',
headers: {
host: username
}
},
{
status: 400
},
function(res, err) {
done(err);
}
);
});
it('should return and an error requesting unsupported image format', function(done) {
assert.response(
server,

6
test/support/test-client.js
View File

@ -1268,9 +1268,6 @@ TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callb
const dbuser = _.template(global.environment.postgres_auth_user, { user_id: 1 });
const publicuser = global.environment.postgres.user;
// we need to guarantee all new connections have the new settings
helper.cleanPGPoolConnections();
const psql = new PSQL({
user: 'postgres',
dbname: dbname,
@ -1278,6 +1275,9 @@ TestClient.prototype.setUserDatabaseTimeoutLimit = function (timeoutLimit, callb
port: global.environment.postgres.port
});
// we need to guarantee all new connections have the new settings
psql.end();
step(
function configureTimeouts () {
const timeoutSQLs = [

9
test/support/test_helper.js
View File

@ -13,7 +13,6 @@ var lzmaWorker = new LZMA();
var redis = require('redis');
var log4js = require('log4js');
var pg = require('pg');
const setICUEnvVariable = require('../../lib/cartodb/utils/icu_data_env_setter');
// set environment specific variables
@ -147,11 +146,6 @@ afterEach(function(done) {
});
});
function cleanPGPoolConnections () {
// TODO: this method will be replaced by psql.end
pg.end();
}
function deleteRedisKeys(keysToDelete, callback) {
if (Object.keys(keysToDelete).length === 0) {
@ -214,6 +208,5 @@ module.exports = {
checkSurrogateKey: checkSurrogateKey,
checkCache: checkCache,
rmdirRecursiveSync: rmdirRecursiveSync,
configureMetadata,
cleanPGPoolConnections
configureMetadata
};

4
yarn.lock
View File

@ -2,6 +2,10 @@
# yarn lockfile v1
"@carto/fqdn-sync@0.2.2":
version "0.2.2"
resolved "https://registry.yarnpkg.com/@carto/fqdn-sync/-/fqdn-sync-0.2.2.tgz#cd7c645ed66690a09249b554d254a0f53963b2dc"
"@carto/mapnik@3.6.2-carto.10":
version "3.6.2-carto.10"
resolved "https://registry.yarnpkg.com/@carto/mapnik/-/mapnik-3.6.2-carto.10.tgz#a97c951dcdac09d0eb35b3ea71e5eeaa206c1af6"