cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into stats-middleware

Browse Source
This commit is contained in:
Daniel García Aubert 2017-10-16 15:06:20 +02:00
commit 9ad6d0cbcc
18 changed files with 685 additions and 553 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
.travis.yml
View File

@ -1,27 +1,14 @@
sudo: required
dist: trusty
addons:
postgresql: "9.5"
apt:
sources:
- ubuntu-toolchain-r-test
packages:
- postgresql-9.5-postgis-2.3
- postgresql-plpython-9.5
- pkg-config
- libcairo2-dev
- libjpeg8-dev
- libgif-dev
- libpango1.0-dev
- g++-4.9
services:
- docker
before_install:
- createdb template_postgis
- createuser publicuser
- psql -c "CREATE EXTENSION postgis" template_postgis
- docker pull cartoimages/windshaft-carto-testing
env:
- NPROCS=1 JOBS=1 PGUSER=postgres CXX=g++-4.9
script:
- docker run -e POSTGIS_VERSION=2.4 -v `pwd`:/srv cartoimages/windshaft-carto-testing
language: generic
language: node_js
node_js:
- "6"

14
config/environments/development.js.example
View File

@ -107,6 +107,20 @@ var config = {
// Milliseconds since last access before renderer cache item expires
cache_ttl: 60000,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mvt: {
//If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
//PostGIS 2.4 is required for this to work
//If disabled it will use Mapnik MVT generation
usePostGIS: false,
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
},
mapnik: {
// The size of the pool of internal mapnik backend
// This pool size is per mapnik renderer created in Windshaft's RendererFactory

14
config/environments/production.js.example
View File

@ -101,6 +101,20 @@ var config = {
// Milliseconds since last access before renderer cache item expires
cache_ttl: 60000,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mvt: {
//If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
//PostGIS 2.4 is required for this to work
//If disabled it will use Mapnik MVT generation
usePostGIS: false,
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
},
mapnik: {
// The size of the pool of internal mapnik backend
// This pool size is per mapnik renderer created in Windshaft's RendererFactory

14
config/environments/staging.js.example
View File

@ -101,6 +101,20 @@ var config = {
// Milliseconds since last access before renderer cache item expires
cache_ttl: 60000,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mvt: {
//If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
//PostGIS 2.4 is required for this to work
//If disabled it will use Mapnik MVT generation
usePostGIS: false,
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
},
mapnik: {
// The size of the pool of internal mapnik backend
// This pool size is per mapnik renderer created in Windshaft's RendererFactory

14
config/environments/test.js.example
View File

@ -100,6 +100,20 @@ var config = {
// Milliseconds since last access before renderer cache item expires
cache_ttl: 60000,
statsInterval: 5000, // milliseconds between each report to statsd about number of renderers and mapnik pool status
mvt: {
//If enabled, MVTs will be generated with PostGIS directly, instead of using Mapnik,
//PostGIS 2.4 is required for this to work
//If disabled it will use Mapnik MVT generation
usePostGIS: false,
dbPoolParams: {
// maximum number of resources to create at any given time
size: 16,
// max milliseconds a resource can go unused before it should be destroyed
idleTimeout: 3000,
// frequency to check for idle resources
reapInterval: 1000
}
},
mapnik: {
// The size of the pool of internal mapnik backend
// This pool size is per mapnik renderer created in Windshaft's RendererFactory

11
docker-test.sh Normal file
View File

@ -0,0 +1,11 @@
export NPROCS=1 && export JOBS=1 && export CXX=g++-4.9 && export PGUSER=postgres
npm install -g yarn@0.27.5
yarn
/etc/init.d/postgresql start
createdb template_postgis && createuser publicuser
psql -c "CREATE EXTENSION postgis" template_postgis
POSTGIS_VERSION=2.4 npm test

6
lib/cartodb/controllers/layergroup.js
View File

@ -294,6 +294,10 @@ LayergroupController.prototype.tileOrLayer = function (req, res, next) {
);
};
function getStatusCode(tile, format){
return tile.length===0 && format==='mvt'? 204:200;
}
// This function is meant for being called as the very last
// step by all endpoints serving tiles or grids
LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, tile, headers, next) {
@ -331,7 +335,7 @@ LayergroupController.prototype.finalizeGetTileOrGrid = function(err, req, res, t
global.statsClient.increment('windshaft.tiles.error');
global.statsClient.increment('windshaft.tiles.' + formatStat + '.error');
} else {
this.sendResponse(req, res, tile, 200, headers);
this.sendResponse(req, res, tile, getStatusCode(tile, formatStat), headers);
global.statsClient.increment('windshaft.tiles.success');
global.statsClient.increment('windshaft.tiles.' + formatStat + '.success');
}

3
lib/cartodb/server.js
View File

@ -158,7 +158,8 @@ module.exports = function(serverOptions) {
grainstore: serverOptions.grainstore,
mapnik: serverOptions.renderer.mapnik
},
http: serverOptions.renderer.http
http: serverOptions.renderer.http,
mvt: serverOptions.renderer.mvt
});
// initialize render cache

1
lib/cartodb/server_options.js
View File

@ -81,6 +81,7 @@ module.exports = {
statsInterval: rendererConfig.statsInterval
},
renderer: {
mvt: rendererConfig.mvt,
mapnik: _.defaults(rendererConfig.mapnik, {
geojson: {
dbPoolParams: {

10
package.json
View File

@ -44,7 +44,7 @@
"step-profiler": "~0.3.0",
"turbo-carto": "0.20.1",
"underscore": "~1.6.0",
"windshaft": "3.3.3",
"windshaft": "~4.0.0",
"yargs": "~5.0.0"
},
"devDependencies": {
@ -59,7 +59,13 @@
"scripts": {
"lint": "jshint lib test",
"preinstall": "make pre-install",
"test": "make test-all"
"test": "make test-all",
"update-internal-deps": "rm -rf node_modules && rm -f yarn.lock && yarn",
"docker-install": "sudo apt install docker.io && sudo usermod -aG docker $(whoami)",
"docker-pull": "docker pull cartoimages/windshaft-testing",
"docker-test": "docker run -v `pwd`:/srv cartoimages/windshaft-testing bash docker-test.sh",
"docker-bash": "docker run -it -v `pwd`:/srv cartoimages/windshaft-testing bash",
"docker-publish": "docker push cartoimages/windshaft-carto-testing"
},
"engines": {
"node": ">=6.9",

2
run_tests.sh
View File

@ -137,7 +137,7 @@ if test x"$OPT_COVERAGE" = xyes; then
./node_modules/.bin/istanbul cover node_modules/.bin/_mocha -- -u tdd -t 5000 ${TESTS}
else
echo "Running tests"
mocha -u tdd -t 5000 ${TESTS}
./node_modules/.bin/_mocha -c -u tdd -t 5000 ${TESTS}
fi
ret=$?

81
test/acceptance/analysis/analysis-layers-geojson.js
View File

@ -1,81 +0,0 @@
require('../../support/test_helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
describe('analysis-layers-dataviews-geojson', function() {
function createMapConfig(layers, dataviews, analysis) {
return {
version: '1.5.0',
layers: layers,
dataviews: dataviews || {},
analyses: analysis || []
};
}
var CARTOCSS = [
"#points {",
" marker-fill-opacity: 1.0;",
" marker-line-color: #FFF;",
" marker-line-width: 0.5;",
" marker-line-opacity: 1.0;",
" marker-placement: point;",
" marker-type: ellipse;",
" marker-width: 8;",
" marker-fill: red;",
" marker-allow-overlap: true;",
"}"
].join('\n');
var mapConfig = createMapConfig(
[
{
"type": "cartodb",
"options": {
"source": {
"id": "2570e105-7b37-40d2-bdf4-1af889598745"
},
"cartocss": CARTOCSS,
"cartocss_version": "2.3.0"
}
}
],
{
pop_max_histogram: {
source: {
id: '2570e105-7b37-40d2-bdf4-1af889598745'
},
type: 'histogram',
options: {
column: 'pop_max'
}
}
},
[
{
"id": "2570e105-7b37-40d2-bdf4-1af889598745",
"type": "source",
"params": {
"query": "select * from populated_places_simple_reduced"
}
}
]
);
it('should get pop_max column from dataview', function(done) {
var testClient = new TestClient(mapConfig, 1234);
testClient.getTile(0, 0, 0, {format: 'geojson', layers: 0}, function(err, res, geojson) {
assert.ok(!err, err);
assert.ok(Array.isArray(geojson.features));
assert.ok(geojson.features.length > 0);
var feature = geojson.features[0];
assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property');
testClient.drain(done);
});
});
});

110
test/acceptance/buffer-size-format.js
View File

@ -3,6 +3,7 @@ require('../support/test_helper');
var fs = require('fs');
var assert = require('../support/assert');
var TestClient = require('../support/test-client');
var serverOptions = require('../../lib/cartodb/server_options');
var mapnik = require('windshaft').mapnik;
var IMAGE_TOLERANCE_PER_MIL = 5;
@ -124,24 +125,37 @@ describe('buffer size per format', function () {
}
];
afterEach(function(done) {
if (this.testClient) {
return this.testClient.drain(done);
}
return done();
});
const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
testCases.forEach(function (test) {
it(test.desc, function (done) {
var testClient = new TestClient(test.mapConfig, 1234);
var coords = test.coords;
var options = {
format: test.format,
layers: test.layers
};
testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
// To generate images use:
// tile.save(test.fixturePath);
test.assert(tile, function (err) {
var testFn = (usePostGIS) => {
it(test.desc, function (done) {
serverOptions.renderer.mvt.usePostGIS = usePostGIS;
this.testClient = new TestClient(test.mapConfig, 1234);
serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
var coords = test.coords;
var options = {
format: test.format,
layers: test.layers
};
this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
testClient.drain(done);
// To generate images use:
// tile.save(test.fixturePath);
test.assert(tile, done);
});
});
});
};
if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){
testFn(true);
}
testFn(false);
});
});
@ -260,23 +274,27 @@ describe('buffer size per format for named maps', function () {
}
];
afterEach(function(done) {
if (this.testClient) {
return this.testClient.drain(done);
}
return done();
});
testCases.forEach(function (test) {
it(test.desc, function (done) {
var testClient = new TestClient(test.template, 1234);
this.testClient = new TestClient(test.template, 1234);
var coords = test.coords;
var options = {
format: test.format,
placeholders: test.placeholders,
layers: test.layers
};
testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
// To generate images use:
//tile.save('./test/fixtures/buffer-size/tile-7.64.48-buffer-size-0-test.png');
test.assert(tile, function (err) {
assert.ifError(err);
testClient.drain(done);
});
test.assert(tile, done);
});
});
});
@ -416,26 +434,40 @@ describe('buffer size per format for named maps w/o placeholders', function () {
];
afterEach(function(done) {
if (this.testClient) {
return this.testClient.drain(done);
}
return done();
});
const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
testCases.forEach(function (test) {
it(test.desc, function (done) {
var testClient = new TestClient(test.template, 1234);
var coords = test.coords;
var options = {
format: test.format,
placeholders: test.placeholders,
layers: test.layers
};
testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
// To generate images use:
//tile.save(test.fixturePath);
// require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile));
// require('fs').writeFileSync(test.fixturePath, tile.getDataSync());
test.assert(tile, function (err) {
assert.ifError(err);
testClient.drain(done);
var testFn = (usePostGIS) => {
it(test.desc + `(${usePostGIS? 'PostGIS':'mapnik'})`, function (done) {
serverOptions.renderer.mvt.usePostGIS = usePostGIS;
test.template.name += '_1';
this.testClient = new TestClient(test.template, 1234);
serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
var coords = test.coords;
var options = {
format: test.format,
placeholders: test.placeholders,
layers: test.layers
};
this.testClient.getTile(coords.z, coords.x, coords.y, options, function (err, res, tile) {
assert.ifError(err);
// To generate images use:
//tile.save(test.fixturePath);
// require('fs').writeFileSync(test.fixturePath, JSON.stringify(tile));
// require('fs').writeFileSync(test.fixturePath, tile.getDataSync());
test.assert(tile, done);
});
});
});
});
};
if (process.env.POSTGIS_VERSION === '2.4' && test.format === 'mvt'){
testFn(true);
}
testFn(false);
});
});

343
test/acceptance/geojson-renderer.js
View File

@ -1,343 +0,0 @@
require('../support/test_helper');
var assert = require('../support/assert');
var TestClient = require('../support/test-client');
describe('use only needed columns', function() {
function getFeatureByCartodbId(features, cartodbId) {
for (var i = 0, len = features.length; i < len; i++) {
if (features[i].properties.cartodb_id === cartodbId) {
return features[i];
}
}
return {};
}
var options = { format: 'geojson', layer: 0 };
afterEach(function(done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
it('with aggregation widget, interactivity and cartocss columns', function(done) {
var widgetMapConfig = {
version: '1.5.0',
layers: [{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: '#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }',
cartocss_version: '2.0.1',
widgets: {
adm0name: {
type: 'aggregation',
options: {
column: 'adm0name',
aggregation: 'sum',
aggregationColumn: 'pop_max'
}
}
},
interactivity: "cartodb_id,pop_min"
}
}]
};
this.testClient = new TestClient(widgetMapConfig);
this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
assert.ok(!err, err);
assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
cartodb_id: 1109,
name: 'Mardin',
adm0name: 'Turkey',
pop_max: 71373,
pop_min: 57586
});
done();
});
});
it('should not duplicate columns', function(done) {
var widgetMapConfig = {
version: '1.5.0',
layers: [{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: ['#layer0 {',
'marker-fill: red;',
'marker-width: 10;',
'[name="Madrid"] { marker-fill: green; } ',
'[pop_max>100000] { marker-fill: black; } ',
'}'].join('\n'),
cartocss_version: '2.3.0',
widgets: {
adm0name: {
type: 'aggregation',
options: {
column: 'adm0name',
aggregation: 'sum',
aggregationColumn: 'pop_max'
}
}
},
interactivity: "cartodb_id,pop_max"
}
}]
};
this.testClient = new TestClient(widgetMapConfig);
this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
assert.ok(!err, err);
assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
cartodb_id: 1109,
name: 'Mardin',
adm0name: 'Turkey',
pop_max: 71373
});
done();
});
});
it('with formula widget, no interactivity and no cartocss columns', function(done) {
var formulaWidgetMapConfig = {
version: '1.5.0',
layers: [{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
cartocss_version: '2.0.1',
interactivity: 'cartodb_id',
widgets: {
pop_max_f: {
type: 'formula',
options: {
column: 'pop_max',
operation: 'count'
}
}
}
}
}]
};
this.testClient = new TestClient(formulaWidgetMapConfig);
this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
assert.ok(!err, err);
assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
cartodb_id: 1109,
pop_max: 71373
});
done();
});
});
it('with cartocss with multiple expressions', function(done) {
var formulaWidgetMapConfig = {
version: '1.5.0',
layers: [{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' +
'#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
'#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
'#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }',
cartocss_version: '2.0.1',
interactivity: 'cartodb_id'
}
}]
};
this.testClient = new TestClient(formulaWidgetMapConfig);
this.testClient.getTile(0, 0, 0, options, function (err, res, geojsonTile) {
assert.ok(!err, err);
assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
cartodb_id: 1109,
pop_max:71373,
name:"Mardin",
adm0name:"Turkey"
});
done();
});
});
it('should work with mapnik substitution tokens', function(done) {
var cartocss = [
"#layer {",
" line-width: 2;",
" line-color: #3B3B58;",
" line-opacity: 1;",
" polygon-opacity: 0.7;",
" polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))",
"}"
].join('\n');
var sql = [
'WITH hgrid AS (',
' SELECT CDB_HexagonGrid(',
' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),',
' greatest(!pixel_width!,!pixel_height!) * 100',
' ) as cell',
')',
'SELECT',
' hgrid.cell as the_geom_webmercator,',
' count(1) as points_count,',
' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,',
' 1 as cartodb_id',
'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i',
'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)',
'GROUP BY hgrid.cell'
].join('\n');
var mapConfig = {
"version": "1.4.0",
"layers": [
{
"type": 'mapnik',
"options": {
"cartocss_version": '2.3.0',
"sql": sql,
"cartocss": cartocss
}
}
]
};
this.testClient = new TestClient(mapConfig);
this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) {
assert.ok(!err, err);
assert.ok(geojson);
assert.equal(geojson.features.length, 5);
done();
});
});
it('should skip empty and null columns for geojson tiles', function(done) {
var mapConfig = {
"analyses": [
{
"id": "a0",
"params": {
"query": "SELECT * FROM test_table"
},
"type": "source"
}
],
"dataviews": {
"4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": {
"options": {
"aggregationColumn": null,
"column": "cartodb_id",
"operation": "avg"
},
"source": {
"id": "a0"
},
"type": "formula"
},
"74f590f8-625c-4e95-922f-34ad3e9919c0": {
"options": {
"aggregation": "sum",
"aggregationColumn": "cartodb_id",
"column": "name"
},
"source": {
"id": "a0"
},
"type": "aggregation"
},
"98a75757-3006-400a-b028-fb613a6c0b69": {
"options": {
"aggregationColumn": null,
"column": "cartodb_id",
"operation": "sum"
},
"source": {
"id": "a0"
},
"type": "formula"
},
"ebbc97b2-87d2-4895-9e1f-2f012df3679d": {
"options": {
"aggregationColumn": null,
"bins": "12",
"column": "cartodb_id"
},
"source": {
"id": "a0"
},
"type": "histogram"
},
"ebc0653f-3581-469c-8b31-c969e440a865": {
"options": {
"aggregationColumn": null,
"column": "cartodb_id",
"operation": "avg"
},
"source": {
"id": "a0"
},
"type": "formula"
}
},
"layers": [
{
"options": {
"subdomains": "abcd",
"urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png"
},
"type": "http"
},
{
"options": {
"attributes": {
"columns": [
"name",
"address"
],
"id": "cartodb_id"
},
"cartocss": "#layer { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0",
"interactivity": "cartodb_id",
"layer_name": "wadus",
"source": {
"id": "a0"
}
},
"type": "cartodb"
},
{
"options": {
"subdomains": "abcd",
"urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png"
},
"type": "http"
}
]
};
this.testClient = new TestClient(mapConfig);
this.testClient.getTile(0, 0, 0, { format: 'geojson', layer: 0 }, function(err, res, geojson) {
assert.ok(!err, err);
assert.ok(geojson);
assert.equal(geojson.features.length, 5);
assert.deepEqual(Object.keys(geojson.features[0].properties), ['cartodb_id', 'name']);
done();
});
});
});

462
test/acceptance/mvt.js
View File

@ -1,9 +1,10 @@
require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
var assert = require('../support/assert');
var TestClient = require('../support/test-client');
var serverOptions = require('../../lib/cartodb/server_options');
function createMapConfig (sql = TestClient.SQL.ONE_POINT) {
function createMapConfig(sql = TestClient.SQL.ONE_POINT) {
return {
version: '1.6.0',
layers: [{
@ -18,7 +19,100 @@ function createMapConfig (sql = TestClient.SQL.ONE_POINT) {
};
}
describe('mvt', function () {
describe('mvt (mapnik)', mvt(false));
if (process.env.POSTGIS_VERSION === '2.4') {
describe('mvt (postgis)', mvt(true));
}
function mvt(usePostGIS) {
return function () {
const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
before(function () {
serverOptions.renderer.mvt.usePostGIS = usePostGIS;
});
after(function (){
serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
});
describe('analysis-layers-dataviews-mvt', function () {
function createMapConfig(layers, dataviews, analysis) {
return {
version: '1.5.0',
layers: layers,
dataviews: dataviews || {},
analyses: analysis || []
};
}
var CARTOCSS = [
"#points {",
" marker-fill-opacity: 1.0;",
" marker-line-color: #FFF;",
" marker-line-width: 0.5;",
" marker-line-opacity: 1.0;",
" marker-placement: point;",
" marker-type: ellipse;",
" marker-width: 8;",
" marker-fill: red;",
" marker-allow-overlap: true;",
"}"
].join('\n');
var mapConfig = createMapConfig(
[
{
"type": "cartodb",
"options": {
"source": {
"id": "2570e105-7b37-40d2-bdf4-1af889598745"
},
"cartocss": CARTOCSS,
"cartocss_version": "2.3.0"
}
}
],
{
pop_max_histogram: {
source: {
id: '2570e105-7b37-40d2-bdf4-1af889598745'
},
type: 'histogram',
options: {
column: 'pop_max'
}
}
},
[
{
"id": "2570e105-7b37-40d2-bdf4-1af889598745",
"type": "source",
"params": {
"query": "select * from populated_places_simple_reduced"
}
}
]
);
it('should get pop_max column from dataview', function (done) {
var testClient = new TestClient(mapConfig);
testClient.getTile(0, 0, 0, { format: 'mvt', layers: 0 }, function (err, res, MVT) {
var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
assert.ok(!err, err);
assert.ok(Array.isArray(geojsonTile.features));
assert.ok(geojsonTile.features.length > 0);
var feature = geojsonTile.features[0];
assert.ok(feature.properties.hasOwnProperty('pop_max'), 'Missing pop_max property');
testClient.drain(done);
});
});
});
const testCases = [
{
desc: 'should get empty mvt with code 204 (no content)',
@ -48,16 +142,366 @@ describe('mvt', function () {
testCases.forEach(function (test) {
it(test.desc, done => {
const testClient = new TestClient(test.mapConfig, 1234);
var testClient = new TestClient(test.mapConfig);
const { z, x, y } = test.coords;
const { format, response } = test;
testClient.getTile(z, x, y, { format, response }, (err, res) => {
testClient.getTile(z, x, y, { format, response }, err => {
assert.ifError(err);
assert.equal(res.statusCode, test.response.status);
testClient.drain(done);
});
});
});
});
if (usePostGIS){
describe('use only needed columns', onlyNeededColumns);
}else{
describe.skip('use only needed columns', onlyNeededColumns);
}
function onlyNeededColumns() {
function getFeatureByCartodbId(features, cartodbId) {
for (var i = 0, len = features.length; i < len; i++) {
if (features[i].properties.cartodb_id === cartodbId) {
return features[i];
}
}
return {};
}
var options = { format: 'mvt', layer: 0 };
afterEach(function (done) {
if (this.testClient) {
this.testClient.drain(done);
} else {
done();
}
});
it('with aggregation widget, interactivity and cartocss columns', function (done) {
var widgetMapConfig = {
version: '1.5.0',
layers: [{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss:
'#layer0 { marker-fill: red; marker-width: 10; [name="Madrid"] { marker-fill: green; } }',
cartocss_version: '2.0.1',
widgets: {
adm0name: {
type: 'aggregation',
options: {
column: 'adm0name',
aggregation: 'sum',
aggregationColumn: 'pop_max'
}
}
},
interactivity: "cartodb_id,pop_min"
}
}]
};
this.testClient = new TestClient(widgetMapConfig);
this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
assert.ok(!err, err);
assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
cartodb_id: 1109,
name: 'Mardin',
adm0name: 'Turkey',
pop_max: 71373,
pop_min: 57586
});
done();
});
});
it('should not duplicate columns', function (done) {
var widgetMapConfig = {
version: '1.5.0',
layers: [{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced',
cartocss: ['#layer0 {',
'marker-fill: red;',
'marker-width: 10;',
'[name="Madrid"] { marker-fill: green; } ',
'[pop_max>100000] { marker-fill: black; } ',
'}'].join('\n'),
cartocss_version: '2.3.0',
widgets: {
adm0name: {
type: 'aggregation',
options: {
column: 'adm0name',
aggregation: 'sum',
aggregationColumn: 'pop_max'
}
}
},
interactivity: "cartodb_id,pop_max"
}
}]
};
this.testClient = new TestClient(widgetMapConfig);
this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
assert.ok(!err, err);
assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
cartodb_id: 1109,
name: 'Mardin',
adm0name: 'Turkey',
pop_max: 71373
});
done();
});
});
it('with formula widget, no interactivity and no cartocss columns', function (done) {
var formulaWidgetMapConfig = {
version: '1.5.0',
layers: [{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
cartocss: '#layer0 { marker-fill: red; marker-width: 10; }',
cartocss_version: '2.0.1',
interactivity: 'cartodb_id',
widgets: {
pop_max_f: {
type: 'formula',
options: {
column: 'pop_max',
operation: 'count'
}
}
}
}
}]
};
this.testClient = new TestClient(formulaWidgetMapConfig);
this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
assert.ok(!err, err);
assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
cartodb_id: 1109,
pop_max: 71373
});
done();
});
});
it('with cartocss with multiple expressions', function (done) {
var formulaWidgetMapConfig = {
version: '1.5.0',
layers: [{
type: 'mapnik',
options: {
sql: 'select * from populated_places_simple_reduced where pop_max > 0 and pop_max < 600000',
cartocss: '#layer0 { marker-fill: red; marker-width: 10; }' +
'#layer0 { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
'#layer0[pop_max>1000] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }' +
'#layer0[adm0name=~".*Turkey*"] { marker-width: 14; [name="Madrid"] { marker-width: 20; } }',
cartocss_version: '2.0.1',
interactivity: 'cartodb_id'
}
}]
};
this.testClient = new TestClient(formulaWidgetMapConfig);
this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
assert.ok(!err, err);
assert.deepEqual(getFeatureByCartodbId(geojsonTile.features, 1109).properties, {
cartodb_id: 1109,
pop_max: 71373,
name: "Mardin",
adm0name: "Turkey"
});
done();
});
});
var skipOnPostGIS = usePostGIS ? it.skip: it;
skipOnPostGIS('should work with mapnik substitution tokens', function (done) {
var cartocss = [
"#layer {",
" line-width: 2;",
" line-color: #3B3B58;",
" line-opacity: 1;",
" polygon-opacity: 0.7;",
" polygon-fill: ramp([points_count], (#E5F5F9,#99D8C9,#2CA25F))",
"}"
].join('\n');
var sql = [
'WITH hgrid AS (',
' SELECT CDB_HexagonGrid(',
' ST_Expand(!bbox!, greatest(!pixel_width!,!pixel_height!) * 100),',
' greatest(!pixel_width!,!pixel_height!) * 100',
' ) as cell',
')',
'SELECT',
' hgrid.cell as the_geom_webmercator,',
' count(1) as points_count,',
' count(1)/power(100 * CDB_XYZ_Resolution(CDB_ZoomFromScale(!scale_denominator!)), 2) as points_density,',
' 1 as cartodb_id',
'FROM hgrid, (SELECT * FROM populated_places_simple_reduced) i',
'where ST_Intersects(i.the_geom_webmercator, hgrid.cell)',
'GROUP BY hgrid.cell'
].join('\n');
var mapConfig = {
"version": "1.4.0",
"layers": [
{
"type": 'mapnik',
"options": {
"cartocss_version": '2.3.0',
"sql": sql,
"cartocss": cartocss
}
}
]
};
this.testClient = new TestClient(mapConfig);
this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
assert.ok(!err, err);
assert.ok(geojsonTile);
assert.equal(geojsonTile.features.length, 5);
done();
});
});
it('should skip empty and null columns for geojson tiles', function (done) {
var mapConfig = {
"analyses": [
{
"id": "a0",
"params": {
"query": "SELECT * FROM test_table"
},
"type": "source"
}
],
"dataviews": {
"4e7b0e07-6d21-4b83-9adb-6d7e17eea6ca": {
"options": {
"aggregationColumn": null,
"column": "cartodb_id",
"operation": "avg"
},
"source": {
"id": "a0"
},
"type": "formula"
},
"74f590f8-625c-4e95-922f-34ad3e9919c0": {
"options": {
"aggregation": "sum",
"aggregationColumn": "cartodb_id",
"column": "name"
},
"source": {
"id": "a0"
},
"type": "aggregation"
},
"98a75757-3006-400a-b028-fb613a6c0b69": {
"options": {
"aggregationColumn": null,
"column": "cartodb_id",
"operation": "sum"
},
"source": {
"id": "a0"
},
"type": "formula"
},
"ebbc97b2-87d2-4895-9e1f-2f012df3679d": {
"options": {
"aggregationColumn": null,
"bins": "12",
"column": "cartodb_id"
},
"source": {
"id": "a0"
},
"type": "histogram"
},
"ebc0653f-3581-469c-8b31-c969e440a865": {
"options": {
"aggregationColumn": null,
"column": "cartodb_id",
"operation": "avg"
},
"source": {
"id": "a0"
},
"type": "formula"
}
},
"layers": [
{
"options": {
"subdomains": "abcd",
"urlTemplate": "http://{s}.basemaps.cartocdn.com/light_nolabels/{z}/{x}/{y}.png"
},
"type": "http"
},
{
"options": {
"attributes": {
"columns": [
"name",
"address"
],
"id": "cartodb_id"
},
"cartocss": "#layer { marker-width: 10; marker-fill: red; }",
"cartocss_version": "2.3.0",
"interactivity": "cartodb_id",
"layer_name": "wadus",
"source": {
"id": "a0"
}
},
"type": "cartodb"
},
{
"options": {
"subdomains": "abcd",
"urlTemplate": "http://{s}.basemaps.cartocdn.com/light_only_labels/{z}/{x}/{y}.png"
},
"type": "http"
}
]
};
this.testClient = new TestClient(mapConfig);
this.testClient.getTile(0, 0, 0, options, function (err, res, MVT) {
var geojsonTile = JSON.parse(MVT.toGeoJSONSync(0));
assert.ok(!err, err);
assert.ok(geojsonTile);
assert.equal(geojsonTile.features.length, 5);
assert.deepEqual(Object.keys(geojsonTile.features[0].properties), ['cartodb_id', 'name']);
done();
});
});
}
};
}

85
test/acceptance/user-render-timeout-limit.js
View File

@ -2,6 +2,7 @@ require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
var serverOptions = require('../../lib/cartodb/server_options');
const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`;
@ -200,49 +201,59 @@ describe('user render timeout limit', function () {
});
});
describe('vector', function () {
beforeEach(function (done) {
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
if (process.env.POSTGIS_VERSION === '2.4') {
describe('vector (PostGIS)', vector(true));
}
afterEach(function (done) {
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
describe('vector (mapnik)', vector(false));
function vector(usePostGIS) {
const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
return function () {
beforeEach(function (done) {
serverOptions.renderer.mvt.usePostGIS = usePostGIS;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(50, done);
});
});
it('layergroup creation works but vector tile request fails due to render timeout', function (done) {
const params = {
format: 'mvt',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
afterEach(function (done) {
serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
assert.deepEqual(tile, {
errors: ['You are over platform\'s limits. Please contact us to know more details'],
errors_with_context: [{
type: 'limit',
subtype: 'render',
message: 'You are over platform\'s limits. Please contact us to know more details'
}]
this.testClient.drain(done);
});
done();
});
});
});
it('layergroup creation works but vector tile request fails due to render timeout', function (done) {
const params = {
format: 'mvt',
response: {
status: 429,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
assert.deepEqual(tile, {
errors: ['You are over platform\'s limits. Please contact us to know more details'],
errors_with_context: [{
type: 'limit',
subtype: 'datasource',
message: 'You are over platform\'s limits. Please contact us to know more details'
}]
});
done();
});
});
};
}
describe('interativity', function () {
beforeEach(function (done) {

1
test/support/test-client.js
View File

@ -20,7 +20,6 @@ const MAPNIK_SUPPORTED_FORMATS = {
'png': true,
'png32': true,
'grid.json': true,
'geojson': true,
'mvt': true
};

36
yarn.lock
View File

@ -2,7 +2,7 @@
# yarn lockfile v1
abaculus@cartodb/abaculus#2.0.3-cdb1:
"abaculus@github:cartodb/abaculus#2.0.3-cdb1":
version "2.0.3-cdb1"
resolved "https://codeload.github.com/cartodb/abaculus/tar.gz/f5f34e1c80cdd8d49edd1d6fe3b2220ab2e23aaf"
dependencies:
@ -210,7 +210,7 @@ camshaft@0.59.2:
dot "^1.0.3"
request "^2.69.0"
canvas@cartodb/node-canvas#1.6.2-cdb2:
"canvas@github:cartodb/node-canvas#1.6.2-cdb2":
version "1.6.2-cdb2"
resolved "https://codeload.github.com/cartodb/node-canvas/tar.gz/8acf04557005c633f9e68524488a2657c04f3766"
dependencies:
@ -228,15 +228,15 @@ carto@0.16.3:
semver "^5.1.0"
yargs "^4.2.0"
carto@CartoDB/carto#0.15.1-cdb1:
"carto@github:cartodb/carto#0.15.1-cdb1":
version "0.15.1-cdb1"
resolved "https://codeload.github.com/CartoDB/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398"
resolved "https://codeload.github.com/cartodb/carto/tar.gz/8050ec843f1f32a6469e5d1cf49602773015d398"
dependencies:
mapnik-reference "~6.0.2"
optimist "~0.6.0"
underscore "~1.6.0"
carto@cartodb/carto#0.15.1-cdb3:
"carto@github:cartodb/carto#0.15.1-cdb3":
version "0.15.1-cdb3"
resolved "https://codeload.github.com/cartodb/carto/tar.gz/945f5efb74fd1af1f5e1f69f409f9567f94fb5a7"
dependencies:
@ -584,8 +584,8 @@ exit@0.1.2, exit@0.1.x:
resolved "https://registry.yarnpkg.com/exit/-/exit-0.1.2.tgz#0632638f8d877cc82107d30a0fff1a17cba1cd0c"
express@~4.16.0:
version "4.16.1"
resolved "https://registry.yarnpkg.com/express/-/express-4.16.1.tgz#6b33b560183c9b253b7b62144df33a4654ac9ed0"
version "4.16.2"
resolved "https://registry.yarnpkg.com/express/-/express-4.16.2.tgz#e35c6dfe2d64b7dca0a5cd4f21781be3299e076c"
dependencies:
accepts "~1.3.4"
array-flatten "1.1.1"
@ -1286,7 +1286,11 @@ mocha@~3.4.1:
mkdirp "0.5.1"
supports-color "3.1.2"
moment@^2.10.6, moment@~2.18.1:
moment@^2.10.6:
version "2.19.0"
resolved "https://registry.yarnpkg.com/moment/-/moment-2.19.0.tgz#44f675ef6b944942762581b1c179fb679e599d67"
moment@~2.18.1:
version "2.18.1"
resolved "https://registry.yarnpkg.com/moment/-/moment-2.18.1.tgz#c36193dd3ce1c2eed2adb7c802dbbc77a81b1c0f"
@ -1522,7 +1526,7 @@ pg-types@1.*:
postgres-date "~1.0.0"
postgres-interval "^1.1.0"
pg@cartodb/node-postgres#6.1.6-cdb1:
"pg@github:cartodb/node-postgres#6.1.6-cdb1":
version "6.1.6"
resolved "https://codeload.github.com/cartodb/node-postgres/tar.gz/3eef52dd1e655f658a4ee8ac5697688b3ecfed44"
dependencies:
@ -1580,8 +1584,8 @@ postcss@5.0.19:
supports-color "^3.1.2"
postcss@^5.0.18, postcss@^5.2.5, postcss@~5.2.8:
version "5.2.17"
resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.17.tgz#cf4f597b864d65c8a492b2eabe9d706c879c388b"
version "5.2.18"
resolved "https://registry.yarnpkg.com/postcss/-/postcss-5.2.18.tgz#badfa1497d46244f6390f58b319830d9107853c5"
dependencies:
chalk "^1.1.3"
js-base64 "^2.1.9"
@ -2089,7 +2093,7 @@ through@2:
version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4:
"tilelive-bridge@github:cartodb/tilelive-bridge#2.3.1-cdb4":
version "2.3.1-cdb4"
resolved "https://codeload.github.com/cartodb/tilelive-bridge/tar.gz/faa2b638da2d119b78281575d40255cb523f6ca6"
dependencies:
@ -2097,7 +2101,7 @@ tilelive-bridge@cartodb/tilelive-bridge#2.3.1-cdb4:
mapnik-pool "~0.1.3"
sphericalmercator "1.0.x"
tilelive-mapnik@cartodb/tilelive-mapnik#0.6.18-cdb3:
"tilelive-mapnik@github:cartodb/tilelive-mapnik#0.6.18-cdb3":
version "0.6.18-cdb3"
resolved "https://codeload.github.com/cartodb/tilelive-mapnik/tar.gz/23bd1c31dd57d0b76c86b9f1eaf62462b3c17d01"
dependencies:
@ -2258,9 +2262,9 @@ window-size@^0.2.0:
version "0.2.0"
resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.2.0.tgz#b4315bb4214a3d7058ebeee892e13fa24d98b075"
windshaft@3.3.3:
version "3.3.3"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-3.3.3.tgz#0582e6a0d9cf91c533134787ace64a3337200e33"
windshaft@~4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/windshaft/-/windshaft-4.0.0.tgz#b28ffb3775db3a4f2578e79d73c9b854a1ae5854"
dependencies:
abaculus cartodb/abaculus#2.0.3-cdb1
canvas cartodb/node-canvas#1.6.2-cdb2