cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into refactor-metadata

Browse Source 
# Conflicts:
#	lib/cartodb/backends/layer-stats/mapnik-layer-stats.js
This commit is contained in:
Javier Goizueta 2018-05-23 12:12:36 +02:00
commit c1feaecbcb
8 changed files with 777 additions and 299 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
NEWS.md
View File

@ -1,17 +1,36 @@
# Changelog
## 6.1.1
## 6.2.0
Released 2018-mm-dd
New features:
- CI tests with Ubuntu Xenial + PostgreSQL 10.1 and Ubuntu Precise + PostgreSQL 9.5
- Upgrades Windshaft to [4.7.3](https://github.com/CartoDB/Windshaft/blob/4.7.3/NEWS.md#version-473) which includes:
- Upgrades Windshaft to [4.8.0](https://github.com/CartoDB/Windshaft/blob/4.8.0/NEWS.md#version-480) which includes:
- Update internal deps.
- A fix in mapnik-vector-tile to avoid grouping together properties with the same value but a different type.
- Performance improvements in the marker symbolizer (local cache, avoid building the collision matrix when possible).
- MVT: Disable simplify_distance to avoid multiple simplifications.
- Fix a bug with zero length lines not being rendered when using the marker symbolizer.
- Upgrades Camshaft to [0.61.9](https://github.com/CartoDB/camshaft/releases/tag/0.61.9):
- Use Dollar-Quoted String Constants to avoid Syntax Error while running moran analyses.
- Update other deps:
- body-parser: 1.18.3
- dot: 1.1.2
- express: 4.16.3
- lru-cache: 4.1.3
- node-statsd: 0.1.1,
- queue-async: 1.1.0
- request: 2.87.0
- semver: 5.5.0
- step: 1.0.0
- yargs: 11.1.0
- Update devel deps:
- istanbul: 0.4.5
- jshint: 2.9.5
- mocha: 3.5.3
- moment: 2.22.1
- nock: 9.2.6
- strftime: 0.10.0
- Optional instantiation metadata stats (https://github.com/CartoDB/Windshaft-cartodb/pull/952)
Bug Fixes:

6
lib/cartodb/backends/layer-stats/mapnik-layer-stats.js
View File

@ -172,12 +172,16 @@ function _columnStats(ctx, columns) {
);
if (columns[name].type === 'string') {
const topN = ctx.metaOptions.columnStats.topCategories || 1024;
const includeNulls = ctx.metaOptions.columnStats.hasOwnProperty('includeNulls') ?
ctx.metaOptions.columnStats.includeNulls :
true;
// TODO: ctx.metaOptions.columnStats.maxCategories
// => use PG stats to dismiss columns with more distinct values
queries.push(
queryPromise(
ctx.dbConnection,
_getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN))
_getSQL(ctx, sql => queryUtils.getQueryTopCategories(sql, name, topN, includeNulls))
).then(res => ({ [name]: { categories: res.rows } }))
);
}

50
package.json
View File

@ -24,41 +24,41 @@
"Simon Martin <simon@carto.com>"
],
"dependencies": {
"basic-auth": "^2.0.0",
"body-parser": "^1.18.2",
"basic-auth": "2.0.0",
"body-parser": "1.18.3",
"camshaft": "0.61.9",
"cartodb-psql": "0.10.2",
"cartodb-query-tables": "0.3.0",
"cartodb-redis": "1.0.0",
"debug": "^3.1.0",
"dot": "~1.0.2",
"express": "~4.16.0",
"fastly-purge": "~1.0.1",
"glob": "^7.1.2",
"debug": "3.1.0",
"dot": "1.1.2",
"express": "4.16.3",
"fastly-purge": "1.0.1",
"glob": "7.1.2",
"log4js": "cartodb/log4js-node#cdb",
"lru-cache": "2.6.5",
"lzma": "~2.3.2",
"node-statsd": "~0.0.7",
"on-headers": "^1.0.1",
"queue-async": "~1.0.7",
"lru-cache": "4.1.3",
"lzma": "2.3.2",
"node-statsd": "0.1.1",
"on-headers": "1.0.1",
"queue-async": "1.1.0",
"redis-mpool": "0.5.0",
"request": "2.85.0",
"semver": "~5.3.0",
"step": "~0.0.6",
"step-profiler": "~0.3.0",
"request": "2.87.0",
"semver": "5.5.0",
"step": "1.0.0",
"step-profiler": "0.3.0",
"turbo-carto": "0.20.2",
"underscore": "~1.6.0",
"windshaft": "4.7.3",
"yargs": "~5.0.0"
"underscore": "1.6.0",
"windshaft": "4.8.0",
"yargs": "11.1.0"
},
"devDependencies": {
"istanbul": "~0.4.3",
"jshint": "~2.9.4",
"mocha": "~3.4.1",
"moment": "~2.18.1",
"nock": "~2.11.0",
"istanbul": "0.4.5",
"jshint": "2.9.5",
"mocha": "3.5.3",
"moment": "2.22.1",
"nock": "9.2.6",
"redis": "2.8.0",
"strftime": "~0.8.2"
"strftime": "0.10.0"
},
"scripts": {
"lint": "jshint lib test",

129
test/acceptance/stats/mapnik_stats_layergroup.js
View File

@ -74,6 +74,26 @@ describe('Create mapnik layergroup', function() {
}
};
var mapnikLayerNullCats = {
type: 'mapnik',
options: {
sql: `
WITH geom AS (
SELECT
'SRID=4326;POINT(0 0)'::geometry AS the_geom,
'SRID=3857;POINT(0 0)'::geometry AS the_geom_webmercator
)
SELECT 1 AS cartodb_id, 'A' As cat, geom.* FROM geom
UNION
SELECT 2 AS cartodb_id, 'B' As cat, geom.* FROM geom
UNION
SELECT 2 AS cartodb_id, NULL::text As cat, geom.* FROM geom
`,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
function mapnikBasicLayerId(index) {
return 'layer' + index;
}
@ -309,6 +329,32 @@ describe('Create mapnik layergroup', function() {
});
});
// metadata categories are ordered only partially by descending frequency;
// this orders them completely to avoid ambiguities when comparing
function withSortedCategories(columns) {
function catOrder(a, b) {
if (a.frequency !== b.frequency) {
return b.frequency - a.frequency;
}
if (a.category < b.category) {
return -1;
}
if (a.category > b.category) {
return +1;
}
return 0;
}
let sorted = {};
Object.keys(columns).forEach(name => {
let data = columns[name];
if (data.hasOwnProperty('categories')) {
data = Object.assign(data, { categories: data.categories.sort(catOrder)});
}
sorted[name] = data;
});
return sorted;
}
it('should provide column stats as optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',
@ -319,32 +365,6 @@ describe('Create mapnik layergroup', function() {
]
});
// metadata categories are ordered only partially by descending frequency;
// this orders them completely to avoid ambiguities when comparing
function withSortedCategories(columns) {
function catOrder(a, b) {
if (a.frequency !== b.frequency) {
return b.frequency - a.frequency;
}
if (a.category < b.category) {
return -1;
}
if (a.category > b.category) {
return +1;
}
return 0;
}
let sorted = {};
Object.keys(columns).forEach(name => {
let data = columns[name];
if (data.hasOwnProperty('categories')) {
data = Object.assign(data, { categories: data.categories.sort(catOrder)});
}
sorted[name] = data;
});
return sorted;
}
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
@ -393,6 +413,63 @@ describe('Create mapnik layergroup', function() {
});
});
it('should limit the number of categories as requested', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
columnStats: { topCategories: 2 }
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns;
assert.equal(columnsMetadata.address.categories.length, 2);
testClient.drain(done);
});
});
it('should include null categories if requested', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayerNullCats, {
columnStats: { includeNulls: true }
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns;
assert.equal(columnsMetadata.cat.categories.length, 3);
testClient.drain(done);
});
});
it('should not include null categories if not requested', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayerNullCats, {
columnStats: { includeNulls: false }
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns;
assert.equal(columnsMetadata.cat.categories.length, 2);
testClient.drain(done);
});
});
it('should provide row count as optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',

72
test/acceptance/user-database-timeout-limit.js
View File

@ -2,6 +2,7 @@ require('../support/test_helper');
const assert = require('../support/assert');
const TestClient = require('../support/test-client');
const serverOptions = require('../../lib/cartodb/server_options');
const timeoutErrorTilePath = `${process.cwd()}/assets/render-timeout-fallback.png`;
@ -438,6 +439,77 @@ describe('user database timeout limit', function () {
});
});
if (process.env.POSTGIS_VERSION === '2.4') {
describe('fetching vector tiles via PostGIS renderer', function() {
const usePostGIS = true;
const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
beforeEach(function (done) {
serverOptions.renderer.mvt.usePostGIS = usePostGIS;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
const expectedResponse = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
this.testClient.getLayergroup({ response: expectedResponse }, (err, res) => {
if (err) {
return done(err);
}
this.layergroupid = res.layergroupid;
done();
});
});
afterEach(function () {
serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
});
describe('with user\'s timeout of 200 ms', function () {
beforeEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(200, done);
});
afterEach(function (done) {
this.testClient.setUserDatabaseTimeoutLimit(0, done);
});
it('"mvt" fails due to statement timeout', function (done) {
const params = {
layergroupid: this.layergroupid,
format: 'mvt',
layers: [ 0 ],
response: {
status: 429,
headers: {
'Content-Type': 'application/x-protobuf'
}
},
cacheBuster: true
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
var tileJSON = tile.toJSON();
assert.equal(Array.isArray(tileJSON), true);
assert.equal(tileJSON.length, 2);
assert.equal(tileJSON[0].name, 'errorTileSquareLayer');
assert.equal(tileJSON[1].name, 'errorTileStripesLayer');
done();
});
});
});
});
}
});
});

101
test/acceptance/user-render-timeout-limit.js
View File

@ -139,7 +139,7 @@ describe('user render timeout limit', function () {
});
it('layergroup creation works but tile request fails due to render timeout', function (done) {
this.testClient.getTile(0, 0, 0, {}, (err, res, tile) => {
this.testClient.getTile(0, 0, 0, { cacheBuster: true }, (err, res, tile) => {
assert.ifError(err);
assert.imageIsSimilarToFile(tile, timeoutErrorTilePath, 0.05, (err) => {
@ -180,7 +180,8 @@ describe('user render timeout limit', function () {
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
},
cacheBuster: true
};
this.testClient.getTile(0, 0, 0, params, (err, res, timeoutError) => {
@ -201,56 +202,52 @@ describe('user render timeout limit', function () {
});
});
if (process.env.POSTGIS_VERSION === '2.4') {
describe('vector (PostGIS)', vector(true));
}
describe('vector (mapnik)', vector(false));
function vector(usePostGIS) {
describe('vector tile via mapnik renderer', function () {
const usePostGIS = false;
const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
return function () {
beforeEach(function (done) {
serverOptions.renderer.mvt.usePostGIS = usePostGIS;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserDatabaseTimeoutLimit(50, done);
});
afterEach(function (done) {
serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
this.testClient.setUserDatabaseTimeoutLimit(0, (err) => {
if (err) {
return done(err);
beforeEach(function (done) {
serverOptions.renderer.mvt.usePostGIS = usePostGIS;
const mapconfig = createMapConfig();
this.testClient = new TestClient(mapconfig, 1234);
this.testClient.setUserRenderTimeoutLimit('localhost', 50, done);
});
afterEach(function (done) {
serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
this.testClient.setUserRenderTimeoutLimit('localhost', 0, (err) => {
if (err) {
return done(err);
}
this.testClient.drain(done);
});
});
it('layergroup creation works but vector tile request fails due to render timeout', function (done) {
const params = {
format: 'mvt',
response: {
status: 429,
headers: {
'Content-Type': 'application/x-protobuf'
}
this.testClient.drain(done);
});
},
cacheBuster: true
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
var tileJSON = tile.toJSON();
assert.equal(Array.isArray(tileJSON), true);
assert.equal(tileJSON.length, 2);
assert.equal(tileJSON[0].name, 'errorTileSquareLayer');
assert.equal(tileJSON[1].name, 'errorTileStripesLayer');
done();
});
it('layergroup creation works but vector tile request fails due to render timeout', function (done) {
const params = {
format: 'mvt',
response: {
status: 429,
headers: {
'Content-Type': 'application/x-protobuf'
}
}
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
assert.ifError(err);
var tileJSON = tile.toJSON();
assert.equal(Array.isArray(tileJSON), true);
assert.equal(tileJSON.length, 2);
assert.equal(tileJSON[0].name, 'errorTileSquareLayer');
assert.equal(tileJSON[1].name, 'errorTileStripesLayer');
done();
});
});
};
}
});
});
describe('interativity', function () {
beforeEach(function (done) {
@ -277,7 +274,8 @@ describe('user render timeout limit', function () {
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
},
cacheBuster: true
};
this.testClient.getTile(0, 0, 0, params, (err, res, tile) => {
@ -365,7 +363,7 @@ describe('user render timeout limit', function () {
});
});
it('layergroup creation works and render tile fails', function (done) {
it('layergroup creation works and render static center tile fails', function (done) {
const params = {
zoom: 0,
lat: 0,
@ -378,7 +376,8 @@ describe('user render timeout limit', function () {
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
},
cacheBuster: true
};
this.testClient.getStaticCenter(params, function (err, res, timeoutError) {

15
test/support/test-client.js
View File

@ -125,6 +125,14 @@ function resErr2errRes(callback) {
};
}
function layergroupidTemplate (layergroupId, params) {
const { token, signer, cacheBuster } = LayergroupToken.parse(layergroupId);
// {user}@{token}:{cache_buster}
// {token}:{cache_buster}
return `${signer ? signer + '@' : ''}${token}:${params.cacheBuster ? Date.now() : cacheBuster }`;
}
TestClient.prototype.getWidget = function(widgetName, params, callback) {
var self = this;
@ -726,7 +734,7 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
url = '/api/v1/map/' + layergroupId + '/';
url = `/api/v1/map/${layergroupidTemplate(layergroupId, params)}/`;
var layers = params.layers;
@ -769,7 +777,6 @@ TestClient.prototype.getTile = function(z, x, y, params, callback) {
}
}, params.response);
var isPng = format.match(/png$/);
if (isPng) {
@ -954,7 +961,9 @@ TestClient.prototype.getStaticCenter = function (params, callback) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(layergroupId).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
url = `/api/v1/map/static/center/${layergroupId}/${zoom}/${lat}/${lng}/${width}/${height}.${format}`;
const layergroupid = layergroupidTemplate(layergroupId, params);
url = `/api/v1/map/static/center/${layergroupid}/${zoom}/${lat}/${lng}/${width}/${height}.${format}`;
if (self.apiKey) {
url += '?' + qs.stringify({api_key: self.apiKey});

680
yarn.lock
View File

File diff suppressed because it is too large Load Diff