Windshaft-cartodb/test/acceptance/stats/mapnik-stats-layergroup-test.js
2019-10-07 11:16:48 +02:00

663 lines
24 KiB
JavaScript

'use strict';
require('../../support/test-helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
const serverOptions = require('../../../lib/server-options');
const suites = [
{
desc: 'mvt (mapnik)',
usePostGIS: false
},
{
desc: 'mvt (postgis)',
usePostGIS: true
}
];
suites.forEach(({desc, usePostGIS}) => {
describe(`[${desc}] Create mapnik layergroup`, function() {
const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
before(function() {
serverOptions.renderer.mvt.usePostGIS = usePostGIS;
this.layerStatsConfig = global.environment.enabledFeatures.layerStats;
global.environment.enabledFeatures.layerStats = true;
});
after(function() {
serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
global.environment.enabledFeatures.layerStats = this.layerStatsConfig;
});
var cartocssVersion = '2.3.0';
var cartocss = '#layer { line-width:16; }';
var mapnikLayer1 = {
type: 'mapnik',
options: {
sql: 'select * from test_table limit 1',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer2 = {
type: 'mapnik',
options: {
sql: 'select * from test_table_2 limit 2',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer3 = {
type: 'mapnik',
options: {
sql: 'select * from test_table_3 limit 3',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer4 = {
type: 'mapnik',
options: {
sql: [
'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address',
' from test_table t1, test_table_2 t2',
' where t1.cartodb_id = t2.cartodb_id'
].join(''),
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer100 = {
type: 'mapnik',
options: {
sql: [
'SELECT * FROM test_table_100'
].join(''),
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var httpLayer = {
type: 'http',
options: {
urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png',
subdomains: ['a','b','c']
}
};
var mapnikLayerGeomColumn = {
type: 'mapnik',
options: {
sql: 'select *, the_geom as my_geom from test_table_3 limit 2',
geom_column: 'my_geom',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayerNullCats = {
type: 'mapnik',
options: {
sql: `
WITH geom AS (
SELECT
'SRID=4326;POINT(0 0)'::geometry AS the_geom,
'SRID=3857;POINT(0 0)'::geometry AS the_geom_webmercator
)
SELECT 1 AS cartodb_id, 'A' As cat, geom.* FROM geom
UNION
SELECT 2 AS cartodb_id, 'B' As cat, geom.* FROM geom
UNION
SELECT 2 AS cartodb_id, NULL::text As cat, geom.* FROM geom
`,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
function mapnikBasicLayerId(index) {
return 'layer' + index;
}
function typeLayerId(type, index) {
return type + '-' + mapnikBasicLayerId(index);
}
it('with one mapnik layer should response with meta-stats for that layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
testClient.drain(done);
});
});
it('with two mapnik layer should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
mapnikLayer2
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2);
testClient.drain(done);
});
});
it('with three mapnik layer should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
mapnikLayer2,
mapnikLayer3
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2);
assert.equal(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2));
assert.equal(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 3);
testClient.drain(done);
});
});
it('with one mapnik layer (sql with join) should response with meta-stats for that layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
testClient.drain(done);
});
});
it('with two mapnik layer (sql with join) should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer4,
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5);
testClient.drain(done);
});
});
it('with two mapnik layer (with & without join) should response with meta-stats for every layer', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer3,
mapnikLayer4
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3);
assert.ok(!layergroup.metadata.layers[0].meta.stats[1]);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5);
assert.ok(!layergroup.metadata.layers[2]);
testClient.drain(done);
});
});
it('with mapnik and layer and httplayer should response with layer metadata with same order', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
httpLayer
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].type, 'mapnik');
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[1].type, 'http');
testClient.drain(done);
});
});
it('with httpLayer and mapnik layer should response with layer metadata with same order', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
httpLayer,
mapnikLayer1
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[0].type, 'http');
assert.ok(!layergroup.metadata.layers[0].meta.cartocss);
assert.equal(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 1);
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[1].type, 'mapnik');
assert.equal(layergroup.metadata.layers[1].meta.cartocss, cartocss);
testClient.drain(done);
});
});
it('should work with different geom_column', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayerGeomColumn
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
// we don't care about stats here as is an aliased column
assert.ok(layergroup.metadata.layers[0].meta.stats.hasOwnProperty('estimatedFeatureCount'));
testClient.drain(done);
});
});
it('should not include the stats part if the FF is disabled', function(done) {
global.environment.enabledFeatures.layerStats = false;
var testClient = new TestClient({
version: '1.4.0',
layers: [
httpLayer,
mapnikLayer1,
httpLayer
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.equal(layergroup.metadata.layers[0].type, 'http');
assert.equal(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[1].type, 'mapnik');
assert.ok(!layergroup.metadata.layers[1].meta.hasOwnProperty('stats'));
assert.equal(layergroup.metadata.layers[2].id, typeLayerId('http', 1));
assert.equal(layergroup.metadata.layers[2].type, 'http');
global.environment.enabledFeatures.layerStats = true;
testClient.drain(done);
});
});
function layerWithMetadata(layer, metadata) {
return Object.assign(layer, {
options: Object.assign(layer.options, { metadata })
});
}
it('should provide columns as optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
columns: true
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
const expectedColumns = {
cartodb_id: { type: 'number' },
the_geom: { type: 'geometry' },
the_geom_webmercator: { type: 'geometry' },
address: { type: 'string' }
};
assert.deepEqual(layergroup.metadata.layers[0].meta.stats.columns, expectedColumns);
testClient.drain(done);
});
});
// metadata categories are ordered only partially by descending frequency;
// this orders them completely to avoid ambiguities when comparing
function withSortedCategories(columns) {
function catOrder(a, b) {
if (a.frequency !== b.frequency) {
return b.frequency - a.frequency;
}
if (a.category < b.category) {
return -1;
}
if (a.category > b.category) {
return +1;
}
return 0;
}
let sorted = {};
Object.keys(columns).forEach(name => {
let data = columns[name];
if (data.hasOwnProperty('categories')) {
data = Object.assign(data, { categories: data.categories.sort(catOrder)});
}
sorted[name] = data;
});
return sorted;
}
it('should provide column stats as optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
columnStats: true
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
const expectedColumns = {
cartodb_id: {
type: 'number',
avg: 3,
max: 5,
min: 1,
sum: 15
},
the_geom: { type: 'geometry' },
the_geom_webmercator: { type: 'geometry' },
address: {
type: 'string',
categories: [
{
category: "Calle de la Palma 72, Madrid, Spain",
frequency: 1
},
{
category: "Calle de Pérez Galdós 9, Madrid, Spain",
frequency: 1
},
{
category: "Calle Divino Pastor 12, Madrid, Spain",
frequency: 1
},
{
category: "Manuel Fernández y González 8, Madrid, Spain",
frequency: 1
},
{
category: "Plaza Conde de Toreno 2, Madrid, Spain",
frequency: 1
}
]
}
};
assert.deepEqual(
withSortedCategories(layergroup.metadata.layers[0].meta.stats.columns),
withSortedCategories(expectedColumns)
);
testClient.drain(done);
});
});
it('should limit the number of categories as requested', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
columnStats: { topCategories: 2 }
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns;
assert.equal(columnsMetadata.address.categories.length, 2);
testClient.drain(done);
});
});
it('should include null categories if requested', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayerNullCats, {
columnStats: { includeNulls: true }
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns;
assert.equal(columnsMetadata.cat.categories.length, 3);
testClient.drain(done);
});
});
it('should not include null categories if not requested', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayerNullCats, {
columnStats: { includeNulls: false }
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns;
assert.equal(columnsMetadata.cat.categories.length, 2);
testClient.drain(done);
});
});
it('should provide row count as optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
featureCount: true
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5);
testClient.drain(done);
});
});
it('should provide geometry type as optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
geometryType: true
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point');
testClient.drain(done);
});
});
it(`should not fail "TypeError: ... 'geom_type' of undefined" for empty results`, function(done) {
var testClient = new TestClient({
version: '1.8.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table where false',
metadata: {
geometryType: true
}
}
}
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, undefined);
testClient.drain(done);
});
});
it('should provide a sample as optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer100, {
sample: { num_rows: 30 }
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 100);
assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0);
const expectedCols = [ 'cartodb_id', 'value', 'the_geom', 'the_geom_webmercator' ].sort();
assert.deepEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols);
testClient.drain(done);
});
});
it('should not provide a sample when the source table is empty', function (done) {
var testClient = new TestClient({
version: "1.4.0",
layers: [
{
type: "mapnik",
options: {
sql: "SELECT * FROM test_table_100 limit 0",
cartocss_version: "2.3.0",
cartocss: "#layer { line-width:16; }",
metadata: {
sample: {
num_rows: 30
}
}
}
}
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.deepStrictEqual(layergroup.metadata.layers[0].meta.stats.sample, {});
testClient.drain(done);
});
});
it('can specify sample columns', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer100, {
sample: {
num_rows: 30,
include_columns: [ 'cartodb_id', 'the_geom' ]
}
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 100);
assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0);
const expectedCols = [ 'cartodb_id', 'the_geom' ].sort();
assert.deepEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols);
testClient.drain(done);
});
});
it('should only provide requested optional metadata', function(done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
geometryType: true,
featureCount: true
})
]
});
testClient.getLayergroup(function(err, layergroup) {
assert.ifError(err);
assert.equal(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.equal(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.equal(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point');
assert.equal(layergroup.metadata.layers[0].meta.stats.featureCount, 5);
assert.equal(layergroup.metadata.layers[0].meta.stats.sample, undefined);
assert.equal(layergroup.metadata.layers[0].meta.stats.columns, undefined);
testClient.drain(done);
});
});
});
});