Windshaft-cartodb/test/acceptance/stats/mapnik-stats-layergroup-test.js

661 lines
27 KiB
JavaScript

'use strict';
require('../../support/test-helper');
var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');
const serverOptions = require('../../../lib/server-options');
const suites = [
{
desc: 'mvt (mapnik)',
usePostGIS: false
},
{
desc: 'mvt (postgis)',
usePostGIS: true
}
];
suites.forEach(({ desc, usePostGIS }) => {
describe(`[${desc}] Create mapnik layergroup`, function () {
const originalUsePostGIS = serverOptions.renderer.mvt.usePostGIS;
before(function () {
serverOptions.renderer.mvt.usePostGIS = usePostGIS;
this.layerStatsConfig = global.environment.enabledFeatures.layerStats;
global.environment.enabledFeatures.layerStats = true;
});
after(function () {
serverOptions.renderer.mvt.usePostGIS = originalUsePostGIS;
global.environment.enabledFeatures.layerStats = this.layerStatsConfig;
});
var cartocssVersion = '2.3.0';
var cartocss = '#layer { line-width:16; }';
var mapnikLayer1 = {
type: 'mapnik',
options: {
sql: 'select * from test_table limit 1',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer2 = {
type: 'mapnik',
options: {
sql: 'select * from test_table_2 limit 2',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer3 = {
type: 'mapnik',
options: {
sql: 'select * from test_table_3 limit 3',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer4 = {
type: 'mapnik',
options: {
sql: [
'select t1.cartodb_id, t1.the_geom, t1.the_geom_webmercator, t2.address',
' from test_table t1, test_table_2 t2',
' where t1.cartodb_id = t2.cartodb_id'
].join(''),
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayer100 = {
type: 'mapnik',
options: {
sql: [
'SELECT * FROM test_table_100'
].join(''),
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var httpLayer = {
type: 'http',
options: {
urlTemplate: 'http://{s}.basemaps.cartocdn.com/dark_nolabels/{z}/{x}/{y}.png',
subdomains: ['a', 'b', 'c']
}
};
var mapnikLayerGeomColumn = {
type: 'mapnik',
options: {
sql: 'select *, the_geom as my_geom from test_table_3 limit 2',
geom_column: 'my_geom',
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
var mapnikLayerNullCats = {
type: 'mapnik',
options: {
sql: `
WITH geom AS (
SELECT
'SRID=4326;POINT(0 0)'::geometry AS the_geom,
'SRID=3857;POINT(0 0)'::geometry AS the_geom_webmercator
)
SELECT 1 AS cartodb_id, 'A' As cat, geom.* FROM geom
UNION
SELECT 2 AS cartodb_id, 'B' As cat, geom.* FROM geom
UNION
SELECT 2 AS cartodb_id, NULL::text As cat, geom.* FROM geom
`,
cartocss_version: cartocssVersion,
cartocss: cartocss
}
};
function mapnikBasicLayerId (index) {
return 'layer' + index;
}
function typeLayerId (type, index) {
return type + '-' + mapnikBasicLayerId(index);
}
it('with one mapnik layer should response with meta-stats for that layer', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
testClient.drain(done);
});
});
it('with two mapnik layer should response with meta-stats for every layer', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
mapnikLayer2
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.strictEqual(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.strictEqual(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2);
testClient.drain(done);
});
});
it('with three mapnik layer should response with meta-stats for every layer', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
mapnikLayer2,
mapnikLayer3
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.strictEqual(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.strictEqual(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 2);
assert.strictEqual(layergroup.metadata.layers[2].id, mapnikBasicLayerId(2));
assert.strictEqual(layergroup.metadata.layers[2].meta.stats.estimatedFeatureCount, 3);
testClient.drain(done);
});
});
it('with one mapnik layer (sql with join) should response with meta-stats for that layer', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer4
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
testClient.drain(done);
});
});
it('with two mapnik layer (sql with join) should response with meta-stats for every layer', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer4,
mapnikLayer4
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.strictEqual(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.strictEqual(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5);
testClient.drain(done);
});
});
it('with two mapnik layer (with & without join) should response with meta-stats for every layer', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer3,
mapnikLayer4
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 3);
assert.ok(!layergroup.metadata.layers[0].meta.stats[1]);
assert.strictEqual(layergroup.metadata.layers[1].id, mapnikBasicLayerId(1));
assert.strictEqual(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 5);
assert.ok(!layergroup.metadata.layers[2]);
testClient.drain(done);
});
});
it('with mapnik and layer and httplayer should response with layer metadata with same order', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayer1,
httpLayer
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].type, 'mapnik');
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 1);
assert.strictEqual(layergroup.metadata.layers[1].id, typeLayerId('http', 0));
assert.strictEqual(layergroup.metadata.layers[1].type, 'http');
testClient.drain(done);
});
});
it('with httpLayer and mapnik layer should response with layer metadata with same order', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
httpLayer,
mapnikLayer1
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.strictEqual(layergroup.metadata.layers[0].type, 'http');
assert.ok(!layergroup.metadata.layers[0].meta.cartocss);
assert.strictEqual(layergroup.metadata.layers[1].meta.stats.estimatedFeatureCount, 1);
assert.strictEqual(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[1].type, 'mapnik');
assert.strictEqual(layergroup.metadata.layers[1].meta.cartocss, cartocss);
testClient.drain(done);
});
});
it('should work with different geom_column', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
mapnikLayerGeomColumn
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
// we don't care about stats here as is an aliased column
assert.ok(Object.prototype.hasOwnProperty.call(layergroup.metadata.layers[0].meta.stats, 'estimatedFeatureCount'));
testClient.drain(done);
});
});
it('should not include the stats part if the FF is disabled', function (done) {
global.environment.enabledFeatures.layerStats = false;
var testClient = new TestClient({
version: '1.4.0',
layers: [
httpLayer,
mapnikLayer1,
httpLayer
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, typeLayerId('http', 0));
assert.strictEqual(layergroup.metadata.layers[0].type, 'http');
assert.strictEqual(layergroup.metadata.layers[1].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[1].type, 'mapnik');
assert.ok(!Object.prototype.hasOwnProperty.call(layergroup.metadata.layers[1].meta, 'stats'));
assert.strictEqual(layergroup.metadata.layers[2].id, typeLayerId('http', 1));
assert.strictEqual(layergroup.metadata.layers[2].type, 'http');
global.environment.enabledFeatures.layerStats = true;
testClient.drain(done);
});
});
function layerWithMetadata (layer, metadata) {
return Object.assign(layer, {
options: Object.assign(layer.options, { metadata })
});
}
it('should provide columns as optional metadata', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
columns: true
})
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
const expectedColumns = {
cartodb_id: { type: 'number' },
the_geom: { type: 'geometry' },
the_geom_webmercator: { type: 'geometry' },
address: { type: 'string' }
};
assert.deepStrictEqual(layergroup.metadata.layers[0].meta.stats.columns, expectedColumns);
testClient.drain(done);
});
});
// metadata categories are ordered only partially by descending frequency;
// this orders them completely to avoid ambiguities when comparing
function withSortedCategories (columns) {
function catOrder (a, b) {
if (a.frequency !== b.frequency) {
return b.frequency - a.frequency;
}
if (a.category < b.category) {
return -1;
}
if (a.category > b.category) {
return +1;
}
return 0;
}
const sorted = {};
Object.keys(columns).forEach(name => {
let data = columns[name];
if (Object.prototype.hasOwnProperty.call(data, 'categories')) {
data = Object.assign(data, { categories: data.categories.sort(catOrder) });
}
sorted[name] = data;
});
return sorted;
}
it('should provide column stats as optional metadata', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
columnStats: true
})
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
const expectedColumns = {
cartodb_id: {
type: 'number',
avg: 3,
max: 5,
min: 1,
sum: 15
},
the_geom: { type: 'geometry' },
the_geom_webmercator: { type: 'geometry' },
address: {
type: 'string',
categories: [
{
category: 'Calle de la Palma 72, Madrid, Spain',
frequency: 1
},
{
category: 'Calle de Pérez Galdós 9, Madrid, Spain',
frequency: 1
},
{
category: 'Calle Divino Pastor 12, Madrid, Spain',
frequency: 1
},
{
category: 'Manuel Fernández y González 8, Madrid, Spain',
frequency: 1
},
{
category: 'Plaza Conde de Toreno 2, Madrid, Spain',
frequency: 1
}
]
}
};
assert.deepStrictEqual(
withSortedCategories(layergroup.metadata.layers[0].meta.stats.columns),
withSortedCategories(expectedColumns)
);
testClient.drain(done);
});
});
it('should limit the number of categories as requested', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
columnStats: { topCategories: 2 }
})
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns;
assert.strictEqual(columnsMetadata.address.categories.length, 2);
testClient.drain(done);
});
});
it('should include null categories if requested', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayerNullCats, {
columnStats: { includeNulls: true }
})
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns;
assert.strictEqual(columnsMetadata.cat.categories.length, 3);
testClient.drain(done);
});
});
it('should not include null categories if not requested', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayerNullCats, {
columnStats: { includeNulls: false }
})
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
const columnsMetadata = layergroup.metadata.layers[0].meta.stats.columns;
assert.strictEqual(columnsMetadata.cat.categories.length, 2);
testClient.drain(done);
});
});
it('should provide row count as optional metadata', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
featureCount: true
})
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.featureCount, 5);
testClient.drain(done);
});
});
it('should provide geometry type as optional metadata', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
geometryType: true
})
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point');
testClient.drain(done);
});
});
it('should not fail "TypeError: ... \'geom_type\' of undefined" for empty results', function (done) {
var testClient = new TestClient({
version: '1.8.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'select * from test_table where false',
metadata: {
geometryType: true
}
}
}
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.geometryType, undefined);
testClient.drain(done);
});
});
it('should provide a sample as optional metadata', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer100, {
sample: { num_rows: 30 }
})
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 100);
assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0);
const expectedCols = ['cartodb_id', 'value', 'the_geom', 'the_geom_webmercator'].sort();
assert.deepStrictEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols);
testClient.drain(done);
});
});
it('should not provide a sample when the source table is empty', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
{
type: 'mapnik',
options: {
sql: 'SELECT * FROM test_table_100 limit 0',
cartocss_version: '2.3.0',
cartocss: '#layer { line-width:16; }',
metadata: {
sample: {
num_rows: 30
}
}
}
}
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.deepStrictEqual(layergroup.metadata.layers[0].meta.stats.sample, {});
testClient.drain(done);
});
});
it('can specify sample columns', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer100, {
sample: {
num_rows: 30,
include_columns: ['cartodb_id', 'the_geom']
}
})
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 100);
assert(layergroup.metadata.layers[0].meta.stats.sample.length > 0);
const expectedCols = ['cartodb_id', 'the_geom'].sort();
assert.deepStrictEqual(Object.keys(layergroup.metadata.layers[0].meta.stats.sample[0]).sort(), expectedCols);
testClient.drain(done);
});
});
it('should only provide requested optional metadata', function (done) {
var testClient = new TestClient({
version: '1.4.0',
layers: [
layerWithMetadata(mapnikLayer4, {
geometryType: true,
featureCount: true
})
]
});
testClient.getLayergroup(function (err, layergroup) {
assert.ifError(err);
assert.strictEqual(layergroup.metadata.layers[0].id, mapnikBasicLayerId(0));
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.estimatedFeatureCount, 5);
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.geometryType, 'ST_Point');
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.featureCount, 5);
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.sample, undefined);
assert.strictEqual(layergroup.metadata.layers[0].meta.stats.columns, undefined);
testClient.drain(done);
});
});
});
});