Windshaft-cartodb/test/acceptance/dataviews/overviews-test.js

'use strict';

require('../../support/test-helper');

var assert = require('../../support/assert');
var TestClient = require('../../support/test-client');

describe('dataviews using tables without overviews', function () {
    var nonOverviewsMapConfig = {
        version: '1.5.0',
        analyses: [
            {
                id: 'data-source',
                type: 'source',
                params: {
                    query: 'select * from populated_places_simple_reduced'
                }
            }
        ],
        dataviews: {
            country_places_count: {
                type: 'formula',
                source: { id: 'data-source' },
                options: {
                    column: 'adm0_a3',
                    operation: 'count'
                }
            },
            country_categories: {
                type: 'aggregation',
                source: { id: 'data-source' },
                options: {
                    column: 'adm0_a3',
                    aggregation: 'count'
                }
            }
        },
        layers: [
            {
                type: 'mapnik',
                options: {
                    sql: 'select * from populated_places_simple_reduced',
                    cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
                    cartocss_version: '2.3.0',
                    source: { id: 'data-source' }
                }
            }
        ]
    };

    it('should expose a formula', function (done) {
        var testClient = new TestClient(nonOverviewsMapConfig);
        testClient.getDataview('country_places_count', { own_filter: 0 }, function (err, formula_result, headers) {
            if (err) {
                return done(err);
            }
            assert.deepStrictEqual(formula_result, { operation: 'count', result: 7313, nulls: 0, type: 'formula' });
            assert(getUsesOverviewsFromHeaders(headers) === false); // Overviews logging

            testClient.drain(done);
        });
    });

    it('should admit a bbox', function (done) {
        var params = {
            bbox: '-170,-80,170,80'
        };
        var testClient = new TestClient(nonOverviewsMapConfig);
        testClient.getDataview('country_places_count', params, function (err, formula_result) {
            if (err) {
                return done(err);
            }
            assert.deepStrictEqual(formula_result, { operation: 'count', result: 7253, nulls: 0, type: 'formula' });

            testClient.drain(done);
        });
    });

    describe('filters', function () {
        describe('category', function () {
            it('should expose a filtered formula', function (done) {
                var params = {
                    filters: {
                        dataviews: { country_categories: { accept: ['CAN'] } }
                    }
                };
                var testClient = new TestClient(nonOverviewsMapConfig);
                testClient.getDataview('country_places_count', params, function (err, formula_result) {
                    if (err) {
                        return done(err);
                    }
                    assert.deepStrictEqual(formula_result, { operation: 'count', result: 256, nulls: 0, type: 'formula' });
                    testClient.drain(done);
                });
            });

            it('should expose a filtered formula and admit a bbox', function (done) {
                var params = {
                    filters: {
                        dataviews: { country_categories: { accept: ['CAN'] } }
                    },
                    bbox: '-170,-80,170,80'
                };
                var testClient = new TestClient(nonOverviewsMapConfig);
                testClient.getDataview('country_places_count', params, function (err, formula_result) {
                    if (err) {
                        return done(err);
                    }
                    assert.deepStrictEqual(formula_result, { operation: 'count', result: 254, nulls: 0, type: 'formula' });
                    testClient.drain(done);
                });
            });
        });
    });
});

describe('dataviews using tables with overviews', function () {
    var overviewsMapConfig = {
        version: '1.5.0',
        analyses: [
            {
                id: 'data-source',
                type: 'source',
                params: {
                    query: 'select * from test_table_overviews'
                }
            },
            {
                id: 'data-source-special-float-values',
                type: 'source',
                params: {
                    query: 'select * from test_special_float_values_table_overviews'
                }
            }
        ],
        dataviews: {
            test_sum: {
                type: 'formula',
                source: { id: 'data-source' },
                options: {
                    column: 'value',
                    operation: 'sum'
                }
            },
            test_categories: {
                type: 'aggregation',
                source: { id: 'data-source' },
                options: {
                    column: 'name',
                    aggregation: 'count',
                    aggregationColumn: 'name'
                }
            },
            test_categories_special_values: {
                type: 'aggregation',
                source: {
                    id: 'data-source-special-float-values'
                },
                options: {
                    column: 'name',
                    aggregation: 'sum',
                    aggregationColumn: 'value'
                }
            },
            test_histogram: {
                type: 'histogram',
                source: { id: 'data-source' },
                options: {
                    column: 'value',
                    bins: 2
                }
            },
            test_histogram_date: {
                type: 'histogram',
                source: { id: 'data-source' },
                options: {
                    column: 'updated_at',
                    bins: 2
                }
            },
            test_histogram_special_values: {
                type: 'histogram',
                source: {
                    id: 'data-source-special-float-values'
                },
                options: {
                    column: 'value',
                    bins: 2
                }
            },
            test_avg: {
                type: 'formula',
                source: { id: 'data-source' },
                options: {
                    column: 'value',
                    operation: 'avg'
                }
            },
            test_formula_sum_special_values: {
                type: 'formula',
                source: {
                    id: 'data-source-special-float-values'
                },
                options: {
                    column: 'value',
                    operation: 'sum'
                }
            },
            test_count: {
                type: 'formula',
                source: { id: 'data-source' },
                options: {
                    column: 'value',
                    operation: 'count'
                }
            },
            test_min: {
                type: 'formula',
                source: { id: 'data-source' },
                options: {
                    column: 'value',
                    operation: 'min'
                }
            },
            test_max: {
                type: 'formula',
                source: { id: 'data-source' },
                options: {
                    column: 'value',
                    operation: 'max'
                }
            }
        },
        layers: [
            {
                type: 'mapnik',
                options: {
                    sql: 'select * from test_table_overviews',
                    cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
                    cartocss_version: '2.3.0',
                    source: { id: 'data-source' }
                }
            },
            {
                type: 'mapnik',
                options: {
                    sql: 'select * from test_special_float_values_table_overviews',
                    cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
                    cartocss_version: '2.3.0',
                    source: {
                        id: 'data-source-special-float-values'
                    }
                }
            }
        ]
    };

    it('should expose a sum formula', function (done) {
        var testClient = new TestClient(overviewsMapConfig);
        testClient.getDataview('test_sum', { own_filter: 0 }, function (err, formula_result, headers) {
            if (err) {
                return done(err);
            }
            assert.deepStrictEqual(formula_result, {
                operation: 'sum',
                result: 15,
                infinities: 0,
                nans: 0,
                nulls: 0,
                type: 'formula'
            });
            assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
            assert(getDataviewTypeFromHeaders(headers) === 'formula'); // Overviews logging

            testClient.drain(done);
        });
    });

    it('should expose an avg formula', function (done) {
        var testClient = new TestClient(overviewsMapConfig);
        testClient.getDataview('test_avg', { own_filter: 0 }, function (err, formula_result, headers) {
            if (err) {
                return done(err);
            }
            assert.deepStrictEqual(formula_result, {
                operation: 'avg',
                result: 3,
                nulls: 0,
                type: 'formula',
                infinities: 0,
                nans: 0
            });
            assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
            assert(getDataviewTypeFromHeaders(headers) === 'formula'); // Overviews logging

            testClient.drain(done);
        });
    });

    it('should expose a count formula', function (done) {
        var testClient = new TestClient(overviewsMapConfig);
        testClient.getDataview('test_count', { own_filter: 0 }, function (err, formula_result, headers) {
            if (err) {
                return done(err);
            }
            assert.deepStrictEqual(formula_result, {
                operation: 'count',
                result: 5,
                nulls: 0,
                type: 'formula',
                infinities: 0,
                nans: 0
            });
            assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
            assert(getDataviewTypeFromHeaders(headers) === 'formula'); // Overviews logging

            testClient.drain(done);
        });
    });

    it('should expose a max formula', function (done) {
        var testClient = new TestClient(overviewsMapConfig);
        testClient.getDataview('test_max', { own_filter: 0 }, function (err, formula_result) {
            if (err) {
                return done(err);
            }
            assert.deepStrictEqual(formula_result, {
                operation: 'max',
                result: 5,
                nulls: 0,
                infinities: 0,
                nans: 0,
                type: 'formula'
            });

            testClient.drain(done);
        });
    });

    it('should expose a min formula', function (done) {
        var testClient = new TestClient(overviewsMapConfig);
        testClient.getDataview('test_min', { own_filter: 0 }, function (err, formula_result) {
            if (err) {
                return done(err);
            }
            assert.deepStrictEqual(formula_result, {
                operation: 'min',
                result: 1,
                nulls: 0,
                infinities: 0,
                nans: 0,
                type: 'formula'
            });

            testClient.drain(done);
        });
    });

    it('should admit a bbox', function (done) {
        var params = {
            bbox: '-170,-80,170,80'
        };
        var testClient = new TestClient(overviewsMapConfig);
        testClient.getDataview('test_sum', params, function (err, formula_result) {
            if (err) {
                return done(err);
            }
            assert.deepStrictEqual(formula_result, {
                operation: 'sum',
                result: 15,
                nulls: 0,
                infinities: 0,
                nans: 0,
                type: 'formula'
            });

            testClient.drain(done);
        });
    });

    it('should expose a histogram', function (done) {
        var testClient = new TestClient(overviewsMapConfig);
        testClient.getDataview('test_histogram', function (err, histogram, headers) {
            if (err) {
                return done(err);
            }
            assert.ok(histogram);
            assert.strictEqual(histogram.type, 'histogram');
            assert.ok(Array.isArray(histogram.bins));
            assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
            assert(getDataviewTypeFromHeaders(headers) === 'histogram'); // Overviews logging

            testClient.drain(done);
        });
    });

    describe('filters', function () {
        describe('histogram', function () {
            it('should expose a filtered histogram', function (done) {
                var params = {
                    filters: {
                        dataviews: { test_histogram: { min: 2 } }
                    },
                    own_filter: 1
                };
                var testClient = new TestClient(overviewsMapConfig);
                testClient.getDataview('test_histogram', params, function (err, histogram) {
                    if (err) {
                        return done(err);
                    }
                    assert.ok(histogram);
                    assert.strictEqual(histogram.type, 'histogram');
                    assert.ok(Array.isArray(histogram.bins));
                    assert.strictEqual(histogram.bins.length, 4);
                    testClient.drain(done);
                });
            });

            it('should expose a filtered histogram with no results', function (done) {
                var params = {
                    filters: {
                        dataviews: { test_histogram: { max: -1 } }
                    },
                    own_filter: 1
                };
                var testClient = new TestClient(overviewsMapConfig);
                testClient.getDataview('test_histogram', params, function (err, histogram) {
                    if (err) {
                        return done(err);
                    }
                    assert.ok(histogram);
                    assert.strictEqual(histogram.type, 'histogram');
                    assert.ok(Array.isArray(histogram.bins));
                    assert.strictEqual(histogram.bins.length, 0);
                    testClient.drain(done);
                });
            });

            it('should expose a filtered date histogram with no results', function (done) {
                // This most likely works because the overviews will pass
                // the responsibility to the normal dataviews.
                var params = {
                    filters: {
                        dataviews: { test_histogram_date: { max: -1 } }
                    },
                    own_filter: 1
                };
                var testClient = new TestClient(overviewsMapConfig);
                testClient.getDataview('test_histogram_date', params, function (err, histogram) {
                    if (err) {
                        return done(err);
                    }
                    assert.ok(histogram);
                    assert.strictEqual(histogram.type, 'histogram');
                    assert.ok(Array.isArray(histogram.bins));
                    assert.strictEqual(histogram.bins.length, 0);
                    testClient.drain(done);
                });
            });
        });

        describe('category', function () {
            var params = {
                filters: {
                    dataviews: { test_categories: { accept: ['Hawai'] } }
                }
            };

            it('should expose a filtered sum formula', function (done) {
                var testClient = new TestClient(overviewsMapConfig);
                testClient.getDataview('test_sum', params, function (err, formula_result, headers) {
                    if (err) {
                        return done(err);
                    }
                    assert.deepStrictEqual(formula_result, {
                        operation: 'sum',
                        result: 1,
                        nulls: 0,
                        infinities: 0,
                        nans: 0,
                        type: 'formula'
                    });
                    assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
                    testClient.drain(done);
                });
            });

            it('should expose a filtered  avg formula', function (done) {
                var testClient = new TestClient(overviewsMapConfig);
                testClient.getDataview('test_avg', params, function (err, formula_result, headers) {
                    if (err) {
                        return done(err);
                    }
                    assert.deepStrictEqual(formula_result, {
                        operation: 'avg',
                        result: 1,
                        nulls: 0,
                        infinities: 0,
                        nans: 0,
                        type: 'formula'
                    });
                    assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging

                    testClient.drain(done);
                });
            });

            it('should expose a filtered count formula', function (done) {
                var testClient = new TestClient(overviewsMapConfig);
                testClient.getDataview('test_count', params, function (err, formula_result, headers) {
                    if (err) {
                        return done(err);
                    }
                    assert.deepStrictEqual(formula_result, {
                        operation: 'count',
                        result: 1,
                        infinities: 0,
                        nans: 0,
                        nulls: 0,
                        type: 'formula'
                    });
                    assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging

                    testClient.drain(done);
                });
            });

            it('should expose a filterd max formula', function (done) {
                var testClient = new TestClient(overviewsMapConfig);
                testClient.getDataview('test_max', params, function (err, formula_result) {
                    if (err) {
                        return done(err);
                    }
                    assert.deepStrictEqual(formula_result, {
                        operation: 'max',
                        result: 1,
                        nulls: 0,
                        infinities: 0,
                        nans: 0,
                        type: 'formula'
                    });

                    testClient.drain(done);
                });
            });

            it('should expose a filterd min formula', function (done) {
                var testClient = new TestClient(overviewsMapConfig);
                testClient.getDataview('test_min', params, function (err, formula_result) {
                    if (err) {
                        return done(err);
                    }
                    assert.deepStrictEqual(formula_result, {
                        operation: 'min',
                        result: 1,
                        nulls: 0,
                        infinities: 0,
                        nans: 0,
                        type: 'formula'
                    });

                    testClient.drain(done);
                });
            });

            it('should expose a filtered sum formula with bbox', function (done) {
                var bboxparams = {
                    filters: {
                        dataviews: { test_categories: { accept: ['Hawai'] } }
                    },
                    bbox: '-170,-80,170,80'
                };
                var testClient = new TestClient(overviewsMapConfig);
                testClient.getDataview('test_sum', bboxparams, function (err, formula_result) {
                    if (err) {
                        return done(err);
                    }
                    assert.deepStrictEqual(formula_result, {
                        operation: 'sum',
                        result: 1,
                        nulls: 0,
                        infinities: 0,
                        nans: 0,
                        type: 'formula'
                    });
                    testClient.drain(done);
                });
            });
        });

        describe('aggregation special float values', function () {
            var params = {};

            it('should expose an aggregation dataview filtering special float values out', function (done) {
                var testClient = new TestClient(overviewsMapConfig);
                testClient.getDataview('test_categories_special_values', params, function (err, dataview, headers) {
                    if (err) {
                        return done(err);
                    }

                    assert.deepStrictEqual(dataview, {
                        aggregation: 'sum',
                        count: 5,
                        nulls: 0,
                        nans: 1,
                        infinities: 1,
                        min: 6,
                        max: 6,
                        categoriesCount: 1,
                        categories: [{ category: 'Hawai', value: 6, agg: false }],
                        type: 'aggregation'
                    });

                    assert.ok(getUsesOverviewsFromHeaders(headers)); // Overviews logging
                    assert(getDataviewTypeFromHeaders(headers) === 'aggregation'); // Overviews logging

                    testClient.drain(done);
                });
            });

            it('should expose a histogram dataview filtering special float values out', function (done) {
                var testClient = new TestClient(overviewsMapConfig);
                testClient.getDataview('test_histogram_special_values', params, function (err, dataview) {
                    if (err) {
                        return done(err);
                    }
                    assert.deepStrictEqual(dataview, {
                        bin_width: 0,
                        bins_count: 1,
                        bins_start: 3,
                        nulls: 0,
                        infinities: 1,
                        nans: 1,
                        avg: 3,
                        bins: [{ bin: 0, min: 3, max: 3, avg: 3, freq: 2 }],
                        type: 'histogram'
                    });
                    testClient.drain(done);
                });
            });

            it('should expose a formula (sum) dataview filtering special float values out', function (done) {
                var testClient = new TestClient(overviewsMapConfig);
                testClient.getDataview('test_formula_sum_special_values', params, function (err, dataview) {
                    if (err) {
                        return done(err);
                    }
                    assert.deepStrictEqual(dataview, {
                        operation: 'sum',
                        result: 6,
                        nulls: 0,
                        nans: 1,
                        infinities: 1,
                        type: 'formula'
                    });
                    testClient.drain(done);
                });
            });
        });

        describe('agreggation validation', function (){
            const params = {
                response: {
                    status: 400,
                    headers: {
                        'Content-Type': 'application/json; charset=utf-8'
                    }
                }
            };

            function createMapConfig(options) {
                return {
                    version: '1.8.0',
                    analyses: [
                        { id: 'data-source',
                            type: 'source',
                            params: {
                                query: 'select * from test_table_overviews'
                            }
                        }
                    ],
                    dataviews:  {
                        test_invalid_aggregation: {
                            type: 'aggregation',
                            source: {id: 'data-source'},
                            options: options
                        }
                    },
                    layers: [
                        {
                            type: 'mapnik',
                            options: {
                                sql: 'select * from test_table_overviews',
                                cartocss: '#layer { marker-fill: red; marker-width: 32; marker-allow-overlap: true; }',
                                cartocss_version: '2.3.0',
                                source: { id: 'data-source' }
                            }
                        }
                    ]
                };
            }

            it('should fail if missing column', function (done) {
                var options = {
                    aggregation: "sum",
                    aggregationColumn: "value"
                };
                var missingColumnMapConfig = createMapConfig(options);

                var testClient = new TestClient(missingColumnMapConfig);
                testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) {
                    if (err) {
                        return done(err);
                    }

                    assert.deepStrictEqual(dataview, {
                        errors: ["Aggregation expects 'column' in dataview options"],
                        errors_with_context: [{
                            type: 'unknown',
                            message: "Aggregation expects 'column' in dataview options"
                        }]
                    });

                    testClient.drain(done);
                });
            });

            it('should fail if no aggregation operation', function (done) {
                var options = {
                    column: "value",
                    aggregationColumn: "value"
                };
                var missingOperationMapConfig = createMapConfig(options);

                var testClient = new TestClient(missingOperationMapConfig);
                testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) {
                    if (err) {
                        return done(err);
                    }

                    assert.deepStrictEqual(dataview, {
                        errors: ["Aggregation expects 'aggregation' operation in dataview options"],
                        errors_with_context: [{
                            type: 'unknown',
                            message: "Aggregation expects 'aggregation' operation in dataview options"
                        }]
                    });

                    testClient.drain(done);
                });
            });

            it('should fail if fake operation', function (done) {
                var options = {
                    column: "value",
                    aggregation: "wadus",
                    aggregationColumn: "value"
                };
                var wrongOperationMapConfig = createMapConfig(options);

                var testClient = new TestClient(wrongOperationMapConfig);
                testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) {
                    if (err) {
                        return done(err);
                    }

                    assert.deepStrictEqual(dataview, {
                        errors: ["Aggregation does not support 'wadus' operation"],
                        errors_with_context: [{
                            type: 'unknown',
                            message: "Aggregation does not support 'wadus' operation"
                        }]
                    });

                    testClient.drain(done);
                });
            });

            it('should fail if invalid operation for overview', function (done) {
                var options = {
                    column: "value",
                    aggregation: "avg",
                    aggregationColumn: "value"
                };
                var wrongOperationMapConfig = createMapConfig(options);

                var testClient = new TestClient(wrongOperationMapConfig);
                testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) {
                    if (err) {
                        return done(err);
                    }

                    assert.deepStrictEqual(dataview, {
                        errors: ["Aggregation does not support 'avg' operation in dataview overview options"],
                        errors_with_context: [{
                            type: 'unknown',
                            message: "Aggregation does not support 'avg' operation in dataview overview options"
                        }]
                    });

                    testClient.drain(done);
                });
            });

            it('should fail if no aggregation column when needed', function (done) {
                var options = {
                    column: "value",
                    aggregation: "sum"
                };
                var missingOptionMapConfig = createMapConfig(options);

                var testClient = new TestClient(missingOptionMapConfig);
                testClient.getDataview('test_invalid_aggregation', params, function (err, dataview) {
                    if (err) {
                        return done(err);
                    }

                    assert.deepStrictEqual(dataview, {
                        errors: ["Aggregation 'sum' is missing some options: aggregationColumn"],
                        errors_with_context: [{
                            type: 'unknown',
                            message: "Aggregation 'sum' is missing some options: aggregationColumn"
                        }]
                    });

                    testClient.drain(done);
                });
            });
        });
    });
});

function getUsesOverviewsFromHeaders (headers) {
    return headers && headers['x-tiler-profiler'] && JSON.parse(headers['x-tiler-profiler']).usesOverviews;
}

function getDataviewTypeFromHeaders (headers) {
    return headers && headers['x-tiler-profiler'] && JSON.parse(headers['x-tiler-profiler']).dataviewType;
}