Windshaft-cartodb/test/unit/overviews-query-rewriter-test.js

'use strict';

require('../support/test-helper');

var assert = require('assert');
var OverviewsQueryRewriter = require('../../lib/utils/overviews-query-rewriter');
var overviewsQueryRewriter = new OverviewsQueryRewriter({
    zoom_level: 'ZoomLevel()'
});

function normalize_whitespace (txt) {
    return txt.replace(/\s+/g, ' ').trim();
}

// compare SQL statements ignoring whitespace
function assertSameSql (sql1, sql2) {
    assert.strictEqual(normalize_whitespace(sql1), normalize_whitespace(sql2));
}

describe('Overviews query rewriter', function () {
    it('does not alter queries if no overviews data is present', function () {
        var sql = 'SELECT * FROM table1';
        var overviews_sql = overviewsQueryRewriter.query(sql);
        assert.strictEqual(overviews_sql, sql);
        overviews_sql = overviewsQueryRewriter.query(sql, {});
        assert.strictEqual(overviews_sql, sql);
        overviews_sql = overviewsQueryRewriter.query(sql, { overviews: {} });
        assert.strictEqual(overviews_sql, sql);
    });

    it('does not alter queries which don\'t use overviews', function () {
        var sql = 'SELECT * FROM table1';
        var data = {
            overviews: {
                table2: {
                    0: { table: 'table2_ov0' },
                    1: { table: 'table2_ov1' },
                    4: { table: 'table2_ov4' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        assert.strictEqual(overviews_sql, sql);
    });

    it('generates query with single overview layer for level 0', function () {
        var sql = 'SELECT * FROM table1';
        var data = {
            overviews: {
                table1: {
                    0: { table: 'table1_ov0' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);

        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT * FROM (
                SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0
                UNION ALL
                SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 0
              ) AS _vovw_table1
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query with single overview layer for level >0', function () {
        var sql = 'SELECT * FROM table1';
        var data = {
            overviews: {
                table1: {
                    2: { table: 'table1_ov2' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT * FROM (
                SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2
                UNION ALL
                SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2
              ) AS _vovw_table1
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query with multiple overview layers for all levels up to N', function () {
        var sql = 'SELECT * FROM table1';
        var data = {
            overviews: {
                table1: {
                    0: { table: 'table1_ov0' },
                    1: { table: 'table1_ov1' },
                    2: { table: 'table1_ov2' },
                    3: { table: 'table1_ov3' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT * FROM  (
                SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0
                UNION ALL
                SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1
                UNION ALL
                SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z = 2
                UNION ALL
                SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3
                UNION ALL
                SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3
              ) AS _vovw_table1
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query with multiple overview layers for random levels', function () {
        var sql = 'SELECT * FROM table1';
        var data = {
            overviews: {
                table1: {
                    0: { table: 'table1_ov0' },
                    1: { table: 'table1_ov1' },
                    6: { table: 'table1_ov6' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT * FROM (
                SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0
                UNION ALL
                SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1
                UNION ALL
                SELECT * FROM table1_ov6, _vovw_scale WHERE _vovw_z > 1 AND _vovw_z <= 6
                UNION ALL
                SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 6
              ) AS _vovw_table1
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query for a table with explicit schema', function () {
        var sql = 'SELECT * FROM public.table1';
        var data = {
            overviews: {
                'public.table1': {
                    2: { table: 'table1_ov2' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT * FROM (
                SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2
                UNION ALL
                SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2
              ) AS _vovw_table1
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query for a table with explicit schema in the overviews info', function () {
        var sql = 'SELECT * FROM public.table1';
        var data = {
            overviews: {
                'public.table1': {
                    2: { table: 'table1_ov2' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT * FROM (
                SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2
                UNION ALL
                SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2
              ) AS _vovw_table1
        `;

        assertSameSql(overviews_sql, expected_sql);
    });

    it('uses schema name from overviews', function () {
        var sql = 'SELECT * FROM public.table1';
        var data = {
            overviews: {
                table1: {
                    schema: 'public',
                    2: { table: 'table1_ov2' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT * FROM (
                SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2
                UNION ALL
                SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2
              ) AS _vovw_table1
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('ignores schema name from overviews if not necessary', function () {
        var sql = 'SELECT * FROM table1';
        var data = {
            overviews: {
                table1: {
                    schema: 'public',
                    2: { table: 'table1_ov2' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT * FROM (
                SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2
                UNION ALL
                SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2
              ) AS _vovw_table1
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('uses redundant schema information', function () {
        var sql = 'SELECT * FROM public.table1';
        var data = {
            overviews: {
                'public.table1': {
                    schema: 'public',
                    2: { table: 'table1_ov2' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT * FROM (
                SELECT * FROM public.table1_ov2, _vovw_scale WHERE _vovw_z <= 2
                UNION ALL
                SELECT * FROM public.table1, _vovw_scale WHERE _vovw_z > 2
              ) AS _vovw_table1
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query for a table that needs quoting with explicit schema', function () {
        var sql = 'SELECT * FROM public."table 1"';
        var data = {
            overviews: {
                'public."table 1"': {
                    2: { table: '"table 1_ov2"' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT * FROM (
                SELECT * FROM public."table 1_ov2", _vovw_scale WHERE _vovw_z <= 2
                UNION ALL
                SELECT * FROM public."table 1", _vovw_scale WHERE _vovw_z > 2
              ) AS "_vovw_table 1"
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query for a table with explicit schema that needs quoting', function () {
        var sql = 'SELECT * FROM "user-1".table1';
        var data = {
            overviews: {
                '"user-1".table1': {
                    2: { table: 'table1_ov2' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT * FROM (
                SELECT * FROM "user-1".table1_ov2, _vovw_scale WHERE _vovw_z <= 2
                UNION ALL
                SELECT * FROM "user-1".table1, _vovw_scale WHERE _vovw_z > 2
              ) AS _vovw_table1
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query for a table with explicit schema both needing quoting', function () {
        var sql = 'SELECT * FROM "user-1"."table 1"';
        var data = {
            overviews: {
                '"user-1"."table 1"': {
                    2: { table: '"table 1_ov2"' }

                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT * FROM (
                SELECT * FROM "user-1"."table 1_ov2", _vovw_scale WHERE _vovw_z <= 2
                UNION ALL
                SELECT * FROM "user-1"."table 1", _vovw_scale WHERE _vovw_z > 2
              ) AS "_vovw_table 1"
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query using overviews for queries with selected columns', function () {
        var sql = 'SELECT column1, column2, column3 FROM table1';
        var data = {
            overviews: {
                table1: {
                    2: { table: 'table1_ov2' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT column1, column2, column3 FROM (
                SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2
                UNION ALL
                SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2
              ) AS _vovw_table1
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query using overviews for queries with a semicolon', function () {
        var sql = 'SELECT column1, column2, column3 FROM table1;';
        var data = {
            overviews: {
                table1: {
                    2: { table: 'table1_ov2' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);

        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT column1, column2, column3 FROM (
                SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2
                UNION ALL
                SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2
              ) AS _vovw_table1;
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query using overviews for queries with extra whitespace', function () {
        var sql = '  SELECT  column1,column2,  column3 FROM  table1  ';
        var data = {
            overviews: {
                table1: {
                    2: { table: 'table1_ov2' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT column1,column2, column3 FROM (
                SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z <= 2
                UNION ALL
                SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2
              ) AS _vovw_table1
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('does not alter queries which have not the simple supported form', function () {
        var sql = "SELECT * FROM table1 WHERE column1='x'";
        var data = {
            overviews: {
                table1: {
                    2: { table: 'table1_ov2' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        assert.strictEqual(overviews_sql, sql);

        sql = 'SELECT * FROM table1 JOIN table2 ON (table1.col1=table2.col1)';
        overviews_sql = overviewsQueryRewriter.query(sql, data);
        assert.strictEqual(overviews_sql, sql);

        sql = 'SELECT a+b AS c FROM table1';
        overviews_sql = overviewsQueryRewriter.query(sql, data);
        assert.strictEqual(overviews_sql, sql);

        sql = 'SELECT f(a) AS b FROM table1';
        overviews_sql = overviewsQueryRewriter.query(sql, data);
        assert.strictEqual(overviews_sql, sql);

        sql = 'SELECT * FROM table1 AS x';
        overviews_sql = overviewsQueryRewriter.query(sql, data);
        assert.strictEqual(overviews_sql, sql);

        sql = 'WITH a AS (1) SELECT * FROM table1';
        overviews_sql = overviewsQueryRewriter.query(sql, data);
        assert.strictEqual(overviews_sql, sql);

        sql = 'SELECT * FROM table1 WHERE a=1';
        overviews_sql = overviewsQueryRewriter.query(sql, data);
        assert.strictEqual(overviews_sql, sql);

        sql = `
            SELECT table1.* FROM table1
                   JOIN areas ON ST_Intersects(table1.the_geom, areas.the_geom)
                   WHERE areas.name='A'
        `;
        overviews_sql = overviewsQueryRewriter.query(sql, data);
        assert.strictEqual(overviews_sql, sql);

        sql = 'SELECT table1.*, column1, column2, column3 FROM table1';
        overviews_sql = overviewsQueryRewriter.query(sql, data);
        assert.strictEqual(overviews_sql, sql);
    });

    it('generates overviews for wrapped query', function () {
        var sql = 'SELECT * FROM (SELECT * FROM table1) AS wrapped_query WHERE 1=1';
        var data = {
            overviews: {
                table1: {
                    0: { table: 'table1_ov0' },
                    1: { table: 'table1_ov1' },
                    2: { table: 'table1_ov2' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            WITH
              _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
              SELECT * FROM (SELECT * FROM  (
                SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0
                UNION ALL
                SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1
                UNION ALL
                SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z = 2
                UNION ALL
                SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 2
              ) AS _vovw_table1) AS wrapped_query WHERE 1=1
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query for specific Z level', function () {
        var sql = 'SELECT * FROM table1';
        var data = {
            overviews: {
                table1: {
                    0: { table: 'table1_ov0' },
                    2: { table: 'table1_ov2' },
                    3: { table: 'table1_ov3' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 3 });
        var expected_sql = 'SELECT * FROM table1_ov3';
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query for specific nonpresent Z level', function () {
        var sql = 'SELECT * FROM table1';
        var data = {
            overviews: {
                table1: {
                    0: { table: 'table1_ov0' },
                    2: { table: 'table1_ov2' },
                    3: { table: 'table1_ov3' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 1 });
        var expected_sql = 'SELECT * FROM table1_ov2';
        assertSameSql(overviews_sql, expected_sql);
    });

    it('does not use overviews for specific out-of-range Z level', function () {
        var sql = 'SELECT * FROM table1';
        var data = {
            overviews: {
                table1: {
                    0: { table: 'table1_ov0' },
                    2: { table: 'table1_ov2' },
                    3: { table: 'table1_ov3' }
                }
            }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 4 });
        var expected_sql = 'SELECT * FROM table1';
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query with filters', function () {
        var sql = `SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name
                   FROM (SELECT *
                   FROM (select * from table1) _analysis_category_filter
                   WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query`;
        var data = {
            overviews: {
                table1: {
                    0: { table: 'table1_ov0' },
                    1: { table: 'table1_ov1' },
                    2: { table: 'table1_ov2' },
                    3: { table: 'table1_ov3' }
                }
            },
            filters: { name_filter: { type: 'category', column: 'name', params: { accept: ['X'] } } },
            unfiltered_query: 'SELECT * FROM table1'
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        var expected_sql = `
            SELECT * FROM (WITH
                _vovw_scale AS ( SELECT ZoomLevel() AS _vovw_z )
                SELECT * FROM  (
                  SELECT * FROM table1_ov0, _vovw_scale WHERE _vovw_z = 0
                  UNION ALL
                  SELECT * FROM table1_ov1, _vovw_scale WHERE _vovw_z = 1
                  UNION ALL
                  SELECT * FROM table1_ov2, _vovw_scale WHERE _vovw_z = 2
                  UNION ALL
                  SELECT * FROM table1_ov3, _vovw_scale WHERE _vovw_z = 3
                  UNION ALL
                  SELECT * FROM table1, _vovw_scale WHERE _vovw_z > 3
                ) AS _vovw_table1) _analysis_category_filter
             WHERE name IN ($escape_0$X$escape_0$)
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('generates query with filters for specific zoom level', function () {
        var sql = `SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name
                   FROM (SELECT *
                   FROM (select * from table1) _analysis_category_filter
                   WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query`;
        var data = {
            overviews: {
                table1: {
                    0: { table: 'table1_ov0' },
                    1: { table: 'table1_ov1' },
                    2: { table: 'table1_ov2' },
                    3: { table: 'table1_ov3' }
                }
            },
            filters: { name_filter: { type: 'category', column: 'name', params: { accept: ['X'] } } },
            unfiltered_query: 'SELECT * FROM table1',
            filter_stats: { unfiltered_rows: 1000, filtered_rows: 900 }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data, { zoom_level: 2 });
        var expected_sql = `
            SELECT * FROM (SELECT * FROM table1_ov2) _analysis_category_filter
            WHERE name IN ($escape_0$X$escape_0$)
        `;
        assertSameSql(overviews_sql, expected_sql);
    });

    it('does not generates query with aggressive filtering', function () {
        var sql = `SELECT ST_Transform(the_geom, 3857) the_geom_webmercator, cartodb_id, name
                   FROM (SELECT *
                   FROM (select * from table1) _analysis_category_filter
                   WHERE name IN ($escape_0$X$escape_0$)) _cdb_analysis_query`;
        var data = {
            overviews: {
                table1: {
                    0: { table: 'table1_ov0' },
                    1: { table: 'table1_ov1' },
                    2: { table: 'table1_ov2' },
                    3: { table: 'table1_ov3' }
                }
            },
            filters: { name_filter: { type: 'category', column: 'name', params: { accept: ['X'] } } },
            unfiltered_query: 'SELECT * FROM table1',
            filter_stats: { unfiltered_rows: 1000, filtered_rows: 10 }
        };
        var overviews_sql = overviewsQueryRewriter.query(sql, data);
        assert.strictEqual(overviews_sql, sql);
    });
});