CartoDB-SQL-API/test/acceptance/export/shapefile.js

require('../../helper');

var server = require('../../../app/server')();
var assert = require('../../support/assert');
var querystring = require('querystring');
var shapefile = require('shapefile');
var _ = require('underscore');
var zipfile = require('zipfile');
var fs      = require('fs');

describe('export.shapefile', function() {

// SHP tests

it('SHP format, unauthenticated', function(done){
    assert.response(server, {
        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp',
        headers: {host: 'vizzuality.cartodb.com'},
        encoding: 'binary',
        method: 'GET'
    },{ }, function(err, res){
        assert.equal(res.statusCode, 200, res.body);
        var cd = res.headers['content-disposition'];
        assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
        assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
        var tmpfile = '/tmp/myshape.zip';
        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
        if (writeErr) {
            return done(writeErr);
        }
        var zf = new zipfile.ZipFile(tmpfile);
        assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
        assert.ok(_.contains(zf.names, 'cartodb-query.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
        assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
        assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
        // TODO: check DBF contents
        fs.unlinkSync(tmpfile);
        done();
    });
});

it('SHP format, unauthenticated, POST', function(done){
    assert.response(server, {
        url: '/api/v1/sql',
        data: 'q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp',
        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
        method: 'POST'
    },{ }, function(err, res){
        assert.equal(res.statusCode, 200, res.body);
        var cd = res.headers['content-disposition'];
        assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
        assert.equal(true, /filename=cartodb-query.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
        done();
    });
});

it('SHP format, big size, POST', function(done){
    assert.response(server, {
        url: '/api/v1/sql',
        data: querystring.stringify({
          q: 'SELECT 0 as fname, st_makepoint(i,i) FROM generate_series(0,81920) i',
          format: 'shp'
        }),
        headers: {host: 'vizzuality.cartodb.com', 'Content-Type': 'application/x-www-form-urlencoded' },
        method: 'POST'
    },{ }, function(err, res){
        assert.equal(res.statusCode, 200, res.body);
        var cd = res.headers['content-disposition'];
        assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
        assert.equal(true, /filename=cartodb-query.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
        assert.ok(res.body.length > 81920, 'SHP smaller than expected: ' + res.body.length);
        done();
    });
});

it('SHP format, unauthenticated, with custom filename', function(done){
    assert.response(server, {
        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=myshape',
        headers: {host: 'vizzuality.cartodb.com'},
        encoding: 'binary',
        method: 'GET'
    },{ }, function(err, res){
        assert.equal(res.statusCode, 200, res.body);
        var cd = res.headers['content-disposition'];
        assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
        assert.equal(true, /filename=myshape.zip/gi.test(cd));
        var tmpfile = '/tmp/myshape.zip';
        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
        if (writeErr) {
            return done(writeErr);
        }
        var zf = new zipfile.ZipFile(tmpfile);
        assert.ok(_.contains(zf.names, 'myshape.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
        assert.ok(_.contains(zf.names, 'myshape.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
        assert.ok(_.contains(zf.names, 'myshape.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
        assert.ok(_.contains(zf.names, 'myshape.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
        fs.unlinkSync(tmpfile);
        done();
    });
});

it('SHP format, unauthenticated, with custom, dangerous filename', function(done){
    assert.response(server, {
        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&filename=b;"%20()[]a',
        headers: {host: 'vizzuality.cartodb.com'},
        encoding: 'binary',
        method: 'GET'
    },{ }, function(err, res){
        assert.equal(res.statusCode, 200, res.body);
        var fname = "b_______a";
        var cd = res.headers['content-disposition'];
        assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
        assert.equal(true, /filename=b_______a.zip/gi.test(cd), 'Unexpected SHP filename: ' + cd);
        var tmpfile = '/tmp/myshape.zip';
        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
        if (writeErr) {
            return done(writeErr);
        }
        var zf = new zipfile.ZipFile(tmpfile);
        assert.ok(_.contains(zf.names, fname + '.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
        assert.ok(_.contains(zf.names, fname + '.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
        assert.ok(_.contains(zf.names, fname + '.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
        assert.ok(_.contains(zf.names, fname+ '.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
        fs.unlinkSync(tmpfile);
        done();
    });
});

it('SHP format, authenticated', function(done){
    assert.response(server, {
        url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp&api_key=1234',
        headers: {host: 'vizzuality.cartodb.com'},
        encoding: 'binary',
        method: 'GET'
    },{ }, function(err, res){
        assert.equal(res.statusCode, 200, res.body);
        var cd = res.headers['content-disposition'];
        assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
        var tmpfile = '/tmp/myshape.zip';
        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
        if (writeErr) {
            return done(writeErr);
        }
        var zf = new zipfile.ZipFile(tmpfile);
        assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
        assert.ok(_.contains(zf.names, 'cartodb-query.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
        assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
        assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
        // TODO: check contents of the DBF
        fs.unlinkSync(tmpfile);
        done();
    });
});


// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/66
it('SHP format, unauthenticated, with utf8 data', function(done){
    var query = querystring.stringify({
        q: "SELECT '♥♦♣♠' as f, st_makepoint(0,0,4326) as the_geom",
        format: 'shp',
        filename: 'myshape'
      });
    assert.response(server, {
        url: '/api/v1/sql?' + query,
        headers: {host: 'vizzuality.cartodb.com'},
        encoding: 'binary',
        method: 'GET'
    },{ }, function(err, res){
        assert.equal(res.statusCode, 200, res.body);
        var tmpfile = '/tmp/myshape.zip';
        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
        if (writeErr) {
            return done(writeErr);
        }
        var zf = new zipfile.ZipFile(tmpfile);
        var buffer = zf.readFileSync('myshape.dbf');
        fs.unlinkSync(tmpfile);
        var strings = buffer.toString();
        assert.ok(/♥♦♣♠/.exec(strings), "Cannot find '♥♦♣♠' in here:\n" + strings);
        done();
    });
});

// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/66
it('mixed type geometry', function(done){
    var query = querystring.stringify({
        q: "SELECT 'POINT(0 0)'::geometry as g UNION ALL SELECT 'LINESTRING(0 0, 1 0)'::geometry",
        format: 'shp'
      });
    assert.response(server, {
        url: '/api/v1/sql?' + query,
        headers: {host: 'vizzuality.cartodb.com'},
        encoding: 'binary',
        method: 'GET'
    },{ }, function(err, res){
        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
        assert.deepEqual(res.headers['content-disposition'], 'inline');
        assert.equal(res.statusCode, 400, res.statusCode + ': ' +res.body);
        var parsedBody = JSON.parse(res.body);
        var error = parsedBody.error[0];
        var expectedError = /Attempt to write non-point \(LINESTRING\) geometry to point shapefile/g;
        assert.ok(expectedError.test(error), error);
        done();
    });
});

// See https://github.com/Vizzuality/CartoDB-SQL-API/issues/87
it('errors are not confused with warnings', function(done){
    var query = querystring.stringify({
        q: [
            "SELECT 'POINT(0 0)'::geometry as g, 1 as a_very_very_very_long_field_name",
            "SELECT 'LINESTRING(0 0, 1 0)'::geometry, 2"
        ].join(" UNION ALL "),
        format: 'shp'
      });
    assert.response(server, {
        url: '/api/v1/sql?' + query,
        headers: {host: 'vizzuality.cartodb.com'},
        encoding: 'binary',
        method: 'GET'
    },{ }, function(err, res){
        assert.deepEqual(res.headers['content-type'], 'application/json; charset=utf-8');
        assert.deepEqual(res.headers['content-disposition'], 'inline');
        assert.equal(res.statusCode, 400, res.statusCode + ': ' +res.body);
        var parsedBody = JSON.parse(res.body);
        var error = parsedBody.error[0];
        var expectedError = /Attempt to write non-point \(LINESTRING\) geometry to point shapefile/g;
        assert.ok(expectedError.test(error), error);
        done();
    });
});

it('skipfields controls fields included in SHP output', function(done){
    var query = querystring.stringify({
        q: "SELECT 111 as skipme, 222 as keepme, 'POINT(0 0)'::geometry as g",
        format: 'shp',
        skipfields: 'skipme',
        filename: 'myshape'
      });
    assert.response(server, {
        url: '/api/v1/sql?' + query,
        headers: {host: 'vizzuality.cartodb.com'},
        encoding: 'binary',
        method: 'GET'
    },{ }, function(err, res){
        assert.equal(res.statusCode, 200, res.body);
        var tmpfile = '/tmp/myshape.zip';
        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
        if (writeErr) {
            return done(writeErr);
        }
        var zf = new zipfile.ZipFile(tmpfile);
        var buffer = zf.readFileSync('myshape.dbf');
        fs.unlinkSync(tmpfile);
        var strings = buffer.toString();
        assert.ok(!/skipme/.exec(strings), "Could not skip 'skipme' field:\n" + strings);
        done();
    });
});

it('SHP format, concurrently', function(done){
    var concurrency = 1;
    var waiting = concurrency;
    function validate(err, res){
        var cd = res.headers['content-disposition'];
        assert.equal(true, /^attachment/.test(cd), 'SHP is not disposed as attachment: ' + cd);
        assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
        var tmpfile = '/tmp/myshape.zip';
        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
        if (writeErr) {
            return done(writeErr);
        }
        var zf = new zipfile.ZipFile(tmpfile);
        assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
        assert.ok(_.contains(zf.names, 'cartodb-query.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
        assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
        assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
        // TODO: check DBF contents
        fs.unlinkSync(tmpfile);
        if ( ! --waiting ) {
            done();
        }
    }
    for (var i=0; i<concurrency; ++i) {
        assert.response(
            server,
            {
                url: '/api/v1/sql?q=SELECT%20*%20FROM%20untitle_table_4%20LIMIT%201&format=shp',
                headers: {host: 'vizzuality.cartodb.com'},
                encoding: 'binary',
                method: 'GET'
            },
            {
                status: 200
            },
            validate
        );
    }
});

// See https://github.com/CartoDB/CartoDB-SQL-API/issues/111
it('point with null first', function(done){
    var query = querystring.stringify({
        q: "SELECT null::geometry as g UNION ALL SELECT 'SRID=4326;POINT(0 0)'::geometry",
        format: 'shp'
      });
    assert.response(server, {
        url: '/api/v1/sql?' + query,
        headers: {host: 'vizzuality.cartodb.com'},
        encoding: 'binary',
        method: 'GET'
    },{ }, function(err, res){
        assert.equal(res.statusCode, 200, res.body);
        var cd = res.headers['content-disposition'];
        assert.equal(true, /filename=cartodb-query.zip/gi.test(cd));
        var tmpfile = '/tmp/myshape.zip';
        var writeErr = fs.writeFileSync(tmpfile, res.body, 'binary');
        if (writeErr) {
            return done(writeErr);
        }
        var zf = new zipfile.ZipFile(tmpfile);
        assert.ok(_.contains(zf.names, 'cartodb-query.shp'), 'SHP zipfile does not contain .shp: ' + zf.names);
        assert.ok(_.contains(zf.names, 'cartodb-query.shx'), 'SHP zipfile does not contain .shx: ' + zf.names);
        assert.ok(_.contains(zf.names, 'cartodb-query.dbf'), 'SHP zipfile does not contain .dbf: ' + zf.names);
        assert.ok(_.contains(zf.names, 'cartodb-query.prj'), 'SHP zipfile does not contain .prj: ' + zf.names);
        // TODO: check contents of the DBF
        fs.unlinkSync(tmpfile);
        done();
    });
});

    var limit = 1200;

    it('expects ' + limit + ' rows in public table', function(done){

        var filename = 'test_1200';

        assert.response(server, {
                url: '/api/v1/sql?' + querystring.stringify({
                    q: "SELECT * from populated_places_simple_reduced limit " + limit,
                    format: 'shp',
                    filename: filename
                }),
                headers: { host: 'vizzuality.cartodb.com' },
                encoding: 'binary',
                method: 'GET'
            },
            {
                status: 200
            },
            function(err, res) {
                if (err) {
                    return done(err);
                }

                var tmpShpPath = '/tmp/'+filename+'.zip';
                err = fs.writeFileSync(tmpShpPath, res.body, 'binary');
                if (err) {
                    return done(err);
                }

                var zf = new zipfile.ZipFile(tmpShpPath);
                zf.names.forEach(function(name) {
                    var buffer = zf.readFileSync(name);
                    var tmpDbfPath = '/tmp/' + name;
                    err = fs.writeFileSync(tmpDbfPath, buffer);
                    if (err) {
                        return done(err);
                    }
                });
                shapefile.read('/tmp/'+filename, function(err, collection) {
                    if (err) {
                        return done(err);
                    }
                    assert.equal(collection.features.length, limit);
                    done();
                });

            }
        );
    });

    it('SHP zip, wrong path for zip command should return error', function(done){
        global.settings.zipCommand = '/wrong/path';
        var query = querystring.stringify({
            q: "SELECT st_makepoint(0,0,4326) as the_geom",
            format: 'shp',
            filename: 'myshape'
        });
        assert.response(server, {
            url: '/api/v1/sql?' + query,
            headers: {host: 'vizzuality.cartodb.com'},
            encoding: 'binary',
            method: 'GET'
        },{ }, function(err, res){
            assert.equal(res.statusCode, 400, res.body);
            var parsedBody = JSON.parse(res.body);
            var respBodyPattern = new RegExp('Error executing zip command,  Error: spawn(.*)ENOENT', 'i');
            assert.equal(respBodyPattern.test(parsedBody.error[0]), true);
            done();
        });
    });
});