cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #510 from CartoDB/copyto-cancel-test

Browse Source 
Copy tests more deterministics
This commit is contained in:
Simon Martín 2018-06-21 11:57:15 +02:00 committed by GitHub
commit 1cb02d0e7f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 346 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

231
test/acceptance/copy-abort.js Normal file
View File

@ -0,0 +1,231 @@
const querystring = require('querystring');
const StatsClient = require('../../app/stats/client');
const statsClient = StatsClient.getInstance(global.settings.statsd);
const server = require('../../app/server')(statsClient);
const request = require('request');
const assert = require('assert');
const copyQuery = `COPY (
INSERT INTO copy_to_test
SELECT updated_at
FROM generate_series(
'1984-06-14 01:00:00'::timestamp,
'2018-06-14 01:00:00'::timestamp,
'1 hour'::interval
) updated_at
RETURNING updated_at
) TO STDOUT`;
const createTableQuery = `CREATE TABLE copy_to_test AS
(SELECT '2018-06-15 14:49:05.126415+00'::timestamp AS updated_at)`;
const dropTableQuery = `DROP TABLE copy_to_test`;
const countQuery = `SELECT count(1) as count FROM copy_to_test`;
function countInsertedRows (host, port, callback) {
setTimeout(function () {
const count = querystring.stringify({ q: countQuery, api_key: 1234 });
const options = {
url: `http://${host}:${port}/api/v1/sql?${count}`,
headers: { host: 'vizzuality.cartodb.com' },
method: 'GET'
};
request(options, function (err, res, body) {
if (err) {
return callback(err);
}
assert.equal(res.statusCode, 200);
const result = JSON.parse(body);
callback(null, result);
});
}, 100);
}
describe('Cancel "copy to" commands', function () {
beforeEach(function (done) {
this.listener = server.listen(0, '127.0.0.1');
this.listener.on('error', done);
this.listener.on('listening', () => {
const { address, port } = this.listener.address();
this.host = address;
this.port = port;
done();
});
});
beforeEach(function (done) {
const { host, port } = this;
const createTable = querystring.stringify({ q: createTableQuery, api_key: 1234});
const createTableOptions = {
url: `http://${host}:${port}/api/v1/sql?${createTable}`,
headers: { host: 'vizzuality.cartodb.com' },
method: 'GET'
};
request(createTableOptions, function (err, res) {
if (err) {
return done(err);
}
assert.equal(res.statusCode, 200);
done();
});
});
afterEach(function (done) {
const { host, port } = this;
const dropTable = querystring.stringify({ q: dropTableQuery, api_key: 1234 });
const dropTableOptions = {
url: `http://${host}:${port}/api/v1/sql?${dropTable}`,
headers: { host: 'vizzuality.cartodb.com' },
method: 'GET'
};
request(dropTableOptions, function (err, res) {
if (err) {
return done(err);
}
assert.equal(res.statusCode, 200);
done();
});
});
afterEach(function (done) {
this.listener.close(done);
});
it('abort on response', function (done) {
const { host, port } = this;
const copy = querystring.stringify({ q: copyQuery, api_key: 1234 });
const options = {
url: `http://${host}:${port}/api/v1/sql/copyto?${copy}`,
headers: { host: 'vizzuality.cartodb.com' },
method: 'GET'
};
const req = request(options);
req.on('response', function () {
req.abort();
countInsertedRows(host, port, function (err, result) {
if (err) {
return done(err);
}
assert.equal(result.rows[0].count, 1);
done();
});
});
});
it('abort on data', function (done) {
const { host, port } = this;
const copy = querystring.stringify({ q: copyQuery, api_key: 1234 });
const options = {
url: `http://${host}:${port}/api/v1/sql/copyto?${copy}`,
headers: { host: 'vizzuality.cartodb.com' },
method: 'GET'
};
const req = request(options);
req.once('data', function () {
req.abort();
countInsertedRows(host, port, function (err, result) {
if (err) {
return done(err);
}
assert.equal(result.rows[0].count, 1);
done();
});
});
});
it('destroy on data', function (done) {
const { host, port } = this;
const copy = querystring.stringify({ q: copyQuery, api_key: 1234 });
const options = {
url: `http://${host}:${port}/api/v1/sql/copyto?${copy}`,
headers: { host: 'vizzuality.cartodb.com' },
method: 'GET'
};
const req = request(options);
let response;
req.on('response', function (res) {
response = res;
});
req.once('data', function () {
response.destroy();
countInsertedRows(host, port, function (err, result) {
if (err) {
return done(err);
}
assert.equal(result.rows[0].count, 1);
done();
});
});
});
it('destroy on response', function (done) {
const { host, port } = this;
const copy = querystring.stringify({ q: copyQuery, api_key: 1234 });
const options = {
url: `http://${host}:${port}/api/v1/sql/copyto?${copy}`,
headers: { host: 'vizzuality.cartodb.com' },
method: 'GET'
};
const req = request(options);
req.on('response', function (response) {
response.destroy();
countInsertedRows(host, port, function (err, result) {
if (err) {
return done(err);
}
assert.equal(result.rows[0].count, 1);
done();
});
});
});
});

186
test/acceptance/copy-endpoints.js
View File

@ -20,21 +20,27 @@ const server = require('../../app/server')(statsClient);
describe('copy-endpoints', function() {
describe('general', function() {
before(function(done) {
const client = new Client({
user: 'postgres',
host: 'localhost',
database: 'cartodb_test_user_1_db',
port: 5432,
});
client.connect();
client.query('TRUNCATE copy_endpoints_test', (err/*, res */) => {
client.end();
done(err);
});
before(function() {
this.client = new Client({
user: 'postgres',
host: 'localhost',
database: 'cartodb_test_user_1_db',
port: 5432,
});
this.client.connect();
});
after(function() {
this.client.end();
});
afterEach(function (done) {
this.client.query('TRUNCATE copy_endpoints_test', err => {
done(err);
});
});
describe('general', function() {
it('should work with copyfrom endpoint', function(done){
assert.response(server, {
url: "/api/v1/sql/copyfrom?" + querystring.stringify({
@ -47,11 +53,11 @@ describe('copy-endpoints', function() {
assert.ifError(err);
const response = JSON.parse(res.body);
assert.equal(!!response.time, true);
assert.strictEqual(response.total_rows, 6);
assert.strictEqual(response.total_rows, 6);
done();
});
});
it('should fail with copyfrom endpoint and unexisting table', function(done){
assert.response(server, {
url: "/api/v1/sql/copyfrom?" + querystring.stringify({
@ -63,7 +69,7 @@ describe('copy-endpoints', function() {
},{}, function(err, res) {
assert.ifError(err);
assert.deepEqual(
JSON.parse(res.body),
JSON.parse(res.body),
{
error:['relation \"unexisting_table\" does not exist']
}
@ -71,7 +77,7 @@ describe('copy-endpoints', function() {
done();
});
});
it('should fail with copyfrom endpoint and without csv', function(done){
assert.response(server, {
url: "/api/v1/sql/copyfrom?" + querystring.stringify({
@ -82,7 +88,7 @@ describe('copy-endpoints', function() {
},{}, function(err, res) {
assert.ifError(err);
assert.deepEqual(
JSON.parse(res.body),
JSON.parse(res.body),
{
error:['No rows copied']
}
@ -90,17 +96,17 @@ describe('copy-endpoints', function() {
done();
});
});
it('should fail with copyfrom endpoint and without q', function(done){
assert.response(server, {
url: "/api/v1/sql/copyfrom",
data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
headers: {host: 'vizzuality.cartodb.com'},
method: 'POST'
},{}, function(err, res) {
assert.ifError(err);
assert.deepEqual(
JSON.parse(res.body),
JSON.parse(res.body),
{
error:["SQL is missing"]
}
@ -108,29 +114,40 @@ describe('copy-endpoints', function() {
done();
});
});
it('should work with copyto endpoint', function(done){
assert.response(server, {
url: "/api/v1/sql/copyto?" + querystring.stringify({
q: 'COPY copy_endpoints_test TO STDOUT',
filename: '/tmp/output.dmp'
url: "/api/v1/sql/copyfrom?" + querystring.stringify({
q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
}),
data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{}, function(err, res) {
method: 'POST'
},{}, function(err) {
assert.ifError(err);
assert.strictEqual(
res.body,
'11\tPaul\t10\n12\tPeter\t10\n13\tMatthew\t10\n14\t\\N\t10\n15\tJames\t10\n16\tJohn\t10\n'
);
assert.equal(res.headers['content-disposition'], 'attachment; filename=%2Ftmp%2Foutput.dmp');
assert.equal(res.headers['content-type'], 'application/octet-stream');
done();
assert.response(server, {
url: "/api/v1/sql/copyto?" + querystring.stringify({
q: 'COPY copy_endpoints_test TO STDOUT',
filename: '/tmp/output.dmp'
}),
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},{}, function(err, res) {
assert.ifError(err);
assert.strictEqual(
res.body,
'11\tPaul\t10\n12\tPeter\t10\n13\tMatthew\t10\n14\t\\N\t10\n15\tJames\t10\n16\tJohn\t10\n'
);
assert.equal(res.headers['content-disposition'], 'attachment; filename=%2Ftmp%2Foutput.dmp');
assert.equal(res.headers['content-type'], 'application/octet-stream');
done();
});
});
});
it('should fail with copyto endpoint and without sql', function(done){
assert.response(server, {
url: "/api/v1/sql/copyto?" + querystring.stringify({
@ -141,7 +158,7 @@ describe('copy-endpoints', function() {
},{}, function(err, res) {
assert.ifError(err);
assert.deepEqual(
JSON.parse(res.body),
JSON.parse(res.body),
{
error:["SQL is missing"]
}
@ -149,15 +166,15 @@ describe('copy-endpoints', function() {
done();
});
});
it('should work with copyfrom and gzip', function(done){
assert.response(server, {
url: "/api/v1/sql/copyfrom?" + querystring.stringify({
q: "COPY copy_endpoints_test2 (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
}),
data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv.gz'),
headers: {
host: 'vizzuality.cartodb.com',
host: 'vizzuality.cartodb.com',
'content-encoding': 'gzip'
},
method: 'POST'
@ -169,11 +186,11 @@ describe('copy-endpoints', function() {
done();
});
});
});
describe('timeout', function() {
describe('timeout', function() {
it('should fail with copyfrom and timeout', function(done){
assert.response(server, {
url: '/api/v1/sql?q=set statement_timeout = 10',
@ -184,7 +201,7 @@ describe('copy-endpoints', function() {
assert.ifError(err);
assert.response(server, {
url: "/api/v1/sql/copyfrom?" + querystring.stringify({
q: `COPY copy_endpoints_test (id, name)
q: `COPY copy_endpoints_test (id, name)
FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)`
}),
data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
@ -204,8 +221,8 @@ describe('copy-endpoints', function() {
context: 'limit',
detail: 'datasource'
});
assert.response(server, {
url: "/api/v1/sql?q=set statement_timeout = 2000",
headers: {host: 'vizzuality.cartodb.com'},
@ -215,7 +232,7 @@ describe('copy-endpoints', function() {
});
});
});
it('should fail with copyto and timeout', function(done){
assert.response(server, {
url: '/api/v1/sql?q=set statement_timeout = 20',
@ -240,7 +257,7 @@ describe('copy-endpoints', function() {
};
const expectedError = res.body.substring(res.body.length - JSON.stringify(error).length);
assert.deepEqual(JSON.parse(expectedError), error);
assert.response(server, {
url: "/api/v1/sql?q=set statement_timeout = 2000",
headers: {host: 'vizzuality.cartodb.com'},
@ -251,24 +268,24 @@ describe('copy-endpoints', function() {
});
});
});
describe('db connections', function() {
before(function() {
this.db_pool_size = global.settings.db_pool_size;
global.settings.db_pool_size = 1;
});
after(function() {
global.settings.db_pool_size = this.db_pool_size;
});
it('copyfrom', function(done) {
function doCopyFrom() {
return new Promise(resolve => {
assert.response(server, {
url: "/api/v1/sql/copyfrom?" + querystring.stringify({
q: `COPY copy_endpoints_test (id, name)
q: `COPY copy_endpoints_test (id, name)
FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)`
}),
data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
@ -282,12 +299,12 @@ describe('copy-endpoints', function() {
});
});
}
Promise.all([doCopyFrom(), doCopyFrom(), doCopyFrom()]).then(function() {
done();
});
});
it('copyto', function(done) {
function doCopyTo() {
return new Promise(resolve => {
@ -305,28 +322,39 @@ describe('copy-endpoints', function() {
});
});
}
Promise.all([doCopyTo(), doCopyTo(), doCopyTo()]).then(function() {
done();
assert.response(server, {
url: "/api/v1/sql/copyfrom?" + querystring.stringify({
q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)"
}),
data: fs.createReadStream(__dirname + '/../support/csv/copy_test_table.csv'),
headers: {host: 'vizzuality.cartodb.com'},
method: 'POST'
},{}, function(err) {
assert.ifError(err);
Promise.all([doCopyTo(), doCopyTo(), doCopyTo()]).then(function() {
done();
});
});
});
});
describe('client disconnection', function() {
// Give it enough time to connect and issue the query
// but not too much so as to disconnect in the middle of the query.
const CLIENT_DISCONNECT_TIMEOUT = 10;
const CLIENT_DISCONNECT_TIMEOUT = 100;
before(function() {
this.db_pool_size = global.settings.db_pool_size;
global.settings.db_pool_size = 1;
});
after(function() {
global.settings.db_pool_size = this.db_pool_size;
});
var assertCanReuseConnection = function (done) {
const assertCanReuseConnection = function (done) {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: 'SELECT 1',
@ -339,7 +367,23 @@ describe('copy-endpoints', function() {
done();
});
};
const assertCanReuseCanceledConnection = function (done) {
assert.response(server, {
url: '/api/v1/sql?' + querystring.stringify({
q: 'SELECT count(*) FROM copy_endpoints_test',
}),
headers: { host: 'vizzuality.cartodb.com' },
method: 'GET'
}, {}, function(err, res) {
assert.ifError(err);
assert.ok(res.statusCode === 200);
const result = JSON.parse(res.body);
assert.strictEqual(result.rows[0].count, 0);
done();
});
};
it('COPY TO returns the connection to the pool if the client disconnects', function(done) {
const listener = server.listen(0, '127.0.0.1');
@ -367,7 +411,7 @@ describe('copy-endpoints', function() {
});
});
});
it('COPY FROM returns the connection to the pool if the client disconnects', function(done) {
const listener = server.listen(0, '127.0.0.1');
@ -390,10 +434,10 @@ describe('copy-endpoints', function() {
setTimeout(() => {
req.abort();
done();
assertCanReuseCanceledConnection(done);
}, CLIENT_DISCONNECT_TIMEOUT);
});
});
});
});

9
test/support/sql/test.sql
View File

@ -226,12 +226,3 @@ CREATE TABLE copy_endpoints_test (
);
GRANT ALL ON TABLE copy_endpoints_test TO :TESTUSER;
GRANT ALL ON TABLE copy_endpoints_test TO :PUBLICUSER;
DROP TABLE IF EXISTS copy_endpoints_test2;
CREATE TABLE copy_endpoints_test2 (
id integer,
name text,
age integer default 10
);
GRANT ALL ON TABLE copy_endpoints_test2 TO :TESTUSER;
GRANT ALL ON TABLE copy_endpoints_test2 TO :PUBLICUSER;