Fix test timeouts (#612)
* Tests: Work with re-runs * Test: Adapt copyfrom test - Reduce timeout from 10 to 1 ms. - Increase the size of the file from 6 to 2016 rows. * Use node-cartodb-psql 0.14.0 Ref: https://github.com/CartoDB/CartoDB-SQL-API/pull/612
This commit is contained in:
Raúl Marín
GitHub
parent
76711c5f22
commit
df523960a2
1
NEWS.md
1
NEWS.md
@ -19,6 +19,7 @@ Announcements:
|
||||
* Middlewarify client abort query checker.
|
||||
* Middlewarify query controller.
|
||||
* Set a hard limit on the size of the X-SQLAPI-Log header.
|
||||
* Update cartodb-psql to 0.14.0 and use the timeout parameter for pg.query.
|
||||
|
||||
## 3.0.0
|
||||
Released 2019-02-22
|
||||
|
||||
@ -24,32 +24,7 @@ QueryRunner.prototype.run = function (job_id, sql, user, timeout, dbparams, call
|
||||
};
|
||||
|
||||
QueryRunner.prototype._run = function (dbparams, job_id, sql, timeout, callback) {
|
||||
var self = this;
|
||||
var pg = new PSQL(dbparams);
|
||||
|
||||
pg.query('SET statement_timeout=' + timeout, function (err) {
|
||||
if(err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
// mark query to allow to users cancel their queries
|
||||
sql = '/* ' + job_id + ' */ ' + sql;
|
||||
|
||||
self.logger.debug('Running query [timeout=%d] %s', timeout, sql);
|
||||
pg.eventedQuery(sql, function (err, query) {
|
||||
if (err) {
|
||||
return callback(err);
|
||||
}
|
||||
|
||||
query.on('error', callback);
|
||||
|
||||
query.on('end', function (result) {
|
||||
// only if result is present then query is done sucessfully otherwise an error has happened
|
||||
// and it was handled by error listener
|
||||
if (result) {
|
||||
callback(null, result);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
this.logger.debug('Running query [timeout=%d] %s', timeout, sql);
|
||||
pg.query(`/* ${job_id} */ ${sql}`, callback, false, timeout);
|
||||
};
|
||||
|
||||
28
package-lock.json
generated
28
package-lock.json
generated
@ -171,7 +171,7 @@
|
||||
"bindings": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.0.tgz",
|
||||
"integrity": "sha1-s0b27PapX1qBXFg5/HzbIlAvHtc="
|
||||
"integrity": "sha512-DpLh5EzMR2kzvX1KIlVC0VkC3iZtHKTgdtZ0a3pglBZdaQFjt5S9g9xd1lE+YvXyfd6mtCeRnrUfOLYiTMlNSw=="
|
||||
},
|
||||
"bintrees": {
|
||||
"version": "1.0.1",
|
||||
@ -231,7 +231,7 @@
|
||||
"brace-expansion": {
|
||||
"version": "1.1.11",
|
||||
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
|
||||
"integrity": "sha1-PH/L9SnYcibz0vUrlm/1Jx60Qd0=",
|
||||
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
|
||||
"requires": {
|
||||
"balanced-match": "^1.0.0",
|
||||
"concat-map": "0.0.1"
|
||||
@ -250,7 +250,7 @@
|
||||
},
|
||||
"buffer-writer": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "http://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz",
|
||||
"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.1.tgz",
|
||||
"integrity": "sha1-Iqk2kB4wKa/NdUfrRIfOtpejvwg="
|
||||
},
|
||||
"builtin-modules": {
|
||||
@ -302,9 +302,9 @@
|
||||
"integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo="
|
||||
},
|
||||
"cartodb-psql": {
|
||||
"version": "0.13.1",
|
||||
"resolved": "https://registry.npmjs.org/cartodb-psql/-/cartodb-psql-0.13.1.tgz",
|
||||
"integrity": "sha512-1z3Dk9G8KQlNGurbcmGBvNj8DVCh1Keue9uzyyvB6hKOYzBHMxixAMG0D+8nSsA7oQmWUsx/xkZZ5ZxT9toEHA==",
|
||||
"version": "0.14.0",
|
||||
"resolved": "https://registry.npmjs.org/cartodb-psql/-/cartodb-psql-0.14.0.tgz",
|
||||
"integrity": "sha512-myLV2xo3q9oTT8m8M+c+UTD/ziDN7hrYtZ9yY00KvMnu2NsVeRQsTe8Yxq1GVS8vF9iYfcelwjVEGObPUdLtHw==",
|
||||
"requires": {
|
||||
"debug": "^3.1.0",
|
||||
"pg": "github:cartodb/node-postgres#6.4.2-cdb2",
|
||||
@ -410,7 +410,7 @@
|
||||
"concat-stream": {
|
||||
"version": "1.6.2",
|
||||
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.2.tgz",
|
||||
"integrity": "sha1-kEvfGUzTEi/Gdcd/xKw9T/D9GjQ=",
|
||||
"integrity": "sha512-27HBghJxjiZtIk3Ycvn/4kbJk/1uZuJFfuPEns6LaEvpvG1f0hTea8lilrouyo9mVc2GWdcEZ8OLoGmSADlrCw==",
|
||||
"requires": {
|
||||
"buffer-from": "^1.0.0",
|
||||
"inherits": "^2.0.3",
|
||||
@ -1685,7 +1685,7 @@
|
||||
"dependencies": {
|
||||
"async": {
|
||||
"version": "1.5.2",
|
||||
"resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz",
|
||||
"resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
|
||||
"integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=",
|
||||
"dev": true
|
||||
},
|
||||
@ -1912,7 +1912,7 @@
|
||||
"minimatch": {
|
||||
"version": "3.0.4",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
|
||||
"integrity": "sha1-UWbihkV/AzBgZL5Ul+jbsMPTIIM=",
|
||||
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
|
||||
"requires": {
|
||||
"brace-expansion": "^1.1.7"
|
||||
}
|
||||
@ -2183,7 +2183,7 @@
|
||||
"normalize-package-data": {
|
||||
"version": "2.4.0",
|
||||
"resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.4.0.tgz",
|
||||
"integrity": "sha1-EvlaMH1YNSB1oEkHuErIvpisAS8=",
|
||||
"integrity": "sha512-9jjUFbTPfEy3R/ad/2oNbKtW9Hgovl5O1FvFWKkKblNXoN/Oou6+9+KKohPK13Yc3/TyunyWhJp6gvRNR/PPAw==",
|
||||
"requires": {
|
||||
"hosted-git-info": "^2.1.4",
|
||||
"is-builtin-module": "^1.0.0",
|
||||
@ -2491,7 +2491,7 @@
|
||||
"process-nextick-args": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz",
|
||||
"integrity": "sha1-o31zL0JxtKsa0HDTVQjoKQeI/6o="
|
||||
"integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw=="
|
||||
},
|
||||
"proxy-addr": {
|
||||
"version": "2.0.4",
|
||||
@ -2847,7 +2847,7 @@
|
||||
"spdx-expression-parse": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.0.tgz",
|
||||
"integrity": "sha1-meEZt6XaAOBUkcn6M4t5BII7QdA=",
|
||||
"integrity": "sha512-Yg6D3XpRD4kkOmTpdgbUiEJFKghJH03fiC1OPll5h/0sO6neh2jqRDVHOQ4o/LMea0tgCkbMgea5ip/e+MkWyg==",
|
||||
"requires": {
|
||||
"spdx-exceptions": "^2.1.0",
|
||||
"spdx-license-ids": "^3.0.0"
|
||||
@ -3016,7 +3016,7 @@
|
||||
},
|
||||
"through": {
|
||||
"version": "2.3.8",
|
||||
"resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz",
|
||||
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
|
||||
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
|
||||
},
|
||||
"topojson": {
|
||||
@ -3068,7 +3068,7 @@
|
||||
"type-is": {
|
||||
"version": "1.6.16",
|
||||
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.16.tgz",
|
||||
"integrity": "sha1-+JzjQVQcZysl7nrjxz3uOyvlAZQ=",
|
||||
"integrity": "sha512-HRkVv/5qY2G6I8iab9cI7v1bOIdhm94dVjQCPFElW9W+3GeDOSHmy2EBYe4VTApuzolPcmgFTN3ftVJRKR2J9Q==",
|
||||
"requires": {
|
||||
"media-typer": "0.3.0",
|
||||
"mime-types": "~2.1.18"
|
||||
|
||||
@ -21,7 +21,7 @@
|
||||
"basic-auth": "^2.0.0",
|
||||
"bintrees": "1.0.1",
|
||||
"bunyan": "1.8.1",
|
||||
"cartodb-psql": "0.13.1",
|
||||
"cartodb-psql": "0.14.0",
|
||||
"cartodb-query-tables": "^0.5.0",
|
||||
"cartodb-redis": "2.1.0",
|
||||
"debug": "^4.1.1",
|
||||
|
||||
@ -40,7 +40,9 @@ describe('batch happy cases', function() {
|
||||
});
|
||||
|
||||
it('should perform job with select into', function (done) {
|
||||
var payload = jobPayload('select * into batch_test_table from (select * from private_table) as job');
|
||||
var payload = jobPayload(`
|
||||
DROP TABLE IF EXISTS batch_test_table;
|
||||
SELECT * INTO batch_test_table FROM (SELECT * from private_table) AS job`);
|
||||
this.batchTestClient.createJob(payload, function(err, jobResult) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
|
||||
@ -43,7 +43,8 @@ describe('Batch API callback templates', function () {
|
||||
};
|
||||
|
||||
self.testClient.getResult(
|
||||
'BEGIN;CREATE TABLE test_batch_errors (job_id text, error_message text);COMMIT', function (err) {
|
||||
'BEGIN; DROP TABLE IF EXISTS test_batch_errors; ' +
|
||||
'CREATE TABLE test_batch_errors (job_id text, error_message text); COMMIT', function (err) {
|
||||
if (err) {
|
||||
return done(err);
|
||||
}
|
||||
@ -78,7 +79,7 @@ describe('Batch API callback templates', function () {
|
||||
"query": {
|
||||
"query": [
|
||||
{
|
||||
query: "create table batch_jobs (job_id text)"
|
||||
query: "drop table if exists batch_jobs; create table batch_jobs (job_id text)"
|
||||
},
|
||||
{
|
||||
"query": "SELECT 1",
|
||||
@ -90,7 +91,7 @@ describe('Batch API callback templates', function () {
|
||||
var expectedQuery = {
|
||||
query: [
|
||||
{
|
||||
query: "create table batch_jobs (job_id text)",
|
||||
query: "drop table if exists batch_jobs; create table batch_jobs (job_id text)",
|
||||
status: 'done'
|
||||
},
|
||||
{
|
||||
|
||||
@ -75,7 +75,7 @@ describe('copy-endpoints', function() {
|
||||
assert.ifError(err);
|
||||
const response = JSON.parse(res.body);
|
||||
assert.equal(!!response.time, true);
|
||||
assert.strictEqual(response.total_rows, 6);
|
||||
assert.strictEqual(response.total_rows, 2016);
|
||||
done();
|
||||
});
|
||||
});
|
||||
@ -157,10 +157,8 @@ describe('copy-endpoints', function() {
|
||||
method: 'GET'
|
||||
},{}, function(err, res) {
|
||||
assert.ifError(err);
|
||||
assert.strictEqual(
|
||||
res.body,
|
||||
'11\tPaul\t10\n12\tPeter\t10\n13\tMatthew\t10\n14\t\\N\t10\n15\tJames\t10\n16\tJohn\t10\n'
|
||||
);
|
||||
const regex = /11\tPaul\t10\n12\tPeter\t10\n13\tMatthew\t10\n14\t\\N\t10\n15\tJames\t10\n16\t*/g;
|
||||
assert.ok(res.body.match(regex));
|
||||
|
||||
assert.equal(res.headers['content-disposition'], 'attachment; filename=%2Ftmp%2Foutput.dmp');
|
||||
assert.equal(res.headers['content-type'], 'application/octet-stream');
|
||||
@ -196,11 +194,8 @@ describe('copy-endpoints', function() {
|
||||
method: 'POST'
|
||||
}, {}, function(err, res) {
|
||||
assert.ifError(err);
|
||||
|
||||
assert.strictEqual(
|
||||
res.body,
|
||||
'11\tPaul\t10\n12\tPeter\t10\n13\tMatthew\t10\n14\t\\N\t10\n15\tJames\t10\n16\tJohn\t10\n'
|
||||
);
|
||||
const regex = /11\tPaul\t10\n12\tPeter\t10\n13\tMatthew\t10\n14\t\\N\t10\n15\tJames\t10\n16\t*/g;
|
||||
assert.ok(res.body.match(regex));
|
||||
|
||||
assert.equal(res.headers['content-disposition'], 'attachment; filename=%2Ftmp%2Foutput.dmp');
|
||||
assert.equal(res.headers['content-type'], 'application/octet-stream');
|
||||
@ -275,9 +270,9 @@ describe('copy-endpoints', function() {
|
||||
|
||||
|
||||
describe('timeout', function() {
|
||||
before('set a 10 ms timeout', function() {
|
||||
before('set a 1 ms timeout', function() {
|
||||
this.previous_timeout = global.settings.copy_timeout;
|
||||
global.settings.copy_timeout = 10;
|
||||
global.settings.copy_timeout = 1;
|
||||
});
|
||||
|
||||
after('restore previous timeout', function() {
|
||||
@ -519,7 +514,7 @@ describe('copy-endpoints', function() {
|
||||
}, function(err, res) {
|
||||
assert.ifError(err);
|
||||
const response = JSON.parse(res.body);
|
||||
assert.strictEqual(response.total_rows, 6);
|
||||
assert.strictEqual(response.total_rows, 2016);
|
||||
done();
|
||||
});
|
||||
});
|
||||
|
||||
@ -23,7 +23,7 @@ describe('special numeric (float) values', function() {
|
||||
var next = this;
|
||||
var opts = {
|
||||
url: URL + querystring.stringify({
|
||||
q: 'create table numbers_test(val float)'
|
||||
q: 'drop table if exists numbers_test; create table numbers_test(val float)'
|
||||
}),
|
||||
headers: HEADERS,
|
||||
method: METHOD
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue
Block a user