From 1a0e2b681bcc443fc07aa7da1b8601b762cdf3fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 26 May 2016 17:37:37 +0200 Subject: [PATCH] Fixes #309, added skipped status to fallback-jobs --- batch/job_status.js | 1 + batch/models/job_base.js | 1 + batch/models/job_fallback.js | 72 +++++- test/acceptance/job.fallback.test.js | 330 +++++++++++++++++++++++++-- 4 files changed, 388 insertions(+), 16 deletions(-) diff --git a/batch/job_status.js b/batch/job_status.js index d212679f..889f1b43 100644 --- a/batch/job_status.js +++ b/batch/job_status.js @@ -6,6 +6,7 @@ var JOB_STATUS_ENUM = { DONE: 'done', CANCELLED: 'cancelled', FAILED: 'failed', + SKIPPED: 'skipped', UNKNOWN: 'unknown' }; diff --git a/batch/models/job_base.js b/batch/models/job_base.js index 5169e560..3b9fc611 100644 --- a/batch/models/job_base.js +++ b/batch/models/job_base.js @@ -7,6 +7,7 @@ var validStatusTransitions = [ [jobStatus.PENDING, jobStatus.RUNNING], [jobStatus.PENDING, jobStatus.CANCELLED], [jobStatus.PENDING, jobStatus.UNKNOWN], + [jobStatus.PENDING, jobStatus.SKIPPED], [jobStatus.RUNNING, jobStatus.DONE], [jobStatus.RUNNING, jobStatus.FAILED], [jobStatus.RUNNING, jobStatus.CANCELLED], diff --git a/batch/models/job_fallback.js b/batch/models/job_fallback.js index 22d08547..ca582512 100644 --- a/batch/models/job_fallback.js +++ b/batch/models/job_fallback.js @@ -185,9 +185,79 @@ JobFallback.prototype.setStatus = function (status, errorMesssage) { throw new Error('Cannot set status from ' + this.data.status + ' to ' + status); } + if (!resultFromQuery.isChangeAppliedToQueryFallback || status === jobStatus.CANCELLED) { + this._setSkipped(status); + } + this.data.updated_at = now; }; +JobFallback.prototype._setSkipped = function (status) { + this._setSkippedQueryStatus(); + this._setSkippedJobStatus(); + + if (status === jobStatus.CANCELLED || status === jobStatus.FAILED) { + this._setRestPendingToSkipped(status); + } +}; + +JobFallback.prototype._setSkippedQueryStatus = function () { + // jshint maxcomplexity: 8 + for (var i = 0; i < this.data.query.query.length; i++) { + if (this.data.query.query[i].status === jobStatus.FAILED && this.data.query.query[i].onsuccess) { + if (this.isValidStatusTransition(this.data.query.query[i].fallback_status, jobStatus.SKIPPED)) { + this.data.query.query[i].fallback_status = jobStatus.SKIPPED; + } + } + + if (this.data.query.query[i].status === jobStatus.DONE && this.data.query.query[i].onerror) { + if (this.isValidStatusTransition(this.data.query.query[i].fallback_status, jobStatus.SKIPPED)) { + this.data.query.query[i].fallback_status = jobStatus.SKIPPED; + } + } + + if (this.data.query.query[i].status === jobStatus.CANCELLED && this.data.query.query[i].fallback_status) { + if (this.isValidStatusTransition(this.data.query.query[i].fallback_status, jobStatus.SKIPPED)) { + this.data.query.query[i].fallback_status = jobStatus.SKIPPED; + } + } + } +}; + +JobFallback.prototype._setSkippedJobStatus = function () { + // jshint maxcomplexity: 7 + + if (this.data.status === jobStatus.FAILED && this.data.query.onsuccess) { + if (this.isValidStatusTransition(this.data.fallback_status, jobStatus.SKIPPED)) { + this.data.fallback_status = jobStatus.SKIPPED; + } + } + + if (this.data.status === jobStatus.DONE && this.data.query.onerror) { + if (this.isValidStatusTransition(this.data.fallback_status, jobStatus.SKIPPED)) { + this.data.fallback_status = jobStatus.SKIPPED; + } + } + + if (this.data.status === jobStatus.CANCELLED && this.data.fallback_status) { + if (this.isValidStatusTransition(this.data.fallback_status, jobStatus.SKIPPED)) { + this.data.fallback_status = jobStatus.SKIPPED; + } + } +}; + +JobFallback.prototype._setRestPendingToSkipped = function (status) { + for (var i = 0; i < this.data.query.query.length; i++) { + if (this.data.query.query[i].status === jobStatus.PENDING) { + this.data.query.query[i].status = jobStatus.SKIPPED; + } + if (this.data.query.query[i].status !== status && + this.data.query.query[i].fallback_status === jobStatus.PENDING) { + this.data.query.query[i].fallback_status = jobStatus.SKIPPED; + } + } +}; + JobFallback.prototype._getLastStatusFromFinishedQuery = function () { var lastStatus = jobStatus.DONE; @@ -263,7 +333,7 @@ JobFallback.prototype._shouldTryToApplyStatusTransitionToQueryFallback = functio }; JobFallback.prototype._setQueryStatus = function (status, errorMesssage) { - // jshint maxcomplexity: 7 + // jshint maxcomplexity: 8 var isValid = false; var isChangeAppliedToQueryFallback = false; diff --git a/test/acceptance/job.fallback.test.js b/test/acceptance/job.fallback.test.js index 9a13491e..d549b279 100644 --- a/test/acceptance/job.fallback.test.js +++ b/test/acceptance/job.fallback.test.js @@ -133,7 +133,7 @@ describe('Batch API fallback job', function () { "query": "SELECT * FROM untitle_table_4", "onerror": "SELECT * FROM untitle_table_4 limit 1", "status": "done", - "fallback_status": "pending" + "fallback_status": "skipped" }] }; var interval = setInterval(function () { @@ -268,7 +268,7 @@ describe('Batch API fallback job', function () { query: 'SELECT * FROM nonexistent_table /* query should fail */', onsuccess: 'SELECT * FROM untitle_table_4 limit 1', status: 'failed', - fallback_status: 'pending', + fallback_status: 'skipped', failed_reason: 'relation "nonexistent_table" does not exist' }] }; @@ -424,7 +424,7 @@ describe('Batch API fallback job', function () { return done(err); } var job = JSON.parse(res.body); - if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.PENDING) { + if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.SKIPPED) { clearInterval(interval); assert.deepEqual(job.query, expectedQuery); done(); @@ -560,7 +560,7 @@ describe('Batch API fallback job', function () { return done(err); } var job = JSON.parse(res.body); - if (job.status === jobStatus.DONE && job.fallback_status === jobStatus.PENDING) { + if (job.status === jobStatus.DONE && job.fallback_status === jobStatus.SKIPPED) { clearInterval(interval); assert.deepEqual(job.query, expectedQuery); done(); @@ -759,13 +759,13 @@ describe('Batch API fallback job', function () { "query": "SELECT * FROM nonexistent_table /* should fail */", "onsuccess": "SELECT * FROM untitle_table_4 limit 1", "status": "failed", - "fallback_status": "pending", + "fallback_status": "skipped", "failed_reason": 'relation "nonexistent_table" does not exist' }, { "query": "SELECT * FROM untitle_table_4 limit 2", "onsuccess": "SELECT * FROM untitle_table_4 limit 3", - "status": "pending", - "fallback_status": "pending" + "status": "skipped", + "fallback_status": "skipped" }] }; @@ -842,7 +842,7 @@ describe('Batch API fallback job', function () { "query": "SELECT * FROM nonexistent_table /* should fail */", "onsuccess": "SELECT * FROM untitle_table_4 limit 3", "status": "failed", - "fallback_status": "pending", + "fallback_status": "skipped", "failed_reason": 'relation "nonexistent_table" does not exist' }] }; @@ -875,7 +875,7 @@ describe('Batch API fallback job', function () { }); }); - describe('"onerror" should not be triggered for any query', function () { + describe('"onerror" should not be triggered for any query and "skipped"', function () { var fallbackJob = {}; it('should create a job', function (done) { @@ -914,12 +914,12 @@ describe('Batch API fallback job', function () { query: 'SELECT * FROM untitle_table_4 limit 1', onerror: 'SELECT * FROM untitle_table_4 limit 2', status: 'done', - fallback_status: 'pending' + fallback_status: 'skipped' }, { query: 'SELECT * FROM untitle_table_4 limit 3', onerror: 'SELECT * FROM untitle_table_4 limit 4', status: 'done', - fallback_status: 'pending' + fallback_status: 'skipped' }] }; @@ -943,6 +943,144 @@ describe('Batch API fallback job', function () { assert.deepEqual(job.query, expectedQuery); done(); } else if (job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be done')); + } + }); + }, 50); + }); + }); + + describe('"onsuccess" should not be triggered for any query and "skipped"', function () { + var fallbackJob = {}; + + it('should create a job', function (done) { + assert.response(app, { + url: '/api/v2/sql/job?api_key=1234', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'POST', + data: querystring.stringify({ + query: { + query: [{ + query: "SELECT * FROM untitle_table_4 limit 1, /* should fail */", + onsuccess: "SELECT * FROM untitle_table_4 limit 2" + }] + } + }) + }, { + status: 201 + }, function (res, err) { + if (err) { + return done(err); + } + fallbackJob = JSON.parse(res.body); + done(); + }); + }); + + it('job should be failed', function (done) { + var expectedQuery = { + query: [{ + query: 'SELECT * FROM untitle_table_4 limit 1, /* should fail */', + onsuccess: 'SELECT * FROM untitle_table_4 limit 2', + status: 'failed', + fallback_status: 'skipped', + failed_reason: 'syntax error at end of input' + }] + }; + + var interval = setInterval(function () { + assert.response(app, { + url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'GET' + }, { + status: 200 + }, function (res, err) { + if (err) { + return done(err); + } + var job = JSON.parse(res.body); + if (job.status === jobStatus.FAILED) { + clearInterval(interval); + assert.deepEqual(job.query, expectedQuery); + done(); + } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be failed')); + } + }); + }, 50); + }); + }); + + + describe('"onsuccess" should not be triggered and "skipped"', function () { + var fallbackJob = {}; + + it('should create a job', function (done) { + assert.response(app, { + url: '/api/v2/sql/job?api_key=1234', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'POST', + data: querystring.stringify({ + query: { + query: [{ + query: "SELECT * FROM untitle_table_4 limit 1, /* should fail */", + }], + onsuccess: "SELECT * FROM untitle_table_4 limit 2" + } + }) + }, { + status: 201 + }, function (res, err) { + if (err) { + return done(err); + } + fallbackJob = JSON.parse(res.body); + done(); + }); + }); + + it('job should be failed', function (done) { + var expectedQuery = { + query: [{ + query: 'SELECT * FROM untitle_table_4 limit 1, /* should fail */', + status: 'failed', + failed_reason: 'syntax error at end of input' + }], + onsuccess: 'SELECT * FROM untitle_table_4 limit 2' + }; + + var interval = setInterval(function () { + assert.response(app, { + url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'GET' + }, { + status: 200 + }, function (res, err) { + if (err) { + return done(err); + } + var job = JSON.parse(res.body); + if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.SKIPPED) { + clearInterval(interval); + assert.deepEqual(job.query, expectedQuery); + done(); + } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { clearInterval(interval); done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be failed')); } @@ -1329,7 +1467,7 @@ describe('Batch API fallback job', function () { job.status === jobStatus.FAILED || job.status === jobStatus.CANCELLED) { clearInterval(interval); - done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be done')); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be running')); } }); }, 50); @@ -1341,7 +1479,7 @@ describe('Batch API fallback job', function () { "query": "SELECT pg_sleep(3)", "onsuccess": "SELECT pg_sleep(0)", "status": "cancelled", - "fallback_status": "pending" + "fallback_status": "skipped" }], "onsuccess": "SELECT pg_sleep(0)" }; @@ -1360,7 +1498,7 @@ describe('Batch API fallback job', function () { return done(err); } var job = JSON.parse(res.body); - if (job.status === jobStatus.CANCELLED && job.fallback_status === jobStatus.PENDING) { + if (job.status === jobStatus.CANCELLED && job.fallback_status === jobStatus.SKIPPED) { assert.deepEqual(job.query, expectedQuery); done(); } else if (job.status === jobStatus.DONE || job.status === jobStatus.FAILED) { @@ -1469,7 +1607,7 @@ describe('Batch API fallback job', function () { return done(err); } var job = JSON.parse(res.body); - if (job.status === jobStatus.CANCELLED && job.fallback_status === jobStatus.PENDING) { + if (job.status === jobStatus.CANCELLED && job.fallback_status === jobStatus.SKIPPED) { assert.deepEqual(job.query, expectedQuery); done(); } else if (job.status === jobStatus.DONE || job.status === jobStatus.FAILED) { @@ -1478,4 +1616,166 @@ describe('Batch API fallback job', function () { }); }); }); + + describe('should run first "onerror" and job "onerror" and skip the other ones', function () { + var fallbackJob = {}; + + it('should create a job', function (done) { + assert.response(app, { + url: '/api/v2/sql/job?api_key=1234', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'POST', + data: querystring.stringify({ + "query": { + "query": [{ + "query": "SELECT * FROM untitle_table_4 limit 1, should fail", + "onerror": "SELECT * FROM untitle_table_4 limit 2" + }, { + "query": "SELECT * FROM untitle_table_4 limit 3", + "onerror": "SELECT * FROM untitle_table_4 limit 4" + }], + "onerror": "SELECT * FROM untitle_table_4 limit 5" + } + }) + }, { + status: 201 + }, function (res, err) { + if (err) { + return done(err); + } + fallbackJob = JSON.parse(res.body); + done(); + }); + }); + + it('job should fail', function (done) { + var expectedQuery = { + "query": [ + { + "query": "SELECT * FROM untitle_table_4 limit 1, should fail", + "onerror": "SELECT * FROM untitle_table_4 limit 2", + "status": "failed", + "fallback_status": "done", + "failed_reason": "LIMIT #,# syntax is not supported" + }, + { + "query": "SELECT * FROM untitle_table_4 limit 3", + "onerror": "SELECT * FROM untitle_table_4 limit 4", + "status": "skipped", + "fallback_status": "skipped" + } + ], + "onerror": "SELECT * FROM untitle_table_4 limit 5" + }; + + var interval = setInterval(function () { + assert.response(app, { + url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'GET' + }, { + status: 200 + }, function (res, err) { + if (err) { + return done(err); + } + var job = JSON.parse(res.body); + if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.DONE) { + clearInterval(interval); + assert.deepEqual(job.query, expectedQuery); + done(); + } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be done')); + } + }); + }, 50); + }); + }); + + + describe('should fail first "onerror" and job "onerror" and skip the other ones', function () { + var fallbackJob = {}; + + it('should create a job', function (done) { + assert.response(app, { + url: '/api/v2/sql/job?api_key=1234', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'POST', + data: querystring.stringify({ + "query": { + "query": [{ + "query": "SELECT * FROM atm_madrid limit 1, should fail", + "onerror": "SELECT * FROM atm_madrid limit 2" + }, { + "query": "SELECT * FROM atm_madrid limit 3", + "onerror": "SELECT * FROM atm_madrid limit 4" + }], + "onerror": "SELECT * FROM atm_madrid limit 5" + } + }) + }, { + status: 201 + }, function (res, err) { + if (err) { + return done(err); + } + fallbackJob = JSON.parse(res.body); + done(); + }); + }); + + it('job should fail', function (done) { + var expectedQuery = { + query: [{ + query: 'SELECT * FROM atm_madrid limit 1, should fail', + onerror: 'SELECT * FROM atm_madrid limit 2', + status: 'failed', + fallback_status: 'failed', + failed_reason: 'relation "atm_madrid" does not exist' + }, { + query: 'SELECT * FROM atm_madrid limit 3', + onerror: 'SELECT * FROM atm_madrid limit 4', + status: 'skipped', + fallback_status: 'skipped' + }], + onerror: 'SELECT * FROM atm_madrid limit 5' + }; + + var interval = setInterval(function () { + assert.response(app, { + url: '/api/v2/sql/job/' + fallbackJob.job_id + '?api_key=1234&', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'host': 'vizzuality.cartodb.com' + }, + method: 'GET' + }, { + status: 200 + }, function (res, err) { + if (err) { + return done(err); + } + var job = JSON.parse(res.body); + if (job.status === jobStatus.FAILED && job.fallback_status === jobStatus.FAILED) { + clearInterval(interval); + assert.deepEqual(job.query, expectedQuery); + done(); + } else if (job.status === jobStatus.DONE || job.status === jobStatus.CANCELLED) { + clearInterval(interval); + done(new Error('Job ' + job.job_id + ' is ' + job.status + ', expected to be done')); + } + }); + }, 50); + }); + }); });