cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fixed conflicts

Browse Source
This commit is contained in:
Daniel García Aubert 2016-05-18 11:48:39 +02:00
commit b94a9c7c0a
10 changed files with 98 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
NEWS.md
View File

@ -1,7 +1,14 @@
1.28.1 - 2016-mm-dd
1.28.2 - 2016-mm-dd
-------------------
1.28.1 - 2016-05-12
-------------------
Bug fixes:
* OGR with _needSRS=true fails for empty tables #299
1.28.0 - 2016-05-11
-------------------

3
app/models/formats/ogr.js
View File

@ -130,6 +130,9 @@ OgrFormat.prototype.toOGR = function(options, out_format, out_filename, callback
var srid = result.rows[0].srid;
var type = result.rows[0].type;
next(null, srid, type);
} else {
// continue as srid and geom type are not critical when there are no results
next(null);
}
});

15
batch/batch.js
View File

@ -2,6 +2,7 @@
var util = require('util');
var EventEmitter = require('events').EventEmitter;
var debug = require('./util/debug')('batch');
var forever = require('./forever');
var queue = require('queue-async');
var jobStatus = require('./job_status');
@ -40,10 +41,10 @@ Batch.prototype._subscribe = function () {
self.jobQueuePool.removeQueue(host);
if (err.name === 'EmptyQueue') {
return console.log(err.message);
return debug(err.message);
}
console.error(err);
debug(err);
});
});
};
@ -69,7 +70,7 @@ Batch.prototype._consumeJobs = function (host, queue, callback) {
self.jobQueuePool.removeCurrentJobId(host);
if (err && err.name === 'JobNotRunnable') {
console.log(err.message);
debug(err.message);
return callback();
}
@ -78,9 +79,9 @@ Batch.prototype._consumeJobs = function (host, queue, callback) {
}
if (job.data.status === jobStatus.FAILED) {
console.log('Job %s %s in %s due to: %s', job_id, job.data.status, host, job.failed_reason);
debug('Job %s %s in %s due to: %s', job_id, job.data.status, host, job.failed_reason);
} else {
console.log('Job %s %s in %s', job_id, job.data.status, host);
debug('Job %s %s in %s', job_id, job.data.status, host);
}
self.emit('job:' + job.data.status, job_id);
@ -101,9 +102,9 @@ Batch.prototype.drain = function (callback) {
batchQueues.awaitAll(function (err) {
if (err) {
console.error('Something went wrong draining', err);
debug('Something went wrong draining', err);
} else {
console.log('Drain complete');
debug('Drain complete');
}
callback();

52
batch/job_backend.js
View File

@ -1,7 +1,10 @@
'use strict';
var queue = require('queue-async');
var debug = require('./util/debug')('job-backend');
var REDIS_PREFIX = 'batch:jobs:';
var REDIS_DB = 5;
var JOBS_TTL_IN_SECONDS = global.settings.jobs_ttl_in_seconds || 48 * 3600; // 48 hours
var jobStatus = require('./job_status');
var finalStatus = [
@ -12,16 +15,15 @@ var finalStatus = [
];
function JobBackend(metadataBackend, jobQueueProducer, jobPublisher, userIndexer) {
this.db = 5;
this.metadataBackend = metadataBackend;
this.jobQueueProducer = jobQueueProducer;
this.jobPublisher = jobPublisher;
this.userIndexer = userIndexer;
}
function toRedisParams(data) {
var redisParams = [REDIS_PREFIX + data.job_id];
var obj = JSON.parse(JSON.stringify(data));
function toRedisParams(job) {
var redisParams = [REDIS_PREFIX + job.job_id];
var obj = JSON.parse(JSON.stringify(job));
delete obj.job_id;
for (var property in obj) {
@ -81,7 +83,7 @@ JobBackend.prototype.get = function (job_id, callback) {
'failed_reason'
];
self.metadataBackend.redisCmd(this.db, 'HMGET', redisParams , function (err, redisValues) {
self.metadataBackend.redisCmd(REDIS_DB, 'HMGET', redisParams , function (err, redisValues) {
if (err) {
return callback(err);
}
@ -98,66 +100,66 @@ JobBackend.prototype.get = function (job_id, callback) {
});
};
JobBackend.prototype.create = function (data, callback) {
JobBackend.prototype.create = function (job, callback) {
var self = this;
self.get(data.job_id, function (err) {
self.get(job.job_id, function (err) {
if (err && err.name !== 'NotFoundError') {
return callback(err);
}
self.save(data, function (err, job) {
self.save(job, function (err, jobSaved) {
if (err) {
return callback(err);
}
self.jobQueueProducer.enqueue(data.job_id, data.host, function (err) {
self.jobQueueProducer.enqueue(job.job_id, job.host, function (err) {
if (err) {
return callback(err);
}
// broadcast to consumers
self.jobPublisher.publish(data.host);
self.jobPublisher.publish(job.host);
self.userIndexer.add(data.user, data.job_id, function (err) {
self.userIndexer.add(job.user, job.job_id, function (err) {
if (err) {
return callback(err);
}
callback(null, job);
callback(null, jobSaved);
});
});
});
});
};
JobBackend.prototype.update = function (data, callback) {
JobBackend.prototype.update = function (job, callback) {
var self = this;
self.get(data.job_id, function (err) {
self.get(job.job_id, function (err) {
if (err) {
return callback(err);
}
self.save(data, callback);
self.save(job, callback);
});
};
JobBackend.prototype.save = function (data, callback) {
JobBackend.prototype.save = function (job, callback) {
var self = this;
var redisParams = toRedisParams(data);
var redisParams = toRedisParams(job);
self.metadataBackend.redisCmd(self.db, 'HMSET', redisParams , function (err) {
self.metadataBackend.redisCmd(REDIS_DB, 'HMSET', redisParams , function (err) {
if (err) {
return callback(err);
}
self.setTTL(data, function (err) {
self.setTTL(job, function (err) {
if (err) {
return callback(err);
}
self.get(data.job_id, function (err, job) {
self.get(job.job_id, function (err, job) {
if (err) {
return callback(err);
}
@ -172,15 +174,15 @@ function isFinalStatus(status) {
return finalStatus.indexOf(status) !== -1;
}
JobBackend.prototype.setTTL = function (data, callback) {
JobBackend.prototype.setTTL = function (job, callback) {
var self = this;
var redisKey = REDIS_PREFIX + data.job_id;
var redisKey = REDIS_PREFIX + job.job_id;
if (!isFinalStatus(data.status)) {
if (!isFinalStatus(job.status)) {
return callback();
}
self.metadataBackend.redisCmd(self.db, 'EXPIRE', [ redisKey, JOBS_TTL_IN_SECONDS ], callback);
self.metadataBackend.redisCmd(REDIS_DB, 'EXPIRE', [ redisKey, JOBS_TTL_IN_SECONDS ], callback);
};
JobBackend.prototype.list = function (user, callback) {
@ -234,7 +236,7 @@ JobBackend.prototype._getIndexedJob = function (job_id, user, callback) {
if (err && err.name === 'NotFoundError') {
return self.userIndexer.remove(user, job_id, function (err) {
if (err) {
console.error('Error removing key %s in user set', job_id, err);
debug('Error removing key %s in user set', job_id, err);
}
callback();
});

5
batch/job_service.js
View File

@ -1,5 +1,6 @@
'use strict';
var debug = require('./util/debug')('job-service');
var JobFactory = require('./job_factory');
var jobStatus = require('./job_status');
@ -40,7 +41,7 @@ JobService.prototype.list = function (user, callback) {
try {
job = JobFactory.create(data);
} catch (err) {
return console.err(err);
return debug(err);
}
return job;
@ -149,7 +150,7 @@ JobService.prototype.drain = function (job_id, callback) {
self.jobCanceller.cancel(job, function (err) {
if (err) {
console.error('There was an error while draining job %s, %s ', job_id, err);
debug('There was an error while draining job %s, %s ', job_id, err);
return callback(err);
}

3
batch/job_subscriber.js
View File

@ -1,11 +1,12 @@
'use strict';
var debug = require('./util/debug')('job-subscriber');
var SUBSCRIBE_INTERVAL_IN_MILLISECONDS = 10 * 60 * 1000; // 10 minutes
function _subscribe(client, channel, queueSeeker, onMessage) {
queueSeeker.seek(onMessage, function (err) {
if (err) {
console.error(err);
debug(err);
}
client.removeAllListeners('message');

7
batch/util/debug.js Normal file
View File

@ -0,0 +1,7 @@
'use strict';
var debug = require('debug');
module.exports = function batchDebug (ns) {
return debug(['batch', ns].join(':'));
};

41
npm-shrinkwrap.json generated
View File

@ -1,6 +1,6 @@
{
"name": "cartodb_sql_api",
"version": "1.28.1",
"version": "1.28.2",
"dependencies": {
"cartodb-psql": {
"version": "0.6.1",
@ -23,18 +23,6 @@
"resolved": "https://registry.npmjs.org/buffer-writer/-/buffer-writer-1.0.0.tgz"
}
}
},
"debug": {
"version": "2.2.0",
"from": "debug@>=2.2.0 <2.3.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
"dependencies": {
"ms": {
"version": "0.7.1",
"from": "ms@0.7.1",
"resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
}
}
}
}
},
@ -94,6 +82,18 @@
}
}
},
"debug": {
"version": "2.2.0",
"from": "debug@2.2.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
"dependencies": {
"ms": {
"version": "0.7.1",
"from": "ms@0.7.1",
"resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
}
}
},
"express": {
"version": "2.5.11",
"from": "express@>=2.5.11 <2.6.0",
@ -256,20 +256,7 @@
"step-profiler": {
"version": "0.3.0",
"from": "step-profiler@>=0.3.0 <0.4.0",
"dependencies": {
"debug": {
"version": "2.2.0",
"from": "debug@>=2.2.0 <2.3.0",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.2.0.tgz",
"dependencies": {
"ms": {
"version": "0.7.1",
"from": "ms@0.7.1",
"resolved": "https://registry.npmjs.org/ms/-/ms-0.7.1.tgz"
}
}
}
}
"resolved": "https://registry.npmjs.org/step-profiler/-/step-profiler-0.3.0.tgz"
},
"topojson": {
"version": "0.0.8",

3
package.json
View File

@ -5,7 +5,7 @@
"keywords": [
"cartodb"
],
"version": "1.28.1",
"version": "1.28.2",
"repository": {
"type": "git",
"url": "git://github.com/CartoDB/CartoDB-SQL-API.git"
@ -19,6 +19,7 @@
"dependencies": {
"cartodb-psql": "~0.6.0",
"cartodb-redis": "~0.11.0",
"debug": "2.2.0",
"express": "~2.5.11",
"log4js": "https://github.com/CartoDB/log4js-node/tarball/cdb",
"lru-cache": "~2.5.0",

24
test/acceptance/export/kml.js
View File

@ -397,4 +397,28 @@ it('check point coordinates, authenticated', function(done){
);
});
it('should work with queries returning no results', function(done) {
assert.response(
app,
{
url: "/api/v1/sql?" + querystring.stringify({
q: "SELECT * FROM populated_places_simple_reduced LIMIT 0",
format: 'kml'
}),
headers: {
host: 'vizzuality.cartodb.com'
},
encoding: 'binary',
method: 'GET'
},
{
status: 200
},
function(res) {
assert.equal(res.body.match(/<Placemark>/g), null);
done();
}
);
});
});