cartodb/CartoDB-SQL-API
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into pythonTimeout

Browse Source
This commit is contained in:
Simon Martín 2018-01-08 11:29:17 +01:00
commit 6e45c39b1f
9 changed files with 225 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS.md
View File

@ -7,6 +7,7 @@ Announcements:
* Change work in progress jobs endpoint from `[..]/job/wip` to `[..]/jobs-wip`
* Documentation updates for Docs repo issue #840, GPKG Export.
* Fix SHP exports, now it uses "the_geom" column by default when a dataset has more than one geometry column.
* Logging all errors
## 1.47.1

34
app/utils/error_handler.js
View File

@ -32,6 +32,8 @@ module.exports = function handleException(err, res) {
res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
}
setErrorHeader(msg, pgErrorHandler.getStatus(), res);
res.header('Content-Type', 'application/json; charset=utf-8');
res.status(getStatusError(pgErrorHandler, req));
if (req.query && req.query.callback) {
@ -56,3 +58,35 @@ function getStatusError(pgErrorHandler, req) {
return statusError;
}
function setErrorHeader(err, statusCode, res) {
let errorsLog = Object.assign({}, err);
errorsLog.statusCode = statusCode || 200;
errorsLog.message = errorsLog.error[0];
delete errorsLog.error;
res.set('X-SQLAPI-Errors', stringifyForLogs(errorsLog));
}
/**
* Remove problematic nested characters
* from object for logs RegEx
*
* @param {Object} object
*/
function stringifyForLogs(object) {
Object.keys(object).map(key => {
if(typeof object[key] === 'string') {
object[key] = object[key].replace(/[^a-zA-Z0-9]/g, ' ');
} else if (typeof object[key] === 'object') {
stringifyForLogs(object[key]);
} else if (object[key] instanceof Array) {
for (let element of object[key]) {
stringifyForLogs(element);
}
}
});
return JSON.stringify(object);
}

2
config/environments/development.js.example
View File

@ -8,7 +8,7 @@ module.exports.base_url = '(?:/api/:version|/user/:user/api/:version)';
// X-SQLAPI-Profile header containing elapsed timing for various
// steps taken for producing the response.
module.exports.useProfiler = true;
module.exports.log_format = '[:date] :remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler])';
module.exports.log_format = '[:date] :remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler]) (:res[X-SQLAPI-Errors])';
// If log_filename is given logs will be written there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
module.exports.log_filename = 'logs/cartodb-sql-api.log';

2
config/environments/production.js.example
View File

@ -8,7 +8,7 @@ module.exports.base_url = '(?:/api/:version|/user/:user/api/:version)';
// X-SQLAPI-Profile header containing elapsed timing for various
// steps taken for producing the response.
module.exports.useProfiler = true;
module.exports.log_format = '[:date] :remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler])';
module.exports.log_format = '[:date] :remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler]) (:res[X-SQLAPI-Errors])';
// If log_filename is given logs will be written there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
module.exports.log_filename = 'logs/cartodb-sql-api.log';

2
config/environments/staging.js.example
View File

@ -8,7 +8,7 @@ module.exports.base_url = '(?:/api/:version|/user/:user/api/:version)';
// X-SQLAPI-Profile header containing elapsed timing for various
// steps taken for producing the response.
module.exports.useProfiler = true;
module.exports.log_format = '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler])';
module.exports.log_format = '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler]) (:res[X-SQLAPI-Errors])';
// If log_filename is given logs will be written there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
module.exports.log_filename = 'logs/cartodb-sql-api.log';

2
config/environments/test.js.example
View File

@ -8,7 +8,7 @@ module.exports.base_url = '(?:/api/:version|/user/:user/api/:version)';
// X-SQLAPI-Profile header containing elapsed timing for various
// steps taken for producing the response.
module.exports.useProfiler = true;
module.exports.log_format = '[:date] :remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler])';
module.exports.log_format = '[:date] :remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler]) (:res[X-SQLAPI-Errors])';
// If log_filename is given logs will be written there, in append mode. Otherwise stdout is used (default).
// Log file will be re-opened on receiving the HUP signal
// module.exports.log_filename = 'logs/cartodb-sql-api.log';

18
doc/tips_and_tricks.md
View File

@ -77,3 +77,21 @@ https://{username}.carto.com/api/v2/sql?format=csv&q=SELECT+*+FROM+tm_world_bord
```
The response creates a direct dataset URL that you can download for use offline.
### Why can't I see my created tables in my CARTO account?
The SQL API automatically displays tables in CARTO if you follow these steps:
- Create a table that has been "cartodbfied", which prepares your table to be compatible with CARTO. View [Creating Tables with the SQL API](https://carto.com/docs/carto-engine/sql-api/creating-tables/#creating-tables-with-the-sql-api) to learn about this function.
- After creating your "cartodbfied" table, you must login to your CARTO account and open _Your datasets_ dashboard. Logging in initiates a check between the database and your account.
**Note:** There is an expected refresh time while the database is checking your account and your tables may not appear at this time, especially if there are a lot of tables or tables with a large amount of rows.
- Once the database updates, CARTO will display your created or changed tables as connected datasets!
### What happens if I remove a table that is used in an existing map?
If you [drop](https://carto.com/docs/carto-engine/sql-api/creating-tables/#remove-a-table) a table using the SQL API, be mindful that there is no warning that the table may be used in an existing map. This is by design. Any maps using a removed table will be missing data and thus, will be deleted.
If you are unsure about which tables are connected to maps, it is suggested to remove tables from _Your datasets_ dashboard in CARTO, which automatically notifies you of any connected maps in use.

31
test/acceptance/error-handler.js Normal file
View File

@ -0,0 +1,31 @@
var server = require('../../app/server')();
var assert = require('../support/assert');
describe('error handler', function () {
it('should returns a errors header', function (done) {
const errorHeader = {
detail: undefined,
hint: undefined,
context: undefined,
statusCode: 400,
message: 'You must indicate a sql query'
};
assert.response(server, {
url: '/api/v1/sql',
headers: {host: 'vizzuality.cartodb.com'},
method: 'GET'
},
{
status: 400,
headers: {
'Content-Type': 'application/json; charset=utf-8',
'X-SQLAPI-Errors': JSON.stringify(errorHeader)
}
},
function(err){
assert.ifError(err);
done();
});
});
});

137
test/unit/error_handler.test.js Normal file
View File

@ -0,0 +1,137 @@
'use strict';
var assert = require('assert');
var errorHandler = require('../../app/utils/error_handler');
describe('error-handler', function() {
it('should return a header with errors', function (done) {
let error = new Error('error test');
error.detail = 'test detail';
error.hint = 'test hint';
error.context = 'test context';
const res = {
req: {},
headers: {},
set (key, value) {
this.headers[key] = value;
},
header (key, value) {
this.set(key, value);
},
statusCode: 0,
status (status) {
this.statusCode = status;
},
json () {}
};
const errorHeader = {
detail: error.detail,
hint: error.hint,
context: error.context,
statusCode: 400,
message: error.message
};
errorHandler(error, res);
assert.ok(res.headers['X-SQLAPI-Errors'].length > 0);
assert.deepEqual(
res.headers['X-SQLAPI-Errors'],
JSON.stringify(errorHeader)
);
done();
});
it('JSONP should return a header with error statuscode', function (done) {
let error = new Error('error test');
error.detail = 'test detail';
error.hint = 'test hint';
error.context = 'test context';
const res = {
req: {
query: { callback: true }
},
headers: {},
set (key, value) {
this.headers[key] = value;
},
header (key, value) {
this.set(key, value);
},
statusCode: 0,
status (status) {
this.statusCode = status;
},
jsonp () {}
};
const errorHeader = {
detail: error.detail,
hint: error.hint,
context: error.context,
statusCode: 400,
message: error.message
};
errorHandler(error, res);
assert.ok(res.headers['X-SQLAPI-Errors'].length > 0);
assert.deepEqual(
res.headers['X-SQLAPI-Errors'],
JSON.stringify(errorHeader)
);
done();
});
it('should escape chars that broke logs regex', function (done) {
const badString = 'error: ( ) = " \" \' * $ & |';
const escapedString = 'error ';
let error = new Error(badString);
error.detail = badString;
error.hint = badString;
error.context = badString;
const res = {
req: {
query: { callback: true }
},
headers: {},
set (key, value) {
this.headers[key] = value;
},
header (key, value) {
this.set(key, value);
},
statusCode: 0,
status (status) {
this.statusCode = status;
},
jsonp () {}
};
const errorHeader = {
detail: escapedString,
hint: escapedString,
context: escapedString,
statusCode: 400,
message: escapedString
};
errorHandler(error, res);
assert.ok(res.headers['X-SQLAPI-Errors'].length > 0);
assert.deepEqual(
res.headers['X-SQLAPI-Errors'],
JSON.stringify(errorHeader)
);
done();
});
});