Merge branch 'master' into pythonTimeout

NEWS.md

@@ -7,6 +7,7 @@ Announcements:
  * Change work in progress jobs endpoint from `[..]/job/wip` to `[..]/jobs-wip`
  * Documentation updates for Docs repo issue #840, GPKG Export.
  * Fix SHP exports, now it uses "the_geom" column by default when a dataset has more than one geometry column.
+ * Logging all errors
 
 
 ## 1.47.1

app/utils/error_handler.js

@@ -32,6 +32,8 @@ module.exports = function handleException(err, res) {
       res.header('X-SQLAPI-Profiler', req.profiler.toJSONString());
     }
 
+    setErrorHeader(msg, pgErrorHandler.getStatus(), res);
+    
     res.header('Content-Type', 'application/json; charset=utf-8');
     res.status(getStatusError(pgErrorHandler, req));
     if (req.query && req.query.callback) {
@@ -56,3 +58,35 @@ function getStatusError(pgErrorHandler, req) {
 
     return statusError;
 }
+
+function setErrorHeader(err, statusCode, res) {
+    let errorsLog = Object.assign({}, err);
+
+    errorsLog.statusCode = statusCode || 200;
+    errorsLog.message = errorsLog.error[0];
+    delete errorsLog.error;
+
+    res.set('X-SQLAPI-Errors', stringifyForLogs(errorsLog));
+}
+
+/**
+ * Remove problematic nested characters 
+ * from object for logs RegEx
+ * 
+ * @param {Object} object 
+ */
+function stringifyForLogs(object) {
+    Object.keys(object).map(key => {
+        if(typeof object[key] === 'string') {
+            object[key] = object[key].replace(/[^a-zA-Z0-9]/g, ' ');
+        } else if (typeof object[key] === 'object') {
+            stringifyForLogs(object[key]);
+        } else if (object[key] instanceof Array) {
+            for (let element of object[key]) {
+                stringifyForLogs(element);
+            }
+        }
+    });
+
+    return JSON.stringify(object);
+}

config/environments/development.js.example

@@ -8,7 +8,7 @@ module.exports.base_url     = '(?:/api/:version|/user/:user/api/:version)';
 // X-SQLAPI-Profile header containing elapsed timing for various
 // steps taken for producing the response.
 module.exports.useProfiler = true;
-module.exports.log_format   = '[:date] :remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler])';
+module.exports.log_format   = '[:date] :remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler]) (:res[X-SQLAPI-Errors])';
 // If log_filename is given logs will be written there, in append mode. Otherwise stdout is used (default).
 // Log file will be re-opened on receiving the HUP signal
 module.exports.log_filename = 'logs/cartodb-sql-api.log';

config/environments/production.js.example

@@ -8,7 +8,7 @@ module.exports.base_url     = '(?:/api/:version|/user/:user/api/:version)';
 // X-SQLAPI-Profile header containing elapsed timing for various
 // steps taken for producing the response.
 module.exports.useProfiler = true;
-module.exports.log_format   = '[:date] :remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler])';
+module.exports.log_format   = '[:date] :remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler]) (:res[X-SQLAPI-Errors])';
 // If log_filename is given logs will be written there, in append mode. Otherwise stdout is used (default).
 // Log file will be re-opened on receiving the HUP signal
 module.exports.log_filename = 'logs/cartodb-sql-api.log';

config/environments/staging.js.example

@@ -8,7 +8,7 @@ module.exports.base_url     = '(?:/api/:version|/user/:user/api/:version)';
 // X-SQLAPI-Profile header containing elapsed timing for various
 // steps taken for producing the response.
 module.exports.useProfiler = true;
-module.exports.log_format   = '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler])';
+module.exports.log_format   = '[:date] :req[X-Real-IP] :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler]) (:res[X-SQLAPI-Errors])';
 // If log_filename is given logs will be written there, in append mode. Otherwise stdout is used (default).
 // Log file will be re-opened on receiving the HUP signal
 module.exports.log_filename = 'logs/cartodb-sql-api.log';

config/environments/test.js.example

@@ -8,7 +8,7 @@ module.exports.base_url     = '(?:/api/:version|/user/:user/api/:version)';
 // X-SQLAPI-Profile header containing elapsed timing for various
 // steps taken for producing the response.
 module.exports.useProfiler = true;
-module.exports.log_format   = '[:date] :remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler])';
+module.exports.log_format   = '[:date] :remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type] (:res[X-SQLAPI-Profiler]) (:res[X-SQLAPI-Errors])';
 // If log_filename is given logs will be written there, in append mode. Otherwise stdout is used (default).
 // Log file will be re-opened on receiving the HUP signal
 // module.exports.log_filename = 'logs/cartodb-sql-api.log';

doc/tips_and_tricks.md

@@ -77,3 +77,21 @@ https://{username}.carto.com/api/v2/sql?format=csv&q=SELECT+*+FROM+tm_world_bord
 ```
 
 The response creates a direct dataset URL that you can download for use offline.
+
+### Why can't I see my created tables in my CARTO account?
+
+The SQL API automatically displays tables in CARTO if you follow these steps:
+
+- Create a table that has been "cartodbfied", which prepares your table to be compatible with CARTO. View [Creating Tables with the SQL API](https://carto.com/docs/carto-engine/sql-api/creating-tables/#creating-tables-with-the-sql-api) to learn about this function.
+
+- After creating your "cartodbfied" table, you must login to your CARTO account and open _Your datasets_ dashboard. Logging in initiates a check between the database and your account.
+
+**Note:** There is an expected refresh time while the database is checking your account and your tables may not appear at this time, especially if there are a lot of tables or tables with a large amount of rows.
+
+- Once the database updates, CARTO will display your created or changed tables as connected datasets!
+
+### What happens if I remove a table that is used in an existing map?
+
+If you [drop](https://carto.com/docs/carto-engine/sql-api/creating-tables/#remove-a-table) a table using the SQL API, be mindful that there is no warning that the table may be used in an existing map. This is by design. Any maps using a removed table will be missing data and thus, will be deleted.
+
+If you are unsure about which tables are connected to maps, it is suggested to remove tables from _Your datasets_ dashboard in CARTO, which automatically notifies you of any connected maps in use.

test/acceptance/error-handler.js

@@ -0,0 +1,31 @@
+var server = require('../../app/server')();
+var assert = require('../support/assert');
+
+describe('error handler', function () { 
+    it('should returns a errors header', function (done) {
+        const errorHeader = {
+            detail: undefined,
+            hint: undefined,
+            context: undefined,
+            statusCode: 400,
+            message: 'You must indicate a sql query'
+        };
+
+        assert.response(server, {
+            url: '/api/v1/sql',
+            headers: {host: 'vizzuality.cartodb.com'},
+            method: 'GET'
+        },
+        {
+            status: 400,
+            headers: {
+                'Content-Type': 'application/json; charset=utf-8',
+                'X-SQLAPI-Errors': JSON.stringify(errorHeader)
+            }
+        }, 
+        function(err){
+            assert.ifError(err);
+            done();
+        });
+    });
+});

test/unit/error_handler.test.js

@@ -0,0 +1,137 @@
+'use strict';
+
+var assert = require('assert');
+var errorHandler = require('../../app/utils/error_handler');
+
+describe('error-handler', function() {
+    it('should return a header with errors', function (done) {
+        let error = new Error('error test');
+        error.detail = 'test detail';
+        error.hint = 'test hint';
+        error.context = 'test context';
+
+        const res = {
+            req: {},
+            headers: {},
+            set (key, value) {
+                this.headers[key] = value;
+            },
+            header (key, value) {
+                this.set(key, value);
+            },
+            statusCode: 0,
+            status (status) {
+                this.statusCode = status;
+            },
+            json () {}
+        };
+
+        const errorHeader = {
+            detail: error.detail,
+            hint: error.hint,
+            context: error.context,
+            statusCode: 400,
+            message: error.message
+        };
+
+        errorHandler(error, res);
+        
+        assert.ok(res.headers['X-SQLAPI-Errors'].length > 0);
+        assert.deepEqual(
+            res.headers['X-SQLAPI-Errors'], 
+            JSON.stringify(errorHeader)
+        );
+
+        done();
+    });
+
+    it('JSONP should return a header with error statuscode', function (done) {
+        let error = new Error('error test');
+        error.detail = 'test detail';
+        error.hint = 'test hint';
+        error.context = 'test context';
+
+        const res = {
+            req: { 
+                query: { callback: true }
+            },
+            headers: {},
+            set (key, value) {
+                this.headers[key] = value;
+            },
+            header (key, value) {
+                this.set(key, value);
+            },
+            statusCode: 0,
+            status (status) {
+                this.statusCode = status;
+            },
+            jsonp () {}
+        };
+
+        const errorHeader = {
+            detail: error.detail,
+            hint: error.hint,
+            context: error.context,
+            statusCode: 400,
+            message: error.message
+        };
+
+        errorHandler(error, res);
+        
+        assert.ok(res.headers['X-SQLAPI-Errors'].length > 0);
+        assert.deepEqual(
+            res.headers['X-SQLAPI-Errors'], 
+            JSON.stringify(errorHeader)
+        );
+
+        done();
+    });
+
+    it('should escape chars that broke logs regex', function (done) {
+        const badString = 'error: ( ) = " \" \' * $ & |';
+        const escapedString = 'error                     ';
+
+        let error = new Error(badString);
+        error.detail = badString;
+        error.hint = badString;
+        error.context = badString;
+
+        const res = {
+            req: { 
+                query: { callback: true }
+            },
+            headers: {},
+            set (key, value) {
+                this.headers[key] = value;
+            },
+            header (key, value) {
+                this.set(key, value);
+            },
+            statusCode: 0,
+            status (status) {
+                this.statusCode = status;
+            },
+            jsonp () {}
+        };
+
+        const errorHeader = {
+            detail: escapedString,
+            hint: escapedString,
+            context: escapedString,
+            statusCode: 400,
+            message: escapedString
+        };
+
+        errorHandler(error, res);
+        
+        assert.ok(res.headers['X-SQLAPI-Errors'].length > 0);
+        assert.deepEqual(
+            res.headers['X-SQLAPI-Errors'], 
+            JSON.stringify(errorHeader)
+        );
+
+        done();
+    });
+  
+});