From 88451f824f934e5be8b07a177dd2ad8f010ed9c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 10:27:17 +0200 Subject: [PATCH 01/37] ES6 class for generic controller --- app/controllers/generic_controller.js | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/app/controllers/generic_controller.js b/app/controllers/generic_controller.js index ab3c5606..b0548c9b 100644 --- a/app/controllers/generic_controller.js +++ b/app/controllers/generic_controller.js @@ -1,14 +1,7 @@ 'use strict'; -function GenericController() { -} - -GenericController.prototype.route = function (app) { - app.options('*', this.handleRequest.bind(this)); +module.exports = class GenericController { + route (app) { + app.options('*', (req, res) => res.end()); + } }; - -GenericController.prototype.handleRequest = function(req, res) { - res.end(); -}; - -module.exports = GenericController; From 51dd9a79991db3769b6cf5da26fe20cd2ca4f26a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 10:52:31 +0200 Subject: [PATCH 02/37] ES6 class and middlewares for healtch-check controller --- app/controllers/health_check_controller.js | 61 ++++++++++++---------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/app/controllers/health_check_controller.js b/app/controllers/health_check_controller.js index a5f84f12..ef19f8a3 100644 --- a/app/controllers/health_check_controller.js +++ b/app/controllers/health_check_controller.js @@ -1,35 +1,40 @@ 'use strict'; -var HealthCheck = require('../monitoring/health_check'); +const HealthCheckBackend = require('../monitoring/health_check'); -function HealthCheckController() { - this.healthCheck = new HealthCheck(global.settings.disabled_file); -} +module.exports = class HealthCheckController { + constructor () { + this.healthCheckBackend = new HealthCheckBackend(global.settings.disabled_file); + } -HealthCheckController.prototype.route = function (app) { - app.get(global.settings.base_url + '/health', this.handleHealthCheck.bind(this)); -}; - -HealthCheckController.prototype.handleHealthCheck = function (req, res) { - var healthConfig = global.settings.health || {}; - if (!!healthConfig.enabled) { - var startTime = Date.now(); - this.healthCheck.check(function(err) { - var ok = !err; - var response = { - enabled: true, - ok: ok, - elapsed: Date.now() - startTime - }; - if (err) { - response.err = err.message; - } - res.status(ok ? 200 : 503).send(response); - - }); - } else { - res.status(200).send({enabled: false, ok: true}); + route (app) { + app.get(`${global.settings.base_url}/health`, healthCheck({ healthCheckBackend: this.healthCheckBackend })); } }; -module.exports = HealthCheckController; +function healthCheck ({ healthCheckBackend }) { + return function healthCheckMiddleware (req, res) { + const healthConfig = global.settings.health || {}; + + if (!healthConfig.enabled) { + return res.status(200).send({enabled: false, ok: true}); + } + + const startTime = Date.now(); + + healthCheckBackend.check((err) => { + const ok = !err; + const response = { + enabled: true, + ok, + elapsed: Date.now() - startTime + }; + + if (err) { + response.err = err.message; + } + + res.status(ok ? 200 : 503).send(response); + }); + }; +} From 645e3a797f3193021cffc97a7b5a216b8b469940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 10:54:26 +0200 Subject: [PATCH 03/37] Middleware for generic controller --- app/controllers/generic_controller.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/controllers/generic_controller.js b/app/controllers/generic_controller.js index b0548c9b..7bf7d8c4 100644 --- a/app/controllers/generic_controller.js +++ b/app/controllers/generic_controller.js @@ -2,6 +2,12 @@ module.exports = class GenericController { route (app) { - app.options('*', (req, res) => res.end()); + app.options('*', emptyResponse()); } }; + +function emptyResponse () { + return function emptyResponseMiddleware (req, res) { + res.end(); + }; +} From ab9a7fd2fac94d34d99ed907b391548ab229a829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 10:59:44 +0200 Subject: [PATCH 04/37] ES6 class for job controller --- app/controllers/job_controller.js | 86 +++++++++++++++---------------- 1 file changed, 43 insertions(+), 43 deletions(-) diff --git a/app/controllers/job_controller.js b/app/controllers/job_controller.js index 54a8be21..a72b9192 100644 --- a/app/controllers/job_controller.js +++ b/app/controllers/job_controller.js @@ -13,51 +13,51 @@ const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware; const params = require('../middlewares/params'); const logMiddleware = require('../middlewares/log'); -function JobController(metadataBackend, userDatabaseService, jobService, statsdClient, userLimitsService) { - this.metadataBackend = metadataBackend; - this.userDatabaseService = userDatabaseService; - this.jobService = jobService; - this.statsdClient = statsdClient; - this.userLimitsService = userLimitsService; -} +module.exports = class JobController { + constructor (metadataBackend, userDatabaseService, jobService, statsdClient, userLimitsService) { + this.metadataBackend = metadataBackend; + this.userDatabaseService = userDatabaseService; + this.jobService = jobService; + this.statsdClient = statsdClient; + this.userLimitsService = userLimitsService; + } -module.exports = JobController; + route (app) { + const { base_url } = global.settings; + const jobMiddlewares = composeJobMiddlewares( + this.metadataBackend, + this.userDatabaseService, + this.jobService, + this.statsdClient, + this.userLimitsService + ); -JobController.prototype.route = function (app) { - const { base_url } = global.settings; - const jobMiddlewares = composeJobMiddlewares( - this.metadataBackend, - this.userDatabaseService, - this.jobService, - this.statsdClient, - this.userLimitsService - ); - - app.get( - `${base_url}/jobs-wip`, - bodyParserMiddleware(), - listWorkInProgressJobs(this.jobService), - sendResponse(), - errorMiddleware() - ); - app.post( - `${base_url}/sql/job`, - bodyParserMiddleware(), - checkBodyPayloadSize(), - params({ strategy: 'job' }), - logMiddleware(logMiddleware.TYPES.JOB), - jobMiddlewares('create', createJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_CREATE) - ); - app.get( - `${base_url}/sql/job/:job_id`, - bodyParserMiddleware(), - jobMiddlewares('retrieve', getJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_GET) - ); - app.delete( - `${base_url}/sql/job/:job_id`, - bodyParserMiddleware(), - jobMiddlewares('cancel', cancelJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_DELETE) - ); + app.get( + `${base_url}/jobs-wip`, + bodyParserMiddleware(), + listWorkInProgressJobs(this.jobService), + sendResponse(), + errorMiddleware() + ); + app.post( + `${base_url}/sql/job`, + bodyParserMiddleware(), + checkBodyPayloadSize(), + params({ strategy: 'job' }), + logMiddleware(logMiddleware.TYPES.JOB), + jobMiddlewares('create', createJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_CREATE) + ); + app.get( + `${base_url}/sql/job/:job_id`, + bodyParserMiddleware(), + jobMiddlewares('retrieve', getJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_GET) + ); + app.delete( + `${base_url}/sql/job/:job_id`, + bodyParserMiddleware(), + jobMiddlewares('cancel', cancelJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_DELETE) + ); + } }; function composeJobMiddlewares (metadataBackend, userDatabaseService, jobService, statsdClient, userLimitsService) { From 158f28c8e748a42cddd12934c0acd014ad1a6bc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 11:11:19 +0200 Subject: [PATCH 05/37] ES6 class and middlewares for version controller --- app/controllers/version_controller.js | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/app/controllers/version_controller.js b/app/controllers/version_controller.js index 853ff891..03c99ba9 100644 --- a/app/controllers/version_controller.js +++ b/app/controllers/version_controller.js @@ -1,18 +1,17 @@ 'use strict'; -var version = { - cartodb_sql_api: require(__dirname + '/../../package.json').version +const versions = { + cartodb_sql_api: require('./../../package.json').version }; -function VersionController() { +module.exports = class VersionController { + route (app) { + app.get(`${global.settings.base_url}/version`, version()); + } +}; + +function version () { + return function versionMiddleware (req, res) { + res.send(versions); + }; } - -VersionController.prototype.route = function (app) { - app.get(global.settings.base_url + '/version', this.handleVersion.bind(this)); -}; - -VersionController.prototype.handleVersion = function (req, res) { - res.send(version); -}; - -module.exports = VersionController; From 6c775c1857d554d92ba7d33baa1e5ae41b4593c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 12:25:22 +0200 Subject: [PATCH 06/37] Create api-router, set route hierarchy --- app/controllers/api-router.js | 70 +++++++++++++++++++++++++++ app/controllers/copy_controller.js | 10 ++-- app/controllers/job_controller.js | 22 +++++---- app/controllers/query_controller.js | 7 ++- app/server.js | 74 +++++------------------------ 5 files changed, 102 insertions(+), 81 deletions(-) create mode 100644 app/controllers/api-router.js diff --git a/app/controllers/api-router.js b/app/controllers/api-router.js new file mode 100644 index 00000000..ad580908 --- /dev/null +++ b/app/controllers/api-router.js @@ -0,0 +1,70 @@ +'use strict'; + +const { Router: router } = require('express'); + +const UserDatabaseService = require('../services/user_database_service'); +const UserLimitsService = require('../services/user_limits'); + +const BatchLogger = require('../../batch/batch-logger'); + +const JobPublisher = require('../../batch/pubsub/job-publisher'); +const JobQueue = require('../../batch/job_queue'); +const JobBackend = require('../../batch/job_backend'); +const JobCanceller = require('../../batch/job_canceller'); +const JobService = require('../../batch/job_service'); + +const QueryController = require('./query_controller'); +const CopyController = require('./copy_controller'); +const JobController = require('./job_controller'); + +module.exports = class ApiRouter { + constructor ({ redisPool, metadataBackend, statsClient, dataIngestionLogger }) { + const userLimitsServiceOptions = { + limits: { + rateLimitsEnabled: global.settings.ratelimits.rateLimitsEnabled + } + }; + + const userDatabaseService = new UserDatabaseService(metadataBackend); + const userLimitsService = new UserLimitsService(metadataBackend, userLimitsServiceOptions); + + this.queryController = new QueryController( + metadataBackend, + userDatabaseService, + statsClient, + userLimitsService + ); + + this.copyController = new CopyController( + metadataBackend, + userDatabaseService, + userLimitsService, + dataIngestionLogger + ); + + const logger = new BatchLogger(global.settings.batch_log_filename, 'batch-queries'); + const jobPublisher = new JobPublisher(redisPool); + const jobQueue = new JobQueue(metadataBackend, jobPublisher, logger); + const jobBackend = new JobBackend(metadataBackend, jobQueue, logger); + const jobCanceller = new JobCanceller(); + const jobService = new JobService(jobBackend, jobCanceller, logger); + + this.jobController = new JobController( + metadataBackend, + userDatabaseService, + jobService, + statsClient, + userLimitsService + ); + } + + route (app) { + const apiRouter = router({ mergeParams: true }); + + this.queryController.route(apiRouter); + this.copyController.route(apiRouter); + this.jobController.route(apiRouter); + + app.use(`${global.settings.base_url}`, apiRouter); + } +}; diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js index 93d544e9..7fe8cb0c 100644 --- a/app/controllers/copy_controller.js +++ b/app/controllers/copy_controller.js @@ -24,9 +24,7 @@ function CopyController(metadataBackend, userDatabaseService, userLimitsService, this.logger = logger; } -CopyController.prototype.route = function (app) { - const { base_url } = global.settings; - +CopyController.prototype.route = function (apiRouter) { const copyFromMiddlewares = endpointGroup => { return [ initializeProfilerMiddleware('copyfrom'), @@ -57,9 +55,9 @@ CopyController.prototype.route = function (app) { ]; }; - app.post(`${base_url}/sql/copyfrom`, copyFromMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_FROM)); - app.get(`${base_url}/sql/copyto`, copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO)); - app.post(`${base_url}/sql/copyto`, copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO)); + apiRouter.post('/sql/copyfrom', copyFromMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_FROM)); + apiRouter.get('/sql/copyto', copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO)); + apiRouter.post('/sql/copyto', copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO)); }; function handleCopyTo (logger) { diff --git a/app/controllers/job_controller.js b/app/controllers/job_controller.js index a72b9192..8dd9e9b8 100644 --- a/app/controllers/job_controller.js +++ b/app/controllers/job_controller.js @@ -22,8 +22,7 @@ module.exports = class JobController { this.userLimitsService = userLimitsService; } - route (app) { - const { base_url } = global.settings; + route (apiRouter) { const jobMiddlewares = composeJobMiddlewares( this.metadataBackend, this.userDatabaseService, @@ -32,28 +31,31 @@ module.exports = class JobController { this.userLimitsService ); - app.get( - `${base_url}/jobs-wip`, + apiRouter.get( + '/jobs-wip', bodyParserMiddleware(), listWorkInProgressJobs(this.jobService), sendResponse(), errorMiddleware() ); - app.post( - `${base_url}/sql/job`, + + apiRouter.post( + '/sql/job', bodyParserMiddleware(), checkBodyPayloadSize(), params({ strategy: 'job' }), logMiddleware(logMiddleware.TYPES.JOB), jobMiddlewares('create', createJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_CREATE) ); - app.get( - `${base_url}/sql/job/:job_id`, + + apiRouter.get( + '/sql/job/:job_id', bodyParserMiddleware(), jobMiddlewares('retrieve', getJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_GET) ); - app.delete( - `${base_url}/sql/job/:job_id`, + + apiRouter.delete( + '/sql/job/:job_id', bodyParserMiddleware(), jobMiddlewares('cancel', cancelJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_DELETE) ); diff --git a/app/controllers/query_controller.js b/app/controllers/query_controller.js index f0798b8a..d36dcdc4 100644 --- a/app/controllers/query_controller.js +++ b/app/controllers/query_controller.js @@ -32,8 +32,7 @@ module.exports = class QueryController { this.userLimitsService = userLimitsService; } - route (app) { - const { base_url } = global.settings; + route (apiRouter) { const forceToBeMaster = false; const queryMiddlewares = () => { @@ -62,8 +61,8 @@ module.exports = class QueryController { ]; }; - app.all(`${base_url}/sql`, queryMiddlewares()); - app.all(`${base_url}/sql.:f`, queryMiddlewares()); + apiRouter.all('/sql', queryMiddlewares()); + apiRouter.all('/sql.:f', queryMiddlewares()); } }; diff --git a/app/server.js b/app/server.js index 03e0a8b1..ffe7086f 100644 --- a/app/server.js +++ b/app/server.js @@ -24,22 +24,12 @@ var mkdirp = require('mkdirp'); var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); -var UserDatabaseService = require('./services/user_database_service'); -var UserLimitsService = require('./services/user_limits'); -var BatchLogger = require('../batch/batch-logger'); -var JobPublisher = require('../batch/pubsub/job-publisher'); -var JobQueue = require('../batch/job_queue'); -var JobBackend = require('../batch/job_backend'); -var JobCanceller = require('../batch/job_canceller'); -var JobService = require('../batch/job_service'); const Logger = require('./services/logger'); var cors = require('./middlewares/cors'); +const ApiRouter = require('./controllers/api-router'); var GenericController = require('./controllers/generic_controller'); -var QueryController = require('./controllers/query_controller'); -var CopyController = require('./controllers/copy_controller'); -var JobController = require('./controllers/job_controller'); var HealthCheckController = require('./controllers/health_check_controller'); var VersionController = require('./controllers/version_controller'); @@ -88,7 +78,7 @@ function App(statsClient) { mkdirp.sync(global.settings.tmpDir); } - if ( global.log4js ) { + if (global.log4js) { var loggerOpts = { buffer: true, // log4js provides a tokens solution as expess but in does not provide the request/response in the callback. @@ -123,12 +113,12 @@ function App(statsClient) { }); // Set connection timeout - if ( global.settings.hasOwnProperty('node_socket_timeout') ) { - var timeout = parseInt(global.settings.node_socket_timeout); - app.use(function(req, res, next) { - req.connection.setTimeout(timeout); - next(); - }); + if (global.settings.hasOwnProperty('node_socket_timeout')) { + var timeout = parseInt(global.settings.node_socket_timeout); + app.use(function(req, res, next) { + req.connection.setTimeout(timeout); + next(); + }); } app.enable('jsonp callback'); @@ -136,68 +126,30 @@ function App(statsClient) { app.disable('x-powered-by'); app.disable('etag'); - // basic routing - - var userDatabaseService = new UserDatabaseService(metadataBackend); - - const userLimitsServiceOptions = { - limits: { - rateLimitsEnabled: global.settings.ratelimits.rateLimitsEnabled - } - }; - const userLimitsService = new UserLimitsService(metadataBackend, userLimitsServiceOptions); - const dataIngestionLogger = new Logger(global.settings.dataIngestionLogPath, 'data-ingestion'); app.dataIngestionLogger = dataIngestionLogger; - var logger = new BatchLogger(global.settings.batch_log_filename, 'batch-queries'); - var jobPublisher = new JobPublisher(redisPool); - var jobQueue = new JobQueue(metadataBackend, jobPublisher, logger); - var jobBackend = new JobBackend(metadataBackend, jobQueue, logger); - var jobCanceller = new JobCanceller(); - var jobService = new JobService(jobBackend, jobCanceller, logger); - var genericController = new GenericController(); genericController.route(app); - var queryController = new QueryController( - metadataBackend, - userDatabaseService, - statsClient, - userLimitsService - ); - queryController.route(app); - - var copyController = new CopyController( - metadataBackend, - userDatabaseService, - userLimitsService, - dataIngestionLogger - ); - copyController.route(app); - - var jobController = new JobController( - metadataBackend, - userDatabaseService, - jobService, - statsClient, - userLimitsService - ); - jobController.route(app); - var healthCheckController = new HealthCheckController(); healthCheckController.route(app); var versionController = new VersionController(); versionController.route(app); + const apiRouter = new ApiRouter({ redisPool, metadataBackend, statsClient, dataIngestionLogger }); + apiRouter.route(app); + var isBatchProcess = process.argv.indexOf('--no-batch') === -1; if (global.settings.environment !== 'test' && isBatchProcess) { var batchName = global.settings.api_hostname || 'batch'; + app.batch = batchFactory( metadataBackend, redisPool, batchName, statsClient, global.settings.batch_log_filename ); + app.batch.start(); } From d85331e97b9f626fda316f16259b637c996ec0e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 12:31:05 +0200 Subject: [PATCH 07/37] Use ES6 class syntax in Copy from/to controller --- app/controllers/copy_controller.js | 80 +++++++++++++++--------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js index 7fe8cb0c..cf1c9b79 100644 --- a/app/controllers/copy_controller.js +++ b/app/controllers/copy_controller.js @@ -17,47 +17,49 @@ const { PassThrough } = require('stream'); const params = require('../middlewares/params'); const bodyParserMiddleware = require('../middlewares/body-parser'); -function CopyController(metadataBackend, userDatabaseService, userLimitsService, logger) { - this.metadataBackend = metadataBackend; - this.userDatabaseService = userDatabaseService; - this.userLimitsService = userLimitsService; - this.logger = logger; -} +module.exports = class CopyController { + constructor (metadataBackend, userDatabaseService, userLimitsService, logger) { + this.metadataBackend = metadataBackend; + this.userDatabaseService = userDatabaseService; + this.userLimitsService = userLimitsService; + this.logger = logger; + } -CopyController.prototype.route = function (apiRouter) { - const copyFromMiddlewares = endpointGroup => { - return [ - initializeProfilerMiddleware('copyfrom'), - userMiddleware(this.metadataBackend), - rateLimitsMiddleware(this.userLimitsService, endpointGroup), - authorizationMiddleware(this.metadataBackend), - connectionParamsMiddleware(this.userDatabaseService), - dbQuotaMiddleware(), - params({ strategy: 'copyfrom' }), - handleCopyFrom(this.logger), - errorHandler(this.logger), - errorMiddleware() - ]; - }; + route (apiRouter) { + const copyFromMiddlewares = endpointGroup => { + return [ + initializeProfilerMiddleware('copyfrom'), + userMiddleware(this.metadataBackend), + rateLimitsMiddleware(this.userLimitsService, endpointGroup), + authorizationMiddleware(this.metadataBackend), + connectionParamsMiddleware(this.userDatabaseService), + dbQuotaMiddleware(), + params({ strategy: 'copyfrom' }), + handleCopyFrom(this.logger), + errorHandler(this.logger), + errorMiddleware() + ]; + }; - const copyToMiddlewares = endpointGroup => { - return [ - bodyParserMiddleware(), - initializeProfilerMiddleware('copyto'), - userMiddleware(this.metadataBackend), - rateLimitsMiddleware(this.userLimitsService, endpointGroup), - authorizationMiddleware(this.metadataBackend), - connectionParamsMiddleware(this.userDatabaseService), - params({ strategy: 'copyto' }), - handleCopyTo(this.logger), - errorHandler(this.logger), - errorMiddleware() - ]; - }; + const copyToMiddlewares = endpointGroup => { + return [ + bodyParserMiddleware(), + initializeProfilerMiddleware('copyto'), + userMiddleware(this.metadataBackend), + rateLimitsMiddleware(this.userLimitsService, endpointGroup), + authorizationMiddleware(this.metadataBackend), + connectionParamsMiddleware(this.userDatabaseService), + params({ strategy: 'copyto' }), + handleCopyTo(this.logger), + errorHandler(this.logger), + errorMiddleware() + ]; + }; - apiRouter.post('/sql/copyfrom', copyFromMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_FROM)); - apiRouter.get('/sql/copyto', copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO)); - apiRouter.post('/sql/copyto', copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO)); + apiRouter.post('/sql/copyfrom', copyFromMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_FROM)); + apiRouter.get('/sql/copyto', copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO)); + apiRouter.post('/sql/copyto', copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO)); + } }; function handleCopyTo (logger) { @@ -178,5 +180,3 @@ function errorHandler (logger) { } }; } - -module.exports = CopyController; From a383785e7c15523ee1665141b83d9271ca47876b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 12:52:54 +0200 Subject: [PATCH 08/37] Remove generic controller, its functionallity fits better in cors middleware --- app/controllers/generic_controller.js | 13 ------------- app/middlewares/cors.js | 4 ++++ app/server.js | 4 ---- 3 files changed, 4 insertions(+), 17 deletions(-) delete mode 100644 app/controllers/generic_controller.js diff --git a/app/controllers/generic_controller.js b/app/controllers/generic_controller.js deleted file mode 100644 index 7bf7d8c4..00000000 --- a/app/controllers/generic_controller.js +++ /dev/null @@ -1,13 +0,0 @@ -'use strict'; - -module.exports = class GenericController { - route (app) { - app.options('*', emptyResponse()); - } -}; - -function emptyResponse () { - return function emptyResponseMiddleware (req, res) { - res.end(); - }; -} diff --git a/app/middlewares/cors.js b/app/middlewares/cors.js index d7699345..a3c80853 100644 --- a/app/middlewares/cors.js +++ b/app/middlewares/cors.js @@ -21,6 +21,10 @@ module.exports = function cors(extraHeaders = []) { res.header('Access-Control-Allow-Headers', headers.join(', ')); res.header('Access-Control-Expose-Headers', exposedHeaders.join(', ')); + if (req.method === 'OPTIONS') { + return res.send(); + } + next(); }; }; diff --git a/app/server.js b/app/server.js index ffe7086f..e3103a69 100644 --- a/app/server.js +++ b/app/server.js @@ -29,7 +29,6 @@ const Logger = require('./services/logger'); var cors = require('./middlewares/cors'); const ApiRouter = require('./controllers/api-router'); -var GenericController = require('./controllers/generic_controller'); var HealthCheckController = require('./controllers/health_check_controller'); var VersionController = require('./controllers/version_controller'); @@ -129,9 +128,6 @@ function App(statsClient) { const dataIngestionLogger = new Logger(global.settings.dataIngestionLogPath, 'data-ingestion'); app.dataIngestionLogger = dataIngestionLogger; - var genericController = new GenericController(); - genericController.route(app); - var healthCheckController = new HealthCheckController(); healthCheckController.route(app); From 1bb1a350e01144adc26d744ea44c542c58b952ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 12:57:03 +0200 Subject: [PATCH 09/37] Move cors middleware to api router --- app/controllers/api-router.js | 4 ++++ app/server.js | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/api-router.js b/app/controllers/api-router.js index ad580908..151a1d8d 100644 --- a/app/controllers/api-router.js +++ b/app/controllers/api-router.js @@ -17,6 +17,8 @@ const QueryController = require('./query_controller'); const CopyController = require('./copy_controller'); const JobController = require('./job_controller'); +const cors = require('../middlewares/cors'); + module.exports = class ApiRouter { constructor ({ redisPool, metadataBackend, statsClient, dataIngestionLogger }) { const userLimitsServiceOptions = { @@ -61,6 +63,8 @@ module.exports = class ApiRouter { route (app) { const apiRouter = router({ mergeParams: true }); + apiRouter.use(cors()); + this.queryController.route(apiRouter); this.copyController.route(apiRouter); this.jobController.route(apiRouter); diff --git a/app/server.js b/app/server.js index e3103a69..6a0b8b55 100644 --- a/app/server.js +++ b/app/server.js @@ -26,8 +26,6 @@ var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); const Logger = require('./services/logger'); -var cors = require('./middlewares/cors'); - const ApiRouter = require('./controllers/api-router'); var HealthCheckController = require('./controllers/health_check_controller'); var VersionController = require('./controllers/version_controller'); @@ -93,8 +91,6 @@ function App(statsClient) { app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level:'info'}))); } - app.use(cors()); - // Use step-profiler app.use(function bootstrap$prepareRequestResponse(req, res, next) { res.locals = res.locals || {}; From 79143e3df6e1a14380d24b33cb6a8b9684c5c159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 13:04:44 +0200 Subject: [PATCH 10/37] Extract 'served-by-host-header' middleware and move it to api router --- app/controllers/api-router.js | 3 +++ app/middlewares/served-by-host-header.js | 13 +++++++++++++ app/server.js | 4 ---- 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 app/middlewares/served-by-host-header.js diff --git a/app/controllers/api-router.js b/app/controllers/api-router.js index 151a1d8d..b1cc5d6a 100644 --- a/app/controllers/api-router.js +++ b/app/controllers/api-router.js @@ -18,6 +18,8 @@ const CopyController = require('./copy_controller'); const JobController = require('./job_controller'); const cors = require('../middlewares/cors'); +const servedByHostHeader = require('../middlewares/served-by-host-header'); + module.exports = class ApiRouter { constructor ({ redisPool, metadataBackend, statsClient, dataIngestionLogger }) { @@ -64,6 +66,7 @@ module.exports = class ApiRouter { const apiRouter = router({ mergeParams: true }); apiRouter.use(cors()); + apiRouter.use(servedByHostHeader()); this.queryController.route(apiRouter); this.copyController.route(apiRouter); diff --git a/app/middlewares/served-by-host-header.js b/app/middlewares/served-by-host-header.js new file mode 100644 index 00000000..d3324817 --- /dev/null +++ b/app/middlewares/served-by-host-header.js @@ -0,0 +1,13 @@ +'use strict'; + +const os = require('os'); + +module.exports = function servedByHostHeader () { + const hostname = global.settings.api_hostname || os.hostname().split('.')[0]; + + return function servedByHostHeaderMiddleware (req, res, next) { + res.set('X-Served-By-Host', hostname); + + next(); + }; +}; diff --git a/app/server.js b/app/server.js index 6a0b8b55..52ed5df4 100644 --- a/app/server.js +++ b/app/server.js @@ -95,10 +95,6 @@ function App(statsClient) { app.use(function bootstrap$prepareRequestResponse(req, res, next) { res.locals = res.locals || {}; - if (global.settings.api_hostname) { - res.header('X-Served-By-Host', global.settings.api_hostname); - } - var profile = global.settings.useProfiler; req.profiler = new Profiler({ profile: profile, From 9c79959f887991853b2b720909564ce8fc9937ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 13:15:13 +0200 Subject: [PATCH 11/37] Add TODO --- app/server.js | 1 + 1 file changed, 1 insertion(+) diff --git a/app/server.js b/app/server.js index 52ed5df4..9637b88e 100644 --- a/app/server.js +++ b/app/server.js @@ -70,6 +70,7 @@ function App(statsClient) { global.settings.ratelimits ); + // TODO: it's here becouse of testing purposes, try to move to top level global.settings.tmpDir = global.settings.tmpDir || '/tmp'; if (!fs.existsSync(global.settings.tmpDir)) { mkdirp.sync(global.settings.tmpDir); From 1a9081135d82ef4d0cdf482024c730cf082e8bae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 14:12:14 +0200 Subject: [PATCH 12/37] Extract logger middleware --- app/controllers/api-router.js | 3 ++- app/middlewares/logger.js | 27 +++++++++++++++++++++++++++ app/server.js | 17 ----------------- 3 files changed, 29 insertions(+), 18 deletions(-) create mode 100644 app/middlewares/logger.js diff --git a/app/controllers/api-router.js b/app/controllers/api-router.js index b1cc5d6a..6a161f5b 100644 --- a/app/controllers/api-router.js +++ b/app/controllers/api-router.js @@ -19,7 +19,7 @@ const JobController = require('./job_controller'); const cors = require('../middlewares/cors'); const servedByHostHeader = require('../middlewares/served-by-host-header'); - +const logger = require('../middlewares/logger'); module.exports = class ApiRouter { constructor ({ redisPool, metadataBackend, statsClient, dataIngestionLogger }) { @@ -65,6 +65,7 @@ module.exports = class ApiRouter { route (app) { const apiRouter = router({ mergeParams: true }); + apiRouter.use(logger()); apiRouter.use(cors()); apiRouter.use(servedByHostHeader()); diff --git a/app/middlewares/logger.js b/app/middlewares/logger.js new file mode 100644 index 00000000..5ada2e61 --- /dev/null +++ b/app/middlewares/logger.js @@ -0,0 +1,27 @@ +'use strict'; + +module.exports = function logger () { + if (!global.log4js) { + return function dummyLoggerMiddleware (req, res, next) { + next(); + }; + } + + const options = { + level: 'info', + buffer: true, + // log4js provides a tokens solution as expess but in does not provide the request/response in the callback. + // Thus it is not possible to extract relevant information from them. + // This is a workaround to be able to access request/response. + format: function (req, res, format) { + const defaultFormat = ':remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type]'; + const logFormat = global.settings.log_format || defaultFormat; + + return format(logFormat); + } + }; + + const logger = global.log4js.getLogger(); + + return global.log4js.connectLogger(logger, options); +}; diff --git a/app/server.js b/app/server.js index 9637b88e..ed45eeb2 100644 --- a/app/server.js +++ b/app/server.js @@ -18,7 +18,6 @@ var express = require('express'); var Profiler = require('./stats/profiler-proxy'); -var _ = require('underscore'); var fs = require('fs'); var mkdirp = require('mkdirp'); @@ -76,22 +75,6 @@ function App(statsClient) { mkdirp.sync(global.settings.tmpDir); } - if (global.log4js) { - var loggerOpts = { - buffer: true, - // log4js provides a tokens solution as expess but in does not provide the request/response in the callback. - // Thus it is not possible to extract relevant information from them. - // This is a workaround to be able to access request/response. - format: function(req, res, format) { - var logFormat = global.settings.log_format || - ':remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type]'; - - return format(logFormat); - } - }; - app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level:'info'}))); - } - // Use step-profiler app.use(function bootstrap$prepareRequestResponse(req, res, next) { res.locals = res.locals || {}; From 9c1a1611d57c5d701ee54135b4bd321167dc965a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 14:31:17 +0200 Subject: [PATCH 13/37] Extract profiler setup to its middleware --- app/controllers/api-router.js | 4 ++++ app/middlewares/profiler.js | 13 +++++++++++++ app/server.js | 7 ------- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/app/controllers/api-router.js b/app/controllers/api-router.js index 6a161f5b..51569385 100644 --- a/app/controllers/api-router.js +++ b/app/controllers/api-router.js @@ -20,9 +20,12 @@ const JobController = require('./job_controller'); const cors = require('../middlewares/cors'); const servedByHostHeader = require('../middlewares/served-by-host-header'); const logger = require('../middlewares/logger'); +const profiler = require('../middlewares/profiler'); module.exports = class ApiRouter { constructor ({ redisPool, metadataBackend, statsClient, dataIngestionLogger }) { + this.statsClient = statsClient; + const userLimitsServiceOptions = { limits: { rateLimitsEnabled: global.settings.ratelimits.rateLimitsEnabled @@ -66,6 +69,7 @@ module.exports = class ApiRouter { const apiRouter = router({ mergeParams: true }); apiRouter.use(logger()); + apiRouter.use(profiler({ statsClient: this.statsClient })); apiRouter.use(cors()); apiRouter.use(servedByHostHeader()); diff --git a/app/middlewares/profiler.js b/app/middlewares/profiler.js index fac63a93..aad91099 100644 --- a/app/middlewares/profiler.js +++ b/app/middlewares/profiler.js @@ -1,5 +1,18 @@ 'use strict'; +const Profiler = require('../stats/profiler-proxy'); + +module.exports = function profiler ({ statsClient }) { + return function profilerMiddleware (req, res, next) { + req.profiler = new Profiler({ + profile: global.settings.useProfiler, + statsd_client: statsClient + }); + + next(); + }; +}; + module.exports.initializeProfilerMiddleware = function initializeProfiler (label) { return function initializeProfilerMiddleware (req, res, next) { if (req.profiler) { diff --git a/app/server.js b/app/server.js index ed45eeb2..0628d12f 100644 --- a/app/server.js +++ b/app/server.js @@ -17,7 +17,6 @@ // var express = require('express'); -var Profiler = require('./stats/profiler-proxy'); var fs = require('fs'); var mkdirp = require('mkdirp'); @@ -75,15 +74,9 @@ function App(statsClient) { mkdirp.sync(global.settings.tmpDir); } - // Use step-profiler app.use(function bootstrap$prepareRequestResponse(req, res, next) { res.locals = res.locals || {}; - var profile = global.settings.useProfiler; - req.profiler = new Profiler({ - profile: profile, - statsd_client: statsClient - }); next(); }); From f9c467a62811a6b492fa54f1df75be47afd71ee1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 14:37:46 +0200 Subject: [PATCH 14/37] Remove dumb initialization --- app/server.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/server.js b/app/server.js index 0628d12f..437f04fe 100644 --- a/app/server.js +++ b/app/server.js @@ -74,12 +74,6 @@ function App(statsClient) { mkdirp.sync(global.settings.tmpDir); } - app.use(function bootstrap$prepareRequestResponse(req, res, next) { - res.locals = res.locals || {}; - - next(); - }); - // Set connection timeout if (global.settings.hasOwnProperty('node_socket_timeout')) { var timeout = parseInt(global.settings.node_socket_timeout); From c60a44252d184fb6b0b016c09450fc4aa196dc80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 15:36:04 +0200 Subject: [PATCH 15/37] Extract socket timeout middleware --- app/controllers/api-router.js | 6 ++++-- app/middlewares/socket-timeout.js | 18 ++++++++++++++++++ app/server.js | 9 --------- 3 files changed, 22 insertions(+), 11 deletions(-) create mode 100644 app/middlewares/socket-timeout.js diff --git a/app/controllers/api-router.js b/app/controllers/api-router.js index 51569385..d38fc645 100644 --- a/app/controllers/api-router.js +++ b/app/controllers/api-router.js @@ -17,10 +17,11 @@ const QueryController = require('./query_controller'); const CopyController = require('./copy_controller'); const JobController = require('./job_controller'); -const cors = require('../middlewares/cors'); -const servedByHostHeader = require('../middlewares/served-by-host-header'); +const socketTimeout = require('../middlewares/socket-timeout'); const logger = require('../middlewares/logger'); const profiler = require('../middlewares/profiler'); +const cors = require('../middlewares/cors'); +const servedByHostHeader = require('../middlewares/served-by-host-header'); module.exports = class ApiRouter { constructor ({ redisPool, metadataBackend, statsClient, dataIngestionLogger }) { @@ -68,6 +69,7 @@ module.exports = class ApiRouter { route (app) { const apiRouter = router({ mergeParams: true }); + apiRouter.use(socketTimeout()); apiRouter.use(logger()); apiRouter.use(profiler({ statsClient: this.statsClient })); apiRouter.use(cors()); diff --git a/app/middlewares/socket-timeout.js b/app/middlewares/socket-timeout.js new file mode 100644 index 00000000..53d14f78 --- /dev/null +++ b/app/middlewares/socket-timeout.js @@ -0,0 +1,18 @@ +'use strict'; + +module.exports = function socketTimeout () { + if (!global.settings.hasOwnProperty('node_socket_timeout')) { + return function dummySocketTimeoutMiddleware (req, res, next) { + next(); + }; + } + + const timeout = parseInt(global.settings.node_socket_timeout); + + return function socketTimeoutMiddleware (req, res, next) { + // Set connection timeout + req.connection.setTimeout(timeout); + + next(); + } +} diff --git a/app/server.js b/app/server.js index 437f04fe..7a6dd68f 100644 --- a/app/server.js +++ b/app/server.js @@ -74,15 +74,6 @@ function App(statsClient) { mkdirp.sync(global.settings.tmpDir); } - // Set connection timeout - if (global.settings.hasOwnProperty('node_socket_timeout')) { - var timeout = parseInt(global.settings.node_socket_timeout); - app.use(function(req, res, next) { - req.connection.setTimeout(timeout); - next(); - }); - } - app.enable('jsonp callback'); app.set("trust proxy", true); app.disable('x-powered-by'); From 6cf92446a7e9be95d429c53b3cfcf7e08deeef52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 15:40:12 +0200 Subject: [PATCH 16/37] Use 'const' instead of 'var' --- app/server.js | 48 +++++++++++++++--------------------------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/app/server.js b/app/server.js index 7a6dd68f..96ebbcdb 100644 --- a/app/server.js +++ b/app/server.js @@ -1,34 +1,18 @@ 'use strict'; -// CartoDB SQL API -// -// all requests expect the following URL args: -// - `sql` {String} SQL to execute -// -// for private (read/write) queries: -// - OAuth. Must have proper OAuth 1.1 headers. For OAuth 1.1 spec see Google -// -// eg. /api/v1/?sql=SELECT 1 as one (with a load of OAuth headers or URL arguments) -// -// for public (read only) queries: -// - sql only, provided the subdomain exists in CartoDB and the table's sharing options are public -// -// eg. vizzuality.cartodb.com/api/v1/?sql=SELECT * from my_table -// +const express = require('express'); +const fs = require('fs'); +const mkdirp = require('mkdirp'); -var express = require('express'); -var fs = require('fs'); -var mkdirp = require('mkdirp'); - -var RedisPool = require('redis-mpool'); -var cartodbRedis = require('cartodb-redis'); +const RedisPool = require('redis-mpool'); +const cartodbRedis = require('cartodb-redis'); const Logger = require('./services/logger'); const ApiRouter = require('./controllers/api-router'); -var HealthCheckController = require('./controllers/health_check_controller'); -var VersionController = require('./controllers/version_controller'); +const HealthCheckController = require('./controllers/health_check_controller'); +const VersionController = require('./controllers/version_controller'); -var batchFactory = require('../batch'); +const batchFactory = require('../batch'); process.env.PGAPPNAME = process.env.PGAPPNAME || 'cartodb_sqlapi'; @@ -37,10 +21,8 @@ require('./utils/date_to_json'); // jshint maxcomplexity:9 function App(statsClient) { - - var app = express(); - - var redisPool = new RedisPool({ + const app = express(); + const redisPool = new RedisPool({ name: 'sql-api', host: global.settings.redis_host, port: global.settings.redis_port, @@ -48,7 +30,7 @@ function App(statsClient) { idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, reapIntervalMillis: global.settings.redisReapIntervalMillis }); - var metadataBackend = cartodbRedis({ pool: redisPool }); + const metadataBackend = cartodbRedis({ pool: redisPool }); // Set default configuration global.settings.db_pubuser = global.settings.db_pubuser || "publicuser"; @@ -82,19 +64,19 @@ function App(statsClient) { const dataIngestionLogger = new Logger(global.settings.dataIngestionLogPath, 'data-ingestion'); app.dataIngestionLogger = dataIngestionLogger; - var healthCheckController = new HealthCheckController(); + const healthCheckController = new HealthCheckController(); healthCheckController.route(app); - var versionController = new VersionController(); + const versionController = new VersionController(); versionController.route(app); const apiRouter = new ApiRouter({ redisPool, metadataBackend, statsClient, dataIngestionLogger }); apiRouter.route(app); - var isBatchProcess = process.argv.indexOf('--no-batch') === -1; + const isBatchProcess = process.argv.indexOf('--no-batch') === -1; if (global.settings.environment !== 'test' && isBatchProcess) { - var batchName = global.settings.api_hostname || 'batch'; + const batchName = global.settings.api_hostname || 'batch'; app.batch = batchFactory( metadataBackend, redisPool, batchName, statsClient, global.settings.batch_log_filename From 92ba3222142395c437546dc951d9f64f1760b072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 15:42:24 +0200 Subject: [PATCH 17/37] Linter --- app/middlewares/socket-timeout.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/middlewares/socket-timeout.js b/app/middlewares/socket-timeout.js index 53d14f78..ff39f1dd 100644 --- a/app/middlewares/socket-timeout.js +++ b/app/middlewares/socket-timeout.js @@ -14,5 +14,5 @@ module.exports = function socketTimeout () { req.connection.setTimeout(timeout); next(); - } -} + }; +}; From 5c95ee0daa1b70d9f55cf94831659e5678707d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 15:50:05 +0200 Subject: [PATCH 18/37] Rename public factory function --- app.js | 3 ++- app/server.js | 7 +------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/app.js b/app.js index cfbf1dfa..39f454ae 100755 --- a/app.js +++ b/app.js @@ -14,6 +14,7 @@ var fs = require('fs'); var path = require('path'); const fqdn = require('@carto/fqdn-sync'); +const serverFactory = require('./app/server'); var argv = require('yargs') .usage('Usage: $0 [options]') @@ -92,7 +93,7 @@ if (global.settings.statsd) { } var statsClient = StatsClient.getInstance(global.settings.statsd); -var server = require('./app/server')(statsClient); +var server = serverFactory(statsClient); var listener = server.listen(global.settings.node_port, global.settings.node_host); listener.on('listening', function() { console.info("Using Node.js %s", process.version); diff --git a/app/server.js b/app/server.js index 96ebbcdb..92557a6e 100644 --- a/app/server.js +++ b/app/server.js @@ -3,15 +3,12 @@ const express = require('express'); const fs = require('fs'); const mkdirp = require('mkdirp'); - const RedisPool = require('redis-mpool'); const cartodbRedis = require('cartodb-redis'); const Logger = require('./services/logger'); - const ApiRouter = require('./controllers/api-router'); const HealthCheckController = require('./controllers/health_check_controller'); const VersionController = require('./controllers/version_controller'); - const batchFactory = require('../batch'); process.env.PGAPPNAME = process.env.PGAPPNAME || 'cartodb_sqlapi'; @@ -20,7 +17,7 @@ process.env.PGAPPNAME = process.env.PGAPPNAME || 'cartodb_sqlapi'; require('./utils/date_to_json'); // jshint maxcomplexity:9 -function App(statsClient) { +module.exports = function serverFactory (statsClient) { const app = express(); const redisPool = new RedisPool({ name: 'sql-api', @@ -87,5 +84,3 @@ function App(statsClient) { return app; } - -module.exports = App; From 8948ad246fab2ab5a011949676cf0bc72d5b6596 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 16:10:47 +0200 Subject: [PATCH 19/37] Linter --- app/server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server.js b/app/server.js index 92557a6e..904d929e 100644 --- a/app/server.js +++ b/app/server.js @@ -83,4 +83,4 @@ module.exports = function serverFactory (statsClient) { } return app; -} +}; From b2f51a4d2da2b3ed5637d267e2ddf13fd2bea83e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 16:11:08 +0200 Subject: [PATCH 20/37] Use 'const' instead of 'var' --- app.js | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/app.js b/app.js index 39f454ae..d9486f01 100755 --- a/app.js +++ b/app.js @@ -11,16 +11,15 @@ * environments: [development, test, production] * */ -var fs = require('fs'); -var path = require('path'); +const fs = require('fs'); +const path = require('path'); const fqdn = require('@carto/fqdn-sync'); -const serverFactory = require('./app/server'); -var argv = require('yargs') - .usage('Usage: $0 [options]') +const argv = require('yargs') + .usage('Usage: node $0 [options]') .help('h') .example( - '$0 production -c /etc/sql-api/config.js', + 'node $0 production -c /etc/sql-api/config.js', 'start server in production environment with /etc/sql-api/config.js as config file' ) .alias('h', 'help') @@ -29,18 +28,19 @@ var argv = require('yargs') .describe('c', 'Load configuration from path') .argv; -var environmentArg = argv._[0] || process.env.NODE_ENV || 'development'; -var configurationFile = path.resolve(argv.config || './config/environments/' + environmentArg + '.js'); +const environmentArg = argv._[0] || process.env.NODE_ENV || 'development'; +const configurationFile = path.resolve(argv.config || './config/environments/' + environmentArg + '.js'); if (!fs.existsSync(configurationFile)) { console.error('Configuration file "%s" does not exist', configurationFile); process.exit(1); } global.settings = require(configurationFile); -var ENVIRONMENT = argv._[0] || process.env.NODE_ENV || global.settings.environment; + +const ENVIRONMENT = argv._[0] || process.env.NODE_ENV || global.settings.environment; process.env.NODE_ENV = ENVIRONMENT; -var availableEnvironments = ['development', 'production', 'test', 'staging']; +const availableEnvironments = ['development', 'production', 'test', 'staging']; // sanity check arguments if (availableEnvironments.indexOf(ENVIRONMENT) === -1) { @@ -52,14 +52,14 @@ if (availableEnvironments.indexOf(ENVIRONMENT) === -1) { global.settings.api_hostname = fqdn.hostname(); global.log4js = require('log4js'); -var log4jsConfig = { +const log4jsConfig = { appenders: [], replaceConsole: true }; if ( global.settings.log_filename ) { - var logFilename = path.resolve(global.settings.log_filename); - var logDirectory = path.dirname(logFilename); + const logFilename = path.resolve(global.settings.log_filename); + const logDirectory = path.dirname(logFilename); if (!fs.existsSync(logDirectory)) { console.error("Log filename directory does not exist: " + logDirectory); process.exit(1); @@ -82,19 +82,21 @@ if ( ! global.settings.base_url ) { global.settings.base_url = '/api/*'; } -var version = require("./package").version; +const version = require("./package").version; -var StatsClient = require('./app/stats/client'); +const StatsClient = require('./app/stats/client'); if (global.settings.statsd) { // Perform keyword substitution in statsd if (global.settings.statsd.prefix) { global.settings.statsd.prefix = global.settings.statsd.prefix.replace(/:host/, fqdn.reverse()); } } -var statsClient = StatsClient.getInstance(global.settings.statsd); +const statsClient = StatsClient.getInstance(global.settings.statsd); -var server = serverFactory(statsClient); -var listener = server.listen(global.settings.node_port, global.settings.node_host); +const serverFactory = require('./app/server'); + +const server = serverFactory(statsClient); +const listener = server.listen(global.settings.node_port, global.settings.node_host); listener.on('listening', function() { console.info("Using Node.js %s", process.version); console.info('Using configuration file "%s"', configurationFile); @@ -182,9 +184,9 @@ function scheduleForcedExit ({ killTimeout }) { } function isGteMinVersion(version, minVersion) { - var versionMatch = /[a-z]?([0-9]*)/.exec(version); + const versionMatch = /[a-z]?([0-9]*)/.exec(version); if (versionMatch) { - var majorVersion = parseInt(versionMatch[1], 10); + const majorVersion = parseInt(versionMatch[1], 10); if (Number.isFinite(majorVersion)) { return majorVersion >= minVersion; } @@ -235,7 +237,7 @@ setInterval(function cpuUsageMetrics () { }, 5000); if (global.gc && isGteMinVersion(process.version, 6)) { - var gcInterval = Number.isFinite(global.settings.gc_interval) ? + const gcInterval = Number.isFinite(global.settings.gc_interval) ? global.settings.gc_interval : 10000; From ec436b9c5ed75f94b415738b0b4ca24dc8a9d5a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 16:11:18 +0200 Subject: [PATCH 21/37] Add contributor --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 3b013e43..c677fd9f 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,8 @@ "author": "Vizzuality (http://vizzuality.com)", "contributors": [ "Simon Tokumine ", - "Sandro Santilli " + "Sandro Santilli ", + "Daniel García Aubert " ], "dependencies": { "@carto/fqdn-sync": "0.2.2", From 7620383f876466422b7222b9f3090ec85f151342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 16:13:19 +0200 Subject: [PATCH 22/37] Remove outdated comment --- app.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/app.js b/app.js index d9486f01..9488bd66 100755 --- a/app.js +++ b/app.js @@ -2,15 +2,6 @@ 'use strict'; -/* -* SQL API loader -* =============== -* -* node app [environment] -* -* environments: [development, test, production] -* -*/ const fs = require('fs'); const path = require('path'); const fqdn = require('@carto/fqdn-sync'); From 07cf020923bdf54cc87f32d6815a6bc25ed89378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 16:17:21 +0200 Subject: [PATCH 23/37] Style --- app.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app.js b/app.js index 9488bd66..1b51e930 100755 --- a/app.js +++ b/app.js @@ -21,6 +21,7 @@ const argv = require('yargs') const environmentArg = argv._[0] || process.env.NODE_ENV || 'development'; const configurationFile = path.resolve(argv.config || './config/environments/' + environmentArg + '.js'); + if (!fs.existsSync(configurationFile)) { console.error('Configuration file "%s" does not exist', configurationFile); process.exit(1); @@ -48,7 +49,7 @@ const log4jsConfig = { replaceConsole: true }; -if ( global.settings.log_filename ) { +if (global.settings.log_filename) { const logFilename = path.resolve(global.settings.log_filename); const logDirectory = path.dirname(logFilename); if (!fs.existsSync(logDirectory)) { @@ -64,6 +65,7 @@ if ( global.settings.log_filename ) { { type: "console", layout: { type:'basic' } } ); } + global.log4js.configure(log4jsConfig); global.logger = global.log4js.getLogger(); From 3dee09a5c50314d9b4ae5b2f3ed80166b72ae02e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 17:27:46 +0200 Subject: [PATCH 24/37] Draft: be able to inject custom middlewares --- app.js | 8 ++++---- app/controllers/api-router.js | 9 +++++++-- app/controllers/health_check_controller.js | 9 +++++++-- app/controllers/version_controller.js | 8 +++++++- app/server.js | 20 +++++++++++++++++--- config/environments/development.js.example | 20 +++++++++++++++++--- config/environments/production.js.example | 20 +++++++++++++++++--- config/environments/staging.js.example | 20 +++++++++++++++++--- config/environments/test.js.example | 20 +++++++++++++++++--- 9 files changed, 110 insertions(+), 24 deletions(-) diff --git a/app.js b/app.js index 1b51e930..f131da42 100755 --- a/app.js +++ b/app.js @@ -69,10 +69,10 @@ if (global.settings.log_filename) { global.log4js.configure(log4jsConfig); global.logger = global.log4js.getLogger(); - -// kick off controller -if ( ! global.settings.base_url ) { - global.settings.base_url = '/api/*'; +if (!global.settings.routes) { + console.error('Missing environment paramenter "routes". Please review your configuration file.'); + console.error("Available environments: " + availableEnvironments.join(', ')); + process.exit(1); } const version = require("./package").version; diff --git a/app/controllers/api-router.js b/app/controllers/api-router.js index d38fc645..6193cb61 100644 --- a/app/controllers/api-router.js +++ b/app/controllers/api-router.js @@ -24,7 +24,8 @@ const cors = require('../middlewares/cors'); const servedByHostHeader = require('../middlewares/served-by-host-header'); module.exports = class ApiRouter { - constructor ({ redisPool, metadataBackend, statsClient, dataIngestionLogger }) { + constructor ({ routes, redisPool, metadataBackend, statsClient, dataIngestionLogger }) { + this.routes = routes; this.statsClient = statsClient; const userLimitsServiceOptions = { @@ -68,6 +69,10 @@ module.exports = class ApiRouter { route (app) { const apiRouter = router({ mergeParams: true }); + const paths = this.routes.paths || []; + const middlewares = this.routes.middlewares || []; + + middlewares.forEach(middleware => apiRouter.use(middleware())); apiRouter.use(socketTimeout()); apiRouter.use(logger()); @@ -79,6 +84,6 @@ module.exports = class ApiRouter { this.copyController.route(apiRouter); this.jobController.route(apiRouter); - app.use(`${global.settings.base_url}`, apiRouter); + paths.forEach(path => app.use(path, apiRouter)); } }; diff --git a/app/controllers/health_check_controller.js b/app/controllers/health_check_controller.js index ef19f8a3..8c6369d7 100644 --- a/app/controllers/health_check_controller.js +++ b/app/controllers/health_check_controller.js @@ -3,12 +3,17 @@ const HealthCheckBackend = require('../monitoring/health_check'); module.exports = class HealthCheckController { - constructor () { + constructor ({ routes }) { + this.routes = routes; this.healthCheckBackend = new HealthCheckBackend(global.settings.disabled_file); } route (app) { - app.get(`${global.settings.base_url}/health`, healthCheck({ healthCheckBackend: this.healthCheckBackend })); + const paths = this.routes.paths || []; + + paths.forEach(path => app.get(`${path}/health`, healthCheck({ + healthCheckBackend: this.healthCheckBackend + }))); } }; diff --git a/app/controllers/version_controller.js b/app/controllers/version_controller.js index 03c99ba9..75c5c019 100644 --- a/app/controllers/version_controller.js +++ b/app/controllers/version_controller.js @@ -5,8 +5,14 @@ const versions = { }; module.exports = class VersionController { + constructor ({ routes }) { + this.routes = routes; + } + route (app) { - app.get(`${global.settings.base_url}/version`, version()); + const paths = this.routes.paths || []; + + paths.forEach(path => app.get(`${path}/version`, version())); } }; diff --git a/app/server.js b/app/server.js index 904d929e..3c15e047 100644 --- a/app/server.js +++ b/app/server.js @@ -61,13 +61,27 @@ module.exports = function serverFactory (statsClient) { const dataIngestionLogger = new Logger(global.settings.dataIngestionLogPath, 'data-ingestion'); app.dataIngestionLogger = dataIngestionLogger; - const healthCheckController = new HealthCheckController(); + // FIXME: health controller should be atached to the main entry point: "/" + // instead of "/api/v1/" + const healthCheckController = new HealthCheckController({ + routes: global.settings.routes + }); healthCheckController.route(app); - const versionController = new VersionController(); + // FIXME: version controller should be atached to the main entry point: "/" + // instead of "/api/v1/" + const versionController = new VersionController({ + routes: global.settings.routes + }); versionController.route(app); - const apiRouter = new ApiRouter({ redisPool, metadataBackend, statsClient, dataIngestionLogger }); + const apiRouter = new ApiRouter({ + routes: global.settings.routes, + redisPool, + metadataBackend, + statsClient, + dataIngestionLogger + }); apiRouter.route(app); const isBatchProcess = process.argv.indexOf('--no-batch') === -1; diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 845f453c..c9dbae48 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -1,9 +1,23 @@ // Time in milliseconds to force GC cycle. // Disable by using <=0 value. module.exports.gc_interval = 10000; -// In case the base_url has a :user param the username will be the one specified in the URL, -// otherwise it will fallback to extract the username from the host header. -module.exports.base_url = '(?:/api/:version|/user/:user/api/:version)'; +module.exports.routes = { + paths: [ + // In case the base_url has a :user param the username will be the one specified in the URL, + // otherwise it will fallback to extract the username from the host header. + '/api/:version', + '/user/:user/api/:version', + ], + // Attach middlewares at the begining of the req/res cycle + // to perform custom operations. + middlewares: [ + function noop () { + return function noopMiddleware (req, res, next) { + next(); + } + } + ] +}; // If useProfiler is true every response will be served with an // X-SQLAPI-Profile header containing elapsed timing for various // steps taken for producing the response. diff --git a/config/environments/production.js.example b/config/environments/production.js.example index ac663399..85b6cf2a 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -1,9 +1,23 @@ // Time in milliseconds to force GC cycle. // Disable by using <=0 value. module.exports.gc_interval = 10000; -// In case the base_url has a :user param the username will be the one specified in the URL, -// otherwise it will fallback to extract the username from the host header. -module.exports.base_url = '(?:/api/:version|/user/:user/api/:version)'; +module.exports.routes = { + paths: [ + // In case the base_url has a :user param the username will be the one specified in the URL, + // otherwise it will fallback to extract the username from the host header. + '/api/:version', + '/user/:user/api/:version', + ], + // Attach middlewares at the begining of the req/res cycle + // to perform custom operations. + middlewares: [ + function noop () { + return function noopMiddleware (req, res, next) { + next(); + } + } + ] +}; // If useProfiler is true every response will be served with an // X-SQLAPI-Profile header containing elapsed timing for various // steps taken for producing the response. diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 06e91ec5..65ca48fe 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -1,9 +1,23 @@ // Time in milliseconds to force GC cycle. // Disable by using <=0 value. module.exports.gc_interval = 10000; -// In case the base_url has a :user param the username will be the one specified in the URL, -// otherwise it will fallback to extract the username from the host header. -module.exports.base_url = '(?:/api/:version|/user/:user/api/:version)'; +module.exports.routes = { + paths: [ + // In case the base_url has a :user param the username will be the one specified in the URL, + // otherwise it will fallback to extract the username from the host header. + '/api/:version', + '/user/:user/api/:version', + ], + // Attach middlewares at the begining of the req/res cycle + // to perform custom operations. + middlewares: [ + function noop () { + return function noopMiddleware (req, res, next) { + next(); + } + } + ] +}; // If useProfiler is true every response will be served with an // X-SQLAPI-Profile header containing elapsed timing for various // steps taken for producing the response. diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 5a123e90..6b55652a 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -1,9 +1,23 @@ // Time in milliseconds to force GC cycle. // Disable by using <=0 value. module.exports.gc_interval = 10000; -// In case the base_url has a :user param the username will be the one specified in the URL, -// otherwise it will fallback to extract the username from the host header. -module.exports.base_url = '(?:/api/:version|/user/:user/api/:version)'; +module.exports.routes = { + paths: [ + // In case the base_url has a :user param the username will be the one specified in the URL, + // otherwise it will fallback to extract the username from the host header. + '/api/:version', + '/user/:user/api/:version', + ], + // Attach middlewares at the begining of the req/res cycle + // to perform custom operations. + middlewares: [ + function noop () { + return function noopMiddleware (req, res, next) { + next(); + } + } + ] +}; // If useProfiler is true every response will be served with an // X-SQLAPI-Profile header containing elapsed timing for various // steps taken for producing the response. From 1d55de3e43be4e30d0fd80ed00f7c9df88a56f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 17:45:12 +0200 Subject: [PATCH 25/37] Explicit routes configuration --- app/server.js | 10 ++++---- config/environments/development.js.example | 30 ++++++++++++---------- config/environments/production.js.example | 30 ++++++++++++---------- config/environments/staging.js.example | 30 ++++++++++++---------- config/environments/test.js.example | 30 ++++++++++++---------- 5 files changed, 69 insertions(+), 61 deletions(-) diff --git a/app/server.js b/app/server.js index 3c15e047..51baedb2 100644 --- a/app/server.js +++ b/app/server.js @@ -62,21 +62,21 @@ module.exports = function serverFactory (statsClient) { app.dataIngestionLogger = dataIngestionLogger; // FIXME: health controller should be atached to the main entry point: "/" - // instead of "/api/v1/" + // instead of "/api/v1/" or "/user/:user/api/:version" const healthCheckController = new HealthCheckController({ - routes: global.settings.routes + routes: global.settings.routes.api }); healthCheckController.route(app); // FIXME: version controller should be atached to the main entry point: "/" - // instead of "/api/v1/" + // instead of "/api/:version" or "/user/:user/api/:version" const versionController = new VersionController({ - routes: global.settings.routes + routes: global.settings.routes.api }); versionController.route(app); const apiRouter = new ApiRouter({ - routes: global.settings.routes, + routes: global.settings.routes.api, redisPool, metadataBackend, statsClient, diff --git a/config/environments/development.js.example b/config/environments/development.js.example index c9dbae48..1ef61235 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -2,21 +2,23 @@ // Disable by using <=0 value. module.exports.gc_interval = 10000; module.exports.routes = { - paths: [ - // In case the base_url has a :user param the username will be the one specified in the URL, - // otherwise it will fallback to extract the username from the host header. - '/api/:version', - '/user/:user/api/:version', - ], - // Attach middlewares at the begining of the req/res cycle - // to perform custom operations. - middlewares: [ - function noop () { - return function noopMiddleware (req, res, next) { - next(); + api: { + paths: [ + // In case the base_url has a :user param the username will be the one specified in the URL, + // otherwise it will fallback to extract the username from the host header. + '/api/:version', + '/user/:user/api/:version', + ], + // Attach middlewares at the begining of the req/res cycle + // to perform custom operations. + middlewares: [ + function noop () { + return function noopMiddleware (req, res, next) { + next(); + } } - } - ] + ] + } }; // If useProfiler is true every response will be served with an // X-SQLAPI-Profile header containing elapsed timing for various diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 85b6cf2a..0d3e43b6 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -2,21 +2,23 @@ // Disable by using <=0 value. module.exports.gc_interval = 10000; module.exports.routes = { - paths: [ - // In case the base_url has a :user param the username will be the one specified in the URL, - // otherwise it will fallback to extract the username from the host header. - '/api/:version', - '/user/:user/api/:version', - ], - // Attach middlewares at the begining of the req/res cycle - // to perform custom operations. - middlewares: [ - function noop () { - return function noopMiddleware (req, res, next) { - next(); + api: { + paths: [ + // In case the base_url has a :user param the username will be the one specified in the URL, + // otherwise it will fallback to extract the username from the host header. + '/api/:version', + '/user/:user/api/:version', + ], + // Attach middlewares at the begining of the req/res cycle + // to perform custom operations. + middlewares: [ + function noop () { + return function noopMiddleware (req, res, next) { + next(); + } } - } - ] + ] + } }; // If useProfiler is true every response will be served with an // X-SQLAPI-Profile header containing elapsed timing for various diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 65ca48fe..c051485d 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -2,21 +2,23 @@ // Disable by using <=0 value. module.exports.gc_interval = 10000; module.exports.routes = { - paths: [ - // In case the base_url has a :user param the username will be the one specified in the URL, - // otherwise it will fallback to extract the username from the host header. - '/api/:version', - '/user/:user/api/:version', - ], - // Attach middlewares at the begining of the req/res cycle - // to perform custom operations. - middlewares: [ - function noop () { - return function noopMiddleware (req, res, next) { - next(); + api: { + paths: [ + // In case the base_url has a :user param the username will be the one specified in the URL, + // otherwise it will fallback to extract the username from the host header. + '/api/:version', + '/user/:user/api/:version', + ], + // Attach middlewares at the begining of the req/res cycle + // to perform custom operations. + middlewares: [ + function noop () { + return function noopMiddleware (req, res, next) { + next(); + } } - } - ] + ] + } }; // If useProfiler is true every response will be served with an // X-SQLAPI-Profile header containing elapsed timing for various diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 6b55652a..ab961c41 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -2,21 +2,23 @@ // Disable by using <=0 value. module.exports.gc_interval = 10000; module.exports.routes = { - paths: [ - // In case the base_url has a :user param the username will be the one specified in the URL, - // otherwise it will fallback to extract the username from the host header. - '/api/:version', - '/user/:user/api/:version', - ], - // Attach middlewares at the begining of the req/res cycle - // to perform custom operations. - middlewares: [ - function noop () { - return function noopMiddleware (req, res, next) { - next(); + api: { + paths: [ + // In case the base_url has a :user param the username will be the one specified in the URL, + // otherwise it will fallback to extract the username from the host header. + '/api/:version', + '/user/:user/api/:version', + ], + // Attach middlewares at the begining of the req/res cycle + // to perform custom operations. + middlewares: [ + function noop () { + return function noopMiddleware (req, res, next) { + next(); + } } - } - ] + ] + } }; // If useProfiler is true every response will be served with an // X-SQLAPI-Profile header containing elapsed timing for various From f71c7bc3a8aaeddb7db870c8d07a7df12a97a259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 17:54:09 +0200 Subject: [PATCH 26/37] Typo --- config/environments/development.js.example | 2 +- config/environments/production.js.example | 2 +- config/environments/staging.js.example | 2 +- config/environments/test.js.example | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 1ef61235..2942417e 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -4,7 +4,7 @@ module.exports.gc_interval = 10000; module.exports.routes = { api: { paths: [ - // In case the base_url has a :user param the username will be the one specified in the URL, + // In case the path has a :user param the username will be the one specified in the URL, // otherwise it will fallback to extract the username from the host header. '/api/:version', '/user/:user/api/:version', diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 0d3e43b6..6c0314d5 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -4,7 +4,7 @@ module.exports.gc_interval = 10000; module.exports.routes = { api: { paths: [ - // In case the base_url has a :user param the username will be the one specified in the URL, + // In case the path has a :user param the username will be the one specified in the URL, // otherwise it will fallback to extract the username from the host header. '/api/:version', '/user/:user/api/:version', diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index c051485d..88e2da44 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -4,7 +4,7 @@ module.exports.gc_interval = 10000; module.exports.routes = { api: { paths: [ - // In case the base_url has a :user param the username will be the one specified in the URL, + // In case the path has a :user param the username will be the one specified in the URL, // otherwise it will fallback to extract the username from the host header. '/api/:version', '/user/:user/api/:version', diff --git a/config/environments/test.js.example b/config/environments/test.js.example index ab961c41..1a960d81 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -4,7 +4,7 @@ module.exports.gc_interval = 10000; module.exports.routes = { api: { paths: [ - // In case the base_url has a :user param the username will be the one specified in the URL, + // In case the path has a :user param the username will be the one specified in the URL, // otherwise it will fallback to extract the username from the host header. '/api/:version', '/user/:user/api/:version', From 54cf5a072fb3899ebe3abc7ab92d029584d84590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Tue, 1 Oct 2019 18:22:13 +0200 Subject: [PATCH 27/37] Improve naming --- app.js | 4 ++-- app/server.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app.js b/app.js index f131da42..061f9839 100755 --- a/app.js +++ b/app.js @@ -86,9 +86,9 @@ if (global.settings.statsd) { } const statsClient = StatsClient.getInstance(global.settings.statsd); -const serverFactory = require('./app/server'); +const createServer = require('./app/server'); -const server = serverFactory(statsClient); +const server = createServer(statsClient); const listener = server.listen(global.settings.node_port, global.settings.node_host); listener.on('listening', function() { console.info("Using Node.js %s", process.version); diff --git a/app/server.js b/app/server.js index 51baedb2..61ed537e 100644 --- a/app/server.js +++ b/app/server.js @@ -17,7 +17,7 @@ process.env.PGAPPNAME = process.env.PGAPPNAME || 'cartodb_sqlapi'; require('./utils/date_to_json'); // jshint maxcomplexity:9 -module.exports = function serverFactory (statsClient) { +module.exports = function createServer (statsClient) { const app = express(); const redisPool = new RedisPool({ name: 'sql-api', From 2951f07ae573e14e56287a1636749d0d3e568584 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 2 Oct 2019 12:10:25 +0200 Subject: [PATCH 28/37] Use Object.values --- app/controllers/api-router.js | 29 ++++++++++++++++------------- app/server.js | 2 +- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/app/controllers/api-router.js b/app/controllers/api-router.js index 6193cb61..28768fac 100644 --- a/app/controllers/api-router.js +++ b/app/controllers/api-router.js @@ -68,22 +68,25 @@ module.exports = class ApiRouter { } route (app) { - const apiRouter = router({ mergeParams: true }); - const paths = this.routes.paths || []; - const middlewares = this.routes.middlewares || []; + Object.values(this.routes).forEach(route => { + const apiRouter = router({ mergeParams: true }); - middlewares.forEach(middleware => apiRouter.use(middleware())); + const paths = route.paths; + const middlewares = route.middlewares || []; - apiRouter.use(socketTimeout()); - apiRouter.use(logger()); - apiRouter.use(profiler({ statsClient: this.statsClient })); - apiRouter.use(cors()); - apiRouter.use(servedByHostHeader()); + middlewares.forEach(middleware => apiRouter.use(middleware())); - this.queryController.route(apiRouter); - this.copyController.route(apiRouter); - this.jobController.route(apiRouter); + apiRouter.use(socketTimeout()); + apiRouter.use(logger()); + apiRouter.use(profiler({ statsClient: this.statsClient })); + apiRouter.use(cors()); + apiRouter.use(servedByHostHeader()); - paths.forEach(path => app.use(path, apiRouter)); + this.queryController.route(apiRouter); + this.copyController.route(apiRouter); + this.jobController.route(apiRouter); + + paths.forEach(path => app.use(path, apiRouter)); + }); } }; diff --git a/app/server.js b/app/server.js index 61ed537e..77f51e5d 100644 --- a/app/server.js +++ b/app/server.js @@ -76,7 +76,7 @@ module.exports = function createServer (statsClient) { versionController.route(app); const apiRouter = new ApiRouter({ - routes: global.settings.routes.api, + routes: global.settings.routes, redisPool, metadataBackend, statsClient, From 51ea2b066d79610535804246fe6a77437ad5ea32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 2 Oct 2019 16:02:13 +0200 Subject: [PATCH 29/37] Create sql-router to encapsulate common stuff from api-router. Extract WIP jobs from job-controller and attatch it to api-router. Note: it modifies routes structure in configuration. --- app/controllers/api-router.js | 80 +++++++--------------- app/controllers/copy_controller.js | 8 +-- app/controllers/health_check_controller.js | 11 +-- app/controllers/job_controller.js | 40 ++--------- app/controllers/jobs_wip_controller.js | 39 +++++++++++ app/controllers/query_controller.js | 6 +- app/controllers/sql-router.js | 75 ++++++++++++++++++++ app/controllers/version_controller.js | 10 +-- app/server.js | 19 +---- config/environments/development.js.example | 11 ++- config/environments/production.js.example | 11 ++- config/environments/staging.js.example | 11 ++- config/environments/test.js.example | 11 ++- 13 files changed, 192 insertions(+), 140 deletions(-) create mode 100644 app/controllers/jobs_wip_controller.js create mode 100644 app/controllers/sql-router.js diff --git a/app/controllers/api-router.js b/app/controllers/api-router.js index 28768fac..3ec37916 100644 --- a/app/controllers/api-router.js +++ b/app/controllers/api-router.js @@ -2,8 +2,11 @@ const { Router: router } = require('express'); -const UserDatabaseService = require('../services/user_database_service'); -const UserLimitsService = require('../services/user_limits'); +const SqlRouter = require('./sql-router'); + +const HealthCheckController = require('./health_check_controller'); +const VersionController = require('./version_controller'); +const JobsWipController = require('./jobs_wip_controller'); const BatchLogger = require('../../batch/batch-logger'); @@ -13,44 +16,8 @@ const JobBackend = require('../../batch/job_backend'); const JobCanceller = require('../../batch/job_canceller'); const JobService = require('../../batch/job_service'); -const QueryController = require('./query_controller'); -const CopyController = require('./copy_controller'); -const JobController = require('./job_controller'); - -const socketTimeout = require('../middlewares/socket-timeout'); -const logger = require('../middlewares/logger'); -const profiler = require('../middlewares/profiler'); -const cors = require('../middlewares/cors'); -const servedByHostHeader = require('../middlewares/served-by-host-header'); - module.exports = class ApiRouter { - constructor ({ routes, redisPool, metadataBackend, statsClient, dataIngestionLogger }) { - this.routes = routes; - this.statsClient = statsClient; - - const userLimitsServiceOptions = { - limits: { - rateLimitsEnabled: global.settings.ratelimits.rateLimitsEnabled - } - }; - - const userDatabaseService = new UserDatabaseService(metadataBackend); - const userLimitsService = new UserLimitsService(metadataBackend, userLimitsServiceOptions); - - this.queryController = new QueryController( - metadataBackend, - userDatabaseService, - statsClient, - userLimitsService - ); - - this.copyController = new CopyController( - metadataBackend, - userDatabaseService, - userLimitsService, - dataIngestionLogger - ); - + constructor ({ redisPool, metadataBackend, statsClient, dataIngestionLogger }) { const logger = new BatchLogger(global.settings.batch_log_filename, 'batch-queries'); const jobPublisher = new JobPublisher(redisPool); const jobQueue = new JobQueue(metadataBackend, jobPublisher, logger); @@ -58,17 +25,20 @@ module.exports = class ApiRouter { const jobCanceller = new JobCanceller(); const jobService = new JobService(jobBackend, jobCanceller, logger); - this.jobController = new JobController( + this.healthCheckController = new HealthCheckController(); + this.versionController = new VersionController(); + this.jobsWipController = new JobsWipController({ jobService }); + + this.sqlRouter = new SqlRouter({ metadataBackend, - userDatabaseService, - jobService, statsClient, - userLimitsService - ); + dataIngestionLogger, + jobService + }); } - route (app) { - Object.values(this.routes).forEach(route => { + route (app, routes) { + routes.forEach(route => { const apiRouter = router({ mergeParams: true }); const paths = route.paths; @@ -76,15 +46,17 @@ module.exports = class ApiRouter { middlewares.forEach(middleware => apiRouter.use(middleware())); - apiRouter.use(socketTimeout()); - apiRouter.use(logger()); - apiRouter.use(profiler({ statsClient: this.statsClient })); - apiRouter.use(cors()); - apiRouter.use(servedByHostHeader()); + // FIXME: version controller should be atached to the main entry point: "/" + // instead of "/api/:version" or "/user/:user/api/:version" + this.healthCheckController.route(apiRouter); - this.queryController.route(apiRouter); - this.copyController.route(apiRouter); - this.jobController.route(apiRouter); + // FIXME: version controller should be atached to the main entry point: "/" + // instead of "/api/:version" or "/user/:user/api/:version" + this.versionController.route(apiRouter); + + this.jobsWipController.route(apiRouter); + + this.sqlRouter.route(apiRouter, route.sql); paths.forEach(path => app.use(path, apiRouter)); }); diff --git a/app/controllers/copy_controller.js b/app/controllers/copy_controller.js index cf1c9b79..ed217627 100644 --- a/app/controllers/copy_controller.js +++ b/app/controllers/copy_controller.js @@ -25,7 +25,7 @@ module.exports = class CopyController { this.logger = logger; } - route (apiRouter) { + route (sqlRouter) { const copyFromMiddlewares = endpointGroup => { return [ initializeProfilerMiddleware('copyfrom'), @@ -56,9 +56,9 @@ module.exports = class CopyController { ]; }; - apiRouter.post('/sql/copyfrom', copyFromMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_FROM)); - apiRouter.get('/sql/copyto', copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO)); - apiRouter.post('/sql/copyto', copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO)); + sqlRouter.post('/copyfrom', copyFromMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_FROM)); + sqlRouter.get('/copyto', copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO)); + sqlRouter.post('/copyto', copyToMiddlewares(RATE_LIMIT_ENDPOINTS_GROUPS.COPY_TO)); } }; diff --git a/app/controllers/health_check_controller.js b/app/controllers/health_check_controller.js index 8c6369d7..39be6633 100644 --- a/app/controllers/health_check_controller.js +++ b/app/controllers/health_check_controller.js @@ -3,17 +3,12 @@ const HealthCheckBackend = require('../monitoring/health_check'); module.exports = class HealthCheckController { - constructor ({ routes }) { - this.routes = routes; + constructor () { this.healthCheckBackend = new HealthCheckBackend(global.settings.disabled_file); } - route (app) { - const paths = this.routes.paths || []; - - paths.forEach(path => app.get(`${path}/health`, healthCheck({ - healthCheckBackend: this.healthCheckBackend - }))); + route (apiRouter) { + apiRouter.get('/health', healthCheck({ healthCheckBackend: this.healthCheckBackend })); } }; diff --git a/app/controllers/job_controller.js b/app/controllers/job_controller.js index 8dd9e9b8..a43e86b6 100644 --- a/app/controllers/job_controller.js +++ b/app/controllers/job_controller.js @@ -22,7 +22,7 @@ module.exports = class JobController { this.userLimitsService = userLimitsService; } - route (apiRouter) { + route (sqlRouter) { const jobMiddlewares = composeJobMiddlewares( this.metadataBackend, this.userDatabaseService, @@ -31,34 +31,23 @@ module.exports = class JobController { this.userLimitsService ); - apiRouter.get( - '/jobs-wip', - bodyParserMiddleware(), - listWorkInProgressJobs(this.jobService), - sendResponse(), - errorMiddleware() - ); - - apiRouter.post( - '/sql/job', + sqlRouter.post('/job', [ bodyParserMiddleware(), checkBodyPayloadSize(), params({ strategy: 'job' }), logMiddleware(logMiddleware.TYPES.JOB), jobMiddlewares('create', createJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_CREATE) - ); + ]); - apiRouter.get( - '/sql/job/:job_id', + sqlRouter.get('/job/:job_id', [ bodyParserMiddleware(), jobMiddlewares('retrieve', getJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_GET) - ); + ]); - apiRouter.delete( - '/sql/job/:job_id', + sqlRouter.delete('/job/:job_id', [ bodyParserMiddleware(), jobMiddlewares('cancel', cancelJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_DELETE) - ); + ]); } }; @@ -155,21 +144,6 @@ function createJob (jobService) { }; } -function listWorkInProgressJobs (jobService) { - return function listWorkInProgressJobsMiddleware (req, res, next) { - jobService.listWorkInProgressJobs((err, list) => { - if (err) { - return next(err); - } - - res.body = list; - - next(); - }); - }; -} - - function checkBodyPayloadSize () { return function checkBodyPayloadSizeMiddleware(req, res, next) { const payload = JSON.stringify(req.body); diff --git a/app/controllers/jobs_wip_controller.js b/app/controllers/jobs_wip_controller.js new file mode 100644 index 00000000..dbe1e8ee --- /dev/null +++ b/app/controllers/jobs_wip_controller.js @@ -0,0 +1,39 @@ +'use strict'; + +const bodyParserMiddleware = require('../middlewares/body-parser'); +const errorMiddleware = require('../middlewares/error'); + +module.exports = class JobsWipController { + constructor ({ jobService }) { + this.jobService = jobService; + } + + route (apiRouter) { + apiRouter.get('/jobs-wip', [ + bodyParserMiddleware(), + listWorkInProgressJobs(this.jobService), + sendResponse(), + errorMiddleware() + ]); + } +}; + +function listWorkInProgressJobs (jobService) { + return function listWorkInProgressJobsMiddleware (req, res, next) { + jobService.listWorkInProgressJobs((err, list) => { + if (err) { + return next(err); + } + + res.body = list; + + next(); + }); + }; +} + +function sendResponse () { + return function sendResponseMiddleware (req, res) { + res.status(res.statusCode || 200).send(res.body); + }; +} diff --git a/app/controllers/query_controller.js b/app/controllers/query_controller.js index d36dcdc4..5e1e9077 100644 --- a/app/controllers/query_controller.js +++ b/app/controllers/query_controller.js @@ -32,7 +32,7 @@ module.exports = class QueryController { this.userLimitsService = userLimitsService; } - route (apiRouter) { + route (sqlRouter) { const forceToBeMaster = false; const queryMiddlewares = () => { @@ -61,8 +61,8 @@ module.exports = class QueryController { ]; }; - apiRouter.all('/sql', queryMiddlewares()); - apiRouter.all('/sql.:f', queryMiddlewares()); + sqlRouter.all('/', queryMiddlewares()); + sqlRouter.all('.:f', queryMiddlewares()); } }; diff --git a/app/controllers/sql-router.js b/app/controllers/sql-router.js new file mode 100644 index 00000000..c073e03b --- /dev/null +++ b/app/controllers/sql-router.js @@ -0,0 +1,75 @@ +'use strict'; + +const { Router: router } = require('express'); + +const UserDatabaseService = require('../services/user_database_service'); +const UserLimitsService = require('../services/user_limits'); + +const socketTimeout = require('../middlewares/socket-timeout'); +const logger = require('../middlewares/logger'); +const profiler = require('../middlewares/profiler'); +const cors = require('../middlewares/cors'); +const servedByHostHeader = require('../middlewares/served-by-host-header'); + +const QueryController = require('./query_controller'); +const CopyController = require('./copy_controller'); +const JobController = require('./job_controller'); + +module.exports = class SqlRouter { + constructor ({ routes, metadataBackend, statsClient, dataIngestionLogger, jobService }) { + this.routes = routes; + + const userLimitsServiceOptions = { + limits: { + rateLimitsEnabled: global.settings.ratelimits.rateLimitsEnabled + } + }; + const userDatabaseService = new UserDatabaseService(metadataBackend); + const userLimitsService = new UserLimitsService(metadataBackend, userLimitsServiceOptions); + + this.queryController = new QueryController( + metadataBackend, + userDatabaseService, + statsClient, + userLimitsService + ); + + this.copyController = new CopyController( + metadataBackend, + userDatabaseService, + userLimitsService, + dataIngestionLogger + ); + + this.jobController = new JobController( + metadataBackend, + userDatabaseService, + jobService, + statsClient, + userLimitsService + ); + } + + route (apiRouter, routes) { + routes.forEach(route => { + const sqlRouter = router({ mergeParams: true }); + + const paths = route.paths; + const middlewares = route.middlewares || []; + + middlewares.forEach(middleware => sqlRouter.use(middleware())); + + sqlRouter.use(socketTimeout()); + sqlRouter.use(logger()); + sqlRouter.use(profiler({ statsClient: this.statsClient })); + sqlRouter.use(cors()); + sqlRouter.use(servedByHostHeader()); + + this.queryController.route(sqlRouter); + this.copyController.route(sqlRouter); + this.jobController.route(sqlRouter); + + paths.forEach(path => apiRouter.use(path, sqlRouter)); + }); + } +}; diff --git a/app/controllers/version_controller.js b/app/controllers/version_controller.js index 75c5c019..3ca0e508 100644 --- a/app/controllers/version_controller.js +++ b/app/controllers/version_controller.js @@ -5,14 +5,8 @@ const versions = { }; module.exports = class VersionController { - constructor ({ routes }) { - this.routes = routes; - } - - route (app) { - const paths = this.routes.paths || []; - - paths.forEach(path => app.get(`${path}/version`, version())); + route (apiRouter) { + apiRouter.get('/version', version()); } }; diff --git a/app/server.js b/app/server.js index 77f51e5d..04c2257d 100644 --- a/app/server.js +++ b/app/server.js @@ -7,8 +7,6 @@ const RedisPool = require('redis-mpool'); const cartodbRedis = require('cartodb-redis'); const Logger = require('./services/logger'); const ApiRouter = require('./controllers/api-router'); -const HealthCheckController = require('./controllers/health_check_controller'); -const VersionController = require('./controllers/version_controller'); const batchFactory = require('../batch'); process.env.PGAPPNAME = process.env.PGAPPNAME || 'cartodb_sqlapi'; @@ -61,28 +59,13 @@ module.exports = function createServer (statsClient) { const dataIngestionLogger = new Logger(global.settings.dataIngestionLogPath, 'data-ingestion'); app.dataIngestionLogger = dataIngestionLogger; - // FIXME: health controller should be atached to the main entry point: "/" - // instead of "/api/v1/" or "/user/:user/api/:version" - const healthCheckController = new HealthCheckController({ - routes: global.settings.routes.api - }); - healthCheckController.route(app); - - // FIXME: version controller should be atached to the main entry point: "/" - // instead of "/api/:version" or "/user/:user/api/:version" - const versionController = new VersionController({ - routes: global.settings.routes.api - }); - versionController.route(app); - const apiRouter = new ApiRouter({ - routes: global.settings.routes, redisPool, metadataBackend, statsClient, dataIngestionLogger }); - apiRouter.route(app); + apiRouter.route(app, global.settings.routes.api); const isBatchProcess = process.argv.indexOf('--no-batch') === -1; diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 2942417e..9fcc8a4c 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -2,7 +2,7 @@ // Disable by using <=0 value. module.exports.gc_interval = 10000; module.exports.routes = { - api: { + api: [{ paths: [ // In case the path has a :user param the username will be the one specified in the URL, // otherwise it will fallback to extract the username from the host header. @@ -17,8 +17,13 @@ module.exports.routes = { next(); } } - ] - } + ], + sql: [{ + paths: [ + '/sql' + ] + }] + }] }; // If useProfiler is true every response will be served with an // X-SQLAPI-Profile header containing elapsed timing for various diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 6c0314d5..660f5d1a 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -2,7 +2,7 @@ // Disable by using <=0 value. module.exports.gc_interval = 10000; module.exports.routes = { - api: { + api: [{ paths: [ // In case the path has a :user param the username will be the one specified in the URL, // otherwise it will fallback to extract the username from the host header. @@ -17,8 +17,13 @@ module.exports.routes = { next(); } } - ] - } + ], + sql: [{ + paths: [ + '/sql' + ] + }] + }] }; // If useProfiler is true every response will be served with an // X-SQLAPI-Profile header containing elapsed timing for various diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 88e2da44..5570b270 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -2,7 +2,7 @@ // Disable by using <=0 value. module.exports.gc_interval = 10000; module.exports.routes = { - api: { + api: [{ paths: [ // In case the path has a :user param the username will be the one specified in the URL, // otherwise it will fallback to extract the username from the host header. @@ -17,8 +17,13 @@ module.exports.routes = { next(); } } - ] - } + ], + sql: [{ + paths: [ + '/sql' + ] + }] + }] }; // If useProfiler is true every response will be served with an // X-SQLAPI-Profile header containing elapsed timing for various diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 1a960d81..91a8f65f 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -2,7 +2,7 @@ // Disable by using <=0 value. module.exports.gc_interval = 10000; module.exports.routes = { - api: { + api: [{ paths: [ // In case the path has a :user param the username will be the one specified in the URL, // otherwise it will fallback to extract the username from the host header. @@ -17,8 +17,13 @@ module.exports.routes = { next(); } } - ] - } + ], + sql: [{ + paths: [ + '/sql' + ] + }] + }] }; // If useProfiler is true every response will be served with an // X-SQLAPI-Profile header containing elapsed timing for various From cec1c609c04d134f45ee74a81f82c07a19bf11bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 2 Oct 2019 17:35:34 +0200 Subject: [PATCH 30/37] Have a default configuration of "routes" when missing environment settings. It prevents errors while deploying (if the new setting is not available yet) --- app/server-options.js | 32 ++++++++++++++++++++++ app/server.js | 4 ++- config/environments/development.js.example | 10 +++++-- config/environments/production.js.example | 10 +++++-- config/environments/staging.js.example | 10 +++++-- config/environments/test.js.example | 10 +++++-- 6 files changed, 67 insertions(+), 9 deletions(-) create mode 100644 app/server-options.js diff --git a/app/server-options.js b/app/server-options.js new file mode 100644 index 00000000..762af298 --- /dev/null +++ b/app/server-options.js @@ -0,0 +1,32 @@ +'use strict' + +module.exports = function getServerOptions () { + const defaults = { + routes: { + // Each entry corresponds with an express' router. + // You must define at least one path. However, middlewares are optional. + api: [{ + // Required: path where other "routers" or "controllers" will be attached to. + paths: [ + // In case the path has a :user param the username will be the one specified in the URL, + // otherwise it will fallback to extract the username from the host header. + '/api/:version', + '/user/:user/api/:version', + ], + // Optional: attach middlewares at the begining of the router + // to perform custom operations. + middlewares: [], + sql: [{ + // Required + paths: [ + '/sql' + ], + // Optional + middlewares: [] + }] + }] + } + }; + + return Object.assign({}, defaults, global.settings); +}; diff --git a/app/server.js b/app/server.js index 04c2257d..3c5d55a7 100644 --- a/app/server.js +++ b/app/server.js @@ -8,6 +8,7 @@ const cartodbRedis = require('cartodb-redis'); const Logger = require('./services/logger'); const ApiRouter = require('./controllers/api-router'); const batchFactory = require('../batch'); +const getServerOptions = require('./server-options'); process.env.PGAPPNAME = process.env.PGAPPNAME || 'cartodb_sqlapi'; @@ -16,6 +17,7 @@ require('./utils/date_to_json'); // jshint maxcomplexity:9 module.exports = function createServer (statsClient) { + const options = getServerOptions(); const app = express(); const redisPool = new RedisPool({ name: 'sql-api', @@ -65,7 +67,7 @@ module.exports = function createServer (statsClient) { statsClient, dataIngestionLogger }); - apiRouter.route(app, global.settings.routes.api); + apiRouter.route(app, options.routes.api); const isBatchProcess = process.argv.indexOf('--no-batch') === -1; diff --git a/config/environments/development.js.example b/config/environments/development.js.example index 9fcc8a4c..343897e0 100644 --- a/config/environments/development.js.example +++ b/config/environments/development.js.example @@ -2,14 +2,17 @@ // Disable by using <=0 value. module.exports.gc_interval = 10000; module.exports.routes = { + // Each entry corresponds with an express' router. + // You must define at least one path. However, middlewares are optional. api: [{ + // Required: path where other "routers" or "controllers" will be attached to. paths: [ // In case the path has a :user param the username will be the one specified in the URL, // otherwise it will fallback to extract the username from the host header. '/api/:version', '/user/:user/api/:version', ], - // Attach middlewares at the begining of the req/res cycle + // Optional: attach middlewares at the begining of the router // to perform custom operations. middlewares: [ function noop () { @@ -19,9 +22,12 @@ module.exports.routes = { } ], sql: [{ + // Required paths: [ '/sql' - ] + ], + // Optional + middlewares: [] }] }] }; diff --git a/config/environments/production.js.example b/config/environments/production.js.example index 660f5d1a..6ca0fc8d 100644 --- a/config/environments/production.js.example +++ b/config/environments/production.js.example @@ -2,14 +2,17 @@ // Disable by using <=0 value. module.exports.gc_interval = 10000; module.exports.routes = { + // Each entry corresponds with an express' router. + // You must define at least one path. However, middlewares are optional. api: [{ + // Required: path where other "routers" or "controllers" will be attached to. paths: [ // In case the path has a :user param the username will be the one specified in the URL, // otherwise it will fallback to extract the username from the host header. '/api/:version', '/user/:user/api/:version', ], - // Attach middlewares at the begining of the req/res cycle + // Optional: attach middlewares at the begining of the router // to perform custom operations. middlewares: [ function noop () { @@ -19,9 +22,12 @@ module.exports.routes = { } ], sql: [{ + // Required paths: [ '/sql' - ] + ], + // Optional + middlewares: [] }] }] }; diff --git a/config/environments/staging.js.example b/config/environments/staging.js.example index 5570b270..abbb7e01 100644 --- a/config/environments/staging.js.example +++ b/config/environments/staging.js.example @@ -2,14 +2,17 @@ // Disable by using <=0 value. module.exports.gc_interval = 10000; module.exports.routes = { + // Each entry corresponds with an express' router. + // You must define at least one path. However, middlewares are optional. api: [{ + // Required: path where other "routers" or "controllers" will be attached to. paths: [ // In case the path has a :user param the username will be the one specified in the URL, // otherwise it will fallback to extract the username from the host header. '/api/:version', '/user/:user/api/:version', ], - // Attach middlewares at the begining of the req/res cycle + // Optional: attach middlewares at the begining of the router // to perform custom operations. middlewares: [ function noop () { @@ -19,9 +22,12 @@ module.exports.routes = { } ], sql: [{ + // Required paths: [ '/sql' - ] + ], + // Optional + middlewares: [] }] }] }; diff --git a/config/environments/test.js.example b/config/environments/test.js.example index 91a8f65f..2f8cddf3 100644 --- a/config/environments/test.js.example +++ b/config/environments/test.js.example @@ -2,14 +2,17 @@ // Disable by using <=0 value. module.exports.gc_interval = 10000; module.exports.routes = { + // Each entry corresponds with an express' router. + // You must define at least one path. However, middlewares are optional. api: [{ + // Required: path where other "routers" or "controllers" will be attached to. paths: [ // In case the path has a :user param the username will be the one specified in the URL, // otherwise it will fallback to extract the username from the host header. '/api/:version', '/user/:user/api/:version', ], - // Attach middlewares at the begining of the req/res cycle + // Optional: attach middlewares at the begining of the router // to perform custom operations. middlewares: [ function noop () { @@ -19,9 +22,12 @@ module.exports.routes = { } ], sql: [{ + // Required paths: [ '/sql' - ] + ], + // Optional + middlewares: [] }] }] }; From ff6a4b063e41d3f8cb7d41eef19213e993b04a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 2 Oct 2019 17:36:18 +0200 Subject: [PATCH 31/37] Linter --- app/server-options.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/server-options.js b/app/server-options.js index 762af298..448e35de 100644 --- a/app/server-options.js +++ b/app/server-options.js @@ -1,4 +1,4 @@ -'use strict' +'use strict'; module.exports = function getServerOptions () { const defaults = { From b5bb06db6c12593d8850b24023c662f6082abf45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Wed, 2 Oct 2019 19:10:32 +0200 Subject: [PATCH 32/37] Add test to validate custom middlewares --- test/acceptance/custom-middlewares.js | 347 ++++++++++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 test/acceptance/custom-middlewares.js diff --git a/test/acceptance/custom-middlewares.js b/test/acceptance/custom-middlewares.js new file mode 100644 index 00000000..396a49e2 --- /dev/null +++ b/test/acceptance/custom-middlewares.js @@ -0,0 +1,347 @@ +'use strict'; + +require('../helper'); + +const querystring = require('querystring'); +const fs = require('fs'); + +const createServer = require('../../app/server'); +const assert = require('../support/assert'); + +describe('custom middlewares', function () { + const RESPONSE_OK = { + statusCode: 200 + }; + const RESPONSE_KO_TEAPOT = { + statusCode: 418 + }; + + const customMiddleware = function teapot () { + return function teapotMiddleware (req, res) { + res.status(418).send('I\'m a teapot'); + }; + }; + + describe('wired in /api/v1/', function () { + before(function () { + this.backupRoutes = global.settings.routes; + + global.settings.routes = { + api: [{ + paths: [ + '/api/:version', + '/user/:user/api/:version', + ], + middlewares: [ + customMiddleware + ], + sql: [{ + paths: [ + '/sql' + ] + }] + }] + }; + + this.server = createServer(); + }); + + after(function () { + global.settings.routes = this.backupRoutes; + }); + + it('GET /api/v1/health returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: '/api/v1/health', + method: 'GET' + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + + it('GET /api/v1/version returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: '/api/v1/version', + method: 'GET' + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + + it('GET /api/v1/sql returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: '/api/v1/sql?q=SELECT%201', + method: 'GET' + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + + it('GET /api/v1/sql/job/:id returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: '/api/v1/sql/job/wadus', + method: 'GET' + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + + it('POST /api/v2/sql/job returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: '/api/v2/sql/job?api_key=1234', + headers: { + host: 'vizzuality.cartodb.com', + 'Content-Type': 'application/x-www-form-urlencoded' + }, + method: 'POST', + data: querystring.stringify({ + query: 'SELECT * FROM untitle_table_4' + }) + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + + it('DELETE /api/v1/sql/job/:id returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: '/api/v1/sql/job/wadus', + method: 'DELETE' + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + + it('POST /api/v1/sql/copyfrom returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: `/api/v1/sql/copyfrom?${querystring.stringify({ + q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)" + })}`, + data: fs.createReadStream(`${__dirname}/../support/csv/copy_test_table.csv`), + headers: { + host: 'vizzuality.cartodb.com' + }, + method: 'POST' + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + + it('GET /api/v1/sql/copyto returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: `/api/v1/sql/copyto?${querystring.stringify({ + filename: '/tmp/output.dmp' + })}`, + headers: { + host: 'vizzuality.cartodb.com' + }, + method: 'GET' + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + }); + + describe('wired in /api/v1/sql', function () { + before(function () { + this.backupRoutes = global.settings.routes; + + global.settings.routes = { + api: [{ + paths: [ + '/api/:version', + '/user/:user/api/:version', + ], + sql: [{ + paths: [ + '/sql' + ], + middlewares: [ customMiddleware ] + }] + }] + }; + + this.server = createServer(); + }); + + after(function () { + global.settings.routes = this.backupRoutes; + }); + + it('GET /api/v1/health returns 200', function (done) { + assert.response(this.server, { + url: '/api/v1/health', + method: 'GET' + }, RESPONSE_OK, function (err, res) { + if (err) { + return done(err); + } + + const parsed = JSON.parse(res.body); + + assert.strictEqual(parsed.enabled, true); + assert.strictEqual(parsed.ok, true); + + done(); + }); + }); + + it('GET /api/v1/version returns 200', function (done) { + assert.response(this.server, { + url: '/api/v1/version', + method: 'GET' + }, RESPONSE_OK, function (err, res) { + if (err) { + return done(err); + } + + const parsed = JSON.parse(res.body); + + assert.ok(parsed.hasOwnProperty('cartodb_sql_api')); + + done(); + }); + }); + + it('GET /api/v1/sql returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: '/api/v1/sql?q=SELECT%201', + method: 'GET' + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + + it('GET /api/v1/sql/job/:id returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: '/api/v1/sql/job/wadus', + method: 'GET' + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + + it('POST /api/v2/sql/job returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: '/api/v2/sql/job?api_key=1234', + headers: { + host: 'vizzuality.cartodb.com', + 'Content-Type': 'application/x-www-form-urlencoded' + }, + method: 'POST', + data: querystring.stringify({ + query: 'SELECT * FROM untitle_table_4' + }) + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + + it('DELETE /api/v1/sql/job/:id returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: '/api/v1/sql/job/wadus', + method: 'DELETE' + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + + it('POST /api/v1/sql/copyfrom returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: `/api/v1/sql/copyfrom?${querystring.stringify({ + q: "COPY copy_endpoints_test (id, name) FROM STDIN WITH (FORMAT CSV, DELIMITER ',', HEADER true)" + })}`, + data: fs.createReadStream(`${__dirname}/../support/csv/copy_test_table.csv`), + headers: { + host: 'vizzuality.cartodb.com' + }, + method: 'POST' + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + + it('GET /api/v1/sql/copyto returns 418: I\'m a teapot', function (done) { + assert.response(this.server, { + url: `/api/v1/sql/copyto?${querystring.stringify({ + filename: '/tmp/output.dmp' + })}`, + headers: { + host: 'vizzuality.cartodb.com' + }, + method: 'GET' + }, RESPONSE_KO_TEAPOT, function (err, res) { + if (err) { + return done(err); + } + + assert.strictEqual(res.body, 'I\'m a teapot'); + done(); + }); + }); + }); +}); From 06a47a8afc853cd7729ce61df88b7a1268647a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Oct 2019 18:24:39 +0200 Subject: [PATCH 33/37] Changed folder structure to reflect application functionallity. Renamed files using hyphens instead of underscore to have a more consistent naming across the whole project --- Makefile | 2 +- app.js | 5 +++-- {app/controllers => lib/api}/api-router.js | 20 +++++++++---------- .../api/health-check-controller.js | 2 +- .../api/jobs-wip-controller.js | 4 ++-- .../api}/middlewares/access-validator.js | 2 +- .../api}/middlewares/affected-tables.js | 0 {app => lib/api}/middlewares/authorization.js | 2 +- {app => lib/api}/middlewares/body-parser.js | 0 {app => lib/api}/middlewares/cache-channel.js | 0 {app => lib/api}/middlewares/cache-control.js | 0 .../middlewares/cancel-on-client-abort.js | 0 .../api}/middlewares/connection-params.js | 0 {app => lib/api}/middlewares/content.js | 2 +- {app => lib/api}/middlewares/cors.js | 0 {app => lib/api}/middlewares/db-quota.js | 0 {app => lib/api}/middlewares/error.js | 4 ++-- {app => lib/api}/middlewares/formatter.js | 2 +- {app => lib/api}/middlewares/last-modified.js | 0 {app => lib/api}/middlewares/log.js | 2 +- {app => lib/api}/middlewares/logger.js | 0 {app => lib/api}/middlewares/params.js | 4 ++-- {app => lib/api}/middlewares/profiler.js | 2 +- .../api}/middlewares/query-may-write.js | 2 +- {app => lib/api}/middlewares/rate-limit.js | 0 .../api}/middlewares/served-by-host-header.js | 0 .../api}/middlewares/socket-timeout.js | 0 {app => lib/api}/middlewares/surrogate-key.js | 0 .../api}/middlewares/timeout-limits.js | 0 {app => lib/api}/middlewares/user.js | 2 +- .../api/sql/copy-controller.js | 8 ++++---- .../api/sql/job-controller.js | 0 .../api/sql/query-controller.js | 0 .../controllers => lib/api/sql}/sql-router.js | 10 +++++----- .../api/version-controller.js | 0 {app => lib}/auth/apikey.js | 0 app/auth/auth_api.js => lib/auth/auth-api.js | 0 {app => lib}/auth/oauth.js | 2 +- {batch => lib/batch}/README.md | 0 {batch => lib/batch}/batch-logger.js | 3 +-- {batch => lib/batch}/batch.js | 0 {batch => lib/batch}/index.js | 14 ++++++------- .../batch/job-backend.js | 2 +- .../batch/job-canceller.js | 0 batch/job_queue.js => lib/batch/job-queue.js | 0 .../job_runner.js => lib/batch/job-runner.js | 4 ++-- .../batch/job-service.js | 4 ++-- .../job_status.js => lib/batch/job-status.js | 0 {batch => lib/batch}/leader/locker.js | 0 .../batch}/leader/provider/redis-distlock.js | 0 .../maintenance/host-user-queue-mover.js | 0 .../maintenance/remove-old-batch-jobs.js | 0 .../batch/models/job-base.js | 4 ++-- .../batch/models/job-factory.js | 6 +++--- .../batch/models/job-fallback.js | 10 +++++----- .../batch/models/job-multiple.js | 4 ++-- .../batch/models/job-simple.js | 4 ++-- .../batch/models/job-state-machine.js | 2 +- {batch => lib/batch}/models/query/fallback.js | 4 ++-- .../batch/models/query/main-fallback.js | 4 ++-- .../batch/models/query/query-base.js | 2 +- .../batch/models/query/query-factory.js | 2 +- .../batch/models/query/query-fallback.js | 4 ++-- {batch => lib/batch}/models/query/query.js | 4 ++-- {batch => lib/batch}/pubsub/channel.js | 0 {batch => lib/batch}/pubsub/job-publisher.js | 0 {batch => lib/batch}/pubsub/job-subscriber.js | 0 .../batch/query-runner.js | 0 .../batch}/scheduler/capacity/fixed.js | 0 .../batch}/scheduler/capacity/http-load.js | 0 .../batch}/scheduler/capacity/http-simple.js | 0 .../batch}/scheduler/host-scheduler.js | 0 {batch => lib/batch}/scheduler/scheduler.js | 0 .../batch/user-database-metadata-service.js | 0 {batch => lib/batch}/util/debug.js | 0 {batch => lib/batch}/util/forever.js | 0 .../models/bin-encoder.js | 0 .../models/cartodb-request.js | 0 {app => lib}/models/formats/README | 0 {app => lib}/models/formats/index.js | 0 {app => lib}/models/formats/ogr.js | 0 {app => lib}/models/formats/ogr/csv.js | 0 {app => lib}/models/formats/ogr/geopackage.js | 0 {app => lib}/models/formats/ogr/kml.js | 0 {app => lib}/models/formats/ogr/shp.js | 0 {app => lib}/models/formats/ogr/spatialite.js | 0 {app => lib}/models/formats/pg.js | 0 {app => lib}/models/formats/pg/arraybuffer.js | 2 +- {app => lib}/models/formats/pg/geojson.js | 4 ++-- {app => lib}/models/formats/pg/json.js | 2 +- {app => lib}/models/formats/pg/svg.js | 0 {app => lib}/models/formats/pg/topojson.js | 6 ++++-- .../monitoring/health-check.js | 0 .../postgresql/error-codes.js | 0 {app => lib}/server-options.js | 0 {app => lib}/server.js | 6 +++--- .../services/error-handler-factory.js | 4 ++-- .../services/error-handler.js | 0 {app => lib}/services/logger.js | 0 .../services/pg-entities-access-validator.js | 0 .../services/stream-copy-metrics.js | 2 +- .../services/stream-copy.js | 0 {app => lib}/services/throttler-stream.js | 0 .../services/user-database-service.js | 0 .../services/user-limits.js | 0 {app => lib}/stats/client.js | 0 {app => lib}/stats/profiler-proxy.js | 0 .../utils/content-disposition.js | 0 .../utils/date-to-json.js | 0 .../utils/filename-sanitizer.js | 0 {app => lib}/utils/logs.js | 0 {app => lib}/utils/md5.js | 0 .../query_info.js => lib/utils/query-info.js | 0 .../utils/query-may-write.js | 0 .../{app.auth.test.js => app-auth-test.js} | 2 +- ...iguration.js => app-configuration-test.js} | 2 +- test/acceptance/{app.test.js => app-test.js} | 2 +- .../{auth-api.js => auth-api-test.js} | 2 +- ...backend_crash.js => backend-crash-test.js} | 2 +- ...atch-drain.test.js => batch-drain-test.js} | 16 +++++++-------- ...ch-limits.test.js => batch-limits-test.js} | 4 ++-- ...query.test.js => batch-multiquery-test.js} | 2 +- .../batch/{batch.test.js => batch-test.js} | 2 +- .../{batch.wip.test.js => batch-wip-test.js} | 2 +- ....test.js => job-callback-template-test.js} | 2 +- ....fallback.test.js => job-fallback-test.js} | 2 +- ....limit.test.js => job-query-limit-test.js} | 6 +++--- ....order.test.js => job-query-order-test.js} | 2 +- ...eout.test.js => job-query-timeout-test.js} | 2 +- .../batch/{job.test.js => job-test.js} | 4 ++-- ...{job.timing.test.js => job-timing-test.js} | 2 +- ...test.js => leader-job-query-order-test.js} | 2 +- ...leader-multiple-users-query-order-test.js} | 2 +- ...imit.test.js => queued-jobs-limit-test.js} | 4 ++-- ...-basic.test.js => scheduler-basic-test.js} | 2 +- .../{use-cases.test.js => use-cases-test.js} | 2 +- ...cache-headers.js => cache-headers-test.js} | 2 +- .../{copy-abort.js => copy-abort-test.js} | 4 ++-- ...py-endpoints.js => copy-endpoints-test.js} | 4 ++-- ...-statements.js => copy-statements-test.js} | 2 +- ...copy-throttle.js => copy-throttle-test.js} | 4 ++-- ...dlewares.js => custom-middlewares-test.js} | 2 +- ...error-handler.js => error-handler-test.js} | 2 +- .../{arraybuffer.js => arraybuffer-test.js} | 2 +- .../acceptance/export/{csv.js => csv-test.js} | 2 +- .../export/{folder.js => folder-test.js} | 2 +- .../export/{geojson.js => geojson-test.js} | 2 +- .../{geopackage.js => geopackage-test.js} | 2 +- .../acceptance/export/{kml.js => kml-test.js} | 2 +- .../{shapefile.js => shapefile-test.js} | 2 +- .../{spatialite.js => spatialite-test.js} | 2 +- .../acceptance/export/{svg.js => svg-test.js} | 2 +- .../export/{timeout.js => timeout-test.js} | 0 .../export/{topojson.js => topojson-test.js} | 2 +- ...ontend_abort.js => frontend_abort-test.js} | 2 +- .../{handle-query.js => handle-query-test.js} | 4 ++-- .../{health_check.js => health-check-test.js} | 2 +- ...header.js => last-modified-header-test.js} | 2 +- test/acceptance/{log.js => log-test.js} | 4 ++-- .../{pagination.js => pagination-test.js} | 2 +- ...s => pg-entities-access-validator-test.js} | 0 .../{pg-types.js => pg-types-test.js} | 2 +- ...t-values.js => query-float-values-test.js} | 2 +- ...y-multipart.js => query-multipart-test.js} | 2 +- ...y-returning.js => query-returning-test.js} | 2 +- .../{rate-limit.js => rate-limit-test.js} | 4 ++-- .../{regressions.js => regressions-test.js} | 2 +- .../{skipfields.js => skipfields-test.js} | 2 +- ...-responses.js => stream-responses-test.js} | 2 +- ...surrogate-key.js => surrogate-key-test.js} | 2 +- ...stem-queries.js => system-queries-test.js} | 2 +- .../{timeout.js => timeout-test.js} | 2 +- .../{transaction.js => transaction-test.js} | 2 +- ...che-channel.js => x-cache-channel-test.js} | 2 +- ...ob_backend.test.js => job-backend-test.js} | 12 +++++------ ...anceller.test.js => job-canceller-test.js} | 14 ++++++------- ...ublisher.test.js => job-publisher-test.js} | 4 ++-- .../{job-queue.test.js => job-queue-test.js} | 14 ++++++------- ...{job_runner.test.js => job-runner-test.js} | 20 +++++++++---------- ...ob_service.test.js => job-service-test.js} | 14 ++++++------- .../batch/{locker.js => locker-test.js} | 4 ++-- .../batch/{scheduler.js => scheduler-test.js} | 6 +++--- ...tream_copy.test.js => stream-copy-test.js} | 2 +- test/support/batch-test-client.js | 8 ++++---- .../{redis_utils.js => redis-utils.js} | 0 test/support/test-client.js | 4 ++-- ...{apikeyauth.test.js => apikeyauth-test.js} | 2 +- ...job_publisher.js => job-publisher-test.js} | 4 ++-- .../batch/{job_queue.js => job-queue-test.js} | 2 +- ...b_subscriber.js => job-subscriber-test.js} | 4 ++-- ....test.js => error-handler-factory-test.js} | 6 +++--- ..._handler.test.js => error-handler-test.js} | 6 +++--- .../{health_check.js => health-check-test.js} | 2 +- .../{bin_encoder.js => bin-encoder-test.js} | 2 +- test/unit/{oauth.test.js => oauth-test.js} | 4 ++-- ...s => pg-entities-access-validator-test.js} | 2 +- ...{query_info.test.js => query-info-test.js} | 2 +- test/websocket_test/public/run_server.sh | 6 +++++- 198 files changed, 241 insertions(+), 235 deletions(-) rename {app/controllers => lib/api}/api-router.js (77%) rename app/controllers/health_check_controller.js => lib/api/health-check-controller.js (94%) rename app/controllers/jobs_wip_controller.js => lib/api/jobs-wip-controller.js (87%) rename {app => lib/api}/middlewares/access-validator.js (83%) rename {app => lib/api}/middlewares/affected-tables.js (100%) rename {app => lib/api}/middlewares/authorization.js (98%) rename {app => lib/api}/middlewares/body-parser.js (100%) rename {app => lib/api}/middlewares/cache-channel.js (100%) rename {app => lib/api}/middlewares/cache-control.js (100%) rename {app => lib/api}/middlewares/cancel-on-client-abort.js (100%) rename {app => lib/api}/middlewares/connection-params.js (100%) rename {app => lib/api}/middlewares/content.js (86%) rename {app => lib/api}/middlewares/cors.js (100%) rename {app => lib/api}/middlewares/db-quota.js (100%) rename {app => lib/api}/middlewares/error.js (93%) rename {app => lib/api}/middlewares/formatter.js (85%) rename {app => lib/api}/middlewares/last-modified.js (100%) rename {app => lib/api}/middlewares/log.js (97%) rename {app => lib/api}/middlewares/logger.js (100%) rename {app => lib/api}/middlewares/params.js (97%) rename {app => lib/api}/middlewares/profiler.js (94%) rename {app => lib/api}/middlewares/query-may-write.js (79%) rename {app => lib/api}/middlewares/rate-limit.js (100%) rename {app => lib/api}/middlewares/served-by-host-header.js (100%) rename {app => lib/api}/middlewares/socket-timeout.js (100%) rename {app => lib/api}/middlewares/surrogate-key.js (100%) rename {app => lib/api}/middlewares/timeout-limits.js (100%) rename {app => lib/api}/middlewares/user.js (94%) rename app/controllers/copy_controller.js => lib/api/sql/copy-controller.js (96%) rename app/controllers/job_controller.js => lib/api/sql/job-controller.js (100%) rename app/controllers/query_controller.js => lib/api/sql/query-controller.js (100%) rename {app/controllers => lib/api/sql}/sql-router.js (88%) rename app/controllers/version_controller.js => lib/api/version-controller.js (100%) rename {app => lib}/auth/apikey.js (100%) rename app/auth/auth_api.js => lib/auth/auth-api.js (100%) rename {app => lib}/auth/oauth.js (99%) rename {batch => lib/batch}/README.md (100%) rename {batch => lib/batch}/batch-logger.js (80%) rename {batch => lib/batch}/batch.js (100%) rename {batch => lib/batch}/index.js (78%) rename batch/job_backend.js => lib/batch/job-backend.js (99%) rename batch/job_canceller.js => lib/batch/job-canceller.js (100%) rename batch/job_queue.js => lib/batch/job-queue.js (100%) rename batch/job_runner.js => lib/batch/job-runner.js (97%) rename batch/job_service.js => lib/batch/job-service.js (97%) rename batch/job_status.js => lib/batch/job-status.js (100%) rename {batch => lib/batch}/leader/locker.js (100%) rename {batch => lib/batch}/leader/provider/redis-distlock.js (100%) rename {batch => lib/batch}/maintenance/host-user-queue-mover.js (100%) rename {batch => lib/batch}/maintenance/remove-old-batch-jobs.js (100%) rename batch/models/job_base.js => lib/batch/models/job-base.js (96%) rename batch/models/job_factory.js => lib/batch/models/job-factory.js (78%) rename batch/models/job_fallback.js => lib/batch/models/job-fallback.js (96%) rename batch/models/job_multiple.js => lib/batch/models/job-multiple.js (97%) rename batch/models/job_simple.js => lib/batch/models/job-simple.js (89%) rename batch/models/job_state_machine.js => lib/batch/models/job-state-machine.js (96%) rename {batch => lib/batch}/models/query/fallback.js (96%) rename batch/models/query/main_fallback.js => lib/batch/models/query/main-fallback.js (95%) rename batch/models/query/query_base.js => lib/batch/models/query/query-base.js (92%) rename batch/models/query/query_factory.js => lib/batch/models/query/query-factory.js (86%) rename batch/models/query/query_fallback.js => lib/batch/models/query/query-fallback.js (95%) rename {batch => lib/batch}/models/query/query.js (94%) rename {batch => lib/batch}/pubsub/channel.js (100%) rename {batch => lib/batch}/pubsub/job-publisher.js (100%) rename {batch => lib/batch}/pubsub/job-subscriber.js (100%) rename batch/query_runner.js => lib/batch/query-runner.js (100%) rename {batch => lib/batch}/scheduler/capacity/fixed.js (100%) rename {batch => lib/batch}/scheduler/capacity/http-load.js (100%) rename {batch => lib/batch}/scheduler/capacity/http-simple.js (100%) rename {batch => lib/batch}/scheduler/host-scheduler.js (100%) rename {batch => lib/batch}/scheduler/scheduler.js (100%) rename batch/user_database_metadata_service.js => lib/batch/user-database-metadata-service.js (100%) rename {batch => lib/batch}/util/debug.js (100%) rename {batch => lib/batch}/util/forever.js (100%) rename app/models/bin_encoder.js => lib/models/bin-encoder.js (100%) rename app/models/cartodb_request.js => lib/models/cartodb-request.js (100%) rename {app => lib}/models/formats/README (100%) rename {app => lib}/models/formats/index.js (100%) rename {app => lib}/models/formats/ogr.js (100%) rename {app => lib}/models/formats/ogr/csv.js (100%) rename {app => lib}/models/formats/ogr/geopackage.js (100%) rename {app => lib}/models/formats/ogr/kml.js (100%) rename {app => lib}/models/formats/ogr/shp.js (100%) rename {app => lib}/models/formats/ogr/spatialite.js (100%) rename {app => lib}/models/formats/pg.js (100%) rename {app => lib}/models/formats/pg/arraybuffer.js (97%) rename {app => lib}/models/formats/pg/geojson.js (96%) rename {app => lib}/models/formats/pg/json.js (98%) rename {app => lib}/models/formats/pg/svg.js (100%) rename {app => lib}/models/formats/pg/topojson.js (98%) rename app/monitoring/health_check.js => lib/monitoring/health-check.js (100%) rename app/postgresql/error_codes.js => lib/postgresql/error-codes.js (100%) rename {app => lib}/server-options.js (100%) rename {app => lib}/server.js (95%) rename app/services/error_handler_factory.js => lib/services/error-handler-factory.js (90%) rename app/services/error_handler.js => lib/services/error-handler.js (100%) rename {app => lib}/services/logger.js (100%) rename {app => lib}/services/pg-entities-access-validator.js (100%) rename app/services/stream_copy_metrics.js => lib/services/stream-copy-metrics.js (96%) rename app/services/stream_copy.js => lib/services/stream-copy.js (100%) rename {app => lib}/services/throttler-stream.js (100%) rename app/services/user_database_service.js => lib/services/user-database-service.js (100%) rename app/services/user_limits.js => lib/services/user-limits.js (100%) rename {app => lib}/stats/client.js (100%) rename {app => lib}/stats/profiler-proxy.js (100%) rename app/utils/content_disposition.js => lib/utils/content-disposition.js (100%) rename app/utils/date_to_json.js => lib/utils/date-to-json.js (100%) rename app/utils/filename_sanitizer.js => lib/utils/filename-sanitizer.js (100%) rename {app => lib}/utils/logs.js (100%) rename {app => lib}/utils/md5.js (100%) rename app/utils/query_info.js => lib/utils/query-info.js (100%) rename app/utils/query_may_write.js => lib/utils/query-may-write.js (100%) rename test/acceptance/{app.auth.test.js => app-auth-test.js} (98%) rename test/acceptance/{app-configuration.js => app-configuration-test.js} (99%) rename test/acceptance/{app.test.js => app-test.js} (99%) rename test/acceptance/{auth-api.js => auth-api-test.js} (99%) rename test/acceptance/{backend_crash.js => backend-crash-test.js} (97%) rename test/acceptance/batch/{batch-drain.test.js => batch-drain-test.js} (82%) rename test/acceptance/batch/{batch-limits.test.js => batch-limits-test.js} (93%) rename test/acceptance/batch/{batch.multiquery.test.js => batch-multiquery-test.js} (99%) rename test/acceptance/batch/{batch.test.js => batch-test.js} (99%) rename test/acceptance/batch/{batch.wip.test.js => batch-wip-test.js} (98%) rename test/acceptance/batch/{job.callback-template.test.js => job-callback-template-test.js} (98%) rename test/acceptance/batch/{job.fallback.test.js => job-fallback-test.js} (99%) rename test/acceptance/batch/{job.query.limit.test.js => job-query-limit-test.js} (95%) rename test/acceptance/batch/{job.query.order.test.js => job-query-order-test.js} (96%) rename test/acceptance/batch/{job.query.timeout.test.js => job-query-timeout-test.js} (98%) rename test/acceptance/batch/{job.test.js => job-test.js} (98%) rename test/acceptance/batch/{job.timing.test.js => job-timing-test.js} (98%) rename test/acceptance/batch/{leader.job.query.order.test.js => leader-job-query-order-test.js} (98%) rename test/acceptance/batch/{leader-multiple-users-query-order.test.js => leader-multiple-users-query-order-test.js} (98%) rename test/acceptance/batch/{queued-jobs-limit.test.js => queued-jobs-limit-test.js} (94%) rename test/acceptance/batch/{scheduler-basic.test.js => scheduler-basic-test.js} (98%) rename test/acceptance/batch/{use-cases.test.js => use-cases-test.js} (98%) rename test/acceptance/{cache-headers.js => cache-headers-test.js} (98%) rename test/acceptance/{copy-abort.js => copy-abort-test.js} (98%) rename test/acceptance/{copy-endpoints.js => copy-endpoints-test.js} (99%) rename test/acceptance/{copy-statements.js => copy-statements-test.js} (98%) rename test/acceptance/{copy-throttle.js => copy-throttle-test.js} (97%) rename test/acceptance/{custom-middlewares.js => custom-middlewares-test.js} (99%) rename test/acceptance/{error-handler.js => error-handler-test.js} (95%) rename test/acceptance/export/{arraybuffer.js => arraybuffer-test.js} (96%) rename test/acceptance/export/{csv.js => csv-test.js} (99%) rename test/acceptance/export/{folder.js => folder-test.js} (95%) rename test/acceptance/export/{geojson.js => geojson-test.js} (99%) rename test/acceptance/export/{geopackage.js => geopackage-test.js} (98%) rename test/acceptance/export/{kml.js => kml-test.js} (99%) rename test/acceptance/export/{shapefile.js => shapefile-test.js} (99%) rename test/acceptance/export/{spatialite.js => spatialite-test.js} (97%) rename test/acceptance/export/{svg.js => svg-test.js} (99%) rename test/acceptance/export/{timeout.js => timeout-test.js} (100%) rename test/acceptance/export/{topojson.js => topojson-test.js} (99%) rename test/acceptance/{frontend_abort.js => frontend_abort-test.js} (97%) rename test/acceptance/{handle-query.js => handle-query-test.js} (97%) rename test/acceptance/{health_check.js => health-check-test.js} (97%) rename test/acceptance/{last-modified-header.js => last-modified-header-test.js} (98%) rename test/acceptance/{log.js => log-test.js} (98%) rename test/acceptance/{pagination.js => pagination-test.js} (99%) rename test/acceptance/{pg-entities-access-validator.js => pg-entities-access-validator-test.js} (100%) rename test/acceptance/{pg-types.js => pg-types-test.js} (99%) rename test/acceptance/{query-float-values.js => query-float-values-test.js} (98%) rename test/acceptance/{query-multipart.js => query-multipart-test.js} (94%) rename test/acceptance/{query-returning.js => query-returning-test.js} (99%) rename test/acceptance/{rate-limit.js => rate-limit-test.js} (96%) rename test/acceptance/{regressions.js => regressions-test.js} (98%) rename test/acceptance/{skipfields.js => skipfields-test.js} (98%) rename test/acceptance/{stream-responses.js => stream-responses-test.js} (97%) rename test/acceptance/{surrogate-key.js => surrogate-key-test.js} (98%) rename test/acceptance/{system-queries.js => system-queries-test.js} (98%) rename test/acceptance/{timeout.js => timeout-test.js} (96%) rename test/acceptance/{transaction.js => transaction-test.js} (97%) rename test/acceptance/{x-cache-channel.js => x-cache-channel-test.js} (98%) rename test/integration/batch/{job_backend.test.js => job-backend-test.js} (95%) rename test/integration/batch/{job_canceller.test.js => job-canceller-test.js} (88%) rename test/integration/batch/{job_publisher.test.js => job-publisher-test.js} (90%) rename test/integration/batch/{job-queue.test.js => job-queue-test.js} (92%) rename test/integration/batch/{job_runner.test.js => job-runner-test.js} (79%) rename test/integration/batch/{job_service.test.js => job-service-test.js} (93%) rename test/integration/batch/{locker.js => locker-test.js} (93%) rename test/integration/batch/{scheduler.js => scheduler-test.js} (96%) rename test/integration/{stream_copy.test.js => stream-copy-test.js} (90%) rename test/support/{redis_utils.js => redis-utils.js} (100%) rename test/unit/{apikeyauth.test.js => apikeyauth-test.js} (98%) rename test/unit/batch/{job_publisher.js => job-publisher-test.js} (89%) rename test/unit/batch/{job_queue.js => job-queue-test.js} (96%) rename test/unit/batch/{job_subscriber.js => job-subscriber-test.js} (94%) rename test/unit/{error_handler_factory.test.js => error-handler-factory-test.js} (91%) rename test/unit/{error_handler.test.js => error-handler-test.js} (97%) rename test/unit/{health_check.js => health-check-test.js} (94%) rename test/unit/model/{bin_encoder.js => bin-encoder-test.js} (97%) rename test/unit/{oauth.test.js => oauth-test.js} (98%) rename test/unit/{pg-entities-access-validator.test.js => pg-entities-access-validator-test.js} (98%) rename test/unit/{query_info.test.js => query-info-test.js} (97%) diff --git a/Makefile b/Makefile index b2c773f0..a7601016 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ check: jshint: @echo "***jshint***" - @./node_modules/.bin/jshint app/ batch/ test/ app.js + @./node_modules/.bin/jshint lib/ test/ app.js TEST_SUITE := $(shell find test/{unit,integration,acceptance} -name "*.js") TEST_SUITE_UNIT := $(shell find test/unit -name "*.js") diff --git a/app.js b/app.js index 061f9839..a9527fc2 100755 --- a/app.js +++ b/app.js @@ -77,7 +77,8 @@ if (!global.settings.routes) { const version = require("./package").version; -const StatsClient = require('./app/stats/client'); +const StatsClient = require('./lib/stats/client'); + if (global.settings.statsd) { // Perform keyword substitution in statsd if (global.settings.statsd.prefix) { @@ -86,7 +87,7 @@ if (global.settings.statsd) { } const statsClient = StatsClient.getInstance(global.settings.statsd); -const createServer = require('./app/server'); +const createServer = require('./lib/server'); const server = createServer(statsClient); const listener = server.listen(global.settings.node_port, global.settings.node_host); diff --git a/app/controllers/api-router.js b/lib/api/api-router.js similarity index 77% rename from app/controllers/api-router.js rename to lib/api/api-router.js index 3ec37916..75501467 100644 --- a/app/controllers/api-router.js +++ b/lib/api/api-router.js @@ -2,19 +2,19 @@ const { Router: router } = require('express'); -const SqlRouter = require('./sql-router'); +const SqlRouter = require('./sql/sql-router'); -const HealthCheckController = require('./health_check_controller'); -const VersionController = require('./version_controller'); -const JobsWipController = require('./jobs_wip_controller'); +const HealthCheckController = require('./health-check-controller'); +const VersionController = require('./version-controller'); +const JobsWipController = require('./jobs-wip-controller'); -const BatchLogger = require('../../batch/batch-logger'); +const BatchLogger = require('../batch/batch-logger'); -const JobPublisher = require('../../batch/pubsub/job-publisher'); -const JobQueue = require('../../batch/job_queue'); -const JobBackend = require('../../batch/job_backend'); -const JobCanceller = require('../../batch/job_canceller'); -const JobService = require('../../batch/job_service'); +const JobPublisher = require('../batch/pubsub/job-publisher'); +const JobQueue = require('../batch/job-queue'); +const JobBackend = require('../batch/job-backend'); +const JobCanceller = require('../batch/job-canceller'); +const JobService = require('../batch/job-service'); module.exports = class ApiRouter { constructor ({ redisPool, metadataBackend, statsClient, dataIngestionLogger }) { diff --git a/app/controllers/health_check_controller.js b/lib/api/health-check-controller.js similarity index 94% rename from app/controllers/health_check_controller.js rename to lib/api/health-check-controller.js index 39be6633..4137d087 100644 --- a/app/controllers/health_check_controller.js +++ b/lib/api/health-check-controller.js @@ -1,6 +1,6 @@ 'use strict'; -const HealthCheckBackend = require('../monitoring/health_check'); +const HealthCheckBackend = require('../monitoring/health-check'); module.exports = class HealthCheckController { constructor () { diff --git a/app/controllers/jobs_wip_controller.js b/lib/api/jobs-wip-controller.js similarity index 87% rename from app/controllers/jobs_wip_controller.js rename to lib/api/jobs-wip-controller.js index dbe1e8ee..885f2604 100644 --- a/app/controllers/jobs_wip_controller.js +++ b/lib/api/jobs-wip-controller.js @@ -1,7 +1,7 @@ 'use strict'; -const bodyParserMiddleware = require('../middlewares/body-parser'); -const errorMiddleware = require('../middlewares/error'); +const bodyParserMiddleware = require('./middlewares/body-parser'); +const errorMiddleware = require('./middlewares/error'); module.exports = class JobsWipController { constructor ({ jobService }) { diff --git a/app/middlewares/access-validator.js b/lib/api/middlewares/access-validator.js similarity index 83% rename from app/middlewares/access-validator.js rename to lib/api/middlewares/access-validator.js index f62624e4..70fd33bf 100644 --- a/app/middlewares/access-validator.js +++ b/lib/api/middlewares/access-validator.js @@ -1,6 +1,6 @@ 'use strict'; -const pgEntitiesAccessValidator = require('../services/pg-entities-access-validator'); +const pgEntitiesAccessValidator = require('../../services/pg-entities-access-validator'); module.exports = function accessValidator () { return function accessValidatorMiddleware (req, res, next) { diff --git a/app/middlewares/affected-tables.js b/lib/api/middlewares/affected-tables.js similarity index 100% rename from app/middlewares/affected-tables.js rename to lib/api/middlewares/affected-tables.js diff --git a/app/middlewares/authorization.js b/lib/api/middlewares/authorization.js similarity index 98% rename from app/middlewares/authorization.js rename to lib/api/middlewares/authorization.js index cbed5455..8a97928c 100644 --- a/app/middlewares/authorization.js +++ b/lib/api/middlewares/authorization.js @@ -1,6 +1,6 @@ 'use strict'; -const AuthApi = require('../auth/auth_api'); +const AuthApi = require('../../auth/auth-api'); const basicAuth = require('basic-auth'); module.exports = function authorization (metadataBackend, forceToBeMaster = false) { diff --git a/app/middlewares/body-parser.js b/lib/api/middlewares/body-parser.js similarity index 100% rename from app/middlewares/body-parser.js rename to lib/api/middlewares/body-parser.js diff --git a/app/middlewares/cache-channel.js b/lib/api/middlewares/cache-channel.js similarity index 100% rename from app/middlewares/cache-channel.js rename to lib/api/middlewares/cache-channel.js diff --git a/app/middlewares/cache-control.js b/lib/api/middlewares/cache-control.js similarity index 100% rename from app/middlewares/cache-control.js rename to lib/api/middlewares/cache-control.js diff --git a/app/middlewares/cancel-on-client-abort.js b/lib/api/middlewares/cancel-on-client-abort.js similarity index 100% rename from app/middlewares/cancel-on-client-abort.js rename to lib/api/middlewares/cancel-on-client-abort.js diff --git a/app/middlewares/connection-params.js b/lib/api/middlewares/connection-params.js similarity index 100% rename from app/middlewares/connection-params.js rename to lib/api/middlewares/connection-params.js diff --git a/app/middlewares/content.js b/lib/api/middlewares/content.js similarity index 86% rename from app/middlewares/content.js rename to lib/api/middlewares/content.js index 3064781f..7aeb7375 100644 --- a/app/middlewares/content.js +++ b/lib/api/middlewares/content.js @@ -1,6 +1,6 @@ 'use strict'; -const getContentDisposition = require('../utils/content_disposition'); +const getContentDisposition = require('../../utils/content-disposition'); module.exports = function content () { return function contentMiddleware (req, res, next) { diff --git a/app/middlewares/cors.js b/lib/api/middlewares/cors.js similarity index 100% rename from app/middlewares/cors.js rename to lib/api/middlewares/cors.js diff --git a/app/middlewares/db-quota.js b/lib/api/middlewares/db-quota.js similarity index 100% rename from app/middlewares/db-quota.js rename to lib/api/middlewares/db-quota.js diff --git a/app/middlewares/error.js b/lib/api/middlewares/error.js similarity index 93% rename from app/middlewares/error.js rename to lib/api/middlewares/error.js index 5c9db4a2..c63876b3 100644 --- a/app/middlewares/error.js +++ b/lib/api/middlewares/error.js @@ -1,7 +1,7 @@ 'use strict'; -const errorHandlerFactory = require('../services/error_handler_factory'); -const { stringifyForLogs } = require('../utils/logs'); +const errorHandlerFactory = require('../../services/error-handler-factory'); +const { stringifyForLogs } = require('../../utils/logs'); const MAX_ERROR_STRING_LENGTH = 1024; module.exports = function error() { diff --git a/app/middlewares/formatter.js b/lib/api/middlewares/formatter.js similarity index 85% rename from app/middlewares/formatter.js rename to lib/api/middlewares/formatter.js index 6216e2a8..baee0347 100644 --- a/app/middlewares/formatter.js +++ b/lib/api/middlewares/formatter.js @@ -1,6 +1,6 @@ 'use strict'; -const formats = require('../models/formats'); +const formats = require('../../models/formats'); module.exports = function formatter () { return function formatterMiddleware (req, res, next) { diff --git a/app/middlewares/last-modified.js b/lib/api/middlewares/last-modified.js similarity index 100% rename from app/middlewares/last-modified.js rename to lib/api/middlewares/last-modified.js diff --git a/app/middlewares/log.js b/lib/api/middlewares/log.js similarity index 97% rename from app/middlewares/log.js rename to lib/api/middlewares/log.js index 32815873..e3faf869 100644 --- a/app/middlewares/log.js +++ b/lib/api/middlewares/log.js @@ -1,6 +1,6 @@ 'use strict'; -const { stringifyForLogs } = require('../utils/logs'); +const { stringifyForLogs } = require('../../utils/logs'); const MAX_SQL_LENGTH = (global.settings.logQueries && global.settings.maxQueriesLogLength) || 1024; diff --git a/app/middlewares/logger.js b/lib/api/middlewares/logger.js similarity index 100% rename from app/middlewares/logger.js rename to lib/api/middlewares/logger.js diff --git a/app/middlewares/params.js b/lib/api/middlewares/params.js similarity index 97% rename from app/middlewares/params.js rename to lib/api/middlewares/params.js index e0d6b01d..a241a17c 100644 --- a/app/middlewares/params.js +++ b/lib/api/middlewares/params.js @@ -1,7 +1,7 @@ 'use strict'; -const sanitizeFilename = require('../utils/filename_sanitizer'); -const formats = require('../models/formats'); +const sanitizeFilename = require('../../utils/filename-sanitizer'); +const formats = require('../../models/formats'); module.exports = function params ({ strategy = 'query' } = {}) { const getParams = getParamsFromStrategy(strategy); diff --git a/app/middlewares/profiler.js b/lib/api/middlewares/profiler.js similarity index 94% rename from app/middlewares/profiler.js rename to lib/api/middlewares/profiler.js index aad91099..3fbc7b2e 100644 --- a/app/middlewares/profiler.js +++ b/lib/api/middlewares/profiler.js @@ -1,6 +1,6 @@ 'use strict'; -const Profiler = require('../stats/profiler-proxy'); +const Profiler = require('../../stats/profiler-proxy'); module.exports = function profiler ({ statsClient }) { return function profilerMiddleware (req, res, next) { diff --git a/app/middlewares/query-may-write.js b/lib/api/middlewares/query-may-write.js similarity index 79% rename from app/middlewares/query-may-write.js rename to lib/api/middlewares/query-may-write.js index bc3436ee..607557c4 100644 --- a/app/middlewares/query-may-write.js +++ b/lib/api/middlewares/query-may-write.js @@ -1,6 +1,6 @@ 'use strict'; -const queryMayWrite = require('../utils/query_may_write'); +const queryMayWrite = require('../../utils/query-may-write'); module.exports = function mayWrite () { return function mayWriteMiddleware (req, res, next) { diff --git a/app/middlewares/rate-limit.js b/lib/api/middlewares/rate-limit.js similarity index 100% rename from app/middlewares/rate-limit.js rename to lib/api/middlewares/rate-limit.js diff --git a/app/middlewares/served-by-host-header.js b/lib/api/middlewares/served-by-host-header.js similarity index 100% rename from app/middlewares/served-by-host-header.js rename to lib/api/middlewares/served-by-host-header.js diff --git a/app/middlewares/socket-timeout.js b/lib/api/middlewares/socket-timeout.js similarity index 100% rename from app/middlewares/socket-timeout.js rename to lib/api/middlewares/socket-timeout.js diff --git a/app/middlewares/surrogate-key.js b/lib/api/middlewares/surrogate-key.js similarity index 100% rename from app/middlewares/surrogate-key.js rename to lib/api/middlewares/surrogate-key.js diff --git a/app/middlewares/timeout-limits.js b/lib/api/middlewares/timeout-limits.js similarity index 100% rename from app/middlewares/timeout-limits.js rename to lib/api/middlewares/timeout-limits.js diff --git a/app/middlewares/user.js b/lib/api/middlewares/user.js similarity index 94% rename from app/middlewares/user.js rename to lib/api/middlewares/user.js index 87e71d0b..5f385337 100644 --- a/app/middlewares/user.js +++ b/lib/api/middlewares/user.js @@ -1,6 +1,6 @@ 'use strict'; -const CdbRequest = require('../models/cartodb_request'); +const CdbRequest = require('../../models/cartodb-request'); module.exports = function user(metadataBackend) { const cdbRequest = new CdbRequest(); diff --git a/app/controllers/copy_controller.js b/lib/api/sql/copy-controller.js similarity index 96% rename from app/controllers/copy_controller.js rename to lib/api/sql/copy-controller.js index ed217627..9b18025e 100644 --- a/app/controllers/copy_controller.js +++ b/lib/api/sql/copy-controller.js @@ -8,10 +8,10 @@ const { initializeProfilerMiddleware } = require('../middlewares/profiler'); const rateLimitsMiddleware = require('../middlewares/rate-limit'); const dbQuotaMiddleware = require('../middlewares/db-quota'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware; -const errorHandlerFactory = require('../services/error_handler_factory'); -const StreamCopy = require('../services/stream_copy'); -const StreamCopyMetrics = require('../services/stream_copy_metrics'); -const Throttler = require('../services/throttler-stream'); +const errorHandlerFactory = require('../../services/error-handler-factory'); +const StreamCopy = require('../../services/stream-copy'); +const StreamCopyMetrics = require('../../services/stream-copy-metrics'); +const Throttler = require('../../services/throttler-stream'); const zlib = require('zlib'); const { PassThrough } = require('stream'); const params = require('../middlewares/params'); diff --git a/app/controllers/job_controller.js b/lib/api/sql/job-controller.js similarity index 100% rename from app/controllers/job_controller.js rename to lib/api/sql/job-controller.js diff --git a/app/controllers/query_controller.js b/lib/api/sql/query-controller.js similarity index 100% rename from app/controllers/query_controller.js rename to lib/api/sql/query-controller.js diff --git a/app/controllers/sql-router.js b/lib/api/sql/sql-router.js similarity index 88% rename from app/controllers/sql-router.js rename to lib/api/sql/sql-router.js index c073e03b..431a91b6 100644 --- a/app/controllers/sql-router.js +++ b/lib/api/sql/sql-router.js @@ -2,8 +2,8 @@ const { Router: router } = require('express'); -const UserDatabaseService = require('../services/user_database_service'); -const UserLimitsService = require('../services/user_limits'); +const UserDatabaseService = require('../../services/user-database-service'); +const UserLimitsService = require('../../services/user-limits'); const socketTimeout = require('../middlewares/socket-timeout'); const logger = require('../middlewares/logger'); @@ -11,9 +11,9 @@ const profiler = require('../middlewares/profiler'); const cors = require('../middlewares/cors'); const servedByHostHeader = require('../middlewares/served-by-host-header'); -const QueryController = require('./query_controller'); -const CopyController = require('./copy_controller'); -const JobController = require('./job_controller'); +const QueryController = require('./query-controller'); +const CopyController = require('./copy-controller'); +const JobController = require('./job-controller'); module.exports = class SqlRouter { constructor ({ routes, metadataBackend, statsClient, dataIngestionLogger, jobService }) { diff --git a/app/controllers/version_controller.js b/lib/api/version-controller.js similarity index 100% rename from app/controllers/version_controller.js rename to lib/api/version-controller.js diff --git a/app/auth/apikey.js b/lib/auth/apikey.js similarity index 100% rename from app/auth/apikey.js rename to lib/auth/apikey.js diff --git a/app/auth/auth_api.js b/lib/auth/auth-api.js similarity index 100% rename from app/auth/auth_api.js rename to lib/auth/auth-api.js diff --git a/app/auth/oauth.js b/lib/auth/oauth.js similarity index 99% rename from app/auth/oauth.js rename to lib/auth/oauth.js index 298ef3c9..812f5458 100644 --- a/app/auth/oauth.js +++ b/lib/auth/oauth.js @@ -4,7 +4,7 @@ var _ = require('underscore'); var OAuthUtil = require('oauth-client'); var step = require('step'); -var CdbRequest = require('../models/cartodb_request'); +var CdbRequest = require('../models/cartodb-request'); var cdbReq = new CdbRequest(); var oAuth = (function(){ diff --git a/batch/README.md b/lib/batch/README.md similarity index 100% rename from batch/README.md rename to lib/batch/README.md diff --git a/batch/batch-logger.js b/lib/batch/batch-logger.js similarity index 80% rename from batch/batch-logger.js rename to lib/batch/batch-logger.js index 11d1ade8..91b6b35e 100644 --- a/batch/batch-logger.js +++ b/lib/batch/batch-logger.js @@ -1,6 +1,6 @@ 'use strict'; -const Logger = require('../app/services/logger'); +const Logger = require('../services/logger'); class BatchLogger extends Logger { constructor (path, name) { @@ -10,7 +10,6 @@ class BatchLogger extends Logger { log (job) { return job.log(this.logger); } - } module.exports = BatchLogger; diff --git a/batch/batch.js b/lib/batch/batch.js similarity index 100% rename from batch/batch.js rename to lib/batch/batch.js diff --git a/batch/index.js b/lib/batch/index.js similarity index 78% rename from batch/index.js rename to lib/batch/index.js index b224b2f6..9268612c 100644 --- a/batch/index.js +++ b/lib/batch/index.js @@ -1,14 +1,14 @@ 'use strict'; -var JobRunner = require('./job_runner'); -var QueryRunner = require('./query_runner'); -var JobCanceller = require('./job_canceller'); +var JobRunner = require('./job-runner'); +var QueryRunner = require('./query-runner'); +var JobCanceller = require('./job-canceller'); var JobSubscriber = require('./pubsub/job-subscriber'); -var UserDatabaseMetadataService = require('./user_database_metadata_service'); +var UserDatabaseMetadataService = require('./user-database-metadata-service'); var JobPublisher = require('./pubsub/job-publisher'); -var JobQueue = require('./job_queue'); -var JobBackend = require('./job_backend'); -var JobService = require('./job_service'); +var JobQueue = require('./job-queue'); +var JobBackend = require('./job-backend'); +var JobService = require('./job-service'); var BatchLogger = require('./batch-logger'); var Batch = require('./batch'); diff --git a/batch/job_backend.js b/lib/batch/job-backend.js similarity index 99% rename from batch/job_backend.js rename to lib/batch/job-backend.js index 45faf02d..ea079736 100644 --- a/batch/job_backend.js +++ b/lib/batch/job-backend.js @@ -2,7 +2,7 @@ var REDIS_PREFIX = 'batch:jobs:'; var REDIS_DB = 5; -var JobStatus = require('./job_status'); +var JobStatus = require('./job-status'); var queue = require('queue-async'); function JobBackend(metadataBackend, jobQueue, logger) { diff --git a/batch/job_canceller.js b/lib/batch/job-canceller.js similarity index 100% rename from batch/job_canceller.js rename to lib/batch/job-canceller.js diff --git a/batch/job_queue.js b/lib/batch/job-queue.js similarity index 100% rename from batch/job_queue.js rename to lib/batch/job-queue.js diff --git a/batch/job_runner.js b/lib/batch/job-runner.js similarity index 97% rename from batch/job_runner.js rename to lib/batch/job-runner.js index 87762fe4..86610808 100644 --- a/batch/job_runner.js +++ b/lib/batch/job-runner.js @@ -1,7 +1,7 @@ 'use strict'; -var errorCodes = require('../app/postgresql/error_codes').codeToCondition; -var jobStatus = require('./job_status'); +var errorCodes = require('../postgresql/error-codes').codeToCondition; +var jobStatus = require('./job-status'); var Profiler = require('step-profiler'); var _ = require('underscore'); diff --git a/batch/job_service.js b/lib/batch/job-service.js similarity index 97% rename from batch/job_service.js rename to lib/batch/job-service.js index 162a4dd6..c4ad55ae 100644 --- a/batch/job_service.js +++ b/lib/batch/job-service.js @@ -1,7 +1,7 @@ 'use strict'; -var JobFactory = require('./models/job_factory'); -var jobStatus = require('./job_status'); +var JobFactory = require('./models/job-factory'); +var jobStatus = require('./job-status'); function JobService(jobBackend, jobCanceller, logger) { this.jobBackend = jobBackend; diff --git a/batch/job_status.js b/lib/batch/job-status.js similarity index 100% rename from batch/job_status.js rename to lib/batch/job-status.js diff --git a/batch/leader/locker.js b/lib/batch/leader/locker.js similarity index 100% rename from batch/leader/locker.js rename to lib/batch/leader/locker.js diff --git a/batch/leader/provider/redis-distlock.js b/lib/batch/leader/provider/redis-distlock.js similarity index 100% rename from batch/leader/provider/redis-distlock.js rename to lib/batch/leader/provider/redis-distlock.js diff --git a/batch/maintenance/host-user-queue-mover.js b/lib/batch/maintenance/host-user-queue-mover.js similarity index 100% rename from batch/maintenance/host-user-queue-mover.js rename to lib/batch/maintenance/host-user-queue-mover.js diff --git a/batch/maintenance/remove-old-batch-jobs.js b/lib/batch/maintenance/remove-old-batch-jobs.js similarity index 100% rename from batch/maintenance/remove-old-batch-jobs.js rename to lib/batch/maintenance/remove-old-batch-jobs.js diff --git a/batch/models/job_base.js b/lib/batch/models/job-base.js similarity index 96% rename from batch/models/job_base.js rename to lib/batch/models/job-base.js index 9eda131a..0827c586 100644 --- a/batch/models/job_base.js +++ b/lib/batch/models/job-base.js @@ -2,8 +2,8 @@ var util = require('util'); var uuid = require('node-uuid'); -var JobStateMachine = require('./job_state_machine'); -var jobStatus = require('../job_status'); +var JobStateMachine = require('./job-state-machine'); +var jobStatus = require('../job-status'); var mandatoryProperties = [ 'job_id', 'status', diff --git a/batch/models/job_factory.js b/lib/batch/models/job-factory.js similarity index 78% rename from batch/models/job_factory.js rename to lib/batch/models/job-factory.js index 7994314a..e802615f 100644 --- a/batch/models/job_factory.js +++ b/lib/batch/models/job-factory.js @@ -1,8 +1,8 @@ 'use strict'; -var JobSimple = require('./job_simple'); -var JobMultiple = require('./job_multiple'); -var JobFallback = require('./job_fallback'); +var JobSimple = require('./job-simple'); +var JobMultiple = require('./job-multiple'); +var JobFallback = require('./job-fallback'); var Models = [ JobSimple, JobMultiple, JobFallback ]; diff --git a/batch/models/job_fallback.js b/lib/batch/models/job-fallback.js similarity index 96% rename from batch/models/job_fallback.js rename to lib/batch/models/job-fallback.js index 9caa0d1d..83d6ff74 100644 --- a/batch/models/job_fallback.js +++ b/lib/batch/models/job-fallback.js @@ -1,11 +1,11 @@ 'use strict'; var util = require('util'); -var JobBase = require('./job_base'); -var JobStatus = require('../job_status'); -var QueryFallback = require('./query/query_fallback'); -var MainFallback = require('./query/main_fallback'); -var QueryFactory = require('./query/query_factory'); +var JobBase = require('./job-base'); +var JobStatus = require('../job-status'); +var QueryFallback = require('./query/query-fallback'); +var MainFallback = require('./query/main-fallback'); +var QueryFactory = require('./query/query-factory'); function JobFallback(jobDefinition) { JobBase.call(this, jobDefinition); diff --git a/batch/models/job_multiple.js b/lib/batch/models/job-multiple.js similarity index 97% rename from batch/models/job_multiple.js rename to lib/batch/models/job-multiple.js index 85cf1d87..16042d69 100644 --- a/batch/models/job_multiple.js +++ b/lib/batch/models/job-multiple.js @@ -1,8 +1,8 @@ 'use strict'; var util = require('util'); -var JobBase = require('./job_base'); -var jobStatus = require('../job_status'); +var JobBase = require('./job-base'); +var jobStatus = require('../job-status'); function JobMultiple(jobDefinition) { JobBase.call(this, jobDefinition); diff --git a/batch/models/job_simple.js b/lib/batch/models/job-simple.js similarity index 89% rename from batch/models/job_simple.js rename to lib/batch/models/job-simple.js index ad982c36..644ea058 100644 --- a/batch/models/job_simple.js +++ b/lib/batch/models/job-simple.js @@ -1,8 +1,8 @@ 'use strict'; var util = require('util'); -var JobBase = require('./job_base'); -var jobStatus = require('../job_status'); +var JobBase = require('./job-base'); +var jobStatus = require('../job-status'); function JobSimple(jobDefinition) { JobBase.call(this, jobDefinition); diff --git a/batch/models/job_state_machine.js b/lib/batch/models/job-state-machine.js similarity index 96% rename from batch/models/job_state_machine.js rename to lib/batch/models/job-state-machine.js index f476b8a8..08bfd538 100644 --- a/batch/models/job_state_machine.js +++ b/lib/batch/models/job-state-machine.js @@ -1,7 +1,7 @@ 'use strict'; var assert = require('assert'); -var JobStatus = require('../job_status'); +var JobStatus = require('../job-status'); var validStatusTransitions = [ [JobStatus.PENDING, JobStatus.RUNNING], [JobStatus.PENDING, JobStatus.CANCELLED], diff --git a/batch/models/query/fallback.js b/lib/batch/models/query/fallback.js similarity index 96% rename from batch/models/query/fallback.js rename to lib/batch/models/query/fallback.js index 1ce5f22f..373c1020 100644 --- a/batch/models/query/fallback.js +++ b/lib/batch/models/query/fallback.js @@ -1,8 +1,8 @@ 'use strict'; var util = require('util'); -var QueryBase = require('./query_base'); -var jobStatus = require('../../job_status'); +var QueryBase = require('./query-base'); +var jobStatus = require('../../job-status'); function Fallback(index) { QueryBase.call(this, index); diff --git a/batch/models/query/main_fallback.js b/lib/batch/models/query/main-fallback.js similarity index 95% rename from batch/models/query/main_fallback.js rename to lib/batch/models/query/main-fallback.js index 7c52194b..8dd1b172 100644 --- a/batch/models/query/main_fallback.js +++ b/lib/batch/models/query/main-fallback.js @@ -1,8 +1,8 @@ 'use strict'; var util = require('util'); -var QueryBase = require('./query_base'); -var jobStatus = require('../../job_status'); +var QueryBase = require('./query-base'); +var jobStatus = require('../../job-status'); function MainFallback() { QueryBase.call(this); diff --git a/batch/models/query/query_base.js b/lib/batch/models/query/query-base.js similarity index 92% rename from batch/models/query/query_base.js rename to lib/batch/models/query/query-base.js index 737e1bf5..cf36c006 100644 --- a/batch/models/query/query_base.js +++ b/lib/batch/models/query/query-base.js @@ -1,7 +1,7 @@ 'use strict'; var util = require('util'); -var JobStateMachine = require('../job_state_machine'); +var JobStateMachine = require('../job-state-machine'); function QueryBase(index) { JobStateMachine.call(this); diff --git a/batch/models/query/query_factory.js b/lib/batch/models/query/query-factory.js similarity index 86% rename from batch/models/query/query_factory.js rename to lib/batch/models/query/query-factory.js index c33534e0..c60a2ab2 100644 --- a/batch/models/query/query_factory.js +++ b/lib/batch/models/query/query-factory.js @@ -1,6 +1,6 @@ 'use strict'; -var QueryFallback = require('./query_fallback'); +var QueryFallback = require('./query-fallback'); function QueryFactory() { } diff --git a/batch/models/query/query_fallback.js b/lib/batch/models/query/query-fallback.js similarity index 95% rename from batch/models/query/query_fallback.js rename to lib/batch/models/query/query-fallback.js index cb0579a3..4dd48a65 100644 --- a/batch/models/query/query_fallback.js +++ b/lib/batch/models/query/query-fallback.js @@ -1,10 +1,10 @@ 'use strict'; var util = require('util'); -var QueryBase = require('./query_base'); +var QueryBase = require('./query-base'); var Query = require('./query'); var Fallback = require('./fallback'); -var jobStatus = require('../../job_status'); +var jobStatus = require('../../job-status'); function QueryFallback(job, index) { QueryBase.call(this, index); diff --git a/batch/models/query/query.js b/lib/batch/models/query/query.js similarity index 94% rename from batch/models/query/query.js rename to lib/batch/models/query/query.js index 6970ae97..2d11ee7f 100644 --- a/batch/models/query/query.js +++ b/lib/batch/models/query/query.js @@ -1,8 +1,8 @@ 'use strict'; var util = require('util'); -var QueryBase = require('./query_base'); -var jobStatus = require('../../job_status'); +var QueryBase = require('./query-base'); +var jobStatus = require('../../job-status'); function Query(index) { QueryBase.call(this, index); diff --git a/batch/pubsub/channel.js b/lib/batch/pubsub/channel.js similarity index 100% rename from batch/pubsub/channel.js rename to lib/batch/pubsub/channel.js diff --git a/batch/pubsub/job-publisher.js b/lib/batch/pubsub/job-publisher.js similarity index 100% rename from batch/pubsub/job-publisher.js rename to lib/batch/pubsub/job-publisher.js diff --git a/batch/pubsub/job-subscriber.js b/lib/batch/pubsub/job-subscriber.js similarity index 100% rename from batch/pubsub/job-subscriber.js rename to lib/batch/pubsub/job-subscriber.js diff --git a/batch/query_runner.js b/lib/batch/query-runner.js similarity index 100% rename from batch/query_runner.js rename to lib/batch/query-runner.js diff --git a/batch/scheduler/capacity/fixed.js b/lib/batch/scheduler/capacity/fixed.js similarity index 100% rename from batch/scheduler/capacity/fixed.js rename to lib/batch/scheduler/capacity/fixed.js diff --git a/batch/scheduler/capacity/http-load.js b/lib/batch/scheduler/capacity/http-load.js similarity index 100% rename from batch/scheduler/capacity/http-load.js rename to lib/batch/scheduler/capacity/http-load.js diff --git a/batch/scheduler/capacity/http-simple.js b/lib/batch/scheduler/capacity/http-simple.js similarity index 100% rename from batch/scheduler/capacity/http-simple.js rename to lib/batch/scheduler/capacity/http-simple.js diff --git a/batch/scheduler/host-scheduler.js b/lib/batch/scheduler/host-scheduler.js similarity index 100% rename from batch/scheduler/host-scheduler.js rename to lib/batch/scheduler/host-scheduler.js diff --git a/batch/scheduler/scheduler.js b/lib/batch/scheduler/scheduler.js similarity index 100% rename from batch/scheduler/scheduler.js rename to lib/batch/scheduler/scheduler.js diff --git a/batch/user_database_metadata_service.js b/lib/batch/user-database-metadata-service.js similarity index 100% rename from batch/user_database_metadata_service.js rename to lib/batch/user-database-metadata-service.js diff --git a/batch/util/debug.js b/lib/batch/util/debug.js similarity index 100% rename from batch/util/debug.js rename to lib/batch/util/debug.js diff --git a/batch/util/forever.js b/lib/batch/util/forever.js similarity index 100% rename from batch/util/forever.js rename to lib/batch/util/forever.js diff --git a/app/models/bin_encoder.js b/lib/models/bin-encoder.js similarity index 100% rename from app/models/bin_encoder.js rename to lib/models/bin-encoder.js diff --git a/app/models/cartodb_request.js b/lib/models/cartodb-request.js similarity index 100% rename from app/models/cartodb_request.js rename to lib/models/cartodb-request.js diff --git a/app/models/formats/README b/lib/models/formats/README similarity index 100% rename from app/models/formats/README rename to lib/models/formats/README diff --git a/app/models/formats/index.js b/lib/models/formats/index.js similarity index 100% rename from app/models/formats/index.js rename to lib/models/formats/index.js diff --git a/app/models/formats/ogr.js b/lib/models/formats/ogr.js similarity index 100% rename from app/models/formats/ogr.js rename to lib/models/formats/ogr.js diff --git a/app/models/formats/ogr/csv.js b/lib/models/formats/ogr/csv.js similarity index 100% rename from app/models/formats/ogr/csv.js rename to lib/models/formats/ogr/csv.js diff --git a/app/models/formats/ogr/geopackage.js b/lib/models/formats/ogr/geopackage.js similarity index 100% rename from app/models/formats/ogr/geopackage.js rename to lib/models/formats/ogr/geopackage.js diff --git a/app/models/formats/ogr/kml.js b/lib/models/formats/ogr/kml.js similarity index 100% rename from app/models/formats/ogr/kml.js rename to lib/models/formats/ogr/kml.js diff --git a/app/models/formats/ogr/shp.js b/lib/models/formats/ogr/shp.js similarity index 100% rename from app/models/formats/ogr/shp.js rename to lib/models/formats/ogr/shp.js diff --git a/app/models/formats/ogr/spatialite.js b/lib/models/formats/ogr/spatialite.js similarity index 100% rename from app/models/formats/ogr/spatialite.js rename to lib/models/formats/ogr/spatialite.js diff --git a/app/models/formats/pg.js b/lib/models/formats/pg.js similarity index 100% rename from app/models/formats/pg.js rename to lib/models/formats/pg.js diff --git a/app/models/formats/pg/arraybuffer.js b/lib/models/formats/pg/arraybuffer.js similarity index 97% rename from app/models/formats/pg/arraybuffer.js rename to lib/models/formats/pg/arraybuffer.js index d0976599..cbb8a7f6 100644 --- a/app/models/formats/pg/arraybuffer.js +++ b/lib/models/formats/pg/arraybuffer.js @@ -3,7 +3,7 @@ var _ = require('underscore'); var Pg = require('./../pg'); -var ArrayBufferSer = require("../../bin_encoder"); +var ArrayBufferSer = require("../../bin-encoder"); function BinaryFormat() {} diff --git a/app/models/formats/pg/geojson.js b/lib/models/formats/pg/geojson.js similarity index 96% rename from app/models/formats/pg/geojson.js rename to lib/models/formats/pg/geojson.js index a6b49807..30f44600 100644 --- a/app/models/formats/pg/geojson.js +++ b/lib/models/formats/pg/geojson.js @@ -2,8 +2,8 @@ var _ = require('underscore'); -var Pg = require('./../pg'); -const errorHandlerFactory = require('../../../services/error_handler_factory'); +var Pg = require('./../pg'); +const errorHandlerFactory = require('../../../services/error-handler-factory'); function GeoJsonFormat() { this.buffer = ''; diff --git a/app/models/formats/pg/json.js b/lib/models/formats/pg/json.js similarity index 98% rename from app/models/formats/pg/json.js rename to lib/models/formats/pg/json.js index 55f49fc4..26fdda71 100644 --- a/app/models/formats/pg/json.js +++ b/lib/models/formats/pg/json.js @@ -3,7 +3,7 @@ var _ = require('underscore'); var Pg = require('./../pg'); -const errorHandlerFactory = require('../../../services/error_handler_factory'); +const errorHandlerFactory = require('../../../services/error-handler-factory'); function JsonFormat() { this.buffer = ''; diff --git a/app/models/formats/pg/svg.js b/lib/models/formats/pg/svg.js similarity index 100% rename from app/models/formats/pg/svg.js rename to lib/models/formats/pg/svg.js diff --git a/app/models/formats/pg/topojson.js b/lib/models/formats/pg/topojson.js similarity index 98% rename from app/models/formats/pg/topojson.js rename to lib/models/formats/pg/topojson.js index f45f740c..cb58ef1e 100644 --- a/app/models/formats/pg/topojson.js +++ b/lib/models/formats/pg/topojson.js @@ -1,5 +1,7 @@ 'use strict'; +// jshint ignore:start + var Pg = require('./../pg'); var _ = require('underscore'); var geojson = require('./geojson'); @@ -133,6 +135,6 @@ TopoJsonFormat.prototype.cancel = function() { } }; - - module.exports = TopoJsonFormat; + +// jshint ignore:end diff --git a/app/monitoring/health_check.js b/lib/monitoring/health-check.js similarity index 100% rename from app/monitoring/health_check.js rename to lib/monitoring/health-check.js diff --git a/app/postgresql/error_codes.js b/lib/postgresql/error-codes.js similarity index 100% rename from app/postgresql/error_codes.js rename to lib/postgresql/error-codes.js diff --git a/app/server-options.js b/lib/server-options.js similarity index 100% rename from app/server-options.js rename to lib/server-options.js diff --git a/app/server.js b/lib/server.js similarity index 95% rename from app/server.js rename to lib/server.js index 3c5d55a7..114fde69 100644 --- a/app/server.js +++ b/lib/server.js @@ -6,14 +6,14 @@ const mkdirp = require('mkdirp'); const RedisPool = require('redis-mpool'); const cartodbRedis = require('cartodb-redis'); const Logger = require('./services/logger'); -const ApiRouter = require('./controllers/api-router'); -const batchFactory = require('../batch'); +const ApiRouter = require('./api/api-router'); +const batchFactory = require('./batch'); const getServerOptions = require('./server-options'); process.env.PGAPPNAME = process.env.PGAPPNAME || 'cartodb_sqlapi'; // override Date.toJSON -require('./utils/date_to_json'); +require('./utils/date-to-json'); // jshint maxcomplexity:9 module.exports = function createServer (statsClient) { diff --git a/app/services/error_handler_factory.js b/lib/services/error-handler-factory.js similarity index 90% rename from app/services/error_handler_factory.js rename to lib/services/error-handler-factory.js index 5d9429c7..2186ea77 100644 --- a/app/services/error_handler_factory.js +++ b/lib/services/error-handler-factory.js @@ -1,7 +1,7 @@ 'use strict'; -const ErrorHandler = require('./error_handler'); -const { codeToCondition } = require('../postgresql/error_codes'); +const ErrorHandler = require('./error-handler'); +const { codeToCondition } = require('../postgresql/error-codes'); module.exports = function ErrorHandlerFactory (err) { if (isTimeoutError(err)) { diff --git a/app/services/error_handler.js b/lib/services/error-handler.js similarity index 100% rename from app/services/error_handler.js rename to lib/services/error-handler.js diff --git a/app/services/logger.js b/lib/services/logger.js similarity index 100% rename from app/services/logger.js rename to lib/services/logger.js diff --git a/app/services/pg-entities-access-validator.js b/lib/services/pg-entities-access-validator.js similarity index 100% rename from app/services/pg-entities-access-validator.js rename to lib/services/pg-entities-access-validator.js diff --git a/app/services/stream_copy_metrics.js b/lib/services/stream-copy-metrics.js similarity index 96% rename from app/services/stream_copy_metrics.js rename to lib/services/stream-copy-metrics.js index 6ac70b39..13f3f417 100644 --- a/app/services/stream_copy_metrics.js +++ b/lib/services/stream-copy-metrics.js @@ -1,6 +1,6 @@ 'use strict'; -const { getFormatFromCopyQuery } = require('../utils/query_info'); +const { getFormatFromCopyQuery } = require('../utils/query-info'); module.exports = class StreamCopyMetrics { constructor(logger, type, sql, user, isGzip = false) { diff --git a/app/services/stream_copy.js b/lib/services/stream-copy.js similarity index 100% rename from app/services/stream_copy.js rename to lib/services/stream-copy.js diff --git a/app/services/throttler-stream.js b/lib/services/throttler-stream.js similarity index 100% rename from app/services/throttler-stream.js rename to lib/services/throttler-stream.js diff --git a/app/services/user_database_service.js b/lib/services/user-database-service.js similarity index 100% rename from app/services/user_database_service.js rename to lib/services/user-database-service.js diff --git a/app/services/user_limits.js b/lib/services/user-limits.js similarity index 100% rename from app/services/user_limits.js rename to lib/services/user-limits.js diff --git a/app/stats/client.js b/lib/stats/client.js similarity index 100% rename from app/stats/client.js rename to lib/stats/client.js diff --git a/app/stats/profiler-proxy.js b/lib/stats/profiler-proxy.js similarity index 100% rename from app/stats/profiler-proxy.js rename to lib/stats/profiler-proxy.js diff --git a/app/utils/content_disposition.js b/lib/utils/content-disposition.js similarity index 100% rename from app/utils/content_disposition.js rename to lib/utils/content-disposition.js diff --git a/app/utils/date_to_json.js b/lib/utils/date-to-json.js similarity index 100% rename from app/utils/date_to_json.js rename to lib/utils/date-to-json.js diff --git a/app/utils/filename_sanitizer.js b/lib/utils/filename-sanitizer.js similarity index 100% rename from app/utils/filename_sanitizer.js rename to lib/utils/filename-sanitizer.js diff --git a/app/utils/logs.js b/lib/utils/logs.js similarity index 100% rename from app/utils/logs.js rename to lib/utils/logs.js diff --git a/app/utils/md5.js b/lib/utils/md5.js similarity index 100% rename from app/utils/md5.js rename to lib/utils/md5.js diff --git a/app/utils/query_info.js b/lib/utils/query-info.js similarity index 100% rename from app/utils/query_info.js rename to lib/utils/query-info.js diff --git a/app/utils/query_may_write.js b/lib/utils/query-may-write.js similarity index 100% rename from app/utils/query_may_write.js rename to lib/utils/query-may-write.js diff --git a/test/acceptance/app.auth.test.js b/test/acceptance/app-auth-test.js similarity index 98% rename from test/acceptance/app.auth.test.js rename to test/acceptance/app-auth-test.js index 34bdc2f5..2299cf93 100644 --- a/test/acceptance/app.auth.test.js +++ b/test/acceptance/app-auth-test.js @@ -2,7 +2,7 @@ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); describe('app.auth', function() { diff --git a/test/acceptance/app-configuration.js b/test/acceptance/app-configuration-test.js similarity index 99% rename from test/acceptance/app-configuration.js rename to test/acceptance/app-configuration-test.js index d42cf5a4..0c48806c 100644 --- a/test/acceptance/app-configuration.js +++ b/test/acceptance/app-configuration-test.js @@ -2,7 +2,7 @@ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); const accessControlHeaders = [ diff --git a/test/acceptance/app.test.js b/test/acceptance/app-test.js similarity index 99% rename from test/acceptance/app.test.js rename to test/acceptance/app-test.js index 8e334c80..12bd6301 100644 --- a/test/acceptance/app.test.js +++ b/test/acceptance/app-test.js @@ -16,7 +16,7 @@ */ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); var _ = require('underscore'); diff --git a/test/acceptance/auth-api.js b/test/acceptance/auth-api-test.js similarity index 99% rename from test/acceptance/auth-api.js rename to test/acceptance/auth-api-test.js index 82304981..4c9654a8 100644 --- a/test/acceptance/auth-api.js +++ b/test/acceptance/auth-api-test.js @@ -3,7 +3,7 @@ const assert = require('../support/assert'); const TestClient = require('../support/test-client'); const BatchTestClient = require('../support/batch-test-client'); -const JobStatus = require('../../batch/job_status'); +const JobStatus = require('../../lib/batch/job-status'); describe('Auth API', function () { const publicSQL = 'select * from untitle_table_4'; diff --git a/test/acceptance/backend_crash.js b/test/acceptance/backend-crash-test.js similarity index 97% rename from test/acceptance/backend_crash.js rename to test/acceptance/backend-crash-test.js index 897638f9..5e11c27f 100644 --- a/test/acceptance/backend_crash.js +++ b/test/acceptance/backend-crash-test.js @@ -25,7 +25,7 @@ it('does not hang server', function(done){ var db_port_backup = global.settings.db_port; global.settings.db_host = 'localhost'; global.settings.db_port = sql_server_port; - var server = require('../../app/server')(); + var server = require('../../lib/server')(); step( function sendQuery() { assert.response(server, { diff --git a/test/acceptance/batch/batch-drain.test.js b/test/acceptance/batch/batch-drain-test.js similarity index 82% rename from test/acceptance/batch/batch-drain.test.js rename to test/acceptance/batch/batch-drain-test.js index e02593ad..33dbdb7e 100644 --- a/test/acceptance/batch/batch-drain.test.js +++ b/test/acceptance/batch/batch-drain-test.js @@ -2,15 +2,15 @@ require('../../helper'); var assert = require('../../support/assert'); -var redisUtils = require('../../support/redis_utils'); -var batchFactory = require('../../../batch/index'); +var redisUtils = require('../../support/redis-utils'); +var batchFactory = require('../../../lib/batch/index'); -var BatchLogger = require('../../../batch/batch-logger'); -var JobPublisher = require('../../../batch/pubsub/job-publisher'); -var JobQueue = require('../../../batch/job_queue'); -var JobBackend = require('../../../batch/job_backend'); -var JobService = require('../../../batch/job_service'); -var JobCanceller = require('../../../batch/job_canceller'); +var BatchLogger = require('../../../lib/batch/batch-logger'); +var JobPublisher = require('../../../lib/batch/pubsub/job-publisher'); +var JobQueue = require('../../../lib/batch/job-queue'); +var JobBackend = require('../../../lib/batch/job-backend'); +var JobService = require('../../../lib/batch/job-service'); +var JobCanceller = require('../../../lib/batch/job-canceller'); var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); describe('batch module', function() { diff --git a/test/acceptance/batch/batch-limits.test.js b/test/acceptance/batch/batch-limits-test.js similarity index 93% rename from test/acceptance/batch/batch-limits.test.js rename to test/acceptance/batch/batch-limits-test.js index fa83d8f4..9b74c7a0 100644 --- a/test/acceptance/batch/batch-limits.test.js +++ b/test/acceptance/batch/batch-limits-test.js @@ -4,8 +4,8 @@ require('../../helper'); var assert = require('../../support/assert'); var BatchTestClient = require('../../support/batch-test-client'); -var JobStatus = require('../../../batch/job_status'); -var redisUtils = require('../../support/redis_utils'); +var JobStatus = require('../../../lib/batch/job-status'); +var redisUtils = require('../../support/redis-utils'); var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); const db_utils = require('../../support/db_utils'); diff --git a/test/acceptance/batch/batch.multiquery.test.js b/test/acceptance/batch/batch-multiquery-test.js similarity index 99% rename from test/acceptance/batch/batch.multiquery.test.js rename to test/acceptance/batch/batch-multiquery-test.js index 1214c4a9..3a52e554 100644 --- a/test/acceptance/batch/batch.multiquery.test.js +++ b/test/acceptance/batch/batch-multiquery-test.js @@ -3,7 +3,7 @@ require('../../helper'); var BatchTestClient = require('../../support/batch-test-client'); -var JobStatus = require('../../../batch/job_status'); +var JobStatus = require('../../../lib/batch/job-status'); var assert = require('../../support/assert'); var queue = require('queue-async'); diff --git a/test/acceptance/batch/batch.test.js b/test/acceptance/batch/batch-test.js similarity index 99% rename from test/acceptance/batch/batch.test.js rename to test/acceptance/batch/batch-test.js index 7f3dec72..9da854af 100644 --- a/test/acceptance/batch/batch.test.js +++ b/test/acceptance/batch/batch-test.js @@ -5,7 +5,7 @@ require('../../helper'); var assert = require('../../support/assert'); var queue = require('queue-async'); var BatchTestClient = require('../../support/batch-test-client'); -var JobStatus = require('../../../batch/job_status'); +var JobStatus = require('../../../lib/batch/job-status'); describe('batch happy cases', function() { diff --git a/test/acceptance/batch/batch.wip.test.js b/test/acceptance/batch/batch-wip-test.js similarity index 98% rename from test/acceptance/batch/batch.wip.test.js rename to test/acceptance/batch/batch-wip-test.js index e41e3ab5..63e0303d 100644 --- a/test/acceptance/batch/batch.wip.test.js +++ b/test/acceptance/batch/batch-wip-test.js @@ -4,7 +4,7 @@ require('../../helper'); var assert = require('../../support/assert'); var BatchTestClient = require('../../support/batch-test-client'); -var JobStatus = require('../../../batch/job_status'); +var JobStatus = require('../../../lib/batch/job-status'); describe('batch work in progress endpoint happy cases', function() { diff --git a/test/acceptance/batch/job.callback-template.test.js b/test/acceptance/batch/job-callback-template-test.js similarity index 98% rename from test/acceptance/batch/job.callback-template.test.js rename to test/acceptance/batch/job-callback-template-test.js index 3369c964..94545650 100644 --- a/test/acceptance/batch/job.callback-template.test.js +++ b/test/acceptance/batch/job-callback-template-test.js @@ -4,7 +4,7 @@ require('../../helper'); var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); -var JobStatus = require('../../../batch/job_status'); +var JobStatus = require('../../../lib/batch/job-status'); var BatchTestClient = require('../../support/batch-test-client'); describe('Batch API callback templates', function () { diff --git a/test/acceptance/batch/job.fallback.test.js b/test/acceptance/batch/job-fallback-test.js similarity index 99% rename from test/acceptance/batch/job.fallback.test.js rename to test/acceptance/batch/job-fallback-test.js index e6b35ce4..a68ec186 100644 --- a/test/acceptance/batch/job.fallback.test.js +++ b/test/acceptance/batch/job-fallback-test.js @@ -3,7 +3,7 @@ require('../../helper'); var assert = require('../../support/assert'); -var JobStatus = require('../../../batch/job_status'); +var JobStatus = require('../../../lib/batch/job-status'); var BatchTestClient = require('../../support/batch-test-client'); describe('Batch API fallback job', function () { diff --git a/test/acceptance/batch/job.query.limit.test.js b/test/acceptance/batch/job-query-limit-test.js similarity index 95% rename from test/acceptance/batch/job.query.limit.test.js rename to test/acceptance/batch/job-query-limit-test.js index b02959f5..ec02c9e0 100644 --- a/test/acceptance/batch/job.query.limit.test.js +++ b/test/acceptance/batch/job-query-limit-test.js @@ -15,9 +15,9 @@ * */ require('../../helper'); -var JobController = require('../../../app/controllers/job_controller'); -var redisUtils = require('../../support/redis_utils'); -var server = require('../../../app/server')(); +var JobController = require('../../../lib/api/sql/job-controller'); +var redisUtils = require('../../support/redis-utils'); +var server = require('../../../lib/server')(); var assert = require('../../support/assert'); var querystring = require('qs'); diff --git a/test/acceptance/batch/job.query.order.test.js b/test/acceptance/batch/job-query-order-test.js similarity index 96% rename from test/acceptance/batch/job.query.order.test.js rename to test/acceptance/batch/job-query-order-test.js index 709692ab..bf280e86 100644 --- a/test/acceptance/batch/job.query.order.test.js +++ b/test/acceptance/batch/job-query-order-test.js @@ -4,7 +4,7 @@ require('../../helper'); var assert = require('../../support/assert'); var BatchTestClient = require('../../support/batch-test-client'); -var JobStatus = require('../../../batch/job_status'); +var JobStatus = require('../../../lib/batch/job-status'); describe('job query order', function() { diff --git a/test/acceptance/batch/job.query.timeout.test.js b/test/acceptance/batch/job-query-timeout-test.js similarity index 98% rename from test/acceptance/batch/job.query.timeout.test.js rename to test/acceptance/batch/job-query-timeout-test.js index e55fe1ce..a449ff5f 100644 --- a/test/acceptance/batch/job.query.timeout.test.js +++ b/test/acceptance/batch/job-query-timeout-test.js @@ -4,7 +4,7 @@ require('../../helper'); var assert = require('../../support/assert'); var BatchTestClient = require('../../support/batch-test-client'); -var JobStatus = require('../../../batch/job_status'); +var JobStatus = require('../../../lib/batch/job-status'); describe('job query timeout', function() { diff --git a/test/acceptance/batch/job.test.js b/test/acceptance/batch/job-test.js similarity index 98% rename from test/acceptance/batch/job.test.js rename to test/acceptance/batch/job-test.js index d8d8288f..fa1d5472 100644 --- a/test/acceptance/batch/job.test.js +++ b/test/acceptance/batch/job-test.js @@ -16,9 +16,9 @@ */ require('../../helper'); -var server = require('../../../app/server')(); +var server = require('../../../lib/server')(); var assert = require('../../support/assert'); -var redisUtils = require('../../support/redis_utils'); +var redisUtils = require('../../support/redis-utils'); var querystring = require('querystring'); describe('job module', function() { diff --git a/test/acceptance/batch/job.timing.test.js b/test/acceptance/batch/job-timing-test.js similarity index 98% rename from test/acceptance/batch/job.timing.test.js rename to test/acceptance/batch/job-timing-test.js index d2482ca4..1391fb14 100644 --- a/test/acceptance/batch/job.timing.test.js +++ b/test/acceptance/batch/job-timing-test.js @@ -3,7 +3,7 @@ require('../../helper'); var BatchTestClient = require('../../support/batch-test-client'); -var JobStatus = require('../../../batch/job_status'); +var JobStatus = require('../../../lib/batch/job-status'); describe('Batch API query timing', function () { before(function() { diff --git a/test/acceptance/batch/leader.job.query.order.test.js b/test/acceptance/batch/leader-job-query-order-test.js similarity index 98% rename from test/acceptance/batch/leader.job.query.order.test.js rename to test/acceptance/batch/leader-job-query-order-test.js index e80b2047..df441c7e 100644 --- a/test/acceptance/batch/leader.job.query.order.test.js +++ b/test/acceptance/batch/leader-job-query-order-test.js @@ -5,7 +5,7 @@ var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); var BatchTestClient = require('../../support/batch-test-client'); -var JobStatus = require('../../../batch/job_status'); +var JobStatus = require('../../../lib/batch/job-status'); describe('multiple batch clients job query order', function() { diff --git a/test/acceptance/batch/leader-multiple-users-query-order.test.js b/test/acceptance/batch/leader-multiple-users-query-order-test.js similarity index 98% rename from test/acceptance/batch/leader-multiple-users-query-order.test.js rename to test/acceptance/batch/leader-multiple-users-query-order-test.js index 0a5c3609..a7b08108 100644 --- a/test/acceptance/batch/leader-multiple-users-query-order.test.js +++ b/test/acceptance/batch/leader-multiple-users-query-order-test.js @@ -5,7 +5,7 @@ var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); var BatchTestClient = require('../../support/batch-test-client'); -var JobStatus = require('../../../batch/job_status'); +var JobStatus = require('../../../lib/batch/job-status'); describe('multiple batch clients and users, job query order', function() { diff --git a/test/acceptance/batch/queued-jobs-limit.test.js b/test/acceptance/batch/queued-jobs-limit-test.js similarity index 94% rename from test/acceptance/batch/queued-jobs-limit.test.js rename to test/acceptance/batch/queued-jobs-limit-test.js index 35a0aa5a..f9d52df2 100644 --- a/test/acceptance/batch/queued-jobs-limit.test.js +++ b/test/acceptance/batch/queued-jobs-limit-test.js @@ -3,7 +3,7 @@ require('../../helper'); var assert = require('../../support/assert'); -var redisUtils = require('../../support/redis_utils'); +var redisUtils = require('../../support/redis-utils'); var TestClient = require('../../support/test-client'); describe('max queued jobs', function() { @@ -11,7 +11,7 @@ describe('max queued jobs', function() { before(function(done) { this.batch_max_queued_jobs = global.settings.batch_max_queued_jobs; global.settings.batch_max_queued_jobs = 1; - this.server = require('../../../app/server')(); + this.server = require('../../../lib/server')(); this.testClient = new TestClient(); this.testClient.getResult( 'drop table if exists max_queued_jobs_inserts; create table max_queued_jobs_inserts (status numeric)', diff --git a/test/acceptance/batch/scheduler-basic.test.js b/test/acceptance/batch/scheduler-basic-test.js similarity index 98% rename from test/acceptance/batch/scheduler-basic.test.js rename to test/acceptance/batch/scheduler-basic-test.js index 7cee1e01..46c35d7a 100644 --- a/test/acceptance/batch/scheduler-basic.test.js +++ b/test/acceptance/batch/scheduler-basic-test.js @@ -5,7 +5,7 @@ var assert = require('../../support/assert'); var TestClient = require('../../support/test-client'); var BatchTestClient = require('../../support/batch-test-client'); -var JobStatus = require('../../../batch/job_status'); +var JobStatus = require('../../../lib/batch/job-status'); describe('basic scheduling', function() { diff --git a/test/acceptance/batch/use-cases.test.js b/test/acceptance/batch/use-cases-test.js similarity index 98% rename from test/acceptance/batch/use-cases.test.js rename to test/acceptance/batch/use-cases-test.js index 255d9056..f1fdf3be 100644 --- a/test/acceptance/batch/use-cases.test.js +++ b/test/acceptance/batch/use-cases-test.js @@ -3,7 +3,7 @@ require('../../helper'); var assert = require('../../support/assert'); -var JobStatus = require('../../../batch/job_status'); +var JobStatus = require('../../../lib/batch/job-status'); var BatchTestClient = require('../../support/batch-test-client'); describe('Use cases', function () { diff --git a/test/acceptance/cache-headers.js b/test/acceptance/cache-headers-test.js similarity index 98% rename from test/acceptance/cache-headers.js rename to test/acceptance/cache-headers-test.js index 8d1b6da5..9a179a94 100644 --- a/test/acceptance/cache-headers.js +++ b/test/acceptance/cache-headers-test.js @@ -1,6 +1,6 @@ 'use strict'; -const server = require('../../app/server')(); +const server = require('../../lib/server')(); const assert = require('../support/assert'); const qs = require('querystring'); diff --git a/test/acceptance/copy-abort.js b/test/acceptance/copy-abort-test.js similarity index 98% rename from test/acceptance/copy-abort.js rename to test/acceptance/copy-abort-test.js index c47368bb..5bdcde46 100644 --- a/test/acceptance/copy-abort.js +++ b/test/acceptance/copy-abort-test.js @@ -1,9 +1,9 @@ 'use strict'; const querystring = require('querystring'); -const StatsClient = require('../../app/stats/client'); +const StatsClient = require('../../lib/stats/client'); const statsClient = StatsClient.getInstance(global.settings.statsd); -const server = require('../../app/server')(statsClient); +const server = require('../../lib/server')(statsClient); const request = require('request'); const assert = require('assert'); diff --git a/test/acceptance/copy-endpoints.js b/test/acceptance/copy-endpoints-test.js similarity index 99% rename from test/acceptance/copy-endpoints.js rename to test/acceptance/copy-endpoints-test.js index 5e4ae92d..2afbbd9d 100644 --- a/test/acceptance/copy-endpoints.js +++ b/test/acceptance/copy-endpoints-test.js @@ -9,7 +9,7 @@ const os = require('os'); const { Client } = require('pg'); const request = require('request'); -const StatsClient = require('../../app/stats/client'); +const StatsClient = require('../../lib/stats/client'); if (global.settings.statsd) { // Perform keyword substitution in statsd if (global.settings.statsd.prefix) { @@ -18,7 +18,7 @@ if (global.settings.statsd) { } } const statsClient = StatsClient.getInstance(global.settings.statsd); -const server = require('../../app/server')(statsClient); +const server = require('../../lib/server')(statsClient); // Give it enough time to connect and issue the query diff --git a/test/acceptance/copy-statements.js b/test/acceptance/copy-statements-test.js similarity index 98% rename from test/acceptance/copy-statements.js rename to test/acceptance/copy-statements-test.js index 6fb5c8f3..5055c7a7 100644 --- a/test/acceptance/copy-statements.js +++ b/test/acceptance/copy-statements-test.js @@ -2,7 +2,7 @@ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); diff --git a/test/acceptance/copy-throttle.js b/test/acceptance/copy-throttle-test.js similarity index 97% rename from test/acceptance/copy-throttle.js rename to test/acceptance/copy-throttle-test.js index 0be5876e..8ff69fcd 100644 --- a/test/acceptance/copy-throttle.js +++ b/test/acceptance/copy-throttle-test.js @@ -1,9 +1,9 @@ 'use strict'; const querystring = require('querystring'); -const StatsClient = require('../../app/stats/client'); +const StatsClient = require('../../lib/stats/client'); const statsClient = StatsClient.getInstance(global.settings.statsd); -const server = require('../../app/server')(statsClient); +const server = require('../../lib/server')(statsClient); const request = require('request'); const assert = require('assert'); const { Readable } = require('stream'); diff --git a/test/acceptance/custom-middlewares.js b/test/acceptance/custom-middlewares-test.js similarity index 99% rename from test/acceptance/custom-middlewares.js rename to test/acceptance/custom-middlewares-test.js index 396a49e2..7635e366 100644 --- a/test/acceptance/custom-middlewares.js +++ b/test/acceptance/custom-middlewares-test.js @@ -5,7 +5,7 @@ require('../helper'); const querystring = require('querystring'); const fs = require('fs'); -const createServer = require('../../app/server'); +const createServer = require('../../lib/server'); const assert = require('../support/assert'); describe('custom middlewares', function () { diff --git a/test/acceptance/error-handler.js b/test/acceptance/error-handler-test.js similarity index 95% rename from test/acceptance/error-handler.js rename to test/acceptance/error-handler-test.js index c6d97542..6bc5043e 100644 --- a/test/acceptance/error-handler.js +++ b/test/acceptance/error-handler-test.js @@ -1,6 +1,6 @@ 'use strict'; -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); describe('error handler', function () { diff --git a/test/acceptance/export/arraybuffer.js b/test/acceptance/export/arraybuffer-test.js similarity index 96% rename from test/acceptance/export/arraybuffer.js rename to test/acceptance/export/arraybuffer-test.js index 8e42679d..b891df3f 100644 --- a/test/acceptance/export/arraybuffer.js +++ b/test/acceptance/export/arraybuffer-test.js @@ -4,7 +4,7 @@ require('../../helper'); require('../../support/assert'); -var server = require('../../../app/server')(); +var server = require('../../../lib/server')(); var assert = require('assert'); var querystring = require('querystring'); diff --git a/test/acceptance/export/csv.js b/test/acceptance/export/csv-test.js similarity index 99% rename from test/acceptance/export/csv.js rename to test/acceptance/export/csv-test.js index 7f37b42c..8a4c5671 100644 --- a/test/acceptance/export/csv.js +++ b/test/acceptance/export/csv-test.js @@ -4,7 +4,7 @@ require('../../helper'); require('../../support/assert'); -var server = require('../../../app/server')(); +var server = require('../../../lib/server')(); var assert = require('assert'); var querystring = require('querystring'); diff --git a/test/acceptance/export/folder.js b/test/acceptance/export/folder-test.js similarity index 95% rename from test/acceptance/export/folder.js rename to test/acceptance/export/folder-test.js index 9aebb8c7..59d0e389 100644 --- a/test/acceptance/export/folder.js +++ b/test/acceptance/export/folder-test.js @@ -4,7 +4,7 @@ require('../../helper'); require('../../support/assert'); const fs = require('fs'); -let server = require('../../../app/server'); +let server = require('../../../lib/server'); const assert = require('assert'); const querystring = require('querystring'); diff --git a/test/acceptance/export/geojson.js b/test/acceptance/export/geojson-test.js similarity index 99% rename from test/acceptance/export/geojson.js rename to test/acceptance/export/geojson-test.js index 725d8678..bf19df34 100644 --- a/test/acceptance/export/geojson.js +++ b/test/acceptance/export/geojson-test.js @@ -2,7 +2,7 @@ require('../../helper'); -var server = require('../../../app/server')(); +var server = require('../../../lib/server')(); var assert = require('../../support/assert'); var querystring = require('querystring'); diff --git a/test/acceptance/export/geopackage.js b/test/acceptance/export/geopackage-test.js similarity index 98% rename from test/acceptance/export/geopackage.js rename to test/acceptance/export/geopackage-test.js index 392171c2..9ae9b7ae 100644 --- a/test/acceptance/export/geopackage.js +++ b/test/acceptance/export/geopackage-test.js @@ -2,7 +2,7 @@ require('../../helper'); -var server = require('../../../app/server')(); +var server = require('../../../lib/server')(); var assert = require('../../support/assert'); var sqlite = require('sqlite3'); var fs = require('fs'); diff --git a/test/acceptance/export/kml.js b/test/acceptance/export/kml-test.js similarity index 99% rename from test/acceptance/export/kml.js rename to test/acceptance/export/kml-test.js index 4447363d..d84f7e40 100644 --- a/test/acceptance/export/kml.js +++ b/test/acceptance/export/kml-test.js @@ -2,7 +2,7 @@ require('../../helper'); -var server = require('../../../app/server')(); +var server = require('../../../lib/server')(); var assert = require('../../support/assert'); var querystring = require('querystring'); var libxmljs = require('libxmljs'); diff --git a/test/acceptance/export/shapefile.js b/test/acceptance/export/shapefile-test.js similarity index 99% rename from test/acceptance/export/shapefile.js rename to test/acceptance/export/shapefile-test.js index 7fd8d1e9..33d26951 100644 --- a/test/acceptance/export/shapefile.js +++ b/test/acceptance/export/shapefile-test.js @@ -2,7 +2,7 @@ require('../../helper'); -var server = require('../../../app/server')(); +var server = require('../../../lib/server')(); var assert = require('../../support/assert'); var querystring = require('querystring'); var shapefile = require('shapefile'); diff --git a/test/acceptance/export/spatialite.js b/test/acceptance/export/spatialite-test.js similarity index 97% rename from test/acceptance/export/spatialite.js rename to test/acceptance/export/spatialite-test.js index e8b245cb..dcfa5854 100644 --- a/test/acceptance/export/spatialite.js +++ b/test/acceptance/export/spatialite-test.js @@ -2,7 +2,7 @@ require('../../helper'); -var server = require('../../../app/server')(); +var server = require('../../../lib/server')(); var assert = require('../../support/assert'); var sqlite = require('sqlite3'); diff --git a/test/acceptance/export/svg.js b/test/acceptance/export/svg-test.js similarity index 99% rename from test/acceptance/export/svg.js rename to test/acceptance/export/svg-test.js index 2c452df3..bc266442 100644 --- a/test/acceptance/export/svg.js +++ b/test/acceptance/export/svg-test.js @@ -2,7 +2,7 @@ require('../../helper'); -var server = require('../../../app/server')(); +var server = require('../../../lib/server')(); var assert = require('../../support/assert'); var querystring = require('querystring'); diff --git a/test/acceptance/export/timeout.js b/test/acceptance/export/timeout-test.js similarity index 100% rename from test/acceptance/export/timeout.js rename to test/acceptance/export/timeout-test.js diff --git a/test/acceptance/export/topojson.js b/test/acceptance/export/topojson-test.js similarity index 99% rename from test/acceptance/export/topojson.js rename to test/acceptance/export/topojson-test.js index 61dd6d79..14053a7e 100644 --- a/test/acceptance/export/topojson.js +++ b/test/acceptance/export/topojson-test.js @@ -2,7 +2,7 @@ require('../../helper'); -var server = require('../../../app/server')(); +var server = require('../../../lib/server')(); var assert = require('../../support/assert'); var querystring = require('querystring'); var _ = require('underscore'); diff --git a/test/acceptance/frontend_abort.js b/test/acceptance/frontend_abort-test.js similarity index 97% rename from test/acceptance/frontend_abort.js rename to test/acceptance/frontend_abort-test.js index 7ba7f63a..fe4f73a9 100644 --- a/test/acceptance/frontend_abort.js +++ b/test/acceptance/frontend_abort-test.js @@ -32,7 +32,7 @@ it('aborts request', function(done){ var db_port_backup = global.settings.db_port; global.settings.db_host = 'localhost'; global.settings.db_port = sql_server_port; - var server = require('../../app/server')(); + var server = require('../../lib/server')(); var timeout; step( function sendQuery() { diff --git a/test/acceptance/handle-query.js b/test/acceptance/handle-query-test.js similarity index 97% rename from test/acceptance/handle-query.js rename to test/acceptance/handle-query-test.js index aaf17560..6adc4579 100644 --- a/test/acceptance/handle-query.js +++ b/test/acceptance/handle-query-test.js @@ -2,11 +2,11 @@ require('../helper'); -const server = require('../../app/server')(); +const server = require('../../lib/server')(); const assert = require('../support/assert'); const qs = require('querystring'); const BatchTestClient = require('../support/batch-test-client'); -const JobStatus = require('../../batch/job_status'); +const JobStatus = require('../../lib/batch/job-status'); const QUERY = `SELECT 14 as foo`; const API_KEY = 1234; diff --git a/test/acceptance/health_check.js b/test/acceptance/health-check-test.js similarity index 97% rename from test/acceptance/health_check.js rename to test/acceptance/health-check-test.js index bd29eb1f..be06eca7 100644 --- a/test/acceptance/health_check.js +++ b/test/acceptance/health-check-test.js @@ -4,7 +4,7 @@ require('../helper'); require('../support/assert'); var assert = require('assert'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); describe('health checks', function() { diff --git a/test/acceptance/last-modified-header.js b/test/acceptance/last-modified-header-test.js similarity index 98% rename from test/acceptance/last-modified-header.js rename to test/acceptance/last-modified-header-test.js index 28799a1b..6c389d66 100644 --- a/test/acceptance/last-modified-header.js +++ b/test/acceptance/last-modified-header-test.js @@ -2,7 +2,7 @@ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); var qs = require('querystring'); var MockDate = require('mockdate'); diff --git a/test/acceptance/log.js b/test/acceptance/log-test.js similarity index 98% rename from test/acceptance/log.js rename to test/acceptance/log-test.js index 4d4281de..efea437b 100644 --- a/test/acceptance/log.js +++ b/test/acceptance/log-test.js @@ -2,11 +2,11 @@ require('../helper'); -const server = require('../../app/server')(); +const server = require('../../lib/server')(); const assert = require('../support/assert'); const qs = require('querystring'); const BatchTestClient = require('../support/batch-test-client'); -const { TYPES } = require('../../app/middlewares/log'); +const { TYPES } = require('../../lib/api/middlewares/log'); const QUERY = `SELECT 14 as foo`; const API_KEY = 1234; diff --git a/test/acceptance/pagination.js b/test/acceptance/pagination-test.js similarity index 99% rename from test/acceptance/pagination.js rename to test/acceptance/pagination-test.js index 5aae8747..a7603b18 100644 --- a/test/acceptance/pagination.js +++ b/test/acceptance/pagination-test.js @@ -2,7 +2,7 @@ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); var step = require('step'); diff --git a/test/acceptance/pg-entities-access-validator.js b/test/acceptance/pg-entities-access-validator-test.js similarity index 100% rename from test/acceptance/pg-entities-access-validator.js rename to test/acceptance/pg-entities-access-validator-test.js diff --git a/test/acceptance/pg-types.js b/test/acceptance/pg-types-test.js similarity index 99% rename from test/acceptance/pg-types.js rename to test/acceptance/pg-types-test.js index 96ba4f24..34a611e2 100644 --- a/test/acceptance/pg-types.js +++ b/test/acceptance/pg-types-test.js @@ -2,7 +2,7 @@ require('../helper'); -const server = require('../../app/server')(); +const server = require('../../lib/server')(); const assert = require('../support/assert'); const querystring = require('querystring'); diff --git a/test/acceptance/query-float-values.js b/test/acceptance/query-float-values-test.js similarity index 98% rename from test/acceptance/query-float-values.js rename to test/acceptance/query-float-values-test.js index 1b711eed..7aafc9d3 100644 --- a/test/acceptance/query-float-values.js +++ b/test/acceptance/query-float-values-test.js @@ -2,7 +2,7 @@ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); var step = require('step'); diff --git a/test/acceptance/query-multipart.js b/test/acceptance/query-multipart-test.js similarity index 94% rename from test/acceptance/query-multipart.js rename to test/acceptance/query-multipart-test.js index a048fae1..90ac65fa 100644 --- a/test/acceptance/query-multipart.js +++ b/test/acceptance/query-multipart-test.js @@ -2,7 +2,7 @@ require('../helper'); -const server = require('../../app/server')(); +const server = require('../../lib/server')(); const assert = require('../support/assert'); describe('query-multipart', function() { diff --git a/test/acceptance/query-returning.js b/test/acceptance/query-returning-test.js similarity index 99% rename from test/acceptance/query-returning.js rename to test/acceptance/query-returning-test.js index cfbf9212..313df9be 100644 --- a/test/acceptance/query-returning.js +++ b/test/acceptance/query-returning-test.js @@ -2,7 +2,7 @@ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); var _ = require('underscore'); diff --git a/test/acceptance/rate-limit.js b/test/acceptance/rate-limit-test.js similarity index 96% rename from test/acceptance/rate-limit.js rename to test/acceptance/rate-limit-test.js index afb3cd97..aa708c08 100644 --- a/test/acceptance/rate-limit.js +++ b/test/acceptance/rate-limit-test.js @@ -5,10 +5,10 @@ require('../helper'); const qs = require('querystring'); const assert = require('../support/assert'); const redis = require('redis'); -const rateLimitMiddleware = require('../../app/middlewares/rate-limit'); +const rateLimitMiddleware = require('../../lib/api/middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitMiddleware; -const app = require('../../app/server'); +const app = require('../../lib/server'); let server; let redisClient; diff --git a/test/acceptance/regressions.js b/test/acceptance/regressions-test.js similarity index 98% rename from test/acceptance/regressions.js rename to test/acceptance/regressions-test.js index adb27e66..31b4c43b 100644 --- a/test/acceptance/regressions.js +++ b/test/acceptance/regressions-test.js @@ -2,7 +2,7 @@ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); var qs = require('querystring'); diff --git a/test/acceptance/skipfields.js b/test/acceptance/skipfields-test.js similarity index 98% rename from test/acceptance/skipfields.js rename to test/acceptance/skipfields-test.js index 7524f22b..9b3429bd 100644 --- a/test/acceptance/skipfields.js +++ b/test/acceptance/skipfields-test.js @@ -2,7 +2,7 @@ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); var _ = require('underscore'); diff --git a/test/acceptance/stream-responses.js b/test/acceptance/stream-responses-test.js similarity index 97% rename from test/acceptance/stream-responses.js rename to test/acceptance/stream-responses-test.js index 126bf997..88b4d068 100644 --- a/test/acceptance/stream-responses.js +++ b/test/acceptance/stream-responses-test.js @@ -2,7 +2,7 @@ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); diff --git a/test/acceptance/surrogate-key.js b/test/acceptance/surrogate-key-test.js similarity index 98% rename from test/acceptance/surrogate-key.js rename to test/acceptance/surrogate-key-test.js index 2fd7ed5c..aac7a758 100644 --- a/test/acceptance/surrogate-key.js +++ b/test/acceptance/surrogate-key-test.js @@ -2,7 +2,7 @@ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); var QueryTables = require('cartodb-query-tables').queryTables; diff --git a/test/acceptance/system-queries.js b/test/acceptance/system-queries-test.js similarity index 98% rename from test/acceptance/system-queries.js rename to test/acceptance/system-queries-test.js index 8526aade..2c6853e1 100644 --- a/test/acceptance/system-queries.js +++ b/test/acceptance/system-queries-test.js @@ -2,7 +2,7 @@ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); diff --git a/test/acceptance/timeout.js b/test/acceptance/timeout-test.js similarity index 96% rename from test/acceptance/timeout.js rename to test/acceptance/timeout-test.js index 99c2d7e5..95270a60 100644 --- a/test/acceptance/timeout.js +++ b/test/acceptance/timeout-test.js @@ -28,7 +28,7 @@ it('after configured milliseconds', function(done){ //console.log("settings:"); console.dir(global.settings); var timeoutBackup = global.settings.node_socket_timeout; global.settings.node_socket_timeout = testTimeout; - var server = require('../../app/server')(); + var server = require('../../lib/server')(); step( function sendLongQuery() { assert.response(server, { diff --git a/test/acceptance/transaction.js b/test/acceptance/transaction-test.js similarity index 97% rename from test/acceptance/transaction.js rename to test/acceptance/transaction-test.js index 0ee54d5a..8fc9d150 100644 --- a/test/acceptance/transaction.js +++ b/test/acceptance/transaction-test.js @@ -12,7 +12,7 @@ describe('transaction', function() { var server; before(function(done) { - server = require('../../app/server')(); + server = require('../../lib/server')(); this.listener = server.listen(SERVER_PORT, '127.0.0.1'); this.listener.on('listening', done); }); diff --git a/test/acceptance/x-cache-channel.js b/test/acceptance/x-cache-channel-test.js similarity index 98% rename from test/acceptance/x-cache-channel.js rename to test/acceptance/x-cache-channel-test.js index dfcc0599..0bf18f16 100644 --- a/test/acceptance/x-cache-channel.js +++ b/test/acceptance/x-cache-channel-test.js @@ -2,7 +2,7 @@ require('../helper'); -var server = require('../../app/server')(); +var server = require('../../lib/server')(); var assert = require('../support/assert'); var querystring = require('querystring'); var _ = require('underscore'); diff --git a/test/integration/batch/job_backend.test.js b/test/integration/batch/job-backend-test.js similarity index 95% rename from test/integration/batch/job_backend.test.js rename to test/integration/batch/job-backend-test.js index fa7534aa..8588a6fa 100644 --- a/test/integration/batch/job_backend.test.js +++ b/test/integration/batch/job-backend-test.js @@ -2,17 +2,17 @@ require('../../helper'); -var BATCH_SOURCE = '../../../batch/'; +var BATCH_SOURCE = '../../../lib/batch/'; var assert = require('../../support/assert'); -var redisUtils = require('../../support/redis_utils'); +var redisUtils = require('../../support/redis-utils'); var BatchLogger = require(BATCH_SOURCE + 'batch-logger'); -var JobQueue = require(BATCH_SOURCE + 'job_queue'); -var JobBackend = require(BATCH_SOURCE + 'job_backend'); +var JobQueue = require(BATCH_SOURCE + 'job-queue'); +var JobBackend = require(BATCH_SOURCE + 'job-backend'); var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher'); -var JobFactory = require(BATCH_SOURCE + 'models/job_factory'); -var jobStatus = require(BATCH_SOURCE + 'job_status'); +var JobFactory = require(BATCH_SOURCE + 'models/job-factory'); +var jobStatus = require(BATCH_SOURCE + 'job-status'); var logger = new BatchLogger(null, 'batch-queries'); var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); diff --git a/test/integration/batch/job_canceller.test.js b/test/integration/batch/job-canceller-test.js similarity index 88% rename from test/integration/batch/job_canceller.test.js rename to test/integration/batch/job-canceller-test.js index d71ac91c..b9c3d6c7 100644 --- a/test/integration/batch/job_canceller.test.js +++ b/test/integration/batch/job-canceller-test.js @@ -2,18 +2,18 @@ require('../../helper'); -var BATCH_SOURCE = '../../../batch/'; +var BATCH_SOURCE = '../../../lib/batch/'; var assert = require('../../support/assert'); -var redisUtils = require('../../support/redis_utils'); +var redisUtils = require('../../support/redis-utils'); var BatchLogger = require(BATCH_SOURCE + 'batch-logger'); -var JobQueue = require(BATCH_SOURCE + 'job_queue'); -var JobBackend = require(BATCH_SOURCE + 'job_backend'); +var JobQueue = require(BATCH_SOURCE + 'job-queue'); +var JobBackend = require(BATCH_SOURCE + 'job-backend'); var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher'); -var jobStatus = require(BATCH_SOURCE + 'job_status'); -var JobCanceller = require(BATCH_SOURCE + 'job_canceller'); -var JobFactory = require(BATCH_SOURCE + 'models/job_factory'); +var jobStatus = require(BATCH_SOURCE + 'job-status'); +var JobCanceller = require(BATCH_SOURCE + 'job-canceller'); +var JobFactory = require(BATCH_SOURCE + 'models/job-factory'); var PSQL = require('cartodb-psql'); var logger = new BatchLogger(null, 'batch-queries'); diff --git a/test/integration/batch/job_publisher.test.js b/test/integration/batch/job-publisher-test.js similarity index 90% rename from test/integration/batch/job_publisher.test.js rename to test/integration/batch/job-publisher-test.js index 50c9f5c7..74b252ae 100644 --- a/test/integration/batch/job_publisher.test.js +++ b/test/integration/batch/job-publisher-test.js @@ -2,11 +2,11 @@ require('../../helper'); -var BATCH_SOURCE = '../../../batch/'; +var BATCH_SOURCE = '../../../lib/batch/'; var assert = require('../../support/assert'); -var redisUtils = require('../../support/redis_utils'); +var redisUtils = require('../../support/redis-utils'); var Channel = require(BATCH_SOURCE + 'pubsub/channel'); diff --git a/test/integration/batch/job-queue.test.js b/test/integration/batch/job-queue-test.js similarity index 92% rename from test/integration/batch/job-queue.test.js rename to test/integration/batch/job-queue-test.js index 735e3d3a..34053ef9 100644 --- a/test/integration/batch/job-queue.test.js +++ b/test/integration/batch/job-queue-test.js @@ -2,16 +2,16 @@ require('../../helper'); var assert = require('../../support/assert'); -var redisUtils = require('../../support/redis_utils'); +var redisUtils = require('../../support/redis-utils'); var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); -var BatchLogger = require('../../../batch/batch-logger'); -var JobPublisher = require('../../../batch/pubsub/job-publisher'); -var JobQueue = require('../../../batch/job_queue'); +var BatchLogger = require('../../../lib/batch/batch-logger'); +var JobPublisher = require('../../../lib/batch/pubsub/job-publisher'); +var JobQueue = require('../../../lib/batch/job-queue'); -var JobBackend = require('../../../batch/job_backend'); -var JobService = require('../../../batch/job_service'); -var JobCanceller = require('../../../batch/job_canceller'); +var JobBackend = require('../../../lib/batch/job-backend'); +var JobService = require('../../../lib/batch/job-service'); +var JobCanceller = require('../../../lib/batch/job-canceller'); var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); describe('job queue', function () { diff --git a/test/integration/batch/job_runner.test.js b/test/integration/batch/job-runner-test.js similarity index 79% rename from test/integration/batch/job_runner.test.js rename to test/integration/batch/job-runner-test.js index cb0eb96d..cfa50a34 100644 --- a/test/integration/batch/job_runner.test.js +++ b/test/integration/batch/job-runner-test.js @@ -2,21 +2,21 @@ require('../../helper'); -var BATCH_SOURCE = '../../../batch/'; +var BATCH_SOURCE = '../../../lib/batch/'; var assert = require('../../support/assert'); -var redisUtils = require('../../support/redis_utils'); +var redisUtils = require('../../support/redis-utils'); var BatchLogger = require(BATCH_SOURCE + 'batch-logger'); -var JobQueue = require(BATCH_SOURCE + 'job_queue'); -var JobBackend = require(BATCH_SOURCE + 'job_backend'); +var JobQueue = require(BATCH_SOURCE + 'job-queue'); +var JobBackend = require(BATCH_SOURCE + 'job-backend'); var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher'); -var jobStatus = require(BATCH_SOURCE + 'job_status'); -var UserDatabaseMetadataService = require(BATCH_SOURCE + 'user_database_metadata_service'); -var JobCanceller = require(BATCH_SOURCE + 'job_canceller'); -var JobService = require(BATCH_SOURCE + 'job_service'); -var JobRunner = require(BATCH_SOURCE + 'job_runner'); -var QueryRunner = require(BATCH_SOURCE + 'query_runner'); +var jobStatus = require(BATCH_SOURCE + 'job-status'); +var UserDatabaseMetadataService = require(BATCH_SOURCE + 'user-database-metadata-service'); +var JobCanceller = require(BATCH_SOURCE + 'job-canceller'); +var JobService = require(BATCH_SOURCE + 'job-service'); +var JobRunner = require(BATCH_SOURCE + 'job-runner'); +var QueryRunner = require(BATCH_SOURCE + 'query-runner'); var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); diff --git a/test/integration/batch/job_service.test.js b/test/integration/batch/job-service-test.js similarity index 93% rename from test/integration/batch/job_service.test.js rename to test/integration/batch/job-service-test.js index b8f45797..efadac90 100644 --- a/test/integration/batch/job_service.test.js +++ b/test/integration/batch/job-service-test.js @@ -2,18 +2,18 @@ require('../../helper'); -var BATCH_SOURCE = '../../../batch/'; +var BATCH_SOURCE = '../../../lib/batch/'; var assert = require('../../support/assert'); -var redisUtils = require('../../support/redis_utils'); +var redisUtils = require('../../support/redis-utils'); var BatchLogger = require(BATCH_SOURCE + 'batch-logger'); -var JobQueue = require(BATCH_SOURCE + 'job_queue'); -var JobBackend = require(BATCH_SOURCE + 'job_backend'); +var JobQueue = require(BATCH_SOURCE + 'job-queue'); +var JobBackend = require(BATCH_SOURCE + 'job-backend'); var JobPublisher = require(BATCH_SOURCE + 'pubsub/job-publisher'); -var jobStatus = require(BATCH_SOURCE + 'job_status'); -var JobCanceller = require(BATCH_SOURCE + 'job_canceller'); -var JobService = require(BATCH_SOURCE + 'job_service'); +var jobStatus = require(BATCH_SOURCE + 'job-status'); +var JobCanceller = require(BATCH_SOURCE + 'job-canceller'); +var JobService = require(BATCH_SOURCE + 'job-service'); var PSQL = require('cartodb-psql'); var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); diff --git a/test/integration/batch/locker.js b/test/integration/batch/locker-test.js similarity index 93% rename from test/integration/batch/locker.js rename to test/integration/batch/locker-test.js index f1f2116f..f1e5c534 100644 --- a/test/integration/batch/locker.js +++ b/test/integration/batch/locker-test.js @@ -3,8 +3,8 @@ require('../../helper'); var assert = require('../../support/assert'); -var redisUtils = require('../../support/redis_utils'); -var Locker = require('../../../batch/leader/locker'); +var redisUtils = require('../../support/redis-utils'); +var Locker = require('../../../lib/batch/leader/locker'); describe('locker', function() { var host = 'localhost'; diff --git a/test/integration/batch/scheduler.js b/test/integration/batch/scheduler-test.js similarity index 96% rename from test/integration/batch/scheduler.js rename to test/integration/batch/scheduler-test.js index 512ef288..2be1807b 100644 --- a/test/integration/batch/scheduler.js +++ b/test/integration/batch/scheduler-test.js @@ -1,10 +1,10 @@ 'use strict'; require('../../helper'); -var debug = require('../../../batch/util/debug')('scheduler-test'); +var debug = require('../../../lib/batch/util/debug')('scheduler-test'); var assert = require('../../support/assert'); -var Scheduler = require('../../../batch/scheduler/scheduler'); -var FixedCapacity = require('../../../batch/scheduler/capacity/fixed'); +var Scheduler = require('../../../lib/batch/scheduler/scheduler'); +var FixedCapacity = require('../../../lib/batch/scheduler/capacity/fixed'); describe('scheduler', function() { diff --git a/test/integration/stream_copy.test.js b/test/integration/stream-copy-test.js similarity index 90% rename from test/integration/stream_copy.test.js rename to test/integration/stream-copy-test.js index 4ee7c816..f73c740b 100644 --- a/test/integration/stream_copy.test.js +++ b/test/integration/stream-copy-test.js @@ -3,7 +3,7 @@ require('../helper'); const assert = require('assert'); -const StreamCopy = require('../../app/services/stream_copy'); +const StreamCopy = require('../../lib/services/stream-copy'); describe('stream copy', function() { it('uses batch api port', function(done) { diff --git a/test/support/batch-test-client.js b/test/support/batch-test-client.js index 3c43735a..82962354 100644 --- a/test/support/batch-test-client.js +++ b/test/support/batch-test-client.js @@ -2,13 +2,13 @@ require('../helper'); var assert = require('assert'); -var appServer = require('../../app/server'); -var redisUtils = require('./redis_utils'); +var appServer = require('../../lib/server'); +var redisUtils = require('./redis-utils'); var debug = require('debug')('batch-test-client'); -var JobStatus = require('../../batch/job_status'); +var JobStatus = require('../../lib/batch/job-status'); var metadataBackend = require('cartodb-redis')({ pool: redisUtils.getPool() }); -var batchFactory = require('../../batch/index'); +var batchFactory = require('../../lib/batch/index'); function response(code) { return { diff --git a/test/support/redis_utils.js b/test/support/redis-utils.js similarity index 100% rename from test/support/redis_utils.js rename to test/support/redis-utils.js diff --git a/test/support/test-client.js b/test/support/test-client.js index bd1b163f..66bbc3aa 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -2,8 +2,8 @@ require('../helper'); var assert = require('assert'); -var appServer = require('../../app/server'); -var redisUtils = require('./redis_utils'); +var appServer = require('../../lib/server'); +var redisUtils = require('./redis-utils'); const step = require('step'); const PSQL = require('cartodb-psql'); const _ = require('underscore'); diff --git a/test/unit/apikeyauth.test.js b/test/unit/apikeyauth-test.js similarity index 98% rename from test/unit/apikeyauth.test.js rename to test/unit/apikeyauth-test.js index 1c47c2fe..1fab72e5 100644 --- a/test/unit/apikeyauth.test.js +++ b/test/unit/apikeyauth-test.js @@ -2,7 +2,7 @@ require('../helper'); -var ApikeyAuth = require('../../app/auth/apikey'); +var ApikeyAuth = require('../../lib/auth/apikey'); var assert = require('assert'); describe.skip('has credentials', function() { diff --git a/test/unit/batch/job_publisher.js b/test/unit/batch/job-publisher-test.js similarity index 89% rename from test/unit/batch/job_publisher.js rename to test/unit/batch/job-publisher-test.js index 82f3bf89..58817213 100644 --- a/test/unit/batch/job_publisher.js +++ b/test/unit/batch/job-publisher-test.js @@ -1,7 +1,7 @@ 'use strict'; -var Channel = require('../../../batch/pubsub/channel'); -var JobPublisher = require('../../../batch/pubsub/job-publisher'); +var Channel = require('../../../lib/batch/pubsub/channel'); +var JobPublisher = require('../../../lib/batch/pubsub/job-publisher'); var assert = require('assert'); describe('batch API job publisher', function () { diff --git a/test/unit/batch/job_queue.js b/test/unit/batch/job-queue-test.js similarity index 96% rename from test/unit/batch/job_queue.js rename to test/unit/batch/job-queue-test.js index 4f7babf2..29b4e43f 100644 --- a/test/unit/batch/job_queue.js +++ b/test/unit/batch/job-queue-test.js @@ -1,6 +1,6 @@ 'use strict'; -var JobQueue = require('../../../batch/job_queue'); +var JobQueue = require('../../../lib/batch/job-queue'); var assert = require('assert'); describe('batch API job queue', function () { diff --git a/test/unit/batch/job_subscriber.js b/test/unit/batch/job-subscriber-test.js similarity index 94% rename from test/unit/batch/job_subscriber.js rename to test/unit/batch/job-subscriber-test.js index 6c63b148..2c3a192a 100644 --- a/test/unit/batch/job_subscriber.js +++ b/test/unit/batch/job-subscriber-test.js @@ -1,7 +1,7 @@ 'use strict'; -var Channel = require('../../../batch/pubsub/channel'); -var JobSubscriber = require('../../../batch/pubsub/job-subscriber'); +var Channel = require('../../../lib/batch/pubsub/channel'); +var JobSubscriber = require('../../../lib/batch/pubsub/job-subscriber'); var assert = require('assert'); describe('batch API job subscriber', function () { diff --git a/test/unit/error_handler_factory.test.js b/test/unit/error-handler-factory-test.js similarity index 91% rename from test/unit/error_handler_factory.test.js rename to test/unit/error-handler-factory-test.js index 97cdfc63..c1d988b5 100644 --- a/test/unit/error_handler_factory.test.js +++ b/test/unit/error-handler-factory-test.js @@ -1,9 +1,9 @@ 'use strict'; const assert = require('assert'); -const errorHandlerFactory = require('../../app/services/error_handler_factory'); -const ErrorHandler = require('../../app/services/error_handler'); -const { codeToCondition } = require('../../app/postgresql/error_codes'); +const errorHandlerFactory = require('../../lib/services/error-handler-factory'); +const ErrorHandler = require('../../lib/services/error-handler'); +const { codeToCondition } = require('../../lib/postgresql/error-codes'); let rateLimitError = new Error( 'You are over platform\'s limits. Please contact us to know more details' diff --git a/test/unit/error_handler.test.js b/test/unit/error-handler-test.js similarity index 97% rename from test/unit/error_handler.test.js rename to test/unit/error-handler-test.js index e3cb5fb3..6f89f95f 100644 --- a/test/unit/error_handler.test.js +++ b/test/unit/error-handler-test.js @@ -1,7 +1,7 @@ 'use strict'; var assert = require('assert'); -var errorMiddleware = require('../../app/middlewares/error'); +var errorMiddleware = require('../../lib/api/middlewares/error'); require('../helper'); const req = { query: { callback: true } }; @@ -75,7 +75,7 @@ describe('error-handler', function() { error.message ); - const res = getRes(); + const res = getRes(); errorMiddleware()(error, req, res, function next () { assert.ok(res.headers['X-SQLAPI-Errors'].length > 0); @@ -105,7 +105,7 @@ describe('error-handler', function() { ); const res = getRes(); - + errorMiddleware()(error, req, res, function () { assert.ok(res.headers['X-SQLAPI-Errors'].length > 0); assert.deepEqual( diff --git a/test/unit/health_check.js b/test/unit/health-check-test.js similarity index 94% rename from test/unit/health_check.js rename to test/unit/health-check-test.js index 1920e977..d776a795 100644 --- a/test/unit/health_check.js +++ b/test/unit/health-check-test.js @@ -3,7 +3,7 @@ require('../helper'); var assert = require('assert'); -var HealthCheck = require('../../app/monitoring/health_check'); +var HealthCheck = require('../../lib/monitoring/health-check'); var metadataBackend = {}; diff --git a/test/unit/model/bin_encoder.js b/test/unit/model/bin-encoder-test.js similarity index 97% rename from test/unit/model/bin_encoder.js rename to test/unit/model/bin-encoder-test.js index a0f7bf4f..a9b4dd8d 100644 --- a/test/unit/model/bin_encoder.js +++ b/test/unit/model/bin-encoder-test.js @@ -3,7 +3,7 @@ require('../../helper'); var assert = require('assert'); -var ArrayBufferSer = require('../../../app/models/bin_encoder'); +var ArrayBufferSer = require('../../../lib/models/bin-encoder'); describe('ArrayBufferSer', function() { diff --git a/test/unit/oauth.test.js b/test/unit/oauth-test.js similarity index 98% rename from test/unit/oauth.test.js rename to test/unit/oauth-test.js index 6b629478..e90b7133 100644 --- a/test/unit/oauth.test.js +++ b/test/unit/oauth-test.js @@ -3,9 +3,9 @@ require('../helper'); var _ = require('underscore'); -var OAuthAuth = require('../../app/auth/oauth'); +var OAuthAuth = require('../../lib/auth/oauth'); var MetadataDB = require('cartodb-redis'); -var oAuth = require('../../app/auth/oauth').backend; +var oAuth = require('../../lib/auth/oauth').backend; var assert = require('assert'); var oauth_data_1 = { oauth_consumer_key: "dpf43f3p2l4k3l03", diff --git a/test/unit/pg-entities-access-validator.test.js b/test/unit/pg-entities-access-validator-test.js similarity index 98% rename from test/unit/pg-entities-access-validator.test.js rename to test/unit/pg-entities-access-validator-test.js index f28a1d67..f859b04b 100644 --- a/test/unit/pg-entities-access-validator.test.js +++ b/test/unit/pg-entities-access-validator-test.js @@ -1,7 +1,7 @@ 'use strict'; const assert = require('assert'); -const pgEntitiesAccessValidator = require('../../app/services/pg-entities-access-validator'); +const pgEntitiesAccessValidator = require('../../lib/services/pg-entities-access-validator'); const fakeAffectedTables = [{ schema_name: 'schema', diff --git a/test/unit/query_info.test.js b/test/unit/query-info-test.js similarity index 97% rename from test/unit/query_info.test.js rename to test/unit/query-info-test.js index 05945b61..d797d8ec 100644 --- a/test/unit/query_info.test.js +++ b/test/unit/query-info-test.js @@ -1,7 +1,7 @@ 'use strict'; const assert = require('assert'); -const queryInfo = require('../../app/utils/query_info'); +const queryInfo = require('../../lib/utils/query-info'); describe('query info', function () { describe('copy format', function () { diff --git a/test/websocket_test/public/run_server.sh b/test/websocket_test/public/run_server.sh index dc1e2ab7..52428779 100755 --- a/test/websocket_test/public/run_server.sh +++ b/test/websocket_test/public/run_server.sh @@ -1 +1,5 @@ -python -m SimpleHTTPServer 8000 \ No newline at end of file +python -m SimpleHTTPServer 8000 + + + +“1-second Burst Limit” means the maximum number of requests per second that the CARTO APIs will serve at the start of a period of activity measured in one-second intervals, and which is based on the generic cell rate and leaky bucket algorithms. Burst Limit takes priority over the Request Limit: when a request arrives in a one-second interval, the request is first checked against the Burst Limit. If the Burst Limit is exceeded, the request is rejected even if the Request Limit has not been exceeded. From 62d98949caed2788aa022959ec7af52886bc6a71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Oct 2019 19:35:18 +0200 Subject: [PATCH 34/37] Consistent middleware naming --- lib/api/jobs-wip-controller.js | 8 +++--- lib/api/middlewares/profiler.js | 4 +-- lib/api/middlewares/user.js | 8 +++--- lib/api/sql/copy-controller.js | 46 ++++++++++++++++----------------- lib/api/sql/job-controller.js | 44 +++++++++++++++---------------- lib/api/sql/query-controller.js | 2 +- 6 files changed, 56 insertions(+), 56 deletions(-) diff --git a/lib/api/jobs-wip-controller.js b/lib/api/jobs-wip-controller.js index 885f2604..58d3090f 100644 --- a/lib/api/jobs-wip-controller.js +++ b/lib/api/jobs-wip-controller.js @@ -1,7 +1,7 @@ 'use strict'; -const bodyParserMiddleware = require('./middlewares/body-parser'); -const errorMiddleware = require('./middlewares/error'); +const bodyParser = require('./middlewares/body-parser'); +const error = require('./middlewares/error'); module.exports = class JobsWipController { constructor ({ jobService }) { @@ -10,10 +10,10 @@ module.exports = class JobsWipController { route (apiRouter) { apiRouter.get('/jobs-wip', [ - bodyParserMiddleware(), + bodyParser(), listWorkInProgressJobs(this.jobService), sendResponse(), - errorMiddleware() + error() ]); } }; diff --git a/lib/api/middlewares/profiler.js b/lib/api/middlewares/profiler.js index 3fbc7b2e..c932300b 100644 --- a/lib/api/middlewares/profiler.js +++ b/lib/api/middlewares/profiler.js @@ -13,7 +13,7 @@ module.exports = function profiler ({ statsClient }) { }; }; -module.exports.initializeProfilerMiddleware = function initializeProfiler (label) { +module.exports.initializeProfiler = function initializeProfiler (label) { return function initializeProfilerMiddleware (req, res, next) { if (req.profiler) { req.profiler.start(`sqlapi.${label}`); @@ -23,7 +23,7 @@ module.exports.initializeProfilerMiddleware = function initializeProfiler (label }; }; -module.exports.finishProfilerMiddleware = function finishProfiler () { +module.exports.finishProfiler = function finishProfiler () { return function finishProfilerMiddleware (req, res, next) { if (req.profiler) { req.profiler.end(); diff --git a/lib/api/middlewares/user.js b/lib/api/middlewares/user.js index 5f385337..6685ca8a 100644 --- a/lib/api/middlewares/user.js +++ b/lib/api/middlewares/user.js @@ -2,7 +2,7 @@ const CdbRequest = require('../../models/cartodb-request'); -module.exports = function user(metadataBackend) { +module.exports = function user (metadataBackend) { const cdbRequest = new CdbRequest(); return function userMiddleware (req, res, next) { @@ -23,16 +23,16 @@ module.exports = function user(metadataBackend) { }; }; -function getUserNameFromRequest(req, cdbRequest) { +function getUserNameFromRequest (req, cdbRequest) { return cdbRequest.userByReq(req); } -function checkUserExists(metadataBackend, userName, callback) { +function checkUserExists (metadataBackend, userName, callback) { metadataBackend.getUserId(userName, function(err) { callback(err, !err); }); } -function errorUserNotFoundMessageTemplate(user) { +function errorUserNotFoundMessageTemplate (user) { return `Sorry, we can't find CARTO user '${user}'. Please check that you have entered the correct domain.`; } diff --git a/lib/api/sql/copy-controller.js b/lib/api/sql/copy-controller.js index 9b18025e..ce5f0dc7 100644 --- a/lib/api/sql/copy-controller.js +++ b/lib/api/sql/copy-controller.js @@ -1,13 +1,14 @@ 'use strict'; -const userMiddleware = require('../middlewares/user'); -const errorMiddleware = require('../middlewares/error'); -const authorizationMiddleware = require('../middlewares/authorization'); -const connectionParamsMiddleware = require('../middlewares/connection-params'); -const { initializeProfilerMiddleware } = require('../middlewares/profiler'); -const rateLimitsMiddleware = require('../middlewares/rate-limit'); -const dbQuotaMiddleware = require('../middlewares/db-quota'); -const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware; +const user = require('../middlewares/user'); +const error = require('../middlewares/error'); +const authorization = require('../middlewares/authorization'); +const connectionParams = require('../middlewares/connection-params'); +const { initializeProfiler } = require('../middlewares/profiler'); +const dbQuota = require('../middlewares/db-quota'); +const bodyParser = require('../middlewares/body-parser'); +const rateLimits = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimits; const errorHandlerFactory = require('../../services/error-handler-factory'); const StreamCopy = require('../../services/stream-copy'); const StreamCopyMetrics = require('../../services/stream-copy-metrics'); @@ -15,7 +16,6 @@ const Throttler = require('../../services/throttler-stream'); const zlib = require('zlib'); const { PassThrough } = require('stream'); const params = require('../middlewares/params'); -const bodyParserMiddleware = require('../middlewares/body-parser'); module.exports = class CopyController { constructor (metadataBackend, userDatabaseService, userLimitsService, logger) { @@ -28,31 +28,31 @@ module.exports = class CopyController { route (sqlRouter) { const copyFromMiddlewares = endpointGroup => { return [ - initializeProfilerMiddleware('copyfrom'), - userMiddleware(this.metadataBackend), - rateLimitsMiddleware(this.userLimitsService, endpointGroup), - authorizationMiddleware(this.metadataBackend), - connectionParamsMiddleware(this.userDatabaseService), - dbQuotaMiddleware(), + initializeProfiler('copyfrom'), + user(this.metadataBackend), + rateLimits(this.userLimitsService, endpointGroup), + authorization(this.metadataBackend), + connectionParams(this.userDatabaseService), + dbQuota(), params({ strategy: 'copyfrom' }), handleCopyFrom(this.logger), errorHandler(this.logger), - errorMiddleware() + error() ]; }; const copyToMiddlewares = endpointGroup => { return [ - bodyParserMiddleware(), - initializeProfilerMiddleware('copyto'), - userMiddleware(this.metadataBackend), - rateLimitsMiddleware(this.userLimitsService, endpointGroup), - authorizationMiddleware(this.metadataBackend), - connectionParamsMiddleware(this.userDatabaseService), + bodyParser(), + initializeProfiler('copyto'), + user(this.metadataBackend), + rateLimits(this.userLimitsService, endpointGroup), + authorization(this.metadataBackend), + connectionParams(this.userDatabaseService), params({ strategy: 'copyto' }), handleCopyTo(this.logger), errorHandler(this.logger), - errorMiddleware() + error() ]; }; diff --git a/lib/api/sql/job-controller.js b/lib/api/sql/job-controller.js index a43e86b6..fd47338c 100644 --- a/lib/api/sql/job-controller.js +++ b/lib/api/sql/job-controller.js @@ -2,16 +2,16 @@ const util = require('util'); -const bodyParserMiddleware = require('../middlewares/body-parser'); -const userMiddleware = require('../middlewares/user'); -const { initializeProfilerMiddleware, finishProfilerMiddleware } = require('../middlewares/profiler'); -const authorizationMiddleware = require('../middlewares/authorization'); -const connectionParamsMiddleware = require('../middlewares/connection-params'); -const errorMiddleware = require('../middlewares/error'); -const rateLimitsMiddleware = require('../middlewares/rate-limit'); -const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimitsMiddleware; +const bodyParser = require('../middlewares/body-parser'); +const user = require('../middlewares/user'); +const { initializeProfiler, finishProfiler } = require('../middlewares/profiler'); +const authorization = require('../middlewares/authorization'); +const connectionParams = require('../middlewares/connection-params'); +const error = require('../middlewares/error'); +const rateLimits = require('../middlewares/rate-limit'); +const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimits; const params = require('../middlewares/params'); -const logMiddleware = require('../middlewares/log'); +const log = require('../middlewares/log'); module.exports = class JobController { constructor (metadataBackend, userDatabaseService, jobService, statsdClient, userLimitsService) { @@ -32,43 +32,43 @@ module.exports = class JobController { ); sqlRouter.post('/job', [ - bodyParserMiddleware(), + bodyParser(), checkBodyPayloadSize(), params({ strategy: 'job' }), - logMiddleware(logMiddleware.TYPES.JOB), + log(log.TYPES.JOB), jobMiddlewares('create', createJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_CREATE) ]); sqlRouter.get('/job/:job_id', [ - bodyParserMiddleware(), + bodyParser(), jobMiddlewares('retrieve', getJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_GET) ]); sqlRouter.delete('/job/:job_id', [ - bodyParserMiddleware(), + bodyParser(), jobMiddlewares('cancel', cancelJob, RATE_LIMIT_ENDPOINTS_GROUPS.JOB_DELETE) ]); } }; function composeJobMiddlewares (metadataBackend, userDatabaseService, jobService, statsdClient, userLimitsService) { - return function jobMiddlewares (action, jobMiddleware, endpointGroup) { + return function jobMiddlewares (action, job, endpointGroup) { const forceToBeMaster = true; return [ - initializeProfilerMiddleware('job'), - userMiddleware(metadataBackend), - rateLimitsMiddleware(userLimitsService, endpointGroup), - authorizationMiddleware(metadataBackend, forceToBeMaster), - connectionParamsMiddleware(userDatabaseService), - jobMiddleware(jobService), + initializeProfiler('job'), + user(metadataBackend), + rateLimits(userLimitsService, endpointGroup), + authorization(metadataBackend, forceToBeMaster), + connectionParams(userDatabaseService), + job(jobService), setServedByDBHostHeader(), - finishProfilerMiddleware(), + finishProfiler(), logJobResult(action), incrementSuccessMetrics(statsdClient), sendResponse(), incrementErrorMetrics(statsdClient), - errorMiddleware() + error() ]; }; } diff --git a/lib/api/sql/query-controller.js b/lib/api/sql/query-controller.js index 5e1e9077..4e88a8c3 100644 --- a/lib/api/sql/query-controller.js +++ b/lib/api/sql/query-controller.js @@ -1,7 +1,7 @@ 'use strict'; const bodyParser = require('../middlewares/body-parser'); -const { initializeProfilerMiddleware: initializeProfiler } = require('../middlewares/profiler'); +const { initializeProfiler } = require('../middlewares/profiler'); const user = require('../middlewares/user'); const rateLimits = require('../middlewares/rate-limit'); const authorization = require('../middlewares/authorization'); From d00d30c3d46b3eef515565745f18b9e6cf421de1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Thu, 3 Oct 2019 20:02:40 +0200 Subject: [PATCH 35/37] Remove unused class member --- lib/api/sql/sql-router.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/api/sql/sql-router.js b/lib/api/sql/sql-router.js index 431a91b6..b0b4397e 100644 --- a/lib/api/sql/sql-router.js +++ b/lib/api/sql/sql-router.js @@ -16,9 +16,7 @@ const CopyController = require('./copy-controller'); const JobController = require('./job-controller'); module.exports = class SqlRouter { - constructor ({ routes, metadataBackend, statsClient, dataIngestionLogger, jobService }) { - this.routes = routes; - + constructor ({ metadataBackend, statsClient, dataIngestionLogger, jobService }) { const userLimitsServiceOptions = { limits: { rateLimitsEnabled: global.settings.ratelimits.rateLimitsEnabled From 23628a40d9278d94a914eda58b9fb85be534e91f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Oct 2019 14:28:17 +0200 Subject: [PATCH 36/37] Remove leftovers --- test/websocket_test/public/run_server.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/websocket_test/public/run_server.sh b/test/websocket_test/public/run_server.sh index 52428779..db92673c 100755 --- a/test/websocket_test/public/run_server.sh +++ b/test/websocket_test/public/run_server.sh @@ -1,5 +1 @@ python -m SimpleHTTPServer 8000 - - - -“1-second Burst Limit” means the maximum number of requests per second that the CARTO APIs will serve at the start of a period of activity measured in one-second intervals, and which is based on the generic cell rate and leaky bucket algorithms. Burst Limit takes priority over the Request Limit: when a request arrives in a one-second interval, the request is first checked against the Burst Limit. If the Burst Limit is exceeded, the request is rejected even if the Request Limit has not been exceeded. From e96ea32ea46c6778d9e76eec5555bab26199a942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Garc=C3=ADa=20Aubert?= Date: Mon, 7 Oct 2019 14:38:21 +0200 Subject: [PATCH 37/37] Rely on defaults (server-options) for routes configuration --- app.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app.js b/app.js index a9527fc2..fe839904 100755 --- a/app.js +++ b/app.js @@ -69,12 +69,6 @@ if (global.settings.log_filename) { global.log4js.configure(log4jsConfig); global.logger = global.log4js.getLogger(); -if (!global.settings.routes) { - console.error('Missing environment paramenter "routes". Please review your configuration file.'); - console.error("Available environments: " + availableEnvironments.join(', ')); - process.exit(1); -} - const version = require("./package").version; const StatsClient = require('./lib/stats/client');