'use strict'; // CartoDB SQL API // // all requests expect the following URL args: // - `sql` {String} SQL to execute // // for private (read/write) queries: // - OAuth. Must have proper OAuth 1.1 headers. For OAuth 1.1 spec see Google // // eg. /api/v1/?sql=SELECT 1 as one (with a load of OAuth headers or URL arguments) // // for public (read only) queries: // - sql only, provided the subdomain exists in CartoDB and the table's sharing options are public // // eg. vizzuality.cartodb.com/api/v1/?sql=SELECT * from my_table // var express = require('express'); var Profiler = require('./stats/profiler-proxy'); var _ = require('underscore'); var fs = require('fs'); var mkdirp = require('mkdirp'); var TableCacheFactory = require('./utils/table_cache_factory'); var RedisPool = require('redis-mpool'); var cartodbRedis = require('cartodb-redis'); var UserDatabaseService = require('./services/user_database_service'); var UserLimitsService = require('./services/user_limits'); var JobPublisher = require('../batch/pubsub/job-publisher'); var JobQueue = require('../batch/job_queue'); var JobBackend = require('../batch/job_backend'); var JobCanceller = require('../batch/job_canceller'); var JobService = require('../batch/job_service'); const Logger = require('./services/logger'); var cors = require('./middlewares/cors'); var GenericController = require('./controllers/generic_controller'); var QueryController = require('./controllers/query_controller'); var CopyController = require('./controllers/copy_controller'); var JobController = require('./controllers/job_controller'); var CacheStatusController = require('./controllers/cache_status_controller'); var HealthCheckController = require('./controllers/health_check_controller'); var VersionController = require('./controllers/version_controller'); var batchFactory = require('../batch'); process.env.PGAPPNAME = process.env.PGAPPNAME || 'cartodb_sqlapi'; // override Date.toJSON require('./utils/date_to_json'); // jshint maxcomplexity:9 function App(statsClient) { var app = express(); var redisPool = new RedisPool({ name: 'sql-api', host: global.settings.redis_host, port: global.settings.redis_port, max: global.settings.redisPool, idleTimeoutMillis: global.settings.redisIdleTimeoutMillis, reapIntervalMillis: global.settings.redisReapIntervalMillis }); var metadataBackend = cartodbRedis({ pool: redisPool }); // Set default configuration global.settings.db_pubuser = global.settings.db_pubuser || "publicuser"; global.settings.bufferedRows = global.settings.bufferedRows || 1000; global.settings.ratelimits = Object.assign( { rateLimitsEnabled: false, endpoints: { query: false, job_create: false, job_get: false, job_delete: false } }, global.settings.ratelimits ); global.settings.tmpDir = global.settings.tmpDir || '/tmp'; if (!fs.existsSync(global.settings.tmpDir)) { mkdirp.sync(global.settings.tmpDir); } var tableCache = new TableCacheFactory().build(global.settings); // Size based on https://github.com/CartoDB/cartodb.js/blob/3.15.2/src/geo/layer_definition.js#L72 var SQL_QUERY_BODY_LOG_MAX_LENGTH = 2000; app.getSqlQueryFromRequestBody = function(req) { var sqlQuery = req.body && req.body.q; if (!sqlQuery) { return ''; } if (sqlQuery.length > SQL_QUERY_BODY_LOG_MAX_LENGTH) { sqlQuery = sqlQuery.substring(0, SQL_QUERY_BODY_LOG_MAX_LENGTH) + ' [...]'; } return JSON.stringify({q: sqlQuery}); }; if ( global.log4js ) { var loggerOpts = { buffer: true, // log4js provides a tokens solution as expess but in does not provide the request/response in the callback. // Thus it is not possible to extract relevant information from them. // This is a workaround to be able to access request/response. format: function(req, res, format) { var logFormat = global.settings.log_format || ':remote-addr :method :req[Host]:url :status :response-time ms -> :res[Content-Type]'; logFormat = logFormat.replace(/:sql/, app.getSqlQueryFromRequestBody(req)); return format(logFormat); } }; app.use(global.log4js.connectLogger(global.log4js.getLogger(), _.defaults(loggerOpts, {level:'info'}))); } app.use(cors()); // Use step-profiler app.use(function bootstrap$prepareRequestResponse(req, res, next) { res.locals = res.locals || {}; if (global.settings.api_hostname) { res.header('X-Served-By-Host', global.settings.api_hostname); } var profile = global.settings.useProfiler; req.profiler = new Profiler({ profile: profile, statsd_client: statsClient }); next(); }); // Set connection timeout if ( global.settings.hasOwnProperty('node_socket_timeout') ) { var timeout = parseInt(global.settings.node_socket_timeout); app.use(function(req, res, next) { req.connection.setTimeout(timeout); next(); }); } app.enable('jsonp callback'); app.set("trust proxy", true); app.disable('x-powered-by'); app.disable('etag'); // basic routing var userDatabaseService = new UserDatabaseService(metadataBackend); const userLimitsServiceOptions = { limits: { rateLimitsEnabled: global.settings.ratelimits.rateLimitsEnabled } }; const userLimitsService = new UserLimitsService(metadataBackend, userLimitsServiceOptions); const dataIngestionLogger = new Logger(global.settings.dataIngestionLogPath, 'data-ingestion'); app.dataIngestionLogger = dataIngestionLogger; var jobPublisher = new JobPublisher(redisPool); var jobQueue = new JobQueue(metadataBackend, jobPublisher); var jobBackend = new JobBackend(metadataBackend, jobQueue); var jobCanceller = new JobCanceller(); var jobService = new JobService(jobBackend, jobCanceller); var genericController = new GenericController(); genericController.route(app); var queryController = new QueryController( metadataBackend, userDatabaseService, tableCache, statsClient, userLimitsService ); queryController.route(app); var copyController = new CopyController( metadataBackend, userDatabaseService, userLimitsService, dataIngestionLogger ); copyController.route(app); var jobController = new JobController( metadataBackend, userDatabaseService, jobService, statsClient, userLimitsService ); jobController.route(app); var cacheStatusController = new CacheStatusController(tableCache); cacheStatusController.route(app); var healthCheckController = new HealthCheckController(); healthCheckController.route(app); var versionController = new VersionController(); versionController.route(app); var isBatchProcess = process.argv.indexOf('--no-batch') === -1; if (global.settings.environment !== 'test' && isBatchProcess) { var batchName = global.settings.api_hostname || 'batch'; app.batch = batchFactory( metadataBackend, redisPool, batchName, statsClient, global.settings.batch_log_filename ); app.batch.start(); } return app; } module.exports = App;