2018-02-17 01:21:06 +08:00
|
|
|
const util = require('util');
|
2015-12-07 16:40:51 +08:00
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
const userMiddleware = require('../middlewares/user');
|
2018-02-19 22:13:36 +08:00
|
|
|
const { initializeProfilerMiddleware, finishProfilerMiddleware } = require('../middlewares/profiler');
|
2018-02-19 22:49:17 +08:00
|
|
|
const authorizationMiddleware = require('../middlewares/authorization');
|
2018-02-22 19:22:39 +08:00
|
|
|
const connectionParamsMiddleware = require('../middlewares/connection-params');
|
2018-02-17 01:21:06 +08:00
|
|
|
const errorMiddleware = require('../middlewares/error');
|
2015-12-07 16:40:51 +08:00
|
|
|
|
2018-02-22 18:46:34 +08:00
|
|
|
function JobController(metadataBackend, userDatabaseService, jobService, statsdClient) {
|
|
|
|
this.metadataBackend = metadataBackend;
|
2016-01-22 00:24:48 +08:00
|
|
|
this.userDatabaseService = userDatabaseService;
|
2016-05-14 00:50:55 +08:00
|
|
|
this.jobService = jobService;
|
2018-02-17 01:21:06 +08:00
|
|
|
this.statsdClient = statsdClient;
|
2016-05-24 20:28:00 +08:00
|
|
|
}
|
|
|
|
|
2016-05-14 00:50:55 +08:00
|
|
|
module.exports = JobController;
|
|
|
|
|
2015-12-07 16:40:51 +08:00
|
|
|
JobController.prototype.route = function (app) {
|
2018-02-17 01:21:06 +08:00
|
|
|
const { base_url } = global.settings;
|
2018-02-20 23:25:16 +08:00
|
|
|
const jobMiddlewares = composeJobMiddlewares(
|
2018-02-22 18:46:34 +08:00
|
|
|
this.metadataBackend,
|
2018-02-20 23:25:16 +08:00
|
|
|
this.userDatabaseService,
|
|
|
|
this.jobService,
|
|
|
|
this.statsdClient
|
2016-10-04 21:40:56 +08:00
|
|
|
);
|
2018-02-17 01:21:06 +08:00
|
|
|
|
2018-02-23 20:19:26 +08:00
|
|
|
app.get(`${base_url}/jobs-wip`, listWorkInProgressJobs(this.jobService), sendResponse(), errorMiddleware());
|
2018-02-20 23:25:16 +08:00
|
|
|
app.post(`${base_url}/sql/job`, checkBodyPayloadSize(), jobMiddlewares('create', createJob));
|
|
|
|
app.get(`${base_url}/sql/job/:job_id`, jobMiddlewares('retrieve', getJob));
|
|
|
|
app.delete(`${base_url}/sql/job/:job_id`, jobMiddlewares('cancel', cancelJob));
|
2015-12-31 03:16:18 +08:00
|
|
|
};
|
|
|
|
|
2018-02-22 18:46:34 +08:00
|
|
|
function composeJobMiddlewares (metadataBackend, userDatabaseService, jobService, statsdClient) {
|
2018-02-20 23:43:43 +08:00
|
|
|
return function jobMiddlewares (action, jobMiddleware) {
|
2018-02-20 23:25:16 +08:00
|
|
|
const forceToBeAuthenticated = true;
|
|
|
|
|
|
|
|
return [
|
|
|
|
initializeProfilerMiddleware('job'),
|
|
|
|
userMiddleware(),
|
2018-02-22 19:22:39 +08:00
|
|
|
authorizationMiddleware(metadataBackend, forceToBeAuthenticated),
|
|
|
|
connectionParamsMiddleware(userDatabaseService),
|
2018-02-20 23:43:43 +08:00
|
|
|
jobMiddleware(jobService),
|
2018-02-20 23:25:16 +08:00
|
|
|
setServedByDBHostHeader(),
|
|
|
|
finishProfilerMiddleware(),
|
|
|
|
logJobResult(action),
|
|
|
|
incrementSuccessMetrics(statsdClient),
|
2018-02-23 20:03:56 +08:00
|
|
|
sendResponse(),
|
2018-02-20 23:25:16 +08:00
|
|
|
incrementErrorMetrics(statsdClient),
|
|
|
|
errorMiddleware()
|
|
|
|
];
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
function cancelJob (jobService) {
|
|
|
|
return function cancelJobMiddleware (req, res, next) {
|
|
|
|
const { job_id } = req.params;
|
2015-12-07 16:40:51 +08:00
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
jobService.cancel(job_id, (err, job) => {
|
2018-02-19 21:42:52 +08:00
|
|
|
if (req.profiler) {
|
|
|
|
req.profiler.done('cancelJob');
|
|
|
|
}
|
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
if (err) {
|
|
|
|
return next(err);
|
|
|
|
}
|
2015-12-25 00:42:49 +08:00
|
|
|
|
2018-02-23 20:03:56 +08:00
|
|
|
res.body = job.serialize();
|
2018-02-17 01:21:06 +08:00
|
|
|
|
|
|
|
next();
|
|
|
|
});
|
2016-10-04 22:07:13 +08:00
|
|
|
};
|
2018-02-17 01:21:06 +08:00
|
|
|
}
|
2016-05-14 00:50:55 +08:00
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
function getJob (jobService) {
|
|
|
|
return function getJobMiddleware (req, res, next) {
|
2018-02-19 18:04:28 +08:00
|
|
|
const { job_id } = req.params;
|
|
|
|
|
|
|
|
jobService.get(job_id, (err, job) => {
|
2018-02-19 21:42:52 +08:00
|
|
|
if (req.profiler) {
|
|
|
|
req.profiler.done('getJob');
|
|
|
|
}
|
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
if (err) {
|
|
|
|
return next(err);
|
|
|
|
}
|
2015-12-07 16:40:51 +08:00
|
|
|
|
2018-02-23 20:03:56 +08:00
|
|
|
res.body = job.serialize();
|
2016-10-28 21:08:42 +08:00
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
next();
|
|
|
|
});
|
|
|
|
};
|
|
|
|
}
|
2016-10-28 21:08:42 +08:00
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
function createJob (jobService) {
|
|
|
|
return function createJobMiddleware (req, res, next) {
|
|
|
|
const params = Object.assign({}, req.query, req.body);
|
2016-10-28 21:08:42 +08:00
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
var data = {
|
|
|
|
user: res.locals.user,
|
2018-02-19 18:04:28 +08:00
|
|
|
query: params.query,
|
2018-02-17 01:21:06 +08:00
|
|
|
host: res.locals.userDbParams.host,
|
|
|
|
port: res.locals.userDbParams.port,
|
|
|
|
pass: res.locals.userDbParams.pass,
|
|
|
|
dbname: res.locals.userDbParams.dbname,
|
|
|
|
dbuser: res.locals.userDbParams.user
|
|
|
|
};
|
2016-10-28 21:08:42 +08:00
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
jobService.create(data, (err, job) => {
|
2018-02-19 21:42:52 +08:00
|
|
|
if (req.profiler) {
|
|
|
|
req.profiler.done('createJob');
|
|
|
|
}
|
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
if (err) {
|
|
|
|
return next(err);
|
|
|
|
}
|
|
|
|
|
|
|
|
res.locals.job_id = job.job_id;
|
|
|
|
|
2018-02-23 20:03:56 +08:00
|
|
|
res.statusCode = 201;
|
|
|
|
res.body = job.serialize();
|
2018-02-17 01:21:06 +08:00
|
|
|
|
|
|
|
next();
|
|
|
|
});
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
function listWorkInProgressJobs (jobService) {
|
|
|
|
return function listWorkInProgressJobsMiddleware (req, res, next) {
|
|
|
|
jobService.listWorkInProgressJobs((err, list) => {
|
|
|
|
if (err) {
|
|
|
|
return next(err);
|
|
|
|
}
|
|
|
|
|
2018-02-23 20:03:56 +08:00
|
|
|
res.body = list;
|
2018-02-17 01:21:06 +08:00
|
|
|
|
|
|
|
next();
|
|
|
|
});
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2018-02-19 21:42:52 +08:00
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
function checkBodyPayloadSize () {
|
|
|
|
return function checkBodyPayloadSizeMiddleware(req, res, next) {
|
|
|
|
const payload = JSON.stringify(req.body);
|
|
|
|
|
|
|
|
if (payload.length > MAX_LIMIT_QUERY_SIZE_IN_BYTES) {
|
|
|
|
return next(new Error(getMaxSizeErrorMessage(payload)), res);
|
2016-10-28 21:08:42 +08:00
|
|
|
}
|
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
next();
|
|
|
|
};
|
|
|
|
}
|
2016-10-28 21:08:42 +08:00
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
const ONE_KILOBYTE_IN_BYTES = 1024;
|
|
|
|
const MAX_LIMIT_QUERY_SIZE_IN_KB = 16;
|
|
|
|
const MAX_LIMIT_QUERY_SIZE_IN_BYTES = MAX_LIMIT_QUERY_SIZE_IN_KB * ONE_KILOBYTE_IN_BYTES;
|
2016-01-07 19:06:01 +08:00
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
function getMaxSizeErrorMessage(sql) {
|
|
|
|
return util.format([
|
|
|
|
'Your payload is too large: %s bytes. Max size allowed is %s bytes (%skb).',
|
|
|
|
'Are you trying to import data?.',
|
|
|
|
'Please, check out import api http://docs.cartodb.com/cartodb-platform/import-api/'
|
|
|
|
].join(' '),
|
|
|
|
sql.length,
|
|
|
|
MAX_LIMIT_QUERY_SIZE_IN_BYTES,
|
|
|
|
Math.round(MAX_LIMIT_QUERY_SIZE_IN_BYTES / ONE_KILOBYTE_IN_BYTES)
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
|
|
|
module.exports.MAX_LIMIT_QUERY_SIZE_IN_BYTES = MAX_LIMIT_QUERY_SIZE_IN_BYTES;
|
|
|
|
module.exports.getMaxSizeErrorMessage = getMaxSizeErrorMessage;
|
|
|
|
|
|
|
|
function setServedByDBHostHeader () {
|
|
|
|
return function setServedByDBHostHeaderMiddleware (req, res, next) {
|
|
|
|
const { userDbParams } = res.locals;
|
|
|
|
|
|
|
|
if (userDbParams.host) {
|
|
|
|
res.header('X-Served-By-DB-Host', res.locals.userDbParams.host);
|
2016-10-04 22:07:13 +08:00
|
|
|
}
|
2016-05-27 01:44:59 +08:00
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
next();
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2018-02-20 20:14:28 +08:00
|
|
|
function logJobResult (action) {
|
|
|
|
return function logJobResultMiddleware (req, res, next) {
|
2016-10-12 07:40:14 +08:00
|
|
|
if (process.env.NODE_ENV !== 'test') {
|
|
|
|
console.info(JSON.stringify({
|
|
|
|
type: 'sql_api_batch_job',
|
2017-11-24 22:49:25 +08:00
|
|
|
username: res.locals.user,
|
2016-10-12 07:40:14 +08:00
|
|
|
action: action,
|
2018-02-17 01:21:06 +08:00
|
|
|
job_id: req.params.job_id || res.locals.job_id
|
2016-10-12 07:40:14 +08:00
|
|
|
}));
|
|
|
|
}
|
2016-10-04 22:07:13 +08:00
|
|
|
|
2018-02-17 01:21:06 +08:00
|
|
|
next();
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
const METRICS_PREFIX = 'sqlapi.job';
|
|
|
|
|
|
|
|
function incrementSuccessMetrics (statsdClient) {
|
|
|
|
return function incrementSuccessMetricsMiddleware (req, res, next) {
|
|
|
|
if (statsdClient !== undefined) {
|
|
|
|
statsdClient.increment(`${METRICS_PREFIX}.success`);
|
|
|
|
}
|
|
|
|
|
|
|
|
next();
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
function incrementErrorMetrics (statsdClient) {
|
|
|
|
return function incrementErrorMetricsMiddleware (err, req, res, next) {
|
|
|
|
if (statsdClient !== undefined) {
|
|
|
|
statsdClient.increment(`${METRICS_PREFIX}.error`);
|
|
|
|
}
|
|
|
|
|
|
|
|
next(err);
|
2016-10-04 22:07:13 +08:00
|
|
|
};
|
|
|
|
}
|
2018-02-23 20:03:56 +08:00
|
|
|
|
|
|
|
function sendResponse () {
|
|
|
|
return function sendResponseMiddleware (req, res) {
|
|
|
|
res.status(res.statusCode || 200).send(res.body);
|
|
|
|
};
|
|
|
|
}
|