cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1175 from CartoDB/dgaubert/ch78384/maps-api-replace-log4js-logger-by-pino-bis-bis-bis

Browse Source 
Tags Middleware
This commit is contained in:
Daniel G. Aubert 2020-06-11 09:38:14 +02:00 committed by GitHub
commit ff00fed43e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 205 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
lib/api/api-router.js
View File

@ -199,20 +199,21 @@ module.exports = class ApiRouter {
const apiRouter = router({ mergeParams: true });
const { paths, middlewares = [] } = route;
middlewares.forEach(middleware => apiRouter.use(middleware()));
apiRouter.use(initLogger({ logger: this.serverOptions.logger }));
apiRouter.use(initializeStatusCode());
apiRouter.use(bodyParser.json());
apiRouter.use(servedByHostHeader());
apiRouter.use(clientHeader());
apiRouter.use(profiler({
enabled: this.serverOptions.useProfiler,
statsClient: global.statsClient
}));
apiRouter.use(user(this.metadataBackend));
middlewares.forEach(middleware => apiRouter.use(middleware()));
apiRouter.use(initializeStatusCode());
apiRouter.use(bodyParser.json());
apiRouter.use(servedByHostHeader());
apiRouter.use(clientHeader());
apiRouter.use(lzmaMiddleware());
apiRouter.use(cors());
apiRouter.use(user(this.metadataBackend));
this.templateRouter.route(apiRouter, route.template);
this.mapRouter.route(apiRouter, route.map);

2
lib/api/map/analyses-catalog-controller.js
View File

@ -1,6 +1,7 @@
'use strict';
const PSQL = require('cartodb-psql');
const tag = require('../middlewares/tag');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const authorize = require('../middlewares/authorize');
@ -23,6 +24,7 @@ module.exports = class AnalysesController {
middlewares () {
return [
tag({ tags: ['analysis', 'catalog'] }),
credentials(),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),

2
lib/api/map/analysis-layergroup-controller.js
View File

@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const layergroupToken = require('../middlewares/layergroup-token');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
@ -23,6 +24,7 @@ module.exports = class AnalysisLayergroupController {
middlewares () {
return [
tag({ tags: ['analysis', 'node'] }),
layergroupToken(),
credentials(),
authorize(this.authBackend),

2
lib/api/map/anonymous-map-controller.js
View File

@ -3,6 +3,7 @@
const windshaft = require('windshaft');
const MapConfig = windshaft.model.MapConfig;
const Datasource = windshaft.model.Datasource;
const tag = require('../middlewares/tag');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
@ -90,6 +91,7 @@ module.exports = class AnonymousMapController {
};
return [
tag({ tags: ['map', 'anonymous'] }),
metrics({
enabled: this.config.pubSubMetrics.enabled,
metricsBackend: this.metricsBackend,

2
lib/api/map/attributes-layergroup-controller.js
View File

@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const layergroupToken = require('../middlewares/layergroup-token');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
@ -38,6 +39,7 @@ module.exports = class AttributesLayergroupController {
middlewares () {
return [
tag({ tags: ['attributes'] }),
layergroupToken(),
credentials(),
authorize(this.authBackend),

2
lib/api/map/clustered-features-layergroup-controller.js
View File

@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const layergroupToken = require('../middlewares/layergroup-token');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
@ -38,6 +39,7 @@ module.exports = class AggregatedFeaturesLayergroupController {
middlewares () {
return [
tag({ tags: ['cluster'] }),
layergroupToken(),
credentials(),
authorize(this.authBackend),

2
lib/api/map/dataview-layergroup-controller.js
View File

@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const layergroupToken = require('../middlewares/layergroup-token');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
@ -76,6 +77,7 @@ module.exports = class DataviewLayergroupController {
middlewares ({ action, rateLimitGroup }) {
return [
tag({ tags: ['dataview', action] }),
layergroupToken(),
credentials(),
authorize(this.authBackend),

2
lib/api/map/preview-layergroup-controller.js
View File

@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const layergroupToken = require('../middlewares/layergroup-token');
const coordinates = require('../middlewares/coordinates');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
@ -61,6 +62,7 @@ module.exports = class PreviewLayergroupController {
}
return [
tag({ tags: ['static', 'tile'] }),
layergroupToken(),
validateZoom ? coordinates({ z: true, x: false, y: false }) : noop(),
credentials(),

2
lib/api/map/preview-template-controller.js
View File

@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
@ -67,6 +68,7 @@ module.exports = class PreviewTemplateController {
};
return [
tag({ tags: ['named', 'static', 'tile'] }),
metrics({
enabled: this.config.pubSubMetrics.enabled,
metricsBackend: this.metricsBackend,

2
lib/api/map/tile-layergroup-controller.js
View File

@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const layergroupToken = require('../middlewares/layergroup-token');
const coordinates = require('../middlewares/coordinates');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
@ -59,6 +60,7 @@ module.exports = class TileLayergroupController {
middlewares () {
return [
tag({ tags: ['tile'] }),
layergroupToken(),
coordinates(),
credentials(),

4
lib/api/middlewares/profiler.js
View File

@ -2,13 +2,13 @@
const Profiler = require('../../stats/profiler-proxy');
const debug = require('debug')('windshaft:cartodb:stats');
const { name: prefix } = require('../../../package.json');
module.exports = function profiler (options) {
const { enabled = true, statsClient } = options;
return function profilerMiddleware (req, res, next) {
const { logger } = res.locals;
const { id } = logger.bindings();
// TODO: stop using profiler and log stats instead of adding them to the profiler
req.profiler = new Profiler({
@ -16,7 +16,7 @@ module.exports = function profiler (options) {
profile: enabled
});
req.profiler.start(id);
req.profiler.start(prefix);
res.on('finish', () => {
req.profiler.done('response');

15
lib/api/middlewares/tag.js Normal file
View File

@ -0,0 +1,15 @@
'use strict';
module.exports = function tag ({ tags }) {
if (!Array.isArray(tags) || !tags.every((tag) => typeof tag === 'string')) {
throw new Error('Required "tags" option must be a valid Array: [string, string, ...]');
}
return function tagMiddleware (req, res, next) {
const { logger } = res.locals;
res.locals.tags = tags;
res.on('finish', () => logger.info({ tags: res.locals.tags }));
next();
};
};

2
lib/api/template/admin-template-controller.js
View File

@ -1,6 +1,7 @@
'use strict';
const { templateName } = require('../../backends/template-maps');
const tag = require('../middlewares/tag');
const credentials = require('../middlewares/credentials');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
@ -76,6 +77,7 @@ module.exports = class AdminTemplateController {
}
return [
tag({ tags: ['named', 'admin', action] }),
credentials(),
authorizedByAPIKey({ authBackend: this.authBackend, action, label }),
rateLimit(this.userLimitsBackend, rateLimitGroup),

2
lib/api/template/named-template-controller.js
View File

@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
const dbConnSetup = require('../middlewares/db-conn-setup');
@ -88,6 +89,7 @@ module.exports = class NamedMapController {
};
return [
tag({ tags: ['map', 'named'] }),
metrics({
enabled: this.config.pubSubMetrics.enabled,
metricsBackend: this.metricsBackend,

2
lib/api/template/tile-template-controller.js
View File

@ -1,5 +1,6 @@
'use strict';
const tag = require('../middlewares/tag');
const coordinates = require('../middlewares/coordinates');
const cleanUpQueryParams = require('../middlewares/clean-up-query-params');
const credentials = require('../middlewares/credentials');
@ -37,6 +38,7 @@ module.exports = class TileTemplateController {
middlewares () {
return [
tag({ tags: ['tile', 'named'] }),
coordinates(),
credentials(),
authorize(this.authBackend),

11
metro/index.js Normal file
View File

@ -0,0 +1,11 @@
'use strict';
const split = require('split2');
const logCollector = require('./log-collector');
const metricsCollector = require('./metrics-collector');
process.stdin
.pipe(split())
.pipe(logCollector())
.pipe(metricsCollector())
.pipe(process.stdout);

13
log-collector.js → metro/log-collector.js
View File

@ -2,7 +2,7 @@
const split = require('split2');
const assingDeep = require('assign-deep');
const { Transform } = require('readable-stream');
const { Transform } = require('stream');
const DEV_ENVS = ['test', 'development'];
const logs = new Map();
@ -15,9 +15,9 @@ const LEVELS = {
60: 'fatal'
}
function logTransport () {
module.exports = function logCollector () {
return new Transform({
transform: function transform (chunk, enc, callback) {
transform (chunk, enc, callback) {
let entry;
try {
@ -25,7 +25,7 @@ function logTransport () {
const { level, time } = entry;
if (level === undefined && time === undefined) {
throw new Error('Entry log is not a valid');
throw new Error('Entry log is not valid');
}
} catch (e) {
if (DEV_ENVS.includes(process.env.NODE_ENV)) {
@ -76,8 +76,3 @@ function logTransport () {
function hasProperty(obj, prop) {
return Object.prototype.hasOwnProperty.call(obj, prop)
}
process.stdin
.pipe(split())
.pipe(logTransport())
.pipe(process.stdout);

119
metro/metrics-collector.js Normal file
View File

@ -0,0 +1,119 @@
'use strict'
const http = require('http');
const { Counter, Histogram, register } = require('prom-client');
const split = require('split2');
const { Transform } = require('stream');
const DEV_ENVS = ['test', 'development'];
const requestCounter = new Counter({
name: 'maps_api_requests_total',
help: 'MAPS API requests total'
});
const requestOkCounter = new Counter({
name: 'maps_api_requests_ok_total',
help: 'MAPS API requests ok total'
});
const requestErrorCounter = new Counter({
name: 'maps_api_requests_errors_total',
help: 'MAPS API requests errors total'
});
const responseTimeHistogram = new Histogram({
name: 'maps_api_response_time_total',
help: 'MAPS API response time total'
});
const userRequestCounter = new Counter({
name: 'maps_api_requests',
help: 'MAPS API requests per user',
labelNames: ['user', 'http_code']
});
const userRequestOkCounter = new Counter({
name: 'maps_api_requests_ok',
help: 'MAPS API requests per user with success HTTP code',
labelNames: ['user', 'http_code']
});
const userRequestErrorCounter = new Counter({
name: 'maps_api_requests_errors',
help: 'MAPS API requests per user with error HTTP code',
labelNames: ['user', 'http_code']
});
const userResponseTimeHistogram = new Histogram({
name: 'maps_api_response_time',
help: 'MAPS API response time total',
labelNames: ['user']
});
module.exports = function metricsCollector () {
return new Transform({
transform (chunk, enc, callback) {
let entry;
try {
entry = JSON.parse(chunk);
const { level, time } = entry;
if (level === undefined && time === undefined) {
throw new Error('Entry log is not valid');
}
} catch (e) {
if (DEV_ENVS.includes(process.env.NODE_ENV)) {
this.push(chunk);
}
return callback();
}
const { request, response, stats } = entry;
if (request === undefined || response === undefined || stats === undefined) {
this.push(chunk);
return callback();
}
const { statusCode, headers } = response;
const { 'carto-user': user } = headers;
if (statusCode === undefined || headers === undefined || user === undefined) {
this.push(chunk);
return callback();
}
requestCounter.inc();
userRequestCounter.labels(user, `${statusCode}`).inc();
if (statusCode >= 200 && statusCode < 400) {
requestOkCounter.inc();
userRequestOkCounter.labels(user, `${statusCode}`).inc();
} else {
requestErrorCounter.inc();
userRequestErrorCounter.labels(user, `${statusCode}`).inc();
}
const { response: responseTime } = stats;
if (Number.isFinite(responseTime)) {
responseTimeHistogram.observe(responseTime);
userResponseTimeHistogram.labels(user).observe(responseTime);
}
this.push(chunk);
callback();
}
})
}
const port = process.env.PORT || 9145;
http
.createServer((req, res) => {
res.writeHead(200, { 'Content-Type': register.contentType });
res.end(register.metrics());
})
.listen(port)
.unref();

21
package-lock.json generated
View File

@ -719,6 +719,11 @@
"integrity": "sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow==",
"dev": true
},
"bintrees": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz",
"integrity": "sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ="
},
"body-parser": {
"version": "1.18.3",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.3.tgz",
@ -5409,6 +5414,14 @@
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
"dev": true
},
"prom-client": {
"version": "12.0.0",
"resolved": "https://registry.npmjs.org/prom-client/-/prom-client-12.0.0.tgz",
"integrity": "sha512-JbzzHnw0VDwCvoqf8y1WDtq4wSBAbthMB1pcVI/0lzdqHGJI3KBJDXle70XK+c7Iv93Gihqo0a5LlOn+g8+DrQ==",
"requires": {
"tdigest": "^0.1.1"
}
},
"propagate": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/propagate/-/propagate-1.0.0.tgz",
@ -6398,6 +6411,14 @@
}
}
},
"tdigest": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/tdigest/-/tdigest-0.1.1.tgz",
"integrity": "sha1-Ljyyw56kSeVdHmzZEReszKRYgCE=",
"requires": {
"bintrees": "1.0.1"
}
},
"test-exclude": {
"version": "5.2.3",
"resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-5.2.3.tgz",

1
package.json
View File

@ -53,6 +53,7 @@
"lzma": "2.3.2",
"node-statsd": "0.1.1",
"pino": "^6.3.1",
"prom-client": "^12.0.0",
"queue-async": "1.1.0",
"redis-mpool": "^0.8.0",
"request": "2.87.0",