diff --git a/NEWS.md b/NEWS.md index 29e94124..14f69a0b 100644 --- a/NEWS.md +++ b/NEWS.md @@ -20,6 +20,7 @@ Announcements: - Rename NamedMapProviderReporter by NamedMapProviderCacheReporter - Remove `bootstrapFonts` at process startup (now done in `windshaft@6.0.0`) - Stop checking the installed version of some dependencies while testing +- Send metrics about `map views` (#1162) Bug Fixes: - Parsing date column in numeric histograms (#1160) diff --git a/lib/api/api-router.js b/lib/api/api-router.js index 7bf7da88..ab2bf332 100644 --- a/lib/api/api-router.js +++ b/lib/api/api-router.js @@ -21,7 +21,7 @@ const OverviewsMetadataBackend = require('../backends/overviews-metadata'); const FilterStatsApi = require('../backends/filter-stats'); const TablesExtentBackend = require('../backends/tables-extent'); const ClusterBackend = require('../backends/cluster'); -const PubSubMetricsBackend = require('../backends/pubsub-metrics'); +const PubSubMetricsBackend = require('../backends/metrics'); const LayergroupAffectedTablesCache = require('../cache/layergroup-affected-tables'); const SurrogateKeysCache = require('../cache/surrogate-keys-cache'); @@ -57,7 +57,6 @@ const user = require('./middlewares/user'); const sendResponse = require('./middlewares/send-response'); const syntaxError = require('./middlewares/syntax-error'); const errorMiddleware = require('./middlewares/error-middleware'); -const pubSubMetrics = require('./middlewares/pubsub-metrics'); const MapRouter = require('./map/map-router'); const TemplateRouter = require('./template/template-router'); @@ -161,7 +160,10 @@ module.exports = class ApiRouter { }); namedMapProviderCacheReporter.start(); + const metricsBackend = new PubSubMetricsBackend(serverOptions.pubSubMetrics); + const collaborators = { + config: serverOptions, analysisStatusBackend, attributesBackend, dataviewBackend, @@ -181,13 +183,13 @@ module.exports = class ApiRouter { layergroupMetadata, namedMapProviderCache, tablesExtentBackend, - clusterBackend + clusterBackend, + metricsBackend }; + this.metadataBackend = metadataBackend; this.mapRouter = new MapRouter({ collaborators }); this.templateRouter = new TemplateRouter({ collaborators }); - this.metadataBackend = metadataBackend; - this.pubSubMetricsBackend = PubSubMetricsBackend.build(); } route (app, routes) { @@ -220,7 +222,6 @@ module.exports = class ApiRouter { apiRouter.use(sendResponse()); apiRouter.use(syntaxError()); apiRouter.use(errorMiddleware()); - apiRouter.use(pubSubMetrics(this.pubSubMetricsBackend)); paths.forEach(path => app.use(path, apiRouter)); }); diff --git a/lib/api/map/anonymous-map-controller.js b/lib/api/map/anonymous-map-controller.js index 508e1cd8..46ed876a 100644 --- a/lib/api/map/anonymous-map-controller.js +++ b/lib/api/map/anonymous-map-controller.js @@ -23,6 +23,7 @@ const mapError = require('../middlewares/map-error'); const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const metrics = require('../middlewares/metrics'); module.exports = class AnonymousMapController { /** @@ -39,6 +40,7 @@ module.exports = class AnonymousMapController { * @constructor */ constructor ( + config, pgConnection, templateMaps, mapBackend, @@ -49,8 +51,10 @@ module.exports = class AnonymousMapController { mapConfigAdapter, statsBackend, authBackend, - layergroupMetadata + layergroupMetadata, + metricsBackend ) { + this.config = config; this.pgConnection = pgConnection; this.templateMaps = templateMaps; this.mapBackend = mapBackend; @@ -62,6 +66,7 @@ module.exports = class AnonymousMapController { this.statsBackend = statsBackend; this.authBackend = authBackend; this.layergroupMetadata = layergroupMetadata; + this.metricsBackend = metricsBackend; } route (mapRouter) { @@ -76,8 +81,23 @@ module.exports = class AnonymousMapController { const includeQuery = true; const label = 'ANONYMOUS LAYERGROUP'; const addContext = true; + const metricsTags = { + event: 'map_view', + attributes: { map_type: 'anonymous' }, + from: { + req: { + query: { client: 'client' } + } + } + }; return [ + metrics({ + enabled: this.config.pubSubMetrics.enabled, + metricsBackend: this.metricsBackend, + logger: global.logger, + tags: metricsTags + }), credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), @@ -207,6 +227,7 @@ function createLayergroup (mapBackend, userLimitsBackend, pgConnection, affected ); res.locals.mapConfig = mapConfig; + res.locals.mapConfigProvider = mapConfigProvider; res.locals.analysesResults = context.analysesResults; const mapParams = { dbuser, dbname, dbpassword, dbhost, dbport }; @@ -220,7 +241,6 @@ function createLayergroup (mapBackend, userLimitsBackend, pgConnection, affected res.statusCode = 200; res.body = layergroup; - res.locals.mapConfigProvider = mapConfigProvider; next(); }); diff --git a/lib/api/map/map-router.js b/lib/api/map/map-router.js index c5070627..2ebccb54 100644 --- a/lib/api/map/map-router.js +++ b/lib/api/map/map-router.js @@ -15,6 +15,7 @@ const ClusteredFeaturesLayergroupController = require('./clustered-features-laye module.exports = class MapRouter { constructor ({ collaborators }) { const { + config, analysisStatusBackend, attributesBackend, dataviewBackend, @@ -34,7 +35,8 @@ module.exports = class MapRouter { layergroupMetadata, namedMapProviderCache, tablesExtentBackend, - clusterBackend + clusterBackend, + metricsBackend } = collaborators; this.analysisLayergroupController = new AnalysisLayergroupController( @@ -85,6 +87,7 @@ module.exports = class MapRouter { ); this.anonymousMapController = new AnonymousMapController( + config, pgConnection, templateMaps, mapBackend, @@ -95,10 +98,12 @@ module.exports = class MapRouter { mapConfigAdapter, statsBackend, authBackend, - layergroupMetadata + layergroupMetadata, + metricsBackend ); this.previewTemplateController = new PreviewTemplateController( + config, namedMapProviderCache, previewBackend, surrogateKeysCache, @@ -106,7 +111,8 @@ module.exports = class MapRouter { metadataBackend, pgConnection, authBackend, - userLimitsBackend + userLimitsBackend, + metricsBackend ); this.analysesController = new AnalysesCatalogController( diff --git a/lib/api/map/preview-template-controller.js b/lib/api/map/preview-template-controller.js index 989bb8e1..d9a64707 100644 --- a/lib/api/map/preview-template-controller.js +++ b/lib/api/map/preview-template-controller.js @@ -12,6 +12,7 @@ const lastModifiedHeader = require('../middlewares/last-modified-header'); const checkStaticImageFormat = require('../middlewares/check-static-image-format'); const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const metrics = require('../middlewares/metrics'); const DEFAULT_ZOOM_CENTER = { zoom: 1, @@ -27,6 +28,7 @@ function numMapper (n) { module.exports = class PreviewTemplateController { constructor ( + config, namedMapProviderCache, previewBackend, surrogateKeysCache, @@ -34,8 +36,10 @@ module.exports = class PreviewTemplateController { metadataBackend, pgConnection, authBackend, - userLimitsBackend + userLimitsBackend, + metricsBackend ) { + this.config = config; this.namedMapProviderCache = namedMapProviderCache; this.previewBackend = previewBackend; this.surrogateKeysCache = surrogateKeysCache; @@ -44,6 +48,7 @@ module.exports = class PreviewTemplateController { this.pgConnection = pgConnection; this.authBackend = authBackend; this.userLimitsBackend = userLimitsBackend; + this.metricsBackend = metricsBackend; } route (mapRouter) { @@ -51,7 +56,23 @@ module.exports = class PreviewTemplateController { } middlewares () { + const metricsTags = { + event: 'map_view', + attributes: { map_type: 'static' }, + from: { + req: { + query: { client: 'client' } + } + } + }; + return [ + metrics({ + enabled: this.config.pubSubMetrics.enabled, + metricsBackend: this.metricsBackend, + logger: global.logger, + tags: metricsTags + }), credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), diff --git a/lib/api/middlewares/last-modified-header.js b/lib/api/middlewares/last-modified-header.js index 4f00c0d2..4afd283f 100644 --- a/lib/api/middlewares/last-modified-header.js +++ b/lib/api/middlewares/last-modified-header.js @@ -36,6 +36,8 @@ module.exports = function setLastModifiedHeader () { res.set('Last-Modified', lastModifiedDate.toUTCString()); + res.locals.cache_buster = lastUpdatedAt; + next(); }); }; diff --git a/lib/api/middlewares/last-updated-time-layergroup.js b/lib/api/middlewares/last-updated-time-layergroup.js index a1c4593a..1c4a3cab 100644 --- a/lib/api/middlewares/last-updated-time-layergroup.js +++ b/lib/api/middlewares/last-updated-time-layergroup.js @@ -22,6 +22,8 @@ module.exports = function setLastUpdatedTimeToLayergroup () { layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime; layergroup.last_updated = new Date(lastUpdateTime).toISOString(); + res.locals.cache_buster = lastUpdateTime; + next(); }); }; diff --git a/lib/api/middlewares/layergroup-id-header.js b/lib/api/middlewares/layergroup-id-header.js index 94470972..edc452ed 100644 --- a/lib/api/middlewares/layergroup-id-header.js +++ b/lib/api/middlewares/layergroup-id-header.js @@ -6,8 +6,9 @@ module.exports = function setLayergroupIdHeader (templateMaps, useTemplateHash) const layergroup = res.body; if (useTemplateHash) { - var templateHash = templateMaps.fingerPrint(template).substring(0, 8); + const templateHash = templateMaps.fingerPrint(template).substring(0, 8); layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`; + res.locals.templateHash = templateHash; } res.set('X-Layergroup-Id', layergroup.layergroupid); diff --git a/lib/api/middlewares/layergroup-token.js b/lib/api/middlewares/layergroup-token.js index 88e9295d..7d8003bb 100644 --- a/lib/api/middlewares/layergroup-token.js +++ b/lib/api/middlewares/layergroup-token.js @@ -13,6 +13,10 @@ module.exports = function layergroupToken () { res.locals.token = layergroupToken.token; res.locals.cache_buster = layergroupToken.cacheBuster; + if (layergroupToken.templateHash) { + res.locals.templateHash = layergroupToken.templateHash; + } + if (layergroupToken.signer) { res.locals.signer = layergroupToken.signer; diff --git a/lib/api/middlewares/metrics.js b/lib/api/middlewares/metrics.js new file mode 100644 index 00000000..e5c36a01 --- /dev/null +++ b/lib/api/middlewares/metrics.js @@ -0,0 +1,178 @@ +'use strict'; + +const EVENT_VERSION = '1'; +const MAX_LENGTH = 100; + +module.exports = function metrics ({ enabled, tags, metricsBackend, logger }) { + if (!enabled) { + return function metricsDisabledMiddleware (req, res, next) { + next(); + }; + } + + if (!tags || !tags.event) { + throw new Error('Missing required "event" parameter to report metrics'); + } + + return function metricsMiddleware (req, res, next) { + res.on('finish', () => { + const { event, attributes } = getEventData(req, res, tags); + + metricsBackend.send(event, attributes) + .catch((error) => logger.error(`Failed to publish event "${event}": ${error.message}`)); + }); + + return next(); + }; +}; + +function getEventData (req, res, tags) { + const event = tags.event; + const extra = {}; + if (tags.from) { + if (tags.from.req) { + Object.assign(extra, getFromReq(req, tags.from.req)); + } + + if (tags.from.res) { + Object.assign(extra, getFromRes(res, tags.from.res)); + } + } + + const attributes = Object.assign({}, { + client_event: normalizedField(req.get('Carto-Event')), + client_event_group_id: normalizedField(req.get('Carto-Event-Group-Id')), + event_source: normalizedField(req.get('Carto-Event-Source')), + event_time: new Date().toISOString(), + user_id: res.locals.userId, + user_agent: req.get('User-Agent'), + map_id: getLayergroupid({ res }), + cache_buster: getCacheBuster({ res }), + template_hash: getTemplateHash({ res }), + stat_tag: getStatTag({ res }), + response_code: res.statusCode.toString(), + response_time: getResponseTime(res), + source_domain: req.hostname, + event_version: EVENT_VERSION + }, tags.attributes, extra); + + // remove undefined properties + Object.keys(attributes).forEach(key => attributes[key] === undefined && delete attributes[key]); + + return { event, attributes }; +} + +function normalizedField (field) { + if (!field) { + return undefined; + } + + return field.toString().trim().substr(0, MAX_LENGTH); +} + +function getLayergroupid ({ res }) { + if (res.locals.token) { + return res.locals.token; + } + + if (res.locals.mapConfig) { + return res.locals.mapConfig.id(); + } + + if (res.locals.mapConfigProvider && res.locals.mapConfigProvider.mapConfig) { + return res.locals.mapConfigProvider.mapConfig.id(); + } +} + +function getCacheBuster ({ res }) { + if (res.locals.cache_buster !== undefined) { + return `${res.locals.cache_buster}`; + } + + if (res.locals.mapConfigProvider) { + return `${res.locals.mapConfigProvider.getCacheBuster()}`; + } +} + +function getTemplateHash ({ res }) { + if (res.locals.templateHash) { + return res.locals.templateHash; + } + + if (res.locals.mapConfigProvider && res.locals.mapConfigProvider.getTemplateHash) { + let templateHash; + + try { + templateHash = res.locals.mapConfigProvider.getTemplateHash().substring(0, 8); + } catch (e) {} + + return templateHash; + } +} + +function getStatTag ({ res }) { + if (res.locals.mapConfig) { + return res.locals.mapConfig.obj().stat_tag; + } + + // FIXME: don't expect that mapConfig is already set + if (res.locals.mapConfigProvider && res.locals.mapConfigProvider.mapConfig) { + return res.locals.mapConfigProvider.mapConfig.obj().stat_tag; + } +} + +// FIXME: 'X-Tiler-Profiler' might not be accurate enough +function getResponseTime (res) { + const profiler = res.get('X-Tiler-Profiler'); + let stats; + + try { + stats = JSON.parse(profiler); + } catch (e) { + return undefined; + } + + return stats && stats.total ? stats.total.toString() : undefined; +} + +function getFromReq (req, { query = {}, body = {}, params = {}, headers = {} } = {}) { + const extra = {}; + + for (const [queryParam, eventName] of Object.entries(query)) { + extra[eventName] = req.query[queryParam]; + } + + for (const [bodyParam, eventName] of Object.entries(body)) { + extra[eventName] = req.body[bodyParam]; + } + + for (const [pathParam, eventName] of Object.entries(params)) { + extra[eventName] = req.params[pathParam]; + } + + for (const [header, eventName] of Object.entries(headers)) { + extra[eventName] = req.get(header); + } + + return extra; +} + +function getFromRes (res, { body = {}, headers = {}, locals = {} } = {}) { + const extra = {}; + + if (res.body) { + for (const [bodyParam, eventName] of Object.entries(body)) { + extra[eventName] = res.body[bodyParam]; + } + } + + for (const [header, eventName] of Object.entries(headers)) { + extra[eventName] = res.get(header); + } + + for (const [localParam, eventName] of Object.entries(locals)) { + extra[eventName] = res.locals[localParam]; + } + + return extra; +} diff --git a/lib/api/middlewares/pubsub-metrics.js b/lib/api/middlewares/pubsub-metrics.js deleted file mode 100644 index e12708b5..00000000 --- a/lib/api/middlewares/pubsub-metrics.js +++ /dev/null @@ -1,74 +0,0 @@ -'use strict'; - -const EVENT_VERSION = '1'; -const MAX_LENGTH = 100; - -function pubSubMetrics (pubSubMetricsBackend) { - if (!pubSubMetricsBackend.isEnabled()) { - return function pubSubMetricsDisabledMiddleware (req, res, next) { next(); }; - } - - return function pubSubMetricsMiddleware (req, res, next) { - const data = getEventData(req, res); - - if (data.event) { - pubSubMetricsBackend.sendEvent(data.event, data.attributes); - } - - return next(); - }; -} - -function getEventData (req, res) { - const event = normalizedField(req.get('Carto-Event')); - const eventSource = normalizedField(req.get('Carto-Event-Source')); - const eventGroupId = normalizedField(req.get('Carto-Event-Group-Id')); - - if (!event || !eventSource) { - return [undefined, undefined]; - } - - const attributes = { - event_source: eventSource, - user_id: res.locals.userId, - response_code: res.statusCode.toString(), - source_domain: req.hostname, - event_time: new Date().toISOString(), - event_version: EVENT_VERSION - }; - - if (eventGroupId) { - attributes.event_group_id = eventGroupId; - } - - const responseTime = getResponseTime(res); - - if (responseTime) { - attributes.response_time = responseTime.toString(); - } - - return { event, attributes }; -} - -function normalizedField (field) { - if (!field) { - return undefined; - } - - return field.toString().trim().substr(0, MAX_LENGTH); -} - -function getResponseTime (res) { - const profiler = res.get('X-Tiler-Profiler'); - let stats; - - try { - stats = JSON.parse(profiler); - } catch (e) { - return undefined; - } - - return stats.total; -} - -module.exports = pubSubMetrics; diff --git a/lib/api/middlewares/user.js b/lib/api/middlewares/user.js index cd3fc85c..57c9b45d 100644 --- a/lib/api/middlewares/user.js +++ b/lib/api/middlewares/user.js @@ -8,10 +8,12 @@ module.exports = function user (metadataBackend) { return function userMiddleware (req, res, next) { res.locals.user = getUserNameFromRequest(req, cdbRequest); - getUserId(metadataBackend, res.locals.user, function (userId) { - if (userId) { - res.locals.userId = userId; + metadataBackend.getUserId(res.locals.user, (err, userId) => { + if (err || !userId) { + return next(); } + + res.locals.userId = userId; return next(); }); }; @@ -20,12 +22,3 @@ module.exports = function user (metadataBackend) { function getUserNameFromRequest (req, cdbRequest) { return cdbRequest.userByReq(req); } - -function getUserId (metadataBackend, userName, callback) { - metadataBackend.getUserId(userName, function (err, userId) { - if (err) { - return callback(); - } - return callback(userId); - }); -} diff --git a/lib/api/template/named-template-controller.js b/lib/api/template/named-template-controller.js index b7b3a865..88270b89 100644 --- a/lib/api/template/named-template-controller.js +++ b/lib/api/template/named-template-controller.js @@ -21,6 +21,7 @@ const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider'); const rateLimit = require('../middlewares/rate-limit'); const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit; +const metrics = require('../middlewares/metrics'); module.exports = class NamedMapController { /** @@ -38,6 +39,7 @@ module.exports = class NamedMapController { * @constructor */ constructor ( + config, pgConnection, templateMaps, mapBackend, @@ -48,8 +50,10 @@ module.exports = class NamedMapController { mapConfigAdapter, statsBackend, authBackend, - layergroupMetadata + layergroupMetadata, + metricsBackend ) { + this.config = config; this.pgConnection = pgConnection; this.templateMaps = templateMaps; this.mapBackend = mapBackend; @@ -61,6 +65,7 @@ module.exports = class NamedMapController { this.statsBackend = statsBackend; this.authBackend = authBackend; this.layergroupMetadata = layergroupMetadata; + this.metricsBackend = metricsBackend; } route (templateRouter) { @@ -74,8 +79,23 @@ module.exports = class NamedMapController { const includeQuery = false; const label = 'NAMED MAP LAYERGROUP'; const addContext = false; + const metricsTags = { + event: 'map_view', + attributes: { map_type: 'named' }, + from: { + req: { + query: { client: 'client' } + } + } + }; return [ + metrics({ + enabled: this.config.pubSubMetrics.enabled, + metricsBackend: this.metricsBackend, + logger: global.logger, + tags: metricsTags + }), credentials(), authorize(this.authBackend), dbConnSetup(this.pgConnection), diff --git a/lib/api/template/template-router.js b/lib/api/template/template-router.js index 81d7973f..7b254f9c 100644 --- a/lib/api/template/template-router.js +++ b/lib/api/template/template-router.js @@ -9,6 +9,7 @@ const TileTemplateController = require('./tile-template-controller'); module.exports = class TemplateRouter { constructor ({ collaborators }) { const { + config, pgConnection, templateMaps, mapBackend, @@ -21,10 +22,12 @@ module.exports = class TemplateRouter { authBackend, layergroupMetadata, namedMapProviderCache, - tileBackend + tileBackend, + metricsBackend } = collaborators; this.namedMapController = new NamedMapController( + config, pgConnection, templateMaps, mapBackend, @@ -35,7 +38,8 @@ module.exports = class TemplateRouter { mapConfigAdapter, statsBackend, authBackend, - layergroupMetadata + layergroupMetadata, + metricsBackend ); this.tileTemplateController = new TileTemplateController( diff --git a/lib/backends/metrics.js b/lib/backends/metrics.js new file mode 100644 index 00000000..ba91b4e6 --- /dev/null +++ b/lib/backends/metrics.js @@ -0,0 +1,17 @@ +'use strict'; + +const { PubSub } = require('@google-cloud/pubsub'); + +module.exports = class MetricsBackend { + constructor (options = {}) { + const { project_id: projectId, credentials: keyFilename, topic } = options; + + this._metricsClient = new PubSub({ projectId, keyFilename }); + this._topicName = topic; + } + + send (event, attributes) { + const data = Buffer.from(event); + return this._metricsClient.topic(this._topicName).publish(data, attributes); + } +}; diff --git a/lib/backends/pubsub-metrics.js b/lib/backends/pubsub-metrics.js deleted file mode 100644 index bc0fb0b7..00000000 --- a/lib/backends/pubsub-metrics.js +++ /dev/null @@ -1,66 +0,0 @@ -'use strict'; - -const { PubSub } = require('@google-cloud/pubsub'); - -/** - * PubSubMetricsBackend - */ -class PubSubMetricsBackend { - static build () { - if (!global.environment.pubSubMetrics || !global.environment.pubSubMetrics.enabled) { - return new PubSubMetricsBackend(undefined, false); - } - - const pubsub = PubSubMetricsBackend.createPubSub(); - - return new PubSubMetricsBackend(pubsub, true); - } - - static createPubSub () { - const projectId = global.environment.pubSubMetrics.project_id; - const credentials = global.environment.pubSubMetrics.credentials; - const config = {}; - - if (projectId) { - config.projectId = projectId; - } - if (credentials) { - config.keyFilename = credentials; - } - return new PubSub(config); - } - - constructor (pubSub, enabled) { - this.pubsub = pubSub; - this.enabled = enabled; - } - - isEnabled () { - return this.enabled; - } - - _getTopic () { - const topicName = global.environment.pubSubMetrics.topic; - - return this.pubsub.topic(topicName); - } - - sendEvent (event, attributes) { - if (!this.enabled) { - return; - } - - const data = Buffer.from(event); - const topic = this._getTopic(); - - topic.publish(data, attributes) - .then(() => { - console.log(`PubSubTracker: event '${event}' published to '${topic.name}'`); - }) - .catch((error) => { - console.error(`ERROR: pubsub middleware failed to publish event '${event}': ${error.message}`); - }); - } -} - -module.exports = PubSubMetricsBackend; diff --git a/lib/backends/template-maps.js b/lib/backends/template-maps.js index 9d09eb9d..f8ff641e 100644 --- a/lib/backends/template-maps.js +++ b/lib/backends/template-maps.js @@ -511,12 +511,15 @@ TemplateMaps.prototype.instance = function (template, params) { // Return a fingerPrint of the object TemplateMaps.prototype.fingerPrint = function (template) { - return crypto.createHash('md5') - .update(JSON.stringify(template)) - .digest('hex') - ; + return fingerPrint(template); }; +function fingerPrint (template) { + return crypto.createHash('md5').update(JSON.stringify(template)).digest('hex'); +} + +module.exports.fingerPrint = fingerPrint; + module.exports.templateName = function templateName (templateId) { var templateIdTokens = templateId.split('@'); var name = templateIdTokens[0]; diff --git a/lib/models/layergroup-token.js b/lib/models/layergroup-token.js index 1e4a6951..5332ce0d 100644 --- a/lib/models/layergroup-token.js +++ b/lib/models/layergroup-token.js @@ -4,7 +4,7 @@ * @param {String} token might match the following pattern: {user}@{tpl_id}@{token}:{cache_buster} */ function parse (token) { - var signer, cacheBuster; + var signer, cacheBuster, templateHash; var tokenSplit = token.split(':'); @@ -17,7 +17,7 @@ function parse (token) { if (tokenSplit.length > 1) { signer = tokenSplit.shift(); if (tokenSplit.length > 1) { - /* var template_hash = */tokenSplit.shift(); // unused + templateHash = tokenSplit.shift(); } token = tokenSplit.shift(); } @@ -25,7 +25,9 @@ function parse (token) { return { token: token, signer: signer, - cacheBuster: cacheBuster + cacheBuster: cacheBuster, + templateHash: templateHash }; } + module.exports.parse = parse; diff --git a/lib/models/mapconfig/provider/named-map-provider.js b/lib/models/mapconfig/provider/named-map-provider.js index d0823f89..63516853 100644 --- a/lib/models/mapconfig/provider/named-map-provider.js +++ b/lib/models/mapconfig/provider/named-map-provider.js @@ -4,7 +4,7 @@ const BaseMapConfigProvider = require('./base-mapconfig-adapter'); const crypto = require('crypto'); const dot = require('dot'); const MapConfig = require('windshaft').model.MapConfig; -const templateName = require('../../../backends/template-maps').templateName; +const { templateName, fingerPrint: templateFingerPrint } = require('../../../backends/template-maps'); // Configure bases for cache keys suitable for string interpolation const baseKey = '{{=it.dbname}}:{{=it.user}}:{{=it.templateName}}'; @@ -40,6 +40,10 @@ module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider { this.authToken = authToken; this.params = params; + // FIXME: why is this different than that: + // this.cacheBuster = params.cache_buster || 0; + // test: "should fail to use template from named map provider after template deletion" + // check named-map-provider-cache invalidation this.cacheBuster = Date.now(); // use template after call to mapConfig @@ -94,17 +98,16 @@ module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider { const { user, rendererParams } = this; - this.mapConfigAdapter.getMapConfig( - user, requestMapConfig, rendererParams, context, (err, mapConfig, stats = {}) => { - if (err) { - return callback(err); - } + this.mapConfigAdapter.getMapConfig(user, requestMapConfig, rendererParams, context, (err, mapConfig, stats = {}) => { + if (err) { + return callback(err); + } - this.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource); - this.analysesResults = context.analysesResults || []; + this.mapConfig = (mapConfig === null) ? null : new MapConfig(mapConfig, context.datasource); + this.analysesResults = context.analysesResults || []; - return callback(null, this.mapConfig, this.rendererParams, this.context, stats); - }); + return callback(null, this.mapConfig, this.rendererParams, this.context, stats); + }); }); }); } @@ -254,6 +257,14 @@ module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider { getTemplateName () { return this.templateName; } + + getTemplateHash () { + if (!this.template) { + throw new Error('Missing template, call "getTemplate()" method first'); + } + + return templateFingerPrint(this.template); + } }; function createConfigHash (config) { diff --git a/lib/server-options.js b/lib/server-options.js index c607d98b..8b3ced76 100644 --- a/lib/server-options.js +++ b/lib/server-options.js @@ -134,5 +134,6 @@ module.exports = { fastly: global.environment.fastly || {}, cache_enabled: global.environment.cache_enabled, log_format: global.environment.log_format, - useProfiler: global.environment.useProfiler + useProfiler: global.environment.useProfiler, + pubSubMetrics: Object.assign({ enabled: false }, global.environment.pubSubMetrics) }; diff --git a/lib/server.js b/lib/server.js index be030631..671045c9 100644 --- a/lib/server.js +++ b/lib/server.js @@ -21,6 +21,7 @@ module.exports = function createServer (serverOptions) { app.disable('etag'); app.set('json replacer', jsonReplacer()); + // FIXME: do not pass 'global.environment' as 'serverOptions' should keep defaults from 'global.environment' const apiRouter = new ApiRouter({ serverOptions, environmentOptions: global.environment }); apiRouter.route(app, serverOptions.routes.api); diff --git a/package-lock.json b/package-lock.json index 80c09e41..b776d336 100644 --- a/package-lock.json +++ b/package-lock.json @@ -331,64 +331,6 @@ "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", "integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=" }, - "@sinonjs/commons": { - "version": "1.7.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.7.1.tgz", - "integrity": "sha512-Debi3Baff1Qu1Unc3mjJ96MgpbwTn43S1+9yJ0llWygPwDNu2aaWBD6yc9y/Z8XDRNhx7U+u2UDg2OGQXkclUQ==", - "dev": true, - "requires": { - "type-detect": "4.0.8" - } - }, - "@sinonjs/fake-timers": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-6.0.0.tgz", - "integrity": "sha512-atR1J/jRXvQAb47gfzSK8zavXy7BcpnYq21ALon0U99etu99vsir0trzIO3wpeLtW+LLVY6X7EkfVTbjGSH8Ww==", - "dev": true, - "requires": { - "@sinonjs/commons": "^1.7.0" - } - }, - "@sinonjs/formatio": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-5.0.0.tgz", - "integrity": "sha512-ejFRrFNMaTAmhg9u1lYKJQxDocowta6KQKFnBE7XtZb/AAPlLkWQQSaqwlGYnDWQ6paXzyM1vbMhLAujSFiVPw==", - "dev": true, - "requires": { - "@sinonjs/commons": "^1", - "@sinonjs/samsam": "^4.2.0" - }, - "dependencies": { - "@sinonjs/samsam": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-4.2.2.tgz", - "integrity": "sha512-z9o4LZUzSD9Hl22zV38aXNykgFeVj8acqfFabCY6FY83n/6s/XwNJyYYldz6/9lBJanpno9h+oL6HTISkviweA==", - "dev": true, - "requires": { - "@sinonjs/commons": "^1.6.0", - "lodash.get": "^4.4.2", - "type-detect": "^4.0.8" - } - } - } - }, - "@sinonjs/samsam": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-5.0.1.tgz", - "integrity": "sha512-iSZdE68szyFvV8ReYve6t4gAA1rLVwGyyhWBg9qrz8VAn1FH141gdg0NJcMrAJ069rD2XM2KQzY8ZNDgmTfBQA==", - "dev": true, - "requires": { - "@sinonjs/commons": "^1.6.0", - "lodash.get": "^4.4.2", - "type-detect": "^4.0.8" - } - }, - "@sinonjs/text-encoding": { - "version": "0.7.1", - "resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz", - "integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==", - "dev": true - }, "@types/duplexify": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.0.tgz", @@ -4078,12 +4020,6 @@ "verror": "1.10.0" } }, - "just-extend": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.0.2.tgz", - "integrity": "sha512-FrLwOgm+iXrPV+5zDU6Jqu4gCRXbWEQg2O3SKONsWE4w7AXFRkryS53bpWdaL9cNol+AmR3AEYz6kn+o0fCPnw==", - "dev": true - }, "jwa": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", @@ -4175,12 +4111,6 @@ "integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=", "dev": true }, - "lodash.get": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", - "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=", - "dev": true - }, "lodash.has": { "version": "4.5.2", "resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz", @@ -4549,52 +4479,6 @@ "integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==", "dev": true }, - "nise": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/nise/-/nise-4.0.1.tgz", - "integrity": "sha512-10PKL272rqg80o2RsWcTT6X9cDYqJ4kXqPTf8yCXPc9hbphZSDmbiG5FqUNeR5nouKCQMM24ld45kgYnBdx2rw==", - "dev": true, - "requires": { - "@sinonjs/commons": "^1.7.0", - "@sinonjs/fake-timers": "^6.0.0", - "@sinonjs/formatio": "^4.0.1", - "@sinonjs/text-encoding": "^0.7.1", - "just-extend": "^4.0.2", - "path-to-regexp": "^1.7.0" - }, - "dependencies": { - "@sinonjs/formatio": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-4.0.1.tgz", - "integrity": "sha512-asIdlLFrla/WZybhm0C8eEzaDNNrzymiTqHMeJl6zPW2881l3uuVRpm0QlRQEjqYWv6CcKMGYME3LbrLJsORBw==", - "dev": true, - "requires": { - "@sinonjs/commons": "^1", - "@sinonjs/samsam": "^4.2.0" - } - }, - "@sinonjs/samsam": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-4.2.2.tgz", - "integrity": "sha512-z9o4LZUzSD9Hl22zV38aXNykgFeVj8acqfFabCY6FY83n/6s/XwNJyYYldz6/9lBJanpno9h+oL6HTISkviweA==", - "dev": true, - "requires": { - "@sinonjs/commons": "^1.6.0", - "lodash.get": "^4.4.2", - "type-detect": "^4.0.8" - } - }, - "path-to-regexp": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz", - "integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==", - "dev": true, - "requires": { - "isarray": "0.0.1" - } - } - } - }, "nock": { "version": "9.2.6", "resolved": "https://registry.npmjs.org/nock/-/nock-9.2.6.tgz", @@ -5811,44 +5695,6 @@ "resolved": "https://registry.npmjs.org/simple-statistics/-/simple-statistics-0.9.2.tgz", "integrity": "sha1-PjXLEDCPx2ljqk7nJS5qbqENKOQ=" }, - "sinon": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.0.tgz", - "integrity": "sha512-c4bREcvuK5VuEGyMW/Oim9I3Rq49Vzb0aMdxouFaA44QCFpilc5LJOugrX+mkrvikbqCimxuK+4cnHVNnLR41g==", - "dev": true, - "requires": { - "@sinonjs/commons": "^1.7.0", - "@sinonjs/fake-timers": "^6.0.0", - "@sinonjs/formatio": "^5.0.0", - "@sinonjs/samsam": "^5.0.1", - "diff": "^4.0.2", - "nise": "^4.0.1", - "supports-color": "^7.1.0" - }, - "dependencies": { - "diff": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", - "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", - "dev": true - }, - "has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true - }, - "supports-color": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz", - "integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==", - "dev": true, - "requires": { - "has-flag": "^4.0.0" - } - } - } - }, "slice-ansi": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", diff --git a/package.json b/package.json index 3ee5f8c1..f062d445 100644 --- a/package.json +++ b/package.json @@ -76,7 +76,6 @@ "nock": "9.2.6", "nyc": "^14.1.1", "redis": "2.8.0", - "sinon": "^9.0.0", "step": "1.0.0", "strftime": "0.10.0" }, diff --git a/test/index.js b/test/index.js index 84f43989..d1e27973 100644 --- a/test/index.js +++ b/test/index.js @@ -77,6 +77,8 @@ async function populateDatabase () { async function populateRedis () { const commands = ` + FLUSHALL + HMSET rails:users:localhost \ id ${TEST_USER_ID} \ database_name "${TEST_DB}" \ diff --git a/test/integration/metrics-test.js b/test/integration/metrics-test.js new file mode 100644 index 00000000..c1b75eec --- /dev/null +++ b/test/integration/metrics-test.js @@ -0,0 +1,583 @@ +'use strict'; + +const assert = require('assert'); +const TestClient = require('../support/test-client'); +const MetricsBackend = require('../../lib/backends/metrics'); +const LayergroupToken = require('../../lib/models/layergroup-token'); +const apikey = 1234; +const mapConfig = { + version: '1.8.0', + layers: [ + { + options: { + sql: TestClient.SQL.ONE_POINT, + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ] +}; + +const mapConfigWithTable = { + version: '1.8.0', + layers: [ + { + options: { + sql: 'select * from test_table', + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ] +}; + +function templateBuilder ({ name }) { + const templateName = `metrics-template-${name}-${Date.now()}`; + + return { + version: '0.0.1', + name: templateName, + layergroup: { + stat_tag: `stat-tag-${templateName}`, + version: '1.8.0', + layers: [ + { + type: 'cartodb', + options: { + sql: TestClient.SQL.ONE_POINT, + cartocss: TestClient.CARTOCSS.POINTS, + cartocss_version: '2.3.0' + } + } + ] + } + }; +} + +function templateMissingCartoCSSVersionBuilder () { + const templateName = `missing-cartocss-version-${Date.now()}`; + + return { + version: '0.0.1', + name: templateName, + layergroup: { + stat_tag: `stat-tag-${templateName}`, + version: '1.8.0', + layers: [ + { + type: 'cartodb', + options: { + sql: TestClient.SQL.ONE_POINT, + cartocss: TestClient.CARTOCSS.POINTS + } + } + ] + } + }; +} + +const suites = [ + { + desc: 'map config with live query', + mapConfig + }, + { + desc: 'map config with query against table', + mapConfig: mapConfigWithTable + } +]; + +suites.forEach(function ({ desc, mapConfig }) { + describe(`metrics: ${desc}`, function () { + beforeEach(function () { + this.originalMetricsBackendSendMethod = MetricsBackend.prototype.send; + this.pubSubMetricsBackendSendMethodCalled = false; + MetricsBackend.prototype.send = (event, attributes) => { + this.pubSubMetricsBackendSendMethodCalled = true; + this.pubSubMetricsBackendSendMethodCalledWith = { event, attributes }; + return Promise.resolve(); + }; + }); + + afterEach(function (done) { + MetricsBackend.prototype.send = this.originalMetricsBackendSendMethod; + return this.testClient.drain(done); + }); + + it('should not send event if not enabled', function (done) { + const extraHeaders = { + 'Carto-Event': 'test-event', + 'Carto-Event-Source': 'test', + 'Carto-Event-Group-Id': '1' + }; + const overrideServerOptions = { pubSubMetrics: { enabled: false } }; + + this.testClient = new TestClient(mapConfig, apikey, extraHeaders, overrideServerOptions); + + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.strictEqual(typeof body.layergroupid, 'string'); + assert.ok(!this.pubSubMetricsBackendSendMethodCalled); + + return done(); + }); + }); + + it('should not send event if headers not present', function (done) { + const extraHeaders = {}; + const overrideServerOptions = { pubSubMetrics: { enabled: false } }; + + this.testClient = new TestClient(mapConfig, apikey, extraHeaders, overrideServerOptions); + + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.strictEqual(typeof body.layergroupid, 'string'); + assert.ok(!this.pubSubMetricsBackendSendMethodCalled); + + return done(); + }); + }); + + it('should send event for map requests', function (done) { + const expectedEvent = 'map_view'; + const expectedMetricsEvent = 'event-test'; + const expectedEventSource = 'event-source-test'; + const expectedEventGroupId = '1'; + const expectedResponseCode = '200'; + const expectedMapType = 'anonymous'; + const extraHeaders = { + 'Carto-Event': expectedMetricsEvent, + 'Carto-Event-Source': expectedEventSource, + 'Carto-Event-Group-Id': expectedEventGroupId + }; + const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } }; + + this.testClient = new TestClient(mapConfig, apikey, extraHeaders, overrideServerOptions); + + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.strictEqual(typeof body.layergroupid, 'string'); + + const { token, cacheBuster } = LayergroupToken.parse(body.layergroupid); + + assert.ok(this.pubSubMetricsBackendSendMethodCalled); + + const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith; + + assert.strictEqual(event, expectedEvent); + assert.strictEqual(attributes.client_event, expectedMetricsEvent); + assert.strictEqual(attributes.event_source, expectedEventSource); + assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId); + assert.strictEqual(attributes.response_code, expectedResponseCode); + assert.strictEqual(attributes.map_type, expectedMapType); + assert.strictEqual(attributes.map_id, token); + assert.strictEqual(attributes.cache_buster, cacheBuster); + + return done(); + }); + }); + + it('should normalized headers type and length', function (done) { + const expectedEvent = 'map_view'; + const eventLong = 'If you are sending a text this long in a header you kind of deserve the worst, honestly. I mean this is not a header, it is almost a novel, and you do not see any Novel cookie here, right?'; + const expectedMetricsEvent = eventLong.trim().substr(0, 100); + const expectedEventGroupId = '1'; + const expectedEventSource = 'test'; + const expectedResponseCode = '200'; + const expectedMapType = 'anonymous'; + const extraHeaders = { + 'Carto-Event': eventLong, + 'Carto-Event-Source': 'test', + 'Carto-Event-Group-Id': 1 + }; + const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } }; + + this.testClient = new TestClient(mapConfig, apikey, extraHeaders, overrideServerOptions); + + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.strictEqual(typeof body.layergroupid, 'string'); + + const { token, cacheBuster } = LayergroupToken.parse(body.layergroupid); + + assert.ok(this.pubSubMetricsBackendSendMethodCalled); + + const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith; + + assert.strictEqual(event, expectedEvent); + assert.strictEqual(attributes.client_event, expectedMetricsEvent); + assert.strictEqual(attributes.event_source, expectedEventSource); + assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId); + assert.strictEqual(attributes.response_code, expectedResponseCode); + assert.strictEqual(attributes.map_type, expectedMapType); + assert.strictEqual(attributes.map_id, token); + assert.strictEqual(attributes.cache_buster, cacheBuster); + + return done(); + }); + }); + + it('should send event when error', function (done) { + const expectedEvent = 'map_view'; + const expectedMetricsEvent = 'event-test'; + const expectedEventSource = 'event-source-test'; + const expectedEventGroupId = '1'; + const expectedResponseCode = '400'; + const expectedMapType = 'anonymous'; + const extraHeaders = { + 'Carto-Event': expectedMetricsEvent, + 'Carto-Event-Source': expectedEventSource, + 'Carto-Event-Group-Id': expectedEventGroupId + }; + const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } }; + const mapConfigMissingCartoCSS = { + version: '1.8.0', + layers: [ + { + options: { + sql: TestClient.SQL.ONE_POINT, + cartocss: TestClient.CARTOCSS.POINTS + } + } + ] + }; + + this.testClient = new TestClient(mapConfigMissingCartoCSS, apikey, extraHeaders, overrideServerOptions); + + const params = { response: { status: 400 } }; + + this.testClient.getLayergroup(params, (err, body) => { + if (err) { + return done(err); + } + + assert.ok(this.pubSubMetricsBackendSendMethodCalled); + + const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith; + + assert.strictEqual(event, expectedEvent); + assert.strictEqual(attributes.client_event, expectedMetricsEvent); + assert.strictEqual(attributes.event_source, expectedEventSource); + assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId); + assert.strictEqual(attributes.response_code, expectedResponseCode); + assert.strictEqual(attributes.map_type, expectedMapType); + assert.strictEqual(typeof attributes.map_id, 'string'); + assert.strictEqual(typeof attributes.cache_buster, 'string'); + + return done(); + }); + }); + + it.skip('should send event for tile requests', function (done) { + const expectedEvent = 'event-tile-test'; + const expectedEventSource = 'event-source-tile-test'; + const expectedEventGroupId = '12345'; + const expectedResponseCode = '200'; + const extraHeaders = { + 'Carto-Event': expectedEvent, + 'Carto-Event-Source': expectedEventSource, + 'Carto-Event-Group-Id': expectedEventGroupId + }; + const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } }; + + this.testClient = new TestClient(mapConfig, apikey, extraHeaders, overrideServerOptions); + + this.testClient.getTile(0, 0, 0, (err, res, tile) => { + if (err) { + return done(err); + } + + assert.ok(this.pubSubMetricsBackendSendMethodCalled); + + const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith; + + assert.strictEqual(event, expectedEvent); + assert.strictEqual(attributes.event_source, expectedEventSource); + assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId); + assert.strictEqual(attributes.response_code, expectedResponseCode); + assert.strictEqual(typeof attributes.map_id, 'string'); + assert.strictEqual(typeof attributes.cache_buster, 'string'); + + return done(); + }); + }); + + it.skip('should send event for errored tile requests', function (done) { + const expectedEvent = 'event-tile-test'; + const expectedEventSource = 'event-source-tile-test'; + const expectedEventGroupId = '12345'; + const expectedResponseCode = '400'; + const extraHeaders = { + 'Carto-Event': expectedEvent, + 'Carto-Event-Source': expectedEventSource, + 'Carto-Event-Group-Id': expectedEventGroupId + }; + const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } }; + + this.testClient = new TestClient(mapConfig, apikey, extraHeaders, overrideServerOptions); + + const params = { + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getTile(0, 0, 2, params, (err, res, tile) => { + if (err) { + return done(err); + } + + assert.ok(this.pubSubMetricsBackendSendMethodCalled); + + const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith; + + assert.strictEqual(event, expectedEvent); + assert.strictEqual(attributes.event_source, expectedEventSource); + assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId); + assert.strictEqual(attributes.response_code, expectedResponseCode); + assert.strictEqual(typeof attributes.map_id, 'string'); + assert.strictEqual(typeof attributes.cache_buster, 'string'); + + return done(); + }); + }); + + it('should send event for named map requests', function (done) { + const expectedEvent = 'map_view'; + const expectedMetricsEvent = 'event-test'; + const expectedEventSource = 'event-source-test'; + const expectedEventGroupId = '1'; + const expectedResponseCode = '200'; + const expectedMapType = 'named'; + const extraHeaders = { + 'Carto-Event': expectedMetricsEvent, + 'Carto-Event-Source': expectedEventSource, + 'Carto-Event-Group-Id': expectedEventGroupId + }; + const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } }; + const template = templateBuilder({ name: 'map' }); + + this.testClient = new TestClient(template, apikey, extraHeaders, overrideServerOptions); + + this.testClient.getLayergroup((err, body) => { + if (err) { + return done(err); + } + + assert.strictEqual(typeof body.layergroupid, 'string'); + + const { token, cacheBuster, templateHash } = LayergroupToken.parse(body.layergroupid); + + assert.ok(this.pubSubMetricsBackendSendMethodCalled); + + const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith; + + assert.strictEqual(event, expectedEvent); + assert.strictEqual(attributes.client_event, expectedMetricsEvent); + assert.strictEqual(attributes.event_source, expectedEventSource); + assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId); + assert.strictEqual(attributes.response_code, expectedResponseCode); + assert.strictEqual(attributes.map_type, expectedMapType); + assert.strictEqual(attributes.map_id, token); + assert.strictEqual(attributes.cache_buster, cacheBuster); + assert.strictEqual(attributes.template_hash, templateHash); + assert.strictEqual(attributes.stat_tag, template.layergroup.stat_tag); + + return done(); + }); + }); + + it('should send event for errored named map requests', function (done) { + const expectedEvent = 'map_view'; + const expectedMetricsEvent = 'event-test'; + const expectedEventSource = 'event-source-test'; + const expectedEventGroupId = '1'; + const expectedResponseCode = '400'; + const expectedMapType = 'named'; + const extraHeaders = { + 'Carto-Event': expectedMetricsEvent, + 'Carto-Event-Source': expectedEventSource, + 'Carto-Event-Group-Id': expectedEventGroupId + }; + const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } }; + const templateMissingCartoCSSVersion = templateMissingCartoCSSVersionBuilder(); + this.testClient = new TestClient(templateMissingCartoCSSVersion, apikey, extraHeaders, overrideServerOptions); + + const params = { + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getLayergroup(params, (err, body) => { + if (err) { + return done(err); + } + + assert.ok(this.pubSubMetricsBackendSendMethodCalled); + + const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith; + + assert.strictEqual(event, expectedEvent); + assert.strictEqual(attributes.client_event, expectedMetricsEvent); + assert.strictEqual(attributes.event_source, expectedEventSource); + assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId); + assert.strictEqual(attributes.response_code, expectedResponseCode); + assert.strictEqual(attributes.map_type, expectedMapType); + assert.strictEqual(typeof attributes.map_id, 'string'); + assert.strictEqual(typeof attributes.cache_buster, 'string'); + assert.strictEqual(typeof attributes.template_hash, 'string'); + assert.strictEqual(attributes.stat_tag, templateMissingCartoCSSVersion.layergroup.stat_tag); + + return done(); + }); + }); + + it.skip('should send event for named map tile requests', function (done) { + const expectedEvent = 'event-named-map-tile-test'; + const expectedEventSource = 'event-source-named-map-tile-test'; + const expectedEventGroupId = '1'; + const expectedResponseCode = '200'; + const extraHeaders = { + 'Carto-Event': expectedEvent, + 'Carto-Event-Source': expectedEventSource, + 'Carto-Event-Group-Id': expectedEventGroupId + }; + const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } }; + const template = templateBuilder({ name: 'tile' }); + + this.testClient = new TestClient(template, apikey, extraHeaders, overrideServerOptions); + + this.testClient.getTile(0, 0, 0, (err, body) => { + if (err) { + return done(err); + } + + assert.ok(this.pubSubMetricsBackendSendMethodCalled); + + const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith; + + assert.strictEqual(event, expectedEvent); + assert.strictEqual(attributes.event_source, expectedEventSource); + assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId); + assert.strictEqual(attributes.response_code, expectedResponseCode); + assert.strictEqual(typeof attributes.map_id, 'string'); + assert.strictEqual(typeof attributes.cache_buster, 'string'); + assert.strictEqual(typeof attributes.template_hash, 'string'); + assert.strictEqual(attributes.stat_tag, template.layergroup.stat_tag); + + return done(); + }); + }); + + it('should send event for static named map requests', function (done) { + const expectedEvent = 'map_view'; + const expectedMetricsEvent = 'event-test'; + const expectedEventSource = 'event-source-test'; + const expectedEventGroupId = '1'; + const expectedResponseCode = '200'; + const expectedMapType = 'static'; + const extraHeaders = { + 'Carto-Event': expectedMetricsEvent, + 'Carto-Event-Source': expectedEventSource, + 'Carto-Event-Group-Id': expectedEventGroupId + }; + const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } }; + const template = templateBuilder({ name: 'preview' }); + + this.testClient = new TestClient(template, apikey, extraHeaders, overrideServerOptions); + + this.testClient.getPreview(640, 480, {}, (err, res, body) => { + if (err) { + return done(err); + } + + assert.ok(this.pubSubMetricsBackendSendMethodCalled); + + const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith; + + assert.strictEqual(event, expectedEvent); + assert.strictEqual(attributes.client_event, expectedMetricsEvent); + assert.strictEqual(attributes.event_source, expectedEventSource); + assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId); + assert.strictEqual(attributes.response_code, expectedResponseCode); + assert.strictEqual(attributes.map_type, expectedMapType); + assert.strictEqual(typeof attributes.map_id, 'string'); + assert.strictEqual(typeof attributes.cache_buster, 'string'); + assert.strictEqual(typeof attributes.template_hash, 'string'); + assert.strictEqual(attributes.stat_tag, template.layergroup.stat_tag); + + return done(); + }); + }); + + it('should send event for errored static named map requests', function (done) { + const expectedEvent = 'map_view'; + const expectedMetricsEvent = 'event-test'; + const expectedEventSource = 'event-source-test'; + const expectedEventGroupId = '1'; + const expectedResponseCode = '400'; + const expectedMapType = 'static'; + const extraHeaders = { + 'Carto-Event': expectedMetricsEvent, + 'Carto-Event-Source': expectedEventSource, + 'Carto-Event-Group-Id': expectedEventGroupId + }; + const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } }; + const template = templateBuilder({ name: 'preview-errored' }); + + this.testClient = new TestClient(template, apikey, extraHeaders, overrideServerOptions); + + const widthTooLarge = 8193; + const params = { + response: { + status: 400, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + } + }; + + this.testClient.getPreview(widthTooLarge, 480, params, (err, res, body) => { + if (err) { + return done(err); + } + + assert.ok(this.pubSubMetricsBackendSendMethodCalled); + + const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith; + + assert.strictEqual(event, expectedEvent); + assert.strictEqual(attributes.client_event, expectedMetricsEvent); + assert.strictEqual(attributes.event_source, expectedEventSource); + assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId); + assert.strictEqual(attributes.response_code, expectedResponseCode); + assert.strictEqual(attributes.map_type, expectedMapType); + assert.strictEqual(typeof attributes.map_id, 'string'); + assert.strictEqual(typeof attributes.cache_buster, 'string'); + assert.strictEqual(typeof attributes.template_hash, 'string'); + assert.strictEqual(attributes.stat_tag, template.layergroup.stat_tag); + + return done(); + }); + }); + }); +}); diff --git a/test/integration/pubsub-metrics-test.js b/test/integration/pubsub-metrics-test.js deleted file mode 100644 index 6bb8110d..00000000 --- a/test/integration/pubsub-metrics-test.js +++ /dev/null @@ -1,166 +0,0 @@ -'use strict'; - -const sinon = require('sinon'); -const assert = require('assert'); -const redis = require('redis'); -const TestClient = require('../support/test-client'); -const PubSubMetricsBackend = require('../../lib/backends/pubsub-metrics'); - -const metricsHeaders = { - 'Carto-Event': 'test-event', - 'Carto-Event-Source': 'test', - 'Carto-Event-Group-Id': '1' -}; - -const tooLongField = ' If you are sending a text this long in a header you kind of deserve the worst, honestly. I mean ' + - 'this is not a header, it is almost a novel, and you do not see any Novel cookie here, right?'; - -const badHeaders = { - 'Carto-Event': tooLongField, - 'Carto-Event-Source': 'test', - 'Carto-Event-Group-Id': 1 -}; - -const mapConfig = { - version: '1.7.0', - layers: [ - { - options: { - sql: 'select * FROM test_table_localhost_regular1', - cartocss: TestClient.CARTOCSS.POINTS, - cartocss_version: '2.3.0' - } - } - ] -}; - -function buildEventAttributes (statusCode) { - return { - event_source: 'test', - user_id: '1', - event_group_id: '1', - response_code: statusCode.toString(), - source_domain: 'localhost', - event_time: new Date().toISOString(), - event_version: '1' - }; -} - -const fakeTopic = { - name: 'test-topic', - publish: sinon.stub().returns(Promise.resolve()) -}; - -const fakePubSub = { - topic: () => fakeTopic -}; - -describe('pubsub metrics middleware', function () { - let redisClient; - let testClient; - let clock; - - before(function () { - redisClient = redis.createClient(global.environment.redis.port); - clock = sinon.useFakeTimers(); - sinon.stub(PubSubMetricsBackend, 'createPubSub').returns(fakePubSub); - }); - - after(function () { - clock.restore(); - PubSubMetricsBackend.createPubSub.restore(); - global.environment.pubSubMetrics.enabled = false; - }); - - afterEach(function (done) { - fakeTopic.publish.resetHistory(); - - redisClient.SELECT(0, () => { - redisClient.del('user:localhost:mapviews:global'); - - redisClient.SELECT(5, () => { - redisClient.del('user:localhost:mapviews:global'); - done(); - }); - }); - }); - - it('should not send event if not enabled', function (done) { - global.environment.pubSubMetrics.enabled = false; - testClient = new TestClient(mapConfig, 1234, metricsHeaders); - - testClient.getLayergroup((err, body) => { - if (err) { - return done(err); - } - - assert.strictEqual(typeof body.metadata, 'object'); - assert(fakeTopic.publish.notCalled); - return done(); - }); - }); - - it('should not send event if headers not present', function (done) { - global.environment.pubSubMetrics.enabled = true; - testClient = new TestClient(mapConfig, 1234); - - testClient.getLayergroup((err, body) => { - if (err) { - return done(err); - } - - assert.strictEqual(typeof body.metadata, 'object'); - assert(fakeTopic.publish.notCalled); - return done(); - }); - }); - - it('should normalized headers type and length', function (done) { - global.environment.pubSubMetrics.enabled = true; - const eventAttributes = buildEventAttributes(200); - const maxLength = 100; - const eventName = tooLongField.trim().substr(0, maxLength); - - testClient = new TestClient(mapConfig, 1234, badHeaders); - - testClient.getLayergroup((err, body) => { - if (err) { - return done(err); - } - - assert.strictEqual(typeof body.metadata, 'object'); - assert(fakeTopic.publish.calledOnceWith(Buffer.from(eventName), eventAttributes)); - return done(); - }); - }); - - it('should send event for map requests', function (done) { - global.environment.pubSubMetrics.enabled = true; - const eventAttributes = buildEventAttributes(200); - testClient = new TestClient(mapConfig, 1234, metricsHeaders); - - testClient.getLayergroup((err, body) => { - if (err) { - return done(err); - } - - assert.strictEqual(typeof body.metadata, 'object'); - assert(fakeTopic.publish.calledOnceWith(Buffer.from('test-event'), eventAttributes)); - return done(); - }); - }); - - it('should send event when error', function (done) { - global.environment.pubSubMetrics.enabled = true; - const eventAttributes = buildEventAttributes(400); - eventAttributes.user_id = undefined; - - testClient = new TestClient({}, 1234, metricsHeaders); - - testClient.getLayergroup(() => { - assert(fakeTopic.publish.calledOnceWith(Buffer.from('test-event'), eventAttributes)); - assert(fakeTopic.publish.calledOnce); - return done(); - }); - }); -}); diff --git a/test/support/test-client.js b/test/support/test-client.js index f0798dad..36183a0d 100644 --- a/test/support/test-client.js +++ b/test/support/test-client.js @@ -23,13 +23,14 @@ const MAPNIK_SUPPORTED_FORMATS = { mvt: true }; -function TestClient (config, apiKey, extraHeaders) { +function TestClient (config, apiKey, extraHeaders = {}, overrideServerOptions = {}) { this.mapConfig = isMapConfig(config) ? config : null; this.template = isTemplate(config) ? config : null; this.apiKey = apiKey; - this.extraHeaders = extraHeaders || {}; + this.extraHeaders = extraHeaders; this.keysToDelete = {}; - this.server = new CartodbWindshaft(serverOptions); + this.serverOptions = Object.assign({}, serverOptions, overrideServerOptions); + this.server = new CartodbWindshaft(this.serverOptions); } module.exports = TestClient; @@ -941,16 +942,8 @@ TestClient.prototype.getLayergroup = function (params, callback) { params = {}; } - if (!params.response) { - params.response = { - status: 200, - headers: { - 'Content-Type': 'application/json; charset=utf-8' - } - }; - } - - var url = '/api/v1/map'; + let url = '/api/v1/map'; + const urlNamed = url + '/named'; const headers = Object.assign({ host: 'localhost', 'Content-Type': 'application/json' }, self.extraHeaders); const queryParams = {}; @@ -967,30 +960,112 @@ TestClient.prototype.getLayergroup = function (params, callback) { url += '?' + qs.stringify(queryParams); } - assert.response(self.server, - { - url: url, - method: 'POST', - headers, - data: JSON.stringify(self.mapConfig) - }, - params.response, - function (res, err) { - var parsedBody; - // If there is a response, we are still interested in catching the created keys - // to be able to delete them on the .drain() method. - if (res) { - parsedBody = JSON.parse(res.body); - if (parsedBody.layergroupid) { - self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; - self.keysToDelete['user:localhost:mapviews:global'] = 5; - } - } - if (err) { - return callback(err); + var layergroupId; + + if (params.layergroupid) { + layergroupId = params.layergroupid; + } + + step( + function createTemplate () { + var next = this; + + if (!self.template) { + return next(); } - return callback(null, parsedBody); + if (!self.apiKey) { + return next(new Error('apiKey param is mandatory to create a new template')); + } + + params.placeholders = params.placeholders || {}; + + assert.response(self.server, + { + url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }), + method: 'POST', + headers, + data: JSON.stringify(self.template) + }, + { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }, + function (res, err) { + if (err) { + return next(err); + } + return next(null, JSON.parse(res.body).template_id); + } + ); + }, + function createLayergroup (err, templateId) { + var next = this; + + if (err) { + return next(err); + } + + if (layergroupId) { + return next(null, layergroupId); + } + + const data = templateId ? params.placeholders : self.mapConfig; + + if (!params.response) { + params.response = { + status: 200, + headers: { + 'Content-Type': 'application/json; charset=utf-8' + } + }; + } + + const queryParams = {}; + + if (self.apiKey) { + queryParams.api_key = self.apiKey; + } + + if (params.aggregation !== undefined) { + queryParams.aggregation = params.aggregation; + } + + const path = templateId + ? urlNamed + '/' + templateId + '?' + qs.stringify(queryParams) + : url; + + assert.response(self.server, + { + url: path, + method: 'POST', + headers, + data: JSON.stringify(data) + }, + params.response, + function (res, err) { + var parsedBody; + // If there is a response, we are still interested in catching the created keys + // to be able to delete them on the .drain() method. + if (res) { + parsedBody = JSON.parse(res.body); + if (parsedBody.layergroupid) { + self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0; + self.keysToDelete['user:localhost:mapviews:global'] = 5; + } + if (res.statusCode === 200 && self.template && self.template.layergroup && self.template.layergroup.stat_tag) { + self.keysToDelete[`user:localhost:mapviews:stat_tag:${self.template.layergroup.stat_tag}`] = 5; + } + } + if (err) { + return callback(err); + } + + return callback(null, parsedBody); + } + ); } ); }; @@ -1686,3 +1761,54 @@ TestClient.prototype.getTemplate = function (params, callback) { return callback(err, res, body); }); }; + +TestClient.prototype.getPreview = function (width, height, params = {}, callback) { + this.createTemplate({}, (err, res, template) => { + if (err) { + return callback(err); + } + + params = Object.assign({ api_key: this.apiKey }, params); + const url = `/api/v1/map/static/named/${template.template_id}/${width}/${height}.png?${qs.stringify(params)}`; + const headers = Object.assign({ host: 'localhost' }, this.extraHeaders); + + const requestOptions = { + url: url, + method: 'GET', + headers, + encoding: 'binary' + }; + + const expectedResponse = Object.assign({ + status: 200, + headers: { + 'Content-Type': 'image/png' + } + }, params.response || {}); + + assert.response(this.server, requestOptions, expectedResponse, (res, err) => { + if (err) { + return callback(err); + } + + let body; + switch (res.headers['content-type']) { + case 'image/png': + this.keysToDelete['user:localhost:mapviews:global'] = 5; + if (this.template.layergroup && this.template.layergroup.stat_tag) { + this.keysToDelete[`user:localhost:mapviews:stat_tag:${this.template.layergroup.stat_tag}`] = 5; + } + body = mapnik.Image.fromBytes(Buffer.from(res.body, 'binary')); + break; + case 'application/json; charset=utf-8': + body = JSON.parse(res.body); + break; + default: + body = res.body; + break; + } + + return callback(null, res, body); + }); + }); +}; diff --git a/test/unit/backends/pubsub-metrics-test.js b/test/unit/backends/pubsub-metrics-test.js deleted file mode 100644 index c08a93e5..00000000 --- a/test/unit/backends/pubsub-metrics-test.js +++ /dev/null @@ -1,40 +0,0 @@ -'use strict'; - -const sinon = require('sinon'); -const assert = require('assert'); -const PubSubMetricsBackend = require('../../../lib/backends/pubsub-metrics'); - -const fakeTopic = { - name: 'test-topic', - publish: sinon.stub().returns(Promise.resolve()) -}; - -const fakePubSub = { - topic: () => fakeTopic -}; - -const eventAttributes = { - event_source: 'test', - user_id: '123', - event_group_id: '1', - response_code: '200', - source_domain: 'localhost', - event_time: new Date().toISOString(), - event_version: '1' -}; - -describe('pubsub metrics backend', function () { - it('should not send event if not enabled', function () { - const pubSubMetricsService = new PubSubMetricsBackend(fakePubSub, false); - - pubSubMetricsService.sendEvent('test-event', eventAttributes); - assert(fakeTopic.publish.notCalled); - }); - - it('should send event if enabled', function () { - const pubSubMetricsService = new PubSubMetricsBackend(fakePubSub, true); - - pubSubMetricsService.sendEvent('test-event', eventAttributes); - assert(fakeTopic.publish.calledOnceWith(Buffer.from('test-event'), eventAttributes)); - }); -});