cartodb4/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1162 from CartoDB/dgaubert/ch71093/update-maps-api-to-new-event-format

Browse Source 
New event format for metrics
This commit is contained in:
Daniel G. Aubert 2020-05-01 13:34:57 +02:00 committed by GitHub
commit 478ea66678
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
28 changed files with 1080 additions and 582 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
NEWS.md
View File

@ -20,6 +20,7 @@ Announcements:
- Rename NamedMapProviderReporter by NamedMapProviderCacheReporter
- Remove `bootstrapFonts` at process startup (now done in `windshaft@6.0.0`)
- Stop checking the installed version of some dependencies while testing
- Send metrics about `map views` (#1162)
Bug Fixes:
- Parsing date column in numeric histograms (#1160)

13
lib/api/api-router.js
View File

@ -21,7 +21,7 @@ const OverviewsMetadataBackend = require('../backends/overviews-metadata');
const FilterStatsApi = require('../backends/filter-stats');
const TablesExtentBackend = require('../backends/tables-extent');
const ClusterBackend = require('../backends/cluster');
const PubSubMetricsBackend = require('../backends/pubsub-metrics');
const PubSubMetricsBackend = require('../backends/metrics');
const LayergroupAffectedTablesCache = require('../cache/layergroup-affected-tables');
const SurrogateKeysCache = require('../cache/surrogate-keys-cache');
@ -57,7 +57,6 @@ const user = require('./middlewares/user');
const sendResponse = require('./middlewares/send-response');
const syntaxError = require('./middlewares/syntax-error');
const errorMiddleware = require('./middlewares/error-middleware');
const pubSubMetrics = require('./middlewares/pubsub-metrics');
const MapRouter = require('./map/map-router');
const TemplateRouter = require('./template/template-router');
@ -161,7 +160,10 @@ module.exports = class ApiRouter {
});
namedMapProviderCacheReporter.start();
const metricsBackend = new PubSubMetricsBackend(serverOptions.pubSubMetrics);
const collaborators = {
config: serverOptions,
analysisStatusBackend,
attributesBackend,
dataviewBackend,
@ -181,13 +183,13 @@ module.exports = class ApiRouter {
layergroupMetadata,
namedMapProviderCache,
tablesExtentBackend,
clusterBackend
clusterBackend,
metricsBackend
};
this.metadataBackend = metadataBackend;
this.mapRouter = new MapRouter({ collaborators });
this.templateRouter = new TemplateRouter({ collaborators });
this.metadataBackend = metadataBackend;
this.pubSubMetricsBackend = PubSubMetricsBackend.build();
}
route (app, routes) {
@ -220,7 +222,6 @@ module.exports = class ApiRouter {
apiRouter.use(sendResponse());
apiRouter.use(syntaxError());
apiRouter.use(errorMiddleware());
apiRouter.use(pubSubMetrics(this.pubSubMetricsBackend));
paths.forEach(path => app.use(path, apiRouter));
});

24
lib/api/map/anonymous-map-controller.js
View File

@ -23,6 +23,7 @@ const mapError = require('../middlewares/map-error');
const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const metrics = require('../middlewares/metrics');
module.exports = class AnonymousMapController {
/**
@ -39,6 +40,7 @@ module.exports = class AnonymousMapController {
* @constructor
*/
constructor (
config,
pgConnection,
templateMaps,
mapBackend,
@ -49,8 +51,10 @@ module.exports = class AnonymousMapController {
mapConfigAdapter,
statsBackend,
authBackend,
layergroupMetadata
layergroupMetadata,
metricsBackend
) {
this.config = config;
this.pgConnection = pgConnection;
this.templateMaps = templateMaps;
this.mapBackend = mapBackend;
@ -62,6 +66,7 @@ module.exports = class AnonymousMapController {
this.statsBackend = statsBackend;
this.authBackend = authBackend;
this.layergroupMetadata = layergroupMetadata;
this.metricsBackend = metricsBackend;
}
route (mapRouter) {
@ -76,8 +81,23 @@ module.exports = class AnonymousMapController {
const includeQuery = true;
const label = 'ANONYMOUS LAYERGROUP';
const addContext = true;
const metricsTags = {
event: 'map_view',
attributes: { map_type: 'anonymous' },
from: {
req: {
query: { client: 'client' }
}
}
};
return [
metrics({
enabled: this.config.pubSubMetrics.enabled,
metricsBackend: this.metricsBackend,
logger: global.logger,
tags: metricsTags
}),
credentials(),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),
@ -207,6 +227,7 @@ function createLayergroup (mapBackend, userLimitsBackend, pgConnection, affected
);
res.locals.mapConfig = mapConfig;
res.locals.mapConfigProvider = mapConfigProvider;
res.locals.analysesResults = context.analysesResults;
const mapParams = { dbuser, dbname, dbpassword, dbhost, dbport };
@ -220,7 +241,6 @@ function createLayergroup (mapBackend, userLimitsBackend, pgConnection, affected
res.statusCode = 200;
res.body = layergroup;
res.locals.mapConfigProvider = mapConfigProvider;
next();
});

12
lib/api/map/map-router.js
View File

@ -15,6 +15,7 @@ const ClusteredFeaturesLayergroupController = require('./clustered-features-laye
module.exports = class MapRouter {
constructor ({ collaborators }) {
const {
config,
analysisStatusBackend,
attributesBackend,
dataviewBackend,
@ -34,7 +35,8 @@ module.exports = class MapRouter {
layergroupMetadata,
namedMapProviderCache,
tablesExtentBackend,
clusterBackend
clusterBackend,
metricsBackend
} = collaborators;
this.analysisLayergroupController = new AnalysisLayergroupController(
@ -85,6 +87,7 @@ module.exports = class MapRouter {
);
this.anonymousMapController = new AnonymousMapController(
config,
pgConnection,
templateMaps,
mapBackend,
@ -95,10 +98,12 @@ module.exports = class MapRouter {
mapConfigAdapter,
statsBackend,
authBackend,
layergroupMetadata
layergroupMetadata,
metricsBackend
);
this.previewTemplateController = new PreviewTemplateController(
config,
namedMapProviderCache,
previewBackend,
surrogateKeysCache,
@ -106,7 +111,8 @@ module.exports = class MapRouter {
metadataBackend,
pgConnection,
authBackend,
userLimitsBackend
userLimitsBackend,
metricsBackend
);
this.analysesController = new AnalysesCatalogController(

23
lib/api/map/preview-template-controller.js
View File

@ -12,6 +12,7 @@ const lastModifiedHeader = require('../middlewares/last-modified-header');
const checkStaticImageFormat = require('../middlewares/check-static-image-format');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const metrics = require('../middlewares/metrics');
const DEFAULT_ZOOM_CENTER = {
zoom: 1,
@ -27,6 +28,7 @@ function numMapper (n) {
module.exports = class PreviewTemplateController {
constructor (
config,
namedMapProviderCache,
previewBackend,
surrogateKeysCache,
@ -34,8 +36,10 @@ module.exports = class PreviewTemplateController {
metadataBackend,
pgConnection,
authBackend,
userLimitsBackend
userLimitsBackend,
metricsBackend
) {
this.config = config;
this.namedMapProviderCache = namedMapProviderCache;
this.previewBackend = previewBackend;
this.surrogateKeysCache = surrogateKeysCache;
@ -44,6 +48,7 @@ module.exports = class PreviewTemplateController {
this.pgConnection = pgConnection;
this.authBackend = authBackend;
this.userLimitsBackend = userLimitsBackend;
this.metricsBackend = metricsBackend;
}
route (mapRouter) {
@ -51,7 +56,23 @@ module.exports = class PreviewTemplateController {
}
middlewares () {
const metricsTags = {
event: 'map_view',
attributes: { map_type: 'static' },
from: {
req: {
query: { client: 'client' }
}
}
};
return [
metrics({
enabled: this.config.pubSubMetrics.enabled,
metricsBackend: this.metricsBackend,
logger: global.logger,
tags: metricsTags
}),
credentials(),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),

2
lib/api/middlewares/last-modified-header.js
View File

@ -36,6 +36,8 @@ module.exports = function setLastModifiedHeader () {
res.set('Last-Modified', lastModifiedDate.toUTCString());
res.locals.cache_buster = lastUpdatedAt;
next();
});
};

2
lib/api/middlewares/last-updated-time-layergroup.js
View File

@ -22,6 +22,8 @@ module.exports = function setLastUpdatedTimeToLayergroup () {
layergroup.layergroupid = layergroup.layergroupid + ':' + lastUpdateTime;
layergroup.last_updated = new Date(lastUpdateTime).toISOString();
res.locals.cache_buster = lastUpdateTime;
next();
});
};

3
lib/api/middlewares/layergroup-id-header.js
View File

@ -6,8 +6,9 @@ module.exports = function setLayergroupIdHeader (templateMaps, useTemplateHash)
const layergroup = res.body;
if (useTemplateHash) {
var templateHash = templateMaps.fingerPrint(template).substring(0, 8);
const templateHash = templateMaps.fingerPrint(template).substring(0, 8);
layergroup.layergroupid = `${user}@${templateHash}@${layergroup.layergroupid}`;
res.locals.templateHash = templateHash;
}
res.set('X-Layergroup-Id', layergroup.layergroupid);

4
lib/api/middlewares/layergroup-token.js
View File

@ -13,6 +13,10 @@ module.exports = function layergroupToken () {
res.locals.token = layergroupToken.token;
res.locals.cache_buster = layergroupToken.cacheBuster;
if (layergroupToken.templateHash) {
res.locals.templateHash = layergroupToken.templateHash;
}
if (layergroupToken.signer) {
res.locals.signer = layergroupToken.signer;

178
lib/api/middlewares/metrics.js Normal file
View File

@ -0,0 +1,178 @@
'use strict';
const EVENT_VERSION = '1';
const MAX_LENGTH = 100;
module.exports = function metrics ({ enabled, tags, metricsBackend, logger }) {
if (!enabled) {
return function metricsDisabledMiddleware (req, res, next) {
next();
};
}
if (!tags || !tags.event) {
throw new Error('Missing required "event" parameter to report metrics');
}
return function metricsMiddleware (req, res, next) {
res.on('finish', () => {
const { event, attributes } = getEventData(req, res, tags);
metricsBackend.send(event, attributes)
.catch((error) => logger.error(`Failed to publish event "${event}": ${error.message}`));
});
return next();
};
};
function getEventData (req, res, tags) {
const event = tags.event;
const extra = {};
if (tags.from) {
if (tags.from.req) {
Object.assign(extra, getFromReq(req, tags.from.req));
}
if (tags.from.res) {
Object.assign(extra, getFromRes(res, tags.from.res));
}
}
const attributes = Object.assign({}, {
client_event: normalizedField(req.get('Carto-Event')),
client_event_group_id: normalizedField(req.get('Carto-Event-Group-Id')),
event_source: normalizedField(req.get('Carto-Event-Source')),
event_time: new Date().toISOString(),
user_id: res.locals.userId,
user_agent: req.get('User-Agent'),
map_id: getLayergroupid({ res }),
cache_buster: getCacheBuster({ res }),
template_hash: getTemplateHash({ res }),
stat_tag: getStatTag({ res }),
response_code: res.statusCode.toString(),
response_time: getResponseTime(res),
source_domain: req.hostname,
event_version: EVENT_VERSION
}, tags.attributes, extra);
// remove undefined properties
Object.keys(attributes).forEach(key => attributes[key] === undefined && delete attributes[key]);
return { event, attributes };
}
function normalizedField (field) {
if (!field) {
return undefined;
}
return field.toString().trim().substr(0, MAX_LENGTH);
}
function getLayergroupid ({ res }) {
if (res.locals.token) {
return res.locals.token;
}
if (res.locals.mapConfig) {
return res.locals.mapConfig.id();
}
if (res.locals.mapConfigProvider && res.locals.mapConfigProvider.mapConfig) {
return res.locals.mapConfigProvider.mapConfig.id();
}
}
function getCacheBuster ({ res }) {
if (res.locals.cache_buster !== undefined) {
return `${res.locals.cache_buster}`;
}
if (res.locals.mapConfigProvider) {
return `${res.locals.mapConfigProvider.getCacheBuster()}`;
}
}
function getTemplateHash ({ res }) {
if (res.locals.templateHash) {
return res.locals.templateHash;
}
if (res.locals.mapConfigProvider && res.locals.mapConfigProvider.getTemplateHash) {
let templateHash;
try {
templateHash = res.locals.mapConfigProvider.getTemplateHash().substring(0, 8);
} catch (e) {}
return templateHash;
}
}
function getStatTag ({ res }) {
if (res.locals.mapConfig) {
return res.locals.mapConfig.obj().stat_tag;
}
// FIXME: don't expect that mapConfig is already set
if (res.locals.mapConfigProvider && res.locals.mapConfigProvider.mapConfig) {
return res.locals.mapConfigProvider.mapConfig.obj().stat_tag;
}
}
// FIXME: 'X-Tiler-Profiler' might not be accurate enough
function getResponseTime (res) {
const profiler = res.get('X-Tiler-Profiler');
let stats;
try {
stats = JSON.parse(profiler);
} catch (e) {
return undefined;
}
return stats && stats.total ? stats.total.toString() : undefined;
}
function getFromReq (req, { query = {}, body = {}, params = {}, headers = {} } = {}) {
const extra = {};
for (const [queryParam, eventName] of Object.entries(query)) {
extra[eventName] = req.query[queryParam];
}
for (const [bodyParam, eventName] of Object.entries(body)) {
extra[eventName] = req.body[bodyParam];
}
for (const [pathParam, eventName] of Object.entries(params)) {
extra[eventName] = req.params[pathParam];
}
for (const [header, eventName] of Object.entries(headers)) {
extra[eventName] = req.get(header);
}
return extra;
}
function getFromRes (res, { body = {}, headers = {}, locals = {} } = {}) {
const extra = {};
if (res.body) {
for (const [bodyParam, eventName] of Object.entries(body)) {
extra[eventName] = res.body[bodyParam];
}
}
for (const [header, eventName] of Object.entries(headers)) {
extra[eventName] = res.get(header);
}
for (const [localParam, eventName] of Object.entries(locals)) {
extra[eventName] = res.locals[localParam];
}
return extra;
}

74
lib/api/middlewares/pubsub-metrics.js
View File

@ -1,74 +0,0 @@
'use strict';
const EVENT_VERSION = '1';
const MAX_LENGTH = 100;
function pubSubMetrics (pubSubMetricsBackend) {
if (!pubSubMetricsBackend.isEnabled()) {
return function pubSubMetricsDisabledMiddleware (req, res, next) { next(); };
}
return function pubSubMetricsMiddleware (req, res, next) {
const data = getEventData(req, res);
if (data.event) {
pubSubMetricsBackend.sendEvent(data.event, data.attributes);
}
return next();
};
}
function getEventData (req, res) {
const event = normalizedField(req.get('Carto-Event'));
const eventSource = normalizedField(req.get('Carto-Event-Source'));
const eventGroupId = normalizedField(req.get('Carto-Event-Group-Id'));
if (!event || !eventSource) {
return [undefined, undefined];
}
const attributes = {
event_source: eventSource,
user_id: res.locals.userId,
response_code: res.statusCode.toString(),
source_domain: req.hostname,
event_time: new Date().toISOString(),
event_version: EVENT_VERSION
};
if (eventGroupId) {
attributes.event_group_id = eventGroupId;
}
const responseTime = getResponseTime(res);
if (responseTime) {
attributes.response_time = responseTime.toString();
}
return { event, attributes };
}
function normalizedField (field) {
if (!field) {
return undefined;
}
return field.toString().trim().substr(0, MAX_LENGTH);
}
function getResponseTime (res) {
const profiler = res.get('X-Tiler-Profiler');
let stats;
try {
stats = JSON.parse(profiler);
} catch (e) {
return undefined;
}
return stats.total;
}
module.exports = pubSubMetrics;

17
lib/api/middlewares/user.js
View File

@ -8,10 +8,12 @@ module.exports = function user (metadataBackend) {
return function userMiddleware (req, res, next) {
res.locals.user = getUserNameFromRequest(req, cdbRequest);
getUserId(metadataBackend, res.locals.user, function (userId) {
if (userId) {
res.locals.userId = userId;
metadataBackend.getUserId(res.locals.user, (err, userId) => {
if (err || !userId) {
return next();
}
res.locals.userId = userId;
return next();
});
};
@ -20,12 +22,3 @@ module.exports = function user (metadataBackend) {
function getUserNameFromRequest (req, cdbRequest) {
return cdbRequest.userByReq(req);
}
function getUserId (metadataBackend, userName, callback) {
metadataBackend.getUserId(userName, function (err, userId) {
if (err) {
return callback();
}
return callback(userId);
});
}

22
lib/api/template/named-template-controller.js
View File

@ -21,6 +21,7 @@ const NamedMapMapConfigProvider = require('../../models/mapconfig/provider/named
const CreateLayergroupMapConfigProvider = require('../../models/mapconfig/provider/create-layergroup-provider');
const rateLimit = require('../middlewares/rate-limit');
const { RATE_LIMIT_ENDPOINTS_GROUPS } = rateLimit;
const metrics = require('../middlewares/metrics');
module.exports = class NamedMapController {
/**
@ -38,6 +39,7 @@ module.exports = class NamedMapController {
* @constructor
*/
constructor (
config,
pgConnection,
templateMaps,
mapBackend,
@ -48,8 +50,10 @@ module.exports = class NamedMapController {
mapConfigAdapter,
statsBackend,
authBackend,
layergroupMetadata
layergroupMetadata,
metricsBackend
) {
this.config = config;
this.pgConnection = pgConnection;
this.templateMaps = templateMaps;
this.mapBackend = mapBackend;
@ -61,6 +65,7 @@ module.exports = class NamedMapController {
this.statsBackend = statsBackend;
this.authBackend = authBackend;
this.layergroupMetadata = layergroupMetadata;
this.metricsBackend = metricsBackend;
}
route (templateRouter) {
@ -74,8 +79,23 @@ module.exports = class NamedMapController {
const includeQuery = false;
const label = 'NAMED MAP LAYERGROUP';
const addContext = false;
const metricsTags = {
event: 'map_view',
attributes: { map_type: 'named' },
from: {
req: {
query: { client: 'client' }
}
}
};
return [
metrics({
enabled: this.config.pubSubMetrics.enabled,
metricsBackend: this.metricsBackend,
logger: global.logger,
tags: metricsTags
}),
credentials(),
authorize(this.authBackend),
dbConnSetup(this.pgConnection),

8
lib/api/template/template-router.js
View File

@ -9,6 +9,7 @@ const TileTemplateController = require('./tile-template-controller');
module.exports = class TemplateRouter {
constructor ({ collaborators }) {
const {
config,
pgConnection,
templateMaps,
mapBackend,
@ -21,10 +22,12 @@ module.exports = class TemplateRouter {
authBackend,
layergroupMetadata,
namedMapProviderCache,
tileBackend
tileBackend,
metricsBackend
} = collaborators;
this.namedMapController = new NamedMapController(
config,
pgConnection,
templateMaps,
mapBackend,
@ -35,7 +38,8 @@ module.exports = class TemplateRouter {
mapConfigAdapter,
statsBackend,
authBackend,
layergroupMetadata
layergroupMetadata,
metricsBackend
);
this.tileTemplateController = new TileTemplateController(

17
lib/backends/metrics.js Normal file
View File

@ -0,0 +1,17 @@
'use strict';
const { PubSub } = require('@google-cloud/pubsub');
module.exports = class MetricsBackend {
constructor (options = {}) {
const { project_id: projectId, credentials: keyFilename, topic } = options;
this._metricsClient = new PubSub({ projectId, keyFilename });
this._topicName = topic;
}
send (event, attributes) {
const data = Buffer.from(event);
return this._metricsClient.topic(this._topicName).publish(data, attributes);
}
};

66
lib/backends/pubsub-metrics.js
View File

@ -1,66 +0,0 @@
'use strict';
const { PubSub } = require('@google-cloud/pubsub');
/**
* PubSubMetricsBackend
*/
class PubSubMetricsBackend {
static build () {
if (!global.environment.pubSubMetrics || !global.environment.pubSubMetrics.enabled) {
return new PubSubMetricsBackend(undefined, false);
}
const pubsub = PubSubMetricsBackend.createPubSub();
return new PubSubMetricsBackend(pubsub, true);
}
static createPubSub () {
const projectId = global.environment.pubSubMetrics.project_id;
const credentials = global.environment.pubSubMetrics.credentials;
const config = {};
if (projectId) {
config.projectId = projectId;
}
if (credentials) {
config.keyFilename = credentials;
}
return new PubSub(config);
}
constructor (pubSub, enabled) {
this.pubsub = pubSub;
this.enabled = enabled;
}
isEnabled () {
return this.enabled;
}
_getTopic () {
const topicName = global.environment.pubSubMetrics.topic;
return this.pubsub.topic(topicName);
}
sendEvent (event, attributes) {
if (!this.enabled) {
return;
}
const data = Buffer.from(event);
const topic = this._getTopic();
topic.publish(data, attributes)
.then(() => {
console.log(`PubSubTracker: event '${event}' published to '${topic.name}'`);
})
.catch((error) => {
console.error(`ERROR: pubsub middleware failed to publish event '${event}': ${error.message}`);
});
}
}
module.exports = PubSubMetricsBackend;

11
lib/backends/template-maps.js
View File

@ -511,12 +511,15 @@ TemplateMaps.prototype.instance = function (template, params) {
// Return a fingerPrint of the object
TemplateMaps.prototype.fingerPrint = function (template) {
return crypto.createHash('md5')
.update(JSON.stringify(template))
.digest('hex')
;
return fingerPrint(template);
};
function fingerPrint (template) {
return crypto.createHash('md5').update(JSON.stringify(template)).digest('hex');
}
module.exports.fingerPrint = fingerPrint;
module.exports.templateName = function templateName (templateId) {
var templateIdTokens = templateId.split('@');
var name = templateIdTokens[0];

8
lib/models/layergroup-token.js
View File

@ -4,7 +4,7 @@
* @param {String} token might match the following pattern: {user}@{tpl_id}@{token}:{cache_buster}
*/
function parse (token) {
var signer, cacheBuster;
var signer, cacheBuster, templateHash;
var tokenSplit = token.split(':');
@ -17,7 +17,7 @@ function parse (token) {
if (tokenSplit.length > 1) {
signer = tokenSplit.shift();
if (tokenSplit.length > 1) {
/* var template_hash = */tokenSplit.shift(); // unused
templateHash = tokenSplit.shift();
}
token = tokenSplit.shift();
}
@ -25,7 +25,9 @@ function parse (token) {
return {
token: token,
signer: signer,
cacheBuster: cacheBuster
cacheBuster: cacheBuster,
templateHash: templateHash
};
}
module.exports.parse = parse;

17
lib/models/mapconfig/provider/named-map-provider.js
View File

@ -4,7 +4,7 @@ const BaseMapConfigProvider = require('./base-mapconfig-adapter');
const crypto = require('crypto');
const dot = require('dot');
const MapConfig = require('windshaft').model.MapConfig;
const templateName = require('../../../backends/template-maps').templateName;
const { templateName, fingerPrint: templateFingerPrint } = require('../../../backends/template-maps');
// Configure bases for cache keys suitable for string interpolation
const baseKey = '{{=it.dbname}}:{{=it.user}}:{{=it.templateName}}';
@ -40,6 +40,10 @@ module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider {
this.authToken = authToken;
this.params = params;
// FIXME: why is this different than that:
// this.cacheBuster = params.cache_buster || 0;
// test: "should fail to use template from named map provider after template deletion"
// check named-map-provider-cache invalidation
this.cacheBuster = Date.now();
// use template after call to mapConfig
@ -94,8 +98,7 @@ module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider {
const { user, rendererParams } = this;
this.mapConfigAdapter.getMapConfig(
user, requestMapConfig, rendererParams, context, (err, mapConfig, stats = {}) => {
this.mapConfigAdapter.getMapConfig(user, requestMapConfig, rendererParams, context, (err, mapConfig, stats = {}) => {
if (err) {
return callback(err);
}
@ -254,6 +257,14 @@ module.exports = class NamedMapMapConfigProvider extends BaseMapConfigProvider {
getTemplateName () {
return this.templateName;
}
getTemplateHash () {
if (!this.template) {
throw new Error('Missing template, call "getTemplate()" method first');
}
return templateFingerPrint(this.template);
}
};
function createConfigHash (config) {

3
lib/server-options.js
View File

@ -134,5 +134,6 @@ module.exports = {
fastly: global.environment.fastly || {},
cache_enabled: global.environment.cache_enabled,
log_format: global.environment.log_format,
useProfiler: global.environment.useProfiler
useProfiler: global.environment.useProfiler,
pubSubMetrics: Object.assign({ enabled: false }, global.environment.pubSubMetrics)
};

1
lib/server.js
View File

@ -21,6 +21,7 @@ module.exports = function createServer (serverOptions) {
app.disable('etag');
app.set('json replacer', jsonReplacer());
// FIXME: do not pass 'global.environment' as 'serverOptions' should keep defaults from 'global.environment'
const apiRouter = new ApiRouter({ serverOptions, environmentOptions: global.environment });
apiRouter.route(app, serverOptions.routes.api);

154
package-lock.json generated
View File

@ -331,64 +331,6 @@
"resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz",
"integrity": "sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA="
},
"@sinonjs/commons": {
"version": "1.7.1",
"resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.7.1.tgz",
"integrity": "sha512-Debi3Baff1Qu1Unc3mjJ96MgpbwTn43S1+9yJ0llWygPwDNu2aaWBD6yc9y/Z8XDRNhx7U+u2UDg2OGQXkclUQ==",
"dev": true,
"requires": {
"type-detect": "4.0.8"
}
},
"@sinonjs/fake-timers": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-6.0.0.tgz",
"integrity": "sha512-atR1J/jRXvQAb47gfzSK8zavXy7BcpnYq21ALon0U99etu99vsir0trzIO3wpeLtW+LLVY6X7EkfVTbjGSH8Ww==",
"dev": true,
"requires": {
"@sinonjs/commons": "^1.7.0"
}
},
"@sinonjs/formatio": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-5.0.0.tgz",
"integrity": "sha512-ejFRrFNMaTAmhg9u1lYKJQxDocowta6KQKFnBE7XtZb/AAPlLkWQQSaqwlGYnDWQ6paXzyM1vbMhLAujSFiVPw==",
"dev": true,
"requires": {
"@sinonjs/commons": "^1",
"@sinonjs/samsam": "^4.2.0"
},
"dependencies": {
"@sinonjs/samsam": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-4.2.2.tgz",
"integrity": "sha512-z9o4LZUzSD9Hl22zV38aXNykgFeVj8acqfFabCY6FY83n/6s/XwNJyYYldz6/9lBJanpno9h+oL6HTISkviweA==",
"dev": true,
"requires": {
"@sinonjs/commons": "^1.6.0",
"lodash.get": "^4.4.2",
"type-detect": "^4.0.8"
}
}
}
},
"@sinonjs/samsam": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-5.0.1.tgz",
"integrity": "sha512-iSZdE68szyFvV8ReYve6t4gAA1rLVwGyyhWBg9qrz8VAn1FH141gdg0NJcMrAJ069rD2XM2KQzY8ZNDgmTfBQA==",
"dev": true,
"requires": {
"@sinonjs/commons": "^1.6.0",
"lodash.get": "^4.4.2",
"type-detect": "^4.0.8"
}
},
"@sinonjs/text-encoding": {
"version": "0.7.1",
"resolved": "https://registry.npmjs.org/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz",
"integrity": "sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==",
"dev": true
},
"@types/duplexify": {
"version": "3.6.0",
"resolved": "https://registry.npmjs.org/@types/duplexify/-/duplexify-3.6.0.tgz",
@ -4078,12 +4020,6 @@
"verror": "1.10.0"
}
},
"just-extend": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/just-extend/-/just-extend-4.0.2.tgz",
"integrity": "sha512-FrLwOgm+iXrPV+5zDU6Jqu4gCRXbWEQg2O3SKONsWE4w7AXFRkryS53bpWdaL9cNol+AmR3AEYz6kn+o0fCPnw==",
"dev": true
},
"jwa": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz",
@ -4175,12 +4111,6 @@
"integrity": "sha1-+wMJF/hqMTTlvJvsDWngAT3f7bI=",
"dev": true
},
"lodash.get": {
"version": "4.4.2",
"resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz",
"integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=",
"dev": true
},
"lodash.has": {
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/lodash.has/-/lodash.has-4.5.2.tgz",
@ -4549,52 +4479,6 @@
"integrity": "sha512-1nh45deeb5olNY7eX82BkPO7SSxR5SSYJiPTrTdFUVYwAl8CKMA5N9PjTYkHiRjisVcxcQ1HXdLhx2qxxJzLNQ==",
"dev": true
},
"nise": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/nise/-/nise-4.0.1.tgz",
"integrity": "sha512-10PKL272rqg80o2RsWcTT6X9cDYqJ4kXqPTf8yCXPc9hbphZSDmbiG5FqUNeR5nouKCQMM24ld45kgYnBdx2rw==",
"dev": true,
"requires": {
"@sinonjs/commons": "^1.7.0",
"@sinonjs/fake-timers": "^6.0.0",
"@sinonjs/formatio": "^4.0.1",
"@sinonjs/text-encoding": "^0.7.1",
"just-extend": "^4.0.2",
"path-to-regexp": "^1.7.0"
},
"dependencies": {
"@sinonjs/formatio": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/@sinonjs/formatio/-/formatio-4.0.1.tgz",
"integrity": "sha512-asIdlLFrla/WZybhm0C8eEzaDNNrzymiTqHMeJl6zPW2881l3uuVRpm0QlRQEjqYWv6CcKMGYME3LbrLJsORBw==",
"dev": true,
"requires": {
"@sinonjs/commons": "^1",
"@sinonjs/samsam": "^4.2.0"
}
},
"@sinonjs/samsam": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/@sinonjs/samsam/-/samsam-4.2.2.tgz",
"integrity": "sha512-z9o4LZUzSD9Hl22zV38aXNykgFeVj8acqfFabCY6FY83n/6s/XwNJyYYldz6/9lBJanpno9h+oL6HTISkviweA==",
"dev": true,
"requires": {
"@sinonjs/commons": "^1.6.0",
"lodash.get": "^4.4.2",
"type-detect": "^4.0.8"
}
},
"path-to-regexp": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-1.8.0.tgz",
"integrity": "sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA==",
"dev": true,
"requires": {
"isarray": "0.0.1"
}
}
}
},
"nock": {
"version": "9.2.6",
"resolved": "https://registry.npmjs.org/nock/-/nock-9.2.6.tgz",
@ -5811,44 +5695,6 @@
"resolved": "https://registry.npmjs.org/simple-statistics/-/simple-statistics-0.9.2.tgz",
"integrity": "sha1-PjXLEDCPx2ljqk7nJS5qbqENKOQ="
},
"sinon": {
"version": "9.0.0",
"resolved": "https://registry.npmjs.org/sinon/-/sinon-9.0.0.tgz",
"integrity": "sha512-c4bREcvuK5VuEGyMW/Oim9I3Rq49Vzb0aMdxouFaA44QCFpilc5LJOugrX+mkrvikbqCimxuK+4cnHVNnLR41g==",
"dev": true,
"requires": {
"@sinonjs/commons": "^1.7.0",
"@sinonjs/fake-timers": "^6.0.0",
"@sinonjs/formatio": "^5.0.0",
"@sinonjs/samsam": "^5.0.1",
"diff": "^4.0.2",
"nise": "^4.0.1",
"supports-color": "^7.1.0"
},
"dependencies": {
"diff": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
"integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
"dev": true
},
"has-flag": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
"dev": true
},
"supports-color": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.1.0.tgz",
"integrity": "sha512-oRSIpR8pxT1Wr2FquTNnGet79b3BWljqOuoW/h4oBhxJ/HUbX5nX6JSruTkvXDCFMwDPvsaTTbvMLKZWSy0R5g==",
"dev": true,
"requires": {
"has-flag": "^4.0.0"
}
}
}
},
"slice-ansi": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz",

1
package.json
View File

@ -76,7 +76,6 @@
"nock": "9.2.6",
"nyc": "^14.1.1",
"redis": "2.8.0",
"sinon": "^9.0.0",
"step": "1.0.0",
"strftime": "0.10.0"
},

2
test/index.js
View File

@ -77,6 +77,8 @@ async function populateDatabase () {
async function populateRedis () {
const commands = `
FLUSHALL
HMSET rails:users:localhost \
id ${TEST_USER_ID} \
database_name "${TEST_DB}" \

583
test/integration/metrics-test.js Normal file
View File

@ -0,0 +1,583 @@
'use strict';
const assert = require('assert');
const TestClient = require('../support/test-client');
const MetricsBackend = require('../../lib/backends/metrics');
const LayergroupToken = require('../../lib/models/layergroup-token');
const apikey = 1234;
const mapConfig = {
version: '1.8.0',
layers: [
{
options: {
sql: TestClient.SQL.ONE_POINT,
cartocss: TestClient.CARTOCSS.POINTS,
cartocss_version: '2.3.0'
}
}
]
};
const mapConfigWithTable = {
version: '1.8.0',
layers: [
{
options: {
sql: 'select * from test_table',
cartocss: TestClient.CARTOCSS.POINTS,
cartocss_version: '2.3.0'
}
}
]
};
function templateBuilder ({ name }) {
const templateName = `metrics-template-${name}-${Date.now()}`;
return {
version: '0.0.1',
name: templateName,
layergroup: {
stat_tag: `stat-tag-${templateName}`,
version: '1.8.0',
layers: [
{
type: 'cartodb',
options: {
sql: TestClient.SQL.ONE_POINT,
cartocss: TestClient.CARTOCSS.POINTS,
cartocss_version: '2.3.0'
}
}
]
}
};
}
function templateMissingCartoCSSVersionBuilder () {
const templateName = `missing-cartocss-version-${Date.now()}`;
return {
version: '0.0.1',
name: templateName,
layergroup: {
stat_tag: `stat-tag-${templateName}`,
version: '1.8.0',
layers: [
{
type: 'cartodb',
options: {
sql: TestClient.SQL.ONE_POINT,
cartocss: TestClient.CARTOCSS.POINTS
}
}
]
}
};
}
const suites = [
{
desc: 'map config with live query',
mapConfig
},
{
desc: 'map config with query against table',
mapConfig: mapConfigWithTable
}
];
suites.forEach(function ({ desc, mapConfig }) {
describe(`metrics: ${desc}`, function () {
beforeEach(function () {
this.originalMetricsBackendSendMethod = MetricsBackend.prototype.send;
this.pubSubMetricsBackendSendMethodCalled = false;
MetricsBackend.prototype.send = (event, attributes) => {
this.pubSubMetricsBackendSendMethodCalled = true;
this.pubSubMetricsBackendSendMethodCalledWith = { event, attributes };
return Promise.resolve();
};
});
afterEach(function (done) {
MetricsBackend.prototype.send = this.originalMetricsBackendSendMethod;
return this.testClient.drain(done);
});
it('should not send event if not enabled', function (done) {
const extraHeaders = {
'Carto-Event': 'test-event',
'Carto-Event-Source': 'test',
'Carto-Event-Group-Id': '1'
};
const overrideServerOptions = { pubSubMetrics: { enabled: false } };
this.testClient = new TestClient(mapConfig, apikey, extraHeaders, overrideServerOptions);
this.testClient.getLayergroup((err, body) => {
if (err) {
return done(err);
}
assert.strictEqual(typeof body.layergroupid, 'string');
assert.ok(!this.pubSubMetricsBackendSendMethodCalled);
return done();
});
});
it('should not send event if headers not present', function (done) {
const extraHeaders = {};
const overrideServerOptions = { pubSubMetrics: { enabled: false } };
this.testClient = new TestClient(mapConfig, apikey, extraHeaders, overrideServerOptions);
this.testClient.getLayergroup((err, body) => {
if (err) {
return done(err);
}
assert.strictEqual(typeof body.layergroupid, 'string');
assert.ok(!this.pubSubMetricsBackendSendMethodCalled);
return done();
});
});
it('should send event for map requests', function (done) {
const expectedEvent = 'map_view';
const expectedMetricsEvent = 'event-test';
const expectedEventSource = 'event-source-test';
const expectedEventGroupId = '1';
const expectedResponseCode = '200';
const expectedMapType = 'anonymous';
const extraHeaders = {
'Carto-Event': expectedMetricsEvent,
'Carto-Event-Source': expectedEventSource,
'Carto-Event-Group-Id': expectedEventGroupId
};
const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } };
this.testClient = new TestClient(mapConfig, apikey, extraHeaders, overrideServerOptions);
this.testClient.getLayergroup((err, body) => {
if (err) {
return done(err);
}
assert.strictEqual(typeof body.layergroupid, 'string');
const { token, cacheBuster } = LayergroupToken.parse(body.layergroupid);
assert.ok(this.pubSubMetricsBackendSendMethodCalled);
const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith;
assert.strictEqual(event, expectedEvent);
assert.strictEqual(attributes.client_event, expectedMetricsEvent);
assert.strictEqual(attributes.event_source, expectedEventSource);
assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId);
assert.strictEqual(attributes.response_code, expectedResponseCode);
assert.strictEqual(attributes.map_type, expectedMapType);
assert.strictEqual(attributes.map_id, token);
assert.strictEqual(attributes.cache_buster, cacheBuster);
return done();
});
});
it('should normalized headers type and length', function (done) {
const expectedEvent = 'map_view';
const eventLong = 'If you are sending a text this long in a header you kind of deserve the worst, honestly. I mean this is not a header, it is almost a novel, and you do not see any Novel cookie here, right?';
const expectedMetricsEvent = eventLong.trim().substr(0, 100);
const expectedEventGroupId = '1';
const expectedEventSource = 'test';
const expectedResponseCode = '200';
const expectedMapType = 'anonymous';
const extraHeaders = {
'Carto-Event': eventLong,
'Carto-Event-Source': 'test',
'Carto-Event-Group-Id': 1
};
const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } };
this.testClient = new TestClient(mapConfig, apikey, extraHeaders, overrideServerOptions);
this.testClient.getLayergroup((err, body) => {
if (err) {
return done(err);
}
assert.strictEqual(typeof body.layergroupid, 'string');
const { token, cacheBuster } = LayergroupToken.parse(body.layergroupid);
assert.ok(this.pubSubMetricsBackendSendMethodCalled);
const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith;
assert.strictEqual(event, expectedEvent);
assert.strictEqual(attributes.client_event, expectedMetricsEvent);
assert.strictEqual(attributes.event_source, expectedEventSource);
assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId);
assert.strictEqual(attributes.response_code, expectedResponseCode);
assert.strictEqual(attributes.map_type, expectedMapType);
assert.strictEqual(attributes.map_id, token);
assert.strictEqual(attributes.cache_buster, cacheBuster);
return done();
});
});
it('should send event when error', function (done) {
const expectedEvent = 'map_view';
const expectedMetricsEvent = 'event-test';
const expectedEventSource = 'event-source-test';
const expectedEventGroupId = '1';
const expectedResponseCode = '400';
const expectedMapType = 'anonymous';
const extraHeaders = {
'Carto-Event': expectedMetricsEvent,
'Carto-Event-Source': expectedEventSource,
'Carto-Event-Group-Id': expectedEventGroupId
};
const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } };
const mapConfigMissingCartoCSS = {
version: '1.8.0',
layers: [
{
options: {
sql: TestClient.SQL.ONE_POINT,
cartocss: TestClient.CARTOCSS.POINTS
}
}
]
};
this.testClient = new TestClient(mapConfigMissingCartoCSS, apikey, extraHeaders, overrideServerOptions);
const params = { response: { status: 400 } };
this.testClient.getLayergroup(params, (err, body) => {
if (err) {
return done(err);
}
assert.ok(this.pubSubMetricsBackendSendMethodCalled);
const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith;
assert.strictEqual(event, expectedEvent);
assert.strictEqual(attributes.client_event, expectedMetricsEvent);
assert.strictEqual(attributes.event_source, expectedEventSource);
assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId);
assert.strictEqual(attributes.response_code, expectedResponseCode);
assert.strictEqual(attributes.map_type, expectedMapType);
assert.strictEqual(typeof attributes.map_id, 'string');
assert.strictEqual(typeof attributes.cache_buster, 'string');
return done();
});
});
it.skip('should send event for tile requests', function (done) {
const expectedEvent = 'event-tile-test';
const expectedEventSource = 'event-source-tile-test';
const expectedEventGroupId = '12345';
const expectedResponseCode = '200';
const extraHeaders = {
'Carto-Event': expectedEvent,
'Carto-Event-Source': expectedEventSource,
'Carto-Event-Group-Id': expectedEventGroupId
};
const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } };
this.testClient = new TestClient(mapConfig, apikey, extraHeaders, overrideServerOptions);
this.testClient.getTile(0, 0, 0, (err, res, tile) => {
if (err) {
return done(err);
}
assert.ok(this.pubSubMetricsBackendSendMethodCalled);
const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith;
assert.strictEqual(event, expectedEvent);
assert.strictEqual(attributes.event_source, expectedEventSource);
assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId);
assert.strictEqual(attributes.response_code, expectedResponseCode);
assert.strictEqual(typeof attributes.map_id, 'string');
assert.strictEqual(typeof attributes.cache_buster, 'string');
return done();
});
});
it.skip('should send event for errored tile requests', function (done) {
const expectedEvent = 'event-tile-test';
const expectedEventSource = 'event-source-tile-test';
const expectedEventGroupId = '12345';
const expectedResponseCode = '400';
const extraHeaders = {
'Carto-Event': expectedEvent,
'Carto-Event-Source': expectedEventSource,
'Carto-Event-Group-Id': expectedEventGroupId
};
const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } };
this.testClient = new TestClient(mapConfig, apikey, extraHeaders, overrideServerOptions);
const params = {
response: {
status: 400,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getTile(0, 0, 2, params, (err, res, tile) => {
if (err) {
return done(err);
}
assert.ok(this.pubSubMetricsBackendSendMethodCalled);
const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith;
assert.strictEqual(event, expectedEvent);
assert.strictEqual(attributes.event_source, expectedEventSource);
assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId);
assert.strictEqual(attributes.response_code, expectedResponseCode);
assert.strictEqual(typeof attributes.map_id, 'string');
assert.strictEqual(typeof attributes.cache_buster, 'string');
return done();
});
});
it('should send event for named map requests', function (done) {
const expectedEvent = 'map_view';
const expectedMetricsEvent = 'event-test';
const expectedEventSource = 'event-source-test';
const expectedEventGroupId = '1';
const expectedResponseCode = '200';
const expectedMapType = 'named';
const extraHeaders = {
'Carto-Event': expectedMetricsEvent,
'Carto-Event-Source': expectedEventSource,
'Carto-Event-Group-Id': expectedEventGroupId
};
const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } };
const template = templateBuilder({ name: 'map' });
this.testClient = new TestClient(template, apikey, extraHeaders, overrideServerOptions);
this.testClient.getLayergroup((err, body) => {
if (err) {
return done(err);
}
assert.strictEqual(typeof body.layergroupid, 'string');
const { token, cacheBuster, templateHash } = LayergroupToken.parse(body.layergroupid);
assert.ok(this.pubSubMetricsBackendSendMethodCalled);
const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith;
assert.strictEqual(event, expectedEvent);
assert.strictEqual(attributes.client_event, expectedMetricsEvent);
assert.strictEqual(attributes.event_source, expectedEventSource);
assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId);
assert.strictEqual(attributes.response_code, expectedResponseCode);
assert.strictEqual(attributes.map_type, expectedMapType);
assert.strictEqual(attributes.map_id, token);
assert.strictEqual(attributes.cache_buster, cacheBuster);
assert.strictEqual(attributes.template_hash, templateHash);
assert.strictEqual(attributes.stat_tag, template.layergroup.stat_tag);
return done();
});
});
it('should send event for errored named map requests', function (done) {
const expectedEvent = 'map_view';
const expectedMetricsEvent = 'event-test';
const expectedEventSource = 'event-source-test';
const expectedEventGroupId = '1';
const expectedResponseCode = '400';
const expectedMapType = 'named';
const extraHeaders = {
'Carto-Event': expectedMetricsEvent,
'Carto-Event-Source': expectedEventSource,
'Carto-Event-Group-Id': expectedEventGroupId
};
const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } };
const templateMissingCartoCSSVersion = templateMissingCartoCSSVersionBuilder();
this.testClient = new TestClient(templateMissingCartoCSSVersion, apikey, extraHeaders, overrideServerOptions);
const params = {
response: {
status: 400,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getLayergroup(params, (err, body) => {
if (err) {
return done(err);
}
assert.ok(this.pubSubMetricsBackendSendMethodCalled);
const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith;
assert.strictEqual(event, expectedEvent);
assert.strictEqual(attributes.client_event, expectedMetricsEvent);
assert.strictEqual(attributes.event_source, expectedEventSource);
assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId);
assert.strictEqual(attributes.response_code, expectedResponseCode);
assert.strictEqual(attributes.map_type, expectedMapType);
assert.strictEqual(typeof attributes.map_id, 'string');
assert.strictEqual(typeof attributes.cache_buster, 'string');
assert.strictEqual(typeof attributes.template_hash, 'string');
assert.strictEqual(attributes.stat_tag, templateMissingCartoCSSVersion.layergroup.stat_tag);
return done();
});
});
it.skip('should send event for named map tile requests', function (done) {
const expectedEvent = 'event-named-map-tile-test';
const expectedEventSource = 'event-source-named-map-tile-test';
const expectedEventGroupId = '1';
const expectedResponseCode = '200';
const extraHeaders = {
'Carto-Event': expectedEvent,
'Carto-Event-Source': expectedEventSource,
'Carto-Event-Group-Id': expectedEventGroupId
};
const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } };
const template = templateBuilder({ name: 'tile' });
this.testClient = new TestClient(template, apikey, extraHeaders, overrideServerOptions);
this.testClient.getTile(0, 0, 0, (err, body) => {
if (err) {
return done(err);
}
assert.ok(this.pubSubMetricsBackendSendMethodCalled);
const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith;
assert.strictEqual(event, expectedEvent);
assert.strictEqual(attributes.event_source, expectedEventSource);
assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId);
assert.strictEqual(attributes.response_code, expectedResponseCode);
assert.strictEqual(typeof attributes.map_id, 'string');
assert.strictEqual(typeof attributes.cache_buster, 'string');
assert.strictEqual(typeof attributes.template_hash, 'string');
assert.strictEqual(attributes.stat_tag, template.layergroup.stat_tag);
return done();
});
});
it('should send event for static named map requests', function (done) {
const expectedEvent = 'map_view';
const expectedMetricsEvent = 'event-test';
const expectedEventSource = 'event-source-test';
const expectedEventGroupId = '1';
const expectedResponseCode = '200';
const expectedMapType = 'static';
const extraHeaders = {
'Carto-Event': expectedMetricsEvent,
'Carto-Event-Source': expectedEventSource,
'Carto-Event-Group-Id': expectedEventGroupId
};
const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } };
const template = templateBuilder({ name: 'preview' });
this.testClient = new TestClient(template, apikey, extraHeaders, overrideServerOptions);
this.testClient.getPreview(640, 480, {}, (err, res, body) => {
if (err) {
return done(err);
}
assert.ok(this.pubSubMetricsBackendSendMethodCalled);
const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith;
assert.strictEqual(event, expectedEvent);
assert.strictEqual(attributes.client_event, expectedMetricsEvent);
assert.strictEqual(attributes.event_source, expectedEventSource);
assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId);
assert.strictEqual(attributes.response_code, expectedResponseCode);
assert.strictEqual(attributes.map_type, expectedMapType);
assert.strictEqual(typeof attributes.map_id, 'string');
assert.strictEqual(typeof attributes.cache_buster, 'string');
assert.strictEqual(typeof attributes.template_hash, 'string');
assert.strictEqual(attributes.stat_tag, template.layergroup.stat_tag);
return done();
});
});
it('should send event for errored static named map requests', function (done) {
const expectedEvent = 'map_view';
const expectedMetricsEvent = 'event-test';
const expectedEventSource = 'event-source-test';
const expectedEventGroupId = '1';
const expectedResponseCode = '400';
const expectedMapType = 'static';
const extraHeaders = {
'Carto-Event': expectedMetricsEvent,
'Carto-Event-Source': expectedEventSource,
'Carto-Event-Group-Id': expectedEventGroupId
};
const overrideServerOptions = { pubSubMetrics: { enabled: true, topic: 'topic-test' } };
const template = templateBuilder({ name: 'preview-errored' });
this.testClient = new TestClient(template, apikey, extraHeaders, overrideServerOptions);
const widthTooLarge = 8193;
const params = {
response: {
status: 400,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
}
};
this.testClient.getPreview(widthTooLarge, 480, params, (err, res, body) => {
if (err) {
return done(err);
}
assert.ok(this.pubSubMetricsBackendSendMethodCalled);
const { event, attributes } = this.pubSubMetricsBackendSendMethodCalledWith;
assert.strictEqual(event, expectedEvent);
assert.strictEqual(attributes.client_event, expectedMetricsEvent);
assert.strictEqual(attributes.event_source, expectedEventSource);
assert.strictEqual(attributes.client_event_group_id, expectedEventGroupId);
assert.strictEqual(attributes.response_code, expectedResponseCode);
assert.strictEqual(attributes.map_type, expectedMapType);
assert.strictEqual(typeof attributes.map_id, 'string');
assert.strictEqual(typeof attributes.cache_buster, 'string');
assert.strictEqual(typeof attributes.template_hash, 'string');
assert.strictEqual(attributes.stat_tag, template.layergroup.stat_tag);
return done();
});
});
});
});

166
test/integration/pubsub-metrics-test.js
View File

@ -1,166 +0,0 @@
'use strict';
const sinon = require('sinon');
const assert = require('assert');
const redis = require('redis');
const TestClient = require('../support/test-client');
const PubSubMetricsBackend = require('../../lib/backends/pubsub-metrics');
const metricsHeaders = {
'Carto-Event': 'test-event',
'Carto-Event-Source': 'test',
'Carto-Event-Group-Id': '1'
};
const tooLongField = ' If you are sending a text this long in a header you kind of deserve the worst, honestly. I mean ' +
'this is not a header, it is almost a novel, and you do not see any Novel cookie here, right?';
const badHeaders = {
'Carto-Event': tooLongField,
'Carto-Event-Source': 'test',
'Carto-Event-Group-Id': 1
};
const mapConfig = {
version: '1.7.0',
layers: [
{
options: {
sql: 'select * FROM test_table_localhost_regular1',
cartocss: TestClient.CARTOCSS.POINTS,
cartocss_version: '2.3.0'
}
}
]
};
function buildEventAttributes (statusCode) {
return {
event_source: 'test',
user_id: '1',
event_group_id: '1',
response_code: statusCode.toString(),
source_domain: 'localhost',
event_time: new Date().toISOString(),
event_version: '1'
};
}
const fakeTopic = {
name: 'test-topic',
publish: sinon.stub().returns(Promise.resolve())
};
const fakePubSub = {
topic: () => fakeTopic
};
describe('pubsub metrics middleware', function () {
let redisClient;
let testClient;
let clock;
before(function () {
redisClient = redis.createClient(global.environment.redis.port);
clock = sinon.useFakeTimers();
sinon.stub(PubSubMetricsBackend, 'createPubSub').returns(fakePubSub);
});
after(function () {
clock.restore();
PubSubMetricsBackend.createPubSub.restore();
global.environment.pubSubMetrics.enabled = false;
});
afterEach(function (done) {
fakeTopic.publish.resetHistory();
redisClient.SELECT(0, () => {
redisClient.del('user:localhost:mapviews:global');
redisClient.SELECT(5, () => {
redisClient.del('user:localhost:mapviews:global');
done();
});
});
});
it('should not send event if not enabled', function (done) {
global.environment.pubSubMetrics.enabled = false;
testClient = new TestClient(mapConfig, 1234, metricsHeaders);
testClient.getLayergroup((err, body) => {
if (err) {
return done(err);
}
assert.strictEqual(typeof body.metadata, 'object');
assert(fakeTopic.publish.notCalled);
return done();
});
});
it('should not send event if headers not present', function (done) {
global.environment.pubSubMetrics.enabled = true;
testClient = new TestClient(mapConfig, 1234);
testClient.getLayergroup((err, body) => {
if (err) {
return done(err);
}
assert.strictEqual(typeof body.metadata, 'object');
assert(fakeTopic.publish.notCalled);
return done();
});
});
it('should normalized headers type and length', function (done) {
global.environment.pubSubMetrics.enabled = true;
const eventAttributes = buildEventAttributes(200);
const maxLength = 100;
const eventName = tooLongField.trim().substr(0, maxLength);
testClient = new TestClient(mapConfig, 1234, badHeaders);
testClient.getLayergroup((err, body) => {
if (err) {
return done(err);
}
assert.strictEqual(typeof body.metadata, 'object');
assert(fakeTopic.publish.calledOnceWith(Buffer.from(eventName), eventAttributes));
return done();
});
});
it('should send event for map requests', function (done) {
global.environment.pubSubMetrics.enabled = true;
const eventAttributes = buildEventAttributes(200);
testClient = new TestClient(mapConfig, 1234, metricsHeaders);
testClient.getLayergroup((err, body) => {
if (err) {
return done(err);
}
assert.strictEqual(typeof body.metadata, 'object');
assert(fakeTopic.publish.calledOnceWith(Buffer.from('test-event'), eventAttributes));
return done();
});
});
it('should send event when error', function (done) {
global.environment.pubSubMetrics.enabled = true;
const eventAttributes = buildEventAttributes(400);
eventAttributes.user_id = undefined;
testClient = new TestClient({}, 1234, metricsHeaders);
testClient.getLayergroup(() => {
assert(fakeTopic.publish.calledOnceWith(Buffer.from('test-event'), eventAttributes));
assert(fakeTopic.publish.calledOnce);
return done();
});
});
});

156
test/support/test-client.js
View File

@ -23,13 +23,14 @@ const MAPNIK_SUPPORTED_FORMATS = {
mvt: true
};
function TestClient (config, apiKey, extraHeaders) {
function TestClient (config, apiKey, extraHeaders = {}, overrideServerOptions = {}) {
this.mapConfig = isMapConfig(config) ? config : null;
this.template = isTemplate(config) ? config : null;
this.apiKey = apiKey;
this.extraHeaders = extraHeaders || {};
this.extraHeaders = extraHeaders;
this.keysToDelete = {};
this.server = new CartodbWindshaft(serverOptions);
this.serverOptions = Object.assign({}, serverOptions, overrideServerOptions);
this.server = new CartodbWindshaft(this.serverOptions);
}
module.exports = TestClient;
@ -941,16 +942,8 @@ TestClient.prototype.getLayergroup = function (params, callback) {
params = {};
}
if (!params.response) {
params.response = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
}
var url = '/api/v1/map';
let url = '/api/v1/map';
const urlNamed = url + '/named';
const headers = Object.assign({ host: 'localhost', 'Content-Type': 'application/json' }, self.extraHeaders);
const queryParams = {};
@ -967,12 +960,89 @@ TestClient.prototype.getLayergroup = function (params, callback) {
url += '?' + qs.stringify(queryParams);
}
var layergroupId;
if (params.layergroupid) {
layergroupId = params.layergroupid;
}
step(
function createTemplate () {
var next = this;
if (!self.template) {
return next();
}
if (!self.apiKey) {
return next(new Error('apiKey param is mandatory to create a new template'));
}
params.placeholders = params.placeholders || {};
assert.response(self.server,
{
url: url,
url: urlNamed + '?' + qs.stringify({ api_key: self.apiKey }),
method: 'POST',
headers,
data: JSON.stringify(self.mapConfig)
data: JSON.stringify(self.template)
},
{
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
},
function (res, err) {
if (err) {
return next(err);
}
return next(null, JSON.parse(res.body).template_id);
}
);
},
function createLayergroup (err, templateId) {
var next = this;
if (err) {
return next(err);
}
if (layergroupId) {
return next(null, layergroupId);
}
const data = templateId ? params.placeholders : self.mapConfig;
if (!params.response) {
params.response = {
status: 200,
headers: {
'Content-Type': 'application/json; charset=utf-8'
}
};
}
const queryParams = {};
if (self.apiKey) {
queryParams.api_key = self.apiKey;
}
if (params.aggregation !== undefined) {
queryParams.aggregation = params.aggregation;
}
const path = templateId
? urlNamed + '/' + templateId + '?' + qs.stringify(queryParams)
: url;
assert.response(self.server,
{
url: path,
method: 'POST',
headers,
data: JSON.stringify(data)
},
params.response,
function (res, err) {
@ -985,6 +1055,9 @@ TestClient.prototype.getLayergroup = function (params, callback) {
self.keysToDelete['map_cfg|' + LayergroupToken.parse(parsedBody.layergroupid).token] = 0;
self.keysToDelete['user:localhost:mapviews:global'] = 5;
}
if (res.statusCode === 200 && self.template && self.template.layergroup && self.template.layergroup.stat_tag) {
self.keysToDelete[`user:localhost:mapviews:stat_tag:${self.template.layergroup.stat_tag}`] = 5;
}
}
if (err) {
return callback(err);
@ -993,6 +1066,8 @@ TestClient.prototype.getLayergroup = function (params, callback) {
return callback(null, parsedBody);
}
);
}
);
};
TestClient.prototype.getStaticCenter = function (params, callback) {
@ -1686,3 +1761,54 @@ TestClient.prototype.getTemplate = function (params, callback) {
return callback(err, res, body);
});
};
TestClient.prototype.getPreview = function (width, height, params = {}, callback) {
this.createTemplate({}, (err, res, template) => {
if (err) {
return callback(err);
}
params = Object.assign({ api_key: this.apiKey }, params);
const url = `/api/v1/map/static/named/${template.template_id}/${width}/${height}.png?${qs.stringify(params)}`;
const headers = Object.assign({ host: 'localhost' }, this.extraHeaders);
const requestOptions = {
url: url,
method: 'GET',
headers,
encoding: 'binary'
};
const expectedResponse = Object.assign({
status: 200,
headers: {
'Content-Type': 'image/png'
}
}, params.response || {});
assert.response(this.server, requestOptions, expectedResponse, (res, err) => {
if (err) {
return callback(err);
}
let body;
switch (res.headers['content-type']) {
case 'image/png':
this.keysToDelete['user:localhost:mapviews:global'] = 5;
if (this.template.layergroup && this.template.layergroup.stat_tag) {
this.keysToDelete[`user:localhost:mapviews:stat_tag:${this.template.layergroup.stat_tag}`] = 5;
}
body = mapnik.Image.fromBytes(Buffer.from(res.body, 'binary'));
break;
case 'application/json; charset=utf-8':
body = JSON.parse(res.body);
break;
default:
body = res.body;
break;
}
return callback(null, res, body);
});
});
};

40
test/unit/backends/pubsub-metrics-test.js
View File

@ -1,40 +0,0 @@
'use strict';
const sinon = require('sinon');
const assert = require('assert');
const PubSubMetricsBackend = require('../../../lib/backends/pubsub-metrics');
const fakeTopic = {
name: 'test-topic',
publish: sinon.stub().returns(Promise.resolve())
};
const fakePubSub = {
topic: () => fakeTopic
};
const eventAttributes = {
event_source: 'test',
user_id: '123',
event_group_id: '1',
response_code: '200',
source_domain: 'localhost',
event_time: new Date().toISOString(),
event_version: '1'
};
describe('pubsub metrics backend', function () {
it('should not send event if not enabled', function () {
const pubSubMetricsService = new PubSubMetricsBackend(fakePubSub, false);
pubSubMetricsService.sendEvent('test-event', eventAttributes);
assert(fakeTopic.publish.notCalled);
});
it('should send event if enabled', function () {
const pubSubMetricsService = new PubSubMetricsBackend(fakePubSub, true);
pubSubMetricsService.sendEvent('test-event', eventAttributes);
assert(fakeTopic.publish.calledOnceWith(Buffer.from('test-event'), eventAttributes));
});
});