cartodb/Windshaft-cartodb
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

- Rename NamedMapProviderReporter by NamedMapProviderCacheReporter

Browse Source 
- Extract getOnTileErrorStrategy to a module
- Stop using MapStore from windshaft while testing and create a custom one instead
This commit is contained in:
Daniel García Aubert 2020-04-04 17:46:08 +02:00
parent 24efc37737
commit a8fb51ba25
9 changed files with 101 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

93
lib/api/api-router.js
View File

@ -1,13 +1,11 @@
'use strict';
const path = require('path');
const { Router: router } = require('express');
const RedisPool = require('redis-mpool');
const cartodbRedis = require('cartodb-redis');
const windshaft = require('windshaft');
const { factory: windshaftFactory } = require('windshaft');
const PgConnection = require('../backends/pg-connection');
const AnalysisBackend = require('../backends/analysis');
@ -22,9 +20,7 @@ const UserLimitsBackend = require('../backends/user-limits');
const OverviewsMetadataBackend = require('../backends/overviews-metadata');
const FilterStatsApi = require('../backends/filter-stats');
const TablesExtentBackend = require('../backends/tables-extent');
const ClusterBackend = require('../backends/cluster');
const PubSubMetricsBackend = require('../backends/pubsub-metrics');
const LayergroupAffectedTablesCache = require('../cache/layergroup-affected-tables');
@ -33,7 +29,7 @@ const VarnishHttpCacheBackend = require('../cache/backend/varnish-http');
const FastlyCacheBackend = require('../cache/backend/fastly');
const NamedMapProviderCache = require('../cache/named-map-provider-cache');
const NamedMapsCacheEntry = require('../cache/model/named-maps-entry');
const NamedMapProviderReporter = require('../stats/reporter/named-map-provider');
const NamedMapProviderCacheReporter = require('../stats/reporter/named-map-provider-cache');
const SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
const MapConfigNamedLayersAdapter = require('../models/mapconfig/adapter/mapconfig-named-layers-adapter');
@ -66,6 +62,8 @@ const pubSubMetrics = require('./middlewares/pubsub-metrics');
const MapRouter = require('./map/map-router');
const TemplateRouter = require('./template/template-router');
const getOnTileErrorStrategy = require('../utils/on-tile-error-strategy');
module.exports = class ApiRouter {
constructor ({ serverOptions, environmentOptions }) {
this.serverOptions = serverOptions;
@ -85,36 +83,22 @@ module.exports = class ApiRouter {
global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
});
const metadataBackend = cartodbRedis({ pool: redisPool });
const pgConnection = new PgConnection(metadataBackend);
const windshaftLogger = environmentOptions.log_windshaft && global.log4js
? global.log4js.getLogger('[windshaft]')
: null;
const mapStore = new windshaft.storage.MapStore({
pool: redisPool,
expire_time: serverOptions.grainstore.default_layergroup_ttl,
const { rendererCache, tileBackend, attributesBackend, previewBackend, mapBackend, mapStore } = windshaftFactory({
rendererOptions: serverOptions,
redisPool,
onTileErrorStrategy: getOnTileErrorStrategy({ enabled: environmentOptions.enabledFeatures.onTileErrorStrategy }),
logger: windshaftLogger
});
const rendererFactory = createRendererFactory({ redisPool, serverOptions, environmentOptions });
const rendererCacheOpts = Object.assign({
ttl: 60000, // 60 seconds TTL by default
statsInterval: 60000 // reports stats every milliseconds defined here
}, serverOptions.renderCache || {});
const rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts);
const rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval);
const rendererStatsReporter = new RendererStatsReporter(rendererCache, serverOptions.statsInterval);
rendererStatsReporter.start();
const tileBackend = new windshaft.backend.Tile(rendererCache);
const attributesBackend = new windshaft.backend.Attributes();
const concurrency = serverOptions.renderer.mapnik.poolSize +
serverOptions.renderer.mapnik.poolMaxWaitingClients;
const previewBackend = new windshaft.backend.Preview(rendererCache, { concurrency });
const mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
const mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
const metadataBackend = cartodbRedis({ pool: redisPool });
const pgConnection = new PgConnection(metadataBackend);
const surrogateKeysCacheBackends = createSurrogateKeysCacheBackends(serverOptions);
const surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends);
@ -171,12 +155,11 @@ module.exports = class ApiRouter {
layergroupAffectedTablesCache
);
const namedMapProviderReporter = new NamedMapProviderReporter({
const namedMapProviderCacheReporter = new NamedMapProviderCacheReporter({
namedMapProviderCache,
intervalInMilliseconds: rendererCacheOpts.statsInterval
intervalInMilliseconds: serverOptions.statsInterval
});
namedMapProviderReporter.start();
namedMapProviderCacheReporter.start();
const collaborators = {
analysisStatusBackend,
@ -291,49 +274,3 @@ function createSurrogateKeysCacheBackends (serverOptions) {
return cacheBackends;
}
const timeoutErrorTilePath = path.join(__dirname, '/../../assets/render-timeout-fallback.png');
const timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, { encoding: null });
function createRendererFactory ({ redisPool, serverOptions, environmentOptions }) {
let onTileErrorStrategy;
if (environmentOptions.enabledFeatures.onTileErrorStrategy !== false) {
onTileErrorStrategy = async function onTileErrorStrategy$TimeoutTile (err, format) {
function isRenderTimeoutError (err) {
return err.message === 'Render timed out';
}
function isDatasourceTimeoutError (err) {
return err.message && err.message.match(/canceling statement due to statement timeout/i);
}
function isTimeoutError (err) {
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
}
function isRasterFormat (format) {
return format === 'png' || format === 'jpg';
}
if (isTimeoutError(err) && isRasterFormat(format)) {
return { buffer: timeoutErrorTile, headers: { 'Content-Type': 'image/png' }, stats: {} };
} else {
throw err;
}
};
}
const rendererFactory = new windshaft.renderer.Factory({
onTileErrorStrategy: onTileErrorStrategy,
mapnik: {
redisPool: redisPool,
grainstore: serverOptions.grainstore,
mapnik: serverOptions.renderer.mapnik
},
http: serverOptions.renderer.http,
mvt: serverOptions.renderer.mvt,
torque: serverOptions.renderer.torque
});
return rendererFactory;
}

12
lib/server.js
View File

@ -1,16 +1,9 @@
'use strict';
const _ = require('underscore');
const semver = require('semver');
const express = require('express');
const windshaft = require('windshaft');
const { mapnik } = windshaft;
const jsonReplacer = require('./utils/json-replacer');
const ApiRouter = require('./api/api-router');
const ServerInfoController = require('./server-info-controller');
const StatsClient = require('./stats/client');
module.exports = function createServer (serverOptions) {
@ -30,11 +23,6 @@ module.exports = function createServer (serverOptions) {
const apiRouter = new ApiRouter({ serverOptions, environmentOptions: global.environment });
// TODO: remove it before releasing next major version
if (!Array.isArray(serverOptions.routes.api)) {
serverOptions.routes.api = [serverOptions.routes.api];
}
apiRouter.route(app, serverOptions.routes.api);
const serverInfoController = new ServerInfoController();

2
lib/stats/reporter/named-map-provider.js → lib/stats/reporter/named-map-provider-cache.js
View File

@ -2,7 +2,7 @@
const statKeyTemplate = ctx => `windshaft.named-map-provider-cache.${ctx.metric}`;
module.exports = class NamedMapProviderReporter {
module.exports = class NamedMapProviderCacheReporter {
constructor ({ namedMapProviderCache, intervalInMilliseconds } = {}) {
this.namedMapProviderCache = namedMapProviderCache;
this.intervalInMilliseconds = intervalInMilliseconds;

37
lib/utils/on-tile-error-strategy.js Normal file
View File

@ -0,0 +1,37 @@
'use strict';
const path = require('path');
const timeoutErrorTilePath = path.join(__dirname, '/../../assets/render-timeout-fallback.png');
const timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, { encoding: null });
module.exports = function getOnTileErrorStrategy ({ enabled }) {
let onTileErrorStrategy;
if (enabled !== false) {
onTileErrorStrategy = async function onTileErrorStrategy$TimeoutTile (err, format) {
function isRenderTimeoutError (err) {
return err.message === 'Render timed out';
}
function isDatasourceTimeoutError (err) {
return err.message && err.message.match(/canceling statement due to statement timeout/i);
}
function isTimeoutError (err) {
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
}
function isRasterFormat (format) {
return format === 'png' || format === 'jpg';
}
if (isTimeoutError(err) && isRasterFormat(format)) {
return { buffer: timeoutErrorTile, headers: { 'Content-Type': 'image/png' }, stats: {} };
} else {
throw err;
}
};
}
return onTileErrorStrategy;
};

7
test/acceptance/overviews-metadata-named-maps-test.js
View File

@ -12,7 +12,7 @@ var RedisPool = require('redis-mpool');
var step = require('step');
var windshaft = require('windshaft');
const MapStore = require('../support/map-store');
describe('overviews metadata for named maps', function () {
var server;
@ -127,10 +127,7 @@ describe('overviews metadata for named maps', function () {
var next = this;
var mapStore = new windshaft.storage.MapStore({
pool: redisPool,
expire_time: 500000
});
const mapStore = new MapStore(redisPool);
mapStore.load(LayergroupToken.parse(layergroupId).token, function (err, mapConfig) {
assert.ifError(err);
assert.deepStrictEqual(nonOverviewsLayer, mapConfig._cfg.layers[1]);

14
test/acceptance/overviews-metadata-test.js
View File

@ -12,7 +12,7 @@ var RedisPool = require('redis-mpool');
var step = require('step');
var windshaft = require('windshaft');
const MapStore = require('../support/map-store');
describe('overviews metadata', function () {
var server;
@ -82,10 +82,8 @@ describe('overviews metadata', function () {
assert.ifError(err);
var next = this;
var mapStore = new windshaft.storage.MapStore({
pool: redisPool,
expire_time: 500000
});
const mapStore = new MapStore(redisPool);
mapStore.load(LayergroupToken.parse(expectedToken).token, function (err, mapConfig) {
assert.ifError(err);
assert.deepStrictEqual(nonOverviewsLayer, mapConfig._cfg.layers[1]);
@ -275,10 +273,8 @@ describe('overviews metadata with filters', function () {
assert.ifError(err);
var next = this;
var mapStore = new windshaft.storage.MapStore({
pool: redisPool,
expire_time: 500000
});
const mapStore = new MapStore(redisPool);
mapStore.load(LayergroupToken.parse(expectedToken).token, function (err, mapConfig) {
assert.ifError(err);
assert.strictEqual(mapConfig._cfg.layers[0].type, 'cartodb');

38
test/support/map-store.js Normal file
View File

@ -0,0 +1,38 @@
'use strict';
const { MapConfig } = require('windshaft').model;
// Windshaft no longer provides the MapStore class to be used just for testing purposes
// This class provides just the method needed to load a map-config from redis
// It should be replaced by a new module @carto/map-config-storage (to be published)
module.exports = class MapStore {
constructor (pool) {
this.pool = pool;
}
load (token, callback) {
const db = 0;
this.pool.acquire(db, (err, client) => {
if (err) {
return callback(err);
}
client.get(`map_cfg|${token}`, (err, data) => {
this.pool.release(db, client);
if (err) {
return callback(err);
}
let mapConfig;
try {
mapConfig = MapConfig.create(JSON.parse(data));
} catch (err) {
return callback(err);
}
return callback(null, mapConfig);
});
});
}
};

5
test/unit/prepare-context-test.js
View File

@ -8,14 +8,13 @@ var cartodbRedis = require('cartodb-redis');
var PgConnection = require('../../lib/backends/pg-connection');
var AuthBackend = require('../../lib/backends/auth');
var TemplateMaps = require('../../lib/backends/template-maps');
const MapStore = require('../support/map-store');
const cleanUpQueryParamsMiddleware = require('../../lib/api/middlewares/clean-up-query-params');
const authorizeMiddleware = require('../../lib/api/middlewares/authorize');
const dbConnSetupMiddleware = require('../../lib/api/middlewares/db-conn-setup');
const credentialsMiddleware = require('../../lib/api/middlewares/credentials');
var windshaft = require('windshaft');
describe('prepare-context', function () {
var testUser = _.template(global.environment.postgres_auth_user, { user_id: 1 });
var testPubuser = global.environment.postgres.user;
@ -28,7 +27,7 @@ describe('prepare-context', function () {
before(function () {
var redisPool = new RedisPool(global.environment.redis);
var mapStore = new windshaft.storage.MapStore();
var mapStore = new MapStore(redisPool);
var metadataBackend = cartodbRedis({ pool: redisPool });
var pgConnection = new PgConnection(metadataBackend);
var templateMaps = new TemplateMaps(redisPool);

2
test/unit/stats/reporter/named-map-provider-test.js
View File

@ -1,7 +1,7 @@
'use strict';
const assert = require('assert');
const NamedMapProviderReporter = require('../../../../lib/stats/reporter/named-map-provider');
const NamedMapProviderReporter = require('../../../../lib/stats/reporter/named-map-provider-cache');
describe('named-map-provider-reporter', function () {
it('should report metrics every 100 ms', function (done) {