diff --git a/lib/api/api-router.js b/lib/api/api-router.js index 6459d48a..fe5733c6 100644 --- a/lib/api/api-router.js +++ b/lib/api/api-router.js @@ -1,13 +1,11 @@ 'use strict'; -const path = require('path'); - const { Router: router } = require('express'); const RedisPool = require('redis-mpool'); const cartodbRedis = require('cartodb-redis'); -const windshaft = require('windshaft'); +const { factory: windshaftFactory } = require('windshaft'); const PgConnection = require('../backends/pg-connection'); const AnalysisBackend = require('../backends/analysis'); @@ -22,9 +20,7 @@ const UserLimitsBackend = require('../backends/user-limits'); const OverviewsMetadataBackend = require('../backends/overviews-metadata'); const FilterStatsApi = require('../backends/filter-stats'); const TablesExtentBackend = require('../backends/tables-extent'); - const ClusterBackend = require('../backends/cluster'); - const PubSubMetricsBackend = require('../backends/pubsub-metrics'); const LayergroupAffectedTablesCache = require('../cache/layergroup-affected-tables'); @@ -33,7 +29,7 @@ const VarnishHttpCacheBackend = require('../cache/backend/varnish-http'); const FastlyCacheBackend = require('../cache/backend/fastly'); const NamedMapProviderCache = require('../cache/named-map-provider-cache'); const NamedMapsCacheEntry = require('../cache/model/named-maps-entry'); -const NamedMapProviderReporter = require('../stats/reporter/named-map-provider'); +const NamedMapProviderCacheReporter = require('../stats/reporter/named-map-provider-cache'); const SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapconfig-adapter'); const MapConfigNamedLayersAdapter = require('../models/mapconfig/adapter/mapconfig-named-layers-adapter'); @@ -66,6 +62,8 @@ const pubSubMetrics = require('./middlewares/pubsub-metrics'); const MapRouter = require('./map/map-router'); const TemplateRouter = require('./template/template-router'); +const getOnTileErrorStrategy = require('../utils/on-tile-error-strategy'); + module.exports = class ApiRouter { constructor ({ serverOptions, environmentOptions }) { this.serverOptions = serverOptions; @@ -85,36 +83,22 @@ module.exports = class ApiRouter { global.statsClient.gauge(keyPrefix + 'waiting', status.waiting); }); - const metadataBackend = cartodbRedis({ pool: redisPool }); - const pgConnection = new PgConnection(metadataBackend); - const windshaftLogger = environmentOptions.log_windshaft && global.log4js ? global.log4js.getLogger('[windshaft]') : null; - const mapStore = new windshaft.storage.MapStore({ - pool: redisPool, - expire_time: serverOptions.grainstore.default_layergroup_ttl, + + const { rendererCache, tileBackend, attributesBackend, previewBackend, mapBackend, mapStore } = windshaftFactory({ + rendererOptions: serverOptions, + redisPool, + onTileErrorStrategy: getOnTileErrorStrategy({ enabled: environmentOptions.enabledFeatures.onTileErrorStrategy }), logger: windshaftLogger }); - const rendererFactory = createRendererFactory({ redisPool, serverOptions, environmentOptions }); - - const rendererCacheOpts = Object.assign({ - ttl: 60000, // 60 seconds TTL by default - statsInterval: 60000 // reports stats every milliseconds defined here - }, serverOptions.renderCache || {}); - - const rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts); - const rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval); + const rendererStatsReporter = new RendererStatsReporter(rendererCache, serverOptions.statsInterval); rendererStatsReporter.start(); - const tileBackend = new windshaft.backend.Tile(rendererCache); - const attributesBackend = new windshaft.backend.Attributes(); - const concurrency = serverOptions.renderer.mapnik.poolSize + - serverOptions.renderer.mapnik.poolMaxWaitingClients; - const previewBackend = new windshaft.backend.Preview(rendererCache, { concurrency }); - const mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend); - const mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend); + const metadataBackend = cartodbRedis({ pool: redisPool }); + const pgConnection = new PgConnection(metadataBackend); const surrogateKeysCacheBackends = createSurrogateKeysCacheBackends(serverOptions); const surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends); @@ -171,12 +155,11 @@ module.exports = class ApiRouter { layergroupAffectedTablesCache ); - const namedMapProviderReporter = new NamedMapProviderReporter({ + const namedMapProviderCacheReporter = new NamedMapProviderCacheReporter({ namedMapProviderCache, - intervalInMilliseconds: rendererCacheOpts.statsInterval + intervalInMilliseconds: serverOptions.statsInterval }); - - namedMapProviderReporter.start(); + namedMapProviderCacheReporter.start(); const collaborators = { analysisStatusBackend, @@ -291,49 +274,3 @@ function createSurrogateKeysCacheBackends (serverOptions) { return cacheBackends; } - -const timeoutErrorTilePath = path.join(__dirname, '/../../assets/render-timeout-fallback.png'); -const timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, { encoding: null }); - -function createRendererFactory ({ redisPool, serverOptions, environmentOptions }) { - let onTileErrorStrategy; - if (environmentOptions.enabledFeatures.onTileErrorStrategy !== false) { - onTileErrorStrategy = async function onTileErrorStrategy$TimeoutTile (err, format) { - function isRenderTimeoutError (err) { - return err.message === 'Render timed out'; - } - - function isDatasourceTimeoutError (err) { - return err.message && err.message.match(/canceling statement due to statement timeout/i); - } - - function isTimeoutError (err) { - return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); - } - - function isRasterFormat (format) { - return format === 'png' || format === 'jpg'; - } - - if (isTimeoutError(err) && isRasterFormat(format)) { - return { buffer: timeoutErrorTile, headers: { 'Content-Type': 'image/png' }, stats: {} }; - } else { - throw err; - } - }; - } - - const rendererFactory = new windshaft.renderer.Factory({ - onTileErrorStrategy: onTileErrorStrategy, - mapnik: { - redisPool: redisPool, - grainstore: serverOptions.grainstore, - mapnik: serverOptions.renderer.mapnik - }, - http: serverOptions.renderer.http, - mvt: serverOptions.renderer.mvt, - torque: serverOptions.renderer.torque - }); - - return rendererFactory; -} diff --git a/lib/server.js b/lib/server.js index d0ac22f5..be030631 100644 --- a/lib/server.js +++ b/lib/server.js @@ -1,16 +1,9 @@ 'use strict'; -const _ = require('underscore'); -const semver = require('semver'); const express = require('express'); -const windshaft = require('windshaft'); -const { mapnik } = windshaft; - const jsonReplacer = require('./utils/json-replacer'); - const ApiRouter = require('./api/api-router'); const ServerInfoController = require('./server-info-controller'); - const StatsClient = require('./stats/client'); module.exports = function createServer (serverOptions) { @@ -30,11 +23,6 @@ module.exports = function createServer (serverOptions) { const apiRouter = new ApiRouter({ serverOptions, environmentOptions: global.environment }); - // TODO: remove it before releasing next major version - if (!Array.isArray(serverOptions.routes.api)) { - serverOptions.routes.api = [serverOptions.routes.api]; - } - apiRouter.route(app, serverOptions.routes.api); const serverInfoController = new ServerInfoController(); diff --git a/lib/stats/reporter/named-map-provider.js b/lib/stats/reporter/named-map-provider-cache.js similarity index 95% rename from lib/stats/reporter/named-map-provider.js rename to lib/stats/reporter/named-map-provider-cache.js index b65893d6..81aea87b 100644 --- a/lib/stats/reporter/named-map-provider.js +++ b/lib/stats/reporter/named-map-provider-cache.js @@ -2,7 +2,7 @@ const statKeyTemplate = ctx => `windshaft.named-map-provider-cache.${ctx.metric}`; -module.exports = class NamedMapProviderReporter { +module.exports = class NamedMapProviderCacheReporter { constructor ({ namedMapProviderCache, intervalInMilliseconds } = {}) { this.namedMapProviderCache = namedMapProviderCache; this.intervalInMilliseconds = intervalInMilliseconds; diff --git a/lib/utils/on-tile-error-strategy.js b/lib/utils/on-tile-error-strategy.js new file mode 100644 index 00000000..b45e7a2e --- /dev/null +++ b/lib/utils/on-tile-error-strategy.js @@ -0,0 +1,37 @@ +'use strict'; + +const path = require('path'); +const timeoutErrorTilePath = path.join(__dirname, '/../../assets/render-timeout-fallback.png'); +const timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, { encoding: null }); + +module.exports = function getOnTileErrorStrategy ({ enabled }) { + let onTileErrorStrategy; + + if (enabled !== false) { + onTileErrorStrategy = async function onTileErrorStrategy$TimeoutTile (err, format) { + function isRenderTimeoutError (err) { + return err.message === 'Render timed out'; + } + + function isDatasourceTimeoutError (err) { + return err.message && err.message.match(/canceling statement due to statement timeout/i); + } + + function isTimeoutError (err) { + return isRenderTimeoutError(err) || isDatasourceTimeoutError(err); + } + + function isRasterFormat (format) { + return format === 'png' || format === 'jpg'; + } + + if (isTimeoutError(err) && isRasterFormat(format)) { + return { buffer: timeoutErrorTile, headers: { 'Content-Type': 'image/png' }, stats: {} }; + } else { + throw err; + } + }; + } + + return onTileErrorStrategy; +}; diff --git a/test/acceptance/overviews-metadata-named-maps-test.js b/test/acceptance/overviews-metadata-named-maps-test.js index cbab9289..60712ec6 100644 --- a/test/acceptance/overviews-metadata-named-maps-test.js +++ b/test/acceptance/overviews-metadata-named-maps-test.js @@ -12,7 +12,7 @@ var RedisPool = require('redis-mpool'); var step = require('step'); -var windshaft = require('windshaft'); +const MapStore = require('../support/map-store'); describe('overviews metadata for named maps', function () { var server; @@ -127,10 +127,7 @@ describe('overviews metadata for named maps', function () { var next = this; - var mapStore = new windshaft.storage.MapStore({ - pool: redisPool, - expire_time: 500000 - }); + const mapStore = new MapStore(redisPool); mapStore.load(LayergroupToken.parse(layergroupId).token, function (err, mapConfig) { assert.ifError(err); assert.deepStrictEqual(nonOverviewsLayer, mapConfig._cfg.layers[1]); diff --git a/test/acceptance/overviews-metadata-test.js b/test/acceptance/overviews-metadata-test.js index b20e699c..d06b8cdf 100644 --- a/test/acceptance/overviews-metadata-test.js +++ b/test/acceptance/overviews-metadata-test.js @@ -12,7 +12,7 @@ var RedisPool = require('redis-mpool'); var step = require('step'); -var windshaft = require('windshaft'); +const MapStore = require('../support/map-store'); describe('overviews metadata', function () { var server; @@ -82,10 +82,8 @@ describe('overviews metadata', function () { assert.ifError(err); var next = this; - var mapStore = new windshaft.storage.MapStore({ - pool: redisPool, - expire_time: 500000 - }); + const mapStore = new MapStore(redisPool); + mapStore.load(LayergroupToken.parse(expectedToken).token, function (err, mapConfig) { assert.ifError(err); assert.deepStrictEqual(nonOverviewsLayer, mapConfig._cfg.layers[1]); @@ -275,10 +273,8 @@ describe('overviews metadata with filters', function () { assert.ifError(err); var next = this; - var mapStore = new windshaft.storage.MapStore({ - pool: redisPool, - expire_time: 500000 - }); + const mapStore = new MapStore(redisPool); + mapStore.load(LayergroupToken.parse(expectedToken).token, function (err, mapConfig) { assert.ifError(err); assert.strictEqual(mapConfig._cfg.layers[0].type, 'cartodb'); diff --git a/test/support/map-store.js b/test/support/map-store.js new file mode 100644 index 00000000..2d916c4b --- /dev/null +++ b/test/support/map-store.js @@ -0,0 +1,38 @@ +'use strict'; + +const { MapConfig } = require('windshaft').model; + +// Windshaft no longer provides the MapStore class to be used just for testing purposes +// This class provides just the method needed to load a map-config from redis +// It should be replaced by a new module @carto/map-config-storage (to be published) +module.exports = class MapStore { + constructor (pool) { + this.pool = pool; + } + + load (token, callback) { + const db = 0; + this.pool.acquire(db, (err, client) => { + if (err) { + return callback(err); + } + + client.get(`map_cfg|${token}`, (err, data) => { + this.pool.release(db, client); + + if (err) { + return callback(err); + } + + let mapConfig; + try { + mapConfig = MapConfig.create(JSON.parse(data)); + } catch (err) { + return callback(err); + } + + return callback(null, mapConfig); + }); + }); + } +}; diff --git a/test/unit/prepare-context-test.js b/test/unit/prepare-context-test.js index 928816d3..9d341515 100644 --- a/test/unit/prepare-context-test.js +++ b/test/unit/prepare-context-test.js @@ -8,14 +8,13 @@ var cartodbRedis = require('cartodb-redis'); var PgConnection = require('../../lib/backends/pg-connection'); var AuthBackend = require('../../lib/backends/auth'); var TemplateMaps = require('../../lib/backends/template-maps'); +const MapStore = require('../support/map-store'); const cleanUpQueryParamsMiddleware = require('../../lib/api/middlewares/clean-up-query-params'); const authorizeMiddleware = require('../../lib/api/middlewares/authorize'); const dbConnSetupMiddleware = require('../../lib/api/middlewares/db-conn-setup'); const credentialsMiddleware = require('../../lib/api/middlewares/credentials'); -var windshaft = require('windshaft'); - describe('prepare-context', function () { var testUser = _.template(global.environment.postgres_auth_user, { user_id: 1 }); var testPubuser = global.environment.postgres.user; @@ -28,7 +27,7 @@ describe('prepare-context', function () { before(function () { var redisPool = new RedisPool(global.environment.redis); - var mapStore = new windshaft.storage.MapStore(); + var mapStore = new MapStore(redisPool); var metadataBackend = cartodbRedis({ pool: redisPool }); var pgConnection = new PgConnection(metadataBackend); var templateMaps = new TemplateMaps(redisPool); diff --git a/test/unit/stats/reporter/named-map-provider-test.js b/test/unit/stats/reporter/named-map-provider-test.js index 0183b69b..3873e59f 100644 --- a/test/unit/stats/reporter/named-map-provider-test.js +++ b/test/unit/stats/reporter/named-map-provider-test.js @@ -1,7 +1,7 @@ 'use strict'; const assert = require('assert'); -const NamedMapProviderReporter = require('../../../../lib/stats/reporter/named-map-provider'); +const NamedMapProviderReporter = require('../../../../lib/stats/reporter/named-map-provider-cache'); describe('named-map-provider-reporter', function () { it('should report metrics every 100 ms', function (done) {