Create a controllers factory where all collaborators are created and controllers are mounted afterwards
This commit is contained in:
Daniel García Aubert
parent
6bf06116df
commit
b6989ac82a
446
lib/cartodb/controllers/factory.js
Normal file
446
lib/cartodb/controllers/factory.js
Normal file
@ -0,0 +1,446 @@
|
||||
const { Router: router } = require('express');
|
||||
|
||||
const RedisPool = require('redis-mpool');
|
||||
const cartodbRedis = require('cartodb-redis');
|
||||
|
||||
const windshaft = require('windshaft');
|
||||
|
||||
const PgConnection = require('../backends/pg_connection');
|
||||
const AnalysisBackend = require('../backends/analysis');
|
||||
const AnalysisStatusBackend = require('../backends/analysis-status');
|
||||
const DataviewBackend = require('../backends/dataview');
|
||||
const TemplateMaps = require('../backends/template_maps.js');
|
||||
const PgQueryRunner = require('../backends/pg_query_runner');
|
||||
const StatsBackend = require('../backends/stats');
|
||||
|
||||
const AuthApi = require('../api/auth_api');
|
||||
const UserLimitsApi = require('../api/user_limits_api');
|
||||
const OverviewsMetadataApi = require('../api/overviews_metadata_api');
|
||||
const FilterStatsApi = require('../api/filter_stats_api');
|
||||
const TablesExtentApi = require('../api/tables_extent_api');
|
||||
|
||||
const LayergroupAffectedTablesCache = require('../cache/layergroup_affected_tables');
|
||||
const SurrogateKeysCache = require('../cache/surrogate_keys_cache');
|
||||
const VarnishHttpCacheBackend = require('../cache/backend/varnish_http');
|
||||
const FastlyCacheBackend = require('../cache/backend/fastly');
|
||||
const NamedMapProviderCache = require('../cache/named_map_provider_cache');
|
||||
const NamedMapsCacheEntry = require('../cache/model/named_maps_entry');
|
||||
|
||||
const SqlWrapMapConfigAdapter = require('../models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
|
||||
const MapConfigNamedLayersAdapter = require('../models/mapconfig/adapter/mapconfig-named-layers-adapter');
|
||||
const MapConfigBufferSizeAdapter = require('../models/mapconfig/adapter/mapconfig-buffer-size-adapter');
|
||||
const AnalysisMapConfigAdapter = require('../models/mapconfig/adapter/analysis-mapconfig-adapter');
|
||||
const MapConfigOverviewsAdapter = require('../models/mapconfig/adapter/mapconfig-overviews-adapter');
|
||||
const TurboCartoAdapter = require('../models/mapconfig/adapter/turbo-carto-adapter');
|
||||
const DataviewsWidgetsAdapter = require('../models/mapconfig/adapter/dataviews-widgets-adapter');
|
||||
const AggregationMapConfigAdapter = require('../models/mapconfig/adapter/aggregation-mapconfig-adapter');
|
||||
const MapConfigAdapter = require('../models/mapconfig/adapter');
|
||||
|
||||
const ResourceLocator = require('../models/resource-locator');
|
||||
const LayergroupMetadata = require('../utils/layergroup-metadata');
|
||||
const RendererStatsReporter = require('../stats/reporter/renderer');
|
||||
|
||||
const AnalysisLayergroupController = require('./layergroup/analysis');
|
||||
const AttributesLayergroupController = require('./layergroup/attributes');
|
||||
const DataviewLayergroupController = require('./layergroup/dataview');
|
||||
const PreviewLayergroupController = require('./layergroup/preview');
|
||||
const TileLayergroupController = require('./layergroup/tile');
|
||||
|
||||
const AnonymousMapController = require('./map/anonymous');
|
||||
const NamedMapController = require('./map/named');
|
||||
|
||||
const AdminTemplateController = require('./template/admin');
|
||||
const PreviewTemplateController = require('./template/preview');
|
||||
const TileTemplateController = require('./template/tile');
|
||||
|
||||
const AnalysesController = require('./analyses');
|
||||
|
||||
const ServerInfoController = require('./server-info');
|
||||
|
||||
module.exports = class ControllersFactory {
|
||||
constructor ({ serverOptions, environmentOptions }) {
|
||||
const redisOptions = Object.assign({}, environmentOptions.redis, {
|
||||
name: 'windshaft-server',
|
||||
unwatchOnRelease: false,
|
||||
noReadyCheck: true
|
||||
});
|
||||
|
||||
const redisPool = new RedisPool(redisOptions);
|
||||
|
||||
redisPool.on('status', function(status) {
|
||||
var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.';
|
||||
global.statsClient.gauge(keyPrefix + 'count', status.count);
|
||||
global.statsClient.gauge(keyPrefix + 'unused', status.unused);
|
||||
global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
|
||||
});
|
||||
|
||||
const metadataBackend = cartodbRedis({ pool: redisPool });
|
||||
const pgConnection = new PgConnection(metadataBackend);
|
||||
|
||||
const mapStore = new windshaft.storage.MapStore({
|
||||
pool: redisPool,
|
||||
expire_time: serverOptions.grainstore.default_layergroup_ttl
|
||||
});
|
||||
|
||||
const rendererFactory = createRendererFactory({ redisPool, serverOptions, environmentOptions });
|
||||
|
||||
const rendererCacheOpts = Object.assign({}, serverOptions.renderCache || {}, {
|
||||
ttl: 60000, // 60 seconds TTL by default
|
||||
statsInterval: 60000 // reports stats every milliseconds defined here
|
||||
});
|
||||
const rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts);
|
||||
const rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval);
|
||||
rendererStatsReporter.start();
|
||||
|
||||
const tileBackend = new windshaft.backend.Tile(rendererCache);
|
||||
const attributesBackend = new windshaft.backend.Attributes();
|
||||
const previewBackend = new windshaft.backend.Preview(rendererCache);
|
||||
const mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
|
||||
const mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
|
||||
|
||||
const surrogateKeysCacheBackends = createSurrogateKeysCacheBackends(serverOptions);
|
||||
const surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends);
|
||||
const templateMaps = createTemplateMaps({ redisPool, surrogateKeysCache });
|
||||
|
||||
const analysisStatusBackend = new AnalysisStatusBackend();
|
||||
const analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
|
||||
const dataviewBackend = new DataviewBackend(analysisBackend);
|
||||
const statsBackend = new StatsBackend();
|
||||
|
||||
const userLimitsApi = new UserLimitsApi(metadataBackend, {
|
||||
limits: {
|
||||
cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
|
||||
render: serverOptions.renderer.mapnik.limits.render || 0,
|
||||
rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled
|
||||
}
|
||||
});
|
||||
const authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps);
|
||||
|
||||
const layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
|
||||
|
||||
if (process.env.NODE_ENV === 'test') {
|
||||
this.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
|
||||
}
|
||||
|
||||
const pgQueryRunner = new PgQueryRunner(pgConnection);
|
||||
const overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
|
||||
|
||||
const filterStatsApi = new FilterStatsApi(pgQueryRunner);
|
||||
const tablesExtentApi = new TablesExtentApi(pgQueryRunner);
|
||||
|
||||
const mapConfigAdapter = new MapConfigAdapter(
|
||||
new MapConfigNamedLayersAdapter(templateMaps, pgConnection),
|
||||
new MapConfigBufferSizeAdapter(),
|
||||
new SqlWrapMapConfigAdapter(),
|
||||
new DataviewsWidgetsAdapter(),
|
||||
new AnalysisMapConfigAdapter(analysisBackend),
|
||||
new AggregationMapConfigAdapter(pgConnection),
|
||||
new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi),
|
||||
new TurboCartoAdapter()
|
||||
);
|
||||
|
||||
const resourceLocator = new ResourceLocator(global.environment);
|
||||
const layergroupMetadata = new LayergroupMetadata(resourceLocator);
|
||||
|
||||
const namedMapProviderCache = new NamedMapProviderCache(
|
||||
templateMaps,
|
||||
pgConnection,
|
||||
metadataBackend,
|
||||
userLimitsApi,
|
||||
mapConfigAdapter,
|
||||
layergroupAffectedTablesCache
|
||||
);
|
||||
|
||||
['update', 'delete'].forEach(function(eventType) {
|
||||
templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
|
||||
});
|
||||
|
||||
const versions = getAndValidateVersions(serverOptions);
|
||||
|
||||
this.mapConfigBasePath = serverOptions.base_url_mapconfig;
|
||||
this.templateBasePath = serverOptions.base_url_templated;
|
||||
|
||||
this.analysisLayergroupController = new AnalysisLayergroupController(
|
||||
analysisStatusBackend,
|
||||
pgConnection,
|
||||
mapStore,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
authApi,
|
||||
surrogateKeysCache
|
||||
);
|
||||
|
||||
this.attributesLayergroupController = new AttributesLayergroupController(
|
||||
attributesBackend,
|
||||
pgConnection,
|
||||
mapStore,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
authApi,
|
||||
surrogateKeysCache
|
||||
);
|
||||
|
||||
this.dataviewLayergroupController = new DataviewLayergroupController(
|
||||
dataviewBackend,
|
||||
pgConnection,
|
||||
mapStore,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
authApi,
|
||||
surrogateKeysCache
|
||||
);
|
||||
|
||||
this.previewLayergroupController = new PreviewLayergroupController(
|
||||
previewBackend,
|
||||
pgConnection,
|
||||
mapStore,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
authApi,
|
||||
surrogateKeysCache
|
||||
);
|
||||
|
||||
this.tileLayergroupController = new TileLayergroupController(
|
||||
tileBackend,
|
||||
pgConnection,
|
||||
mapStore,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
authApi,
|
||||
surrogateKeysCache
|
||||
);
|
||||
|
||||
this.anonymousMapController = new AnonymousMapController(
|
||||
pgConnection,
|
||||
templateMaps,
|
||||
mapBackend,
|
||||
metadataBackend,
|
||||
surrogateKeysCache,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
mapConfigAdapter,
|
||||
statsBackend,
|
||||
authApi,
|
||||
layergroupMetadata
|
||||
);
|
||||
|
||||
this.namedMapController = new NamedMapController(
|
||||
pgConnection,
|
||||
templateMaps,
|
||||
mapBackend,
|
||||
metadataBackend,
|
||||
surrogateKeysCache,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
mapConfigAdapter,
|
||||
statsBackend,
|
||||
authApi,
|
||||
layergroupMetadata
|
||||
);
|
||||
|
||||
this.tileTemplateController = new TileTemplateController(
|
||||
namedMapProviderCache,
|
||||
tileBackend,
|
||||
surrogateKeysCache,
|
||||
pgConnection,
|
||||
authApi,
|
||||
userLimitsApi
|
||||
);
|
||||
|
||||
this.previewTemplateController = new PreviewTemplateController(
|
||||
namedMapProviderCache,
|
||||
previewBackend,
|
||||
surrogateKeysCache,
|
||||
tablesExtentApi,
|
||||
metadataBackend,
|
||||
pgConnection,
|
||||
authApi,
|
||||
userLimitsApi
|
||||
);
|
||||
|
||||
this.adminTemplateController = new AdminTemplateController(
|
||||
authApi,
|
||||
templateMaps,
|
||||
userLimitsApi
|
||||
);
|
||||
|
||||
this.analysesController = new AnalysesController(
|
||||
pgConnection,
|
||||
authApi,
|
||||
userLimitsApi
|
||||
);
|
||||
|
||||
this.serverInfoController = new ServerInfoController(versions);
|
||||
}
|
||||
|
||||
regist (app) {
|
||||
if (process.env.NODE_ENV === 'test') {
|
||||
app.layergroupAffectedTablesCache = this.layergroupAffectedTablesCache;
|
||||
}
|
||||
|
||||
const mapRouter = router();
|
||||
|
||||
this.analysisLayergroupController.register(mapRouter);
|
||||
this.attributesLayergroupController.register(mapRouter);
|
||||
this.dataviewLayergroupController.register(mapRouter);
|
||||
this.previewLayergroupController.register(mapRouter);
|
||||
this.tileLayergroupController.register(mapRouter);
|
||||
this.anonymousMapController.register(mapRouter);
|
||||
this.previewTemplateController.register(mapRouter);
|
||||
this.analysesController.register(mapRouter);
|
||||
|
||||
app.use(this.mapConfigBasePath, mapRouter);
|
||||
|
||||
const templateRouter = router();
|
||||
|
||||
this.namedMapController.register(templateRouter);
|
||||
this.tileTemplateController.register(templateRouter);
|
||||
this.adminTemplateController.register(templateRouter);
|
||||
|
||||
app.use(this.templateBasePath, templateRouter);
|
||||
|
||||
const monitorRouter = router();
|
||||
|
||||
this.serverInfoController.register(monitorRouter);
|
||||
|
||||
app.use('/', monitorRouter);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
function createTemplateMaps ({ redisPool, surrogateKeysCache }) {
|
||||
const templateMaps = new TemplateMaps(redisPool, {
|
||||
max_user_templates: global.environment.maxUserTemplates
|
||||
});
|
||||
|
||||
function invalidateNamedMap (owner, templateName) {
|
||||
var startTime = Date.now();
|
||||
surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) {
|
||||
var logMessage = JSON.stringify({
|
||||
username: owner,
|
||||
type: 'named_map_invalidation',
|
||||
elapsed: Date.now() - startTime,
|
||||
error: !!err ? JSON.stringify(err.message) : undefined
|
||||
});
|
||||
if (err) {
|
||||
global.logger.warn(logMessage);
|
||||
} else {
|
||||
global.logger.info(logMessage);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
['update', 'delete'].forEach(function(eventType) {
|
||||
templateMaps.on(eventType, invalidateNamedMap);
|
||||
});
|
||||
|
||||
return templateMaps;
|
||||
}
|
||||
|
||||
function createSurrogateKeysCacheBackends(serverOptions) {
|
||||
var cacheBackends = [];
|
||||
|
||||
if (serverOptions.varnish_purge_enabled) {
|
||||
cacheBackends.push(
|
||||
new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port)
|
||||
);
|
||||
}
|
||||
|
||||
if (serverOptions.fastly &&
|
||||
!!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) {
|
||||
cacheBackends.push(
|
||||
new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId)
|
||||
);
|
||||
}
|
||||
|
||||
return cacheBackends;
|
||||
}
|
||||
|
||||
const timeoutErrorTilePath = __dirname + '/../../../assets/render-timeout-fallback.png';
|
||||
const timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
|
||||
|
||||
function createRendererFactory ({ redisPool, serverOptions, environmentOptions }) {
|
||||
var onTileErrorStrategy;
|
||||
if (environmentOptions.enabledFeatures.onTileErrorStrategy !== false) {
|
||||
onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) {
|
||||
|
||||
function isRenderTimeoutError (err) {
|
||||
return err.message === 'Render timed out';
|
||||
}
|
||||
|
||||
function isDatasourceTimeoutError (err) {
|
||||
return err.message && err.message.match(/canceling statement due to statement timeout/i);
|
||||
}
|
||||
|
||||
function isTimeoutError (err) {
|
||||
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
|
||||
}
|
||||
|
||||
function isRasterFormat (format) {
|
||||
return format === 'png' || format === 'jpg';
|
||||
}
|
||||
|
||||
if (isTimeoutError(err) && isRasterFormat(format)) {
|
||||
return callback(null, timeoutErrorTile, {
|
||||
'Content-Type': 'image/png',
|
||||
}, {});
|
||||
} else {
|
||||
return callback(err, tile, headers, stats);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const rendererFactory = new windshaft.renderer.Factory({
|
||||
onTileErrorStrategy: onTileErrorStrategy,
|
||||
mapnik: {
|
||||
redisPool: redisPool,
|
||||
grainstore: serverOptions.grainstore,
|
||||
mapnik: serverOptions.renderer.mapnik
|
||||
},
|
||||
http: serverOptions.renderer.http,
|
||||
mvt: serverOptions.renderer.mvt
|
||||
});
|
||||
|
||||
|
||||
return rendererFactory;
|
||||
}
|
||||
|
||||
function getAndValidateVersions(options) {
|
||||
// jshint undef:false
|
||||
var warn = console.warn.bind(console);
|
||||
// jshint undef:true
|
||||
|
||||
var packageDefinition = require('../../../package.json');
|
||||
|
||||
var declaredDependencies = packageDefinition.dependencies || {};
|
||||
var installedDependenciesVersions = {
|
||||
camshaft: require('camshaft').version,
|
||||
grainstore: windshaft.grainstore.version(),
|
||||
mapnik: windshaft.mapnik.versions.mapnik,
|
||||
node_mapnik: windshaft.mapnik.version,
|
||||
'turbo-carto': require('turbo-carto').version,
|
||||
windshaft: windshaft.version,
|
||||
windshaft_cartodb: packageDefinition.version
|
||||
};
|
||||
|
||||
var dependenciesToValidate = ['camshaft', 'turbo-carto', 'windshaft'];
|
||||
dependenciesToValidate.forEach(function(depName) {
|
||||
var declaredDependencyVersion = declaredDependencies[depName];
|
||||
var installedDependencyVersion = installedDependenciesVersions[depName];
|
||||
if (declaredDependencyVersion !== installedDependencyVersion) {
|
||||
warn(
|
||||
'Dependency="%s" installed version="%s" does not match declared version="%s". Check your installation.',
|
||||
depName, installedDependencyVersion, declaredDependencyVersion
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// Be nice and warn if configured mapnik version is != installed mapnik version
|
||||
if (windshaft.mapnik.versions.mapnik !== options.grainstore.mapnik_version) {
|
||||
warn('WARNING: detected mapnik version (' + windshaft.mapnik.versions.mapnik + ')' +
|
||||
' != configured mapnik version (' + options.grainstore.mapnik_version + ')');
|
||||
}
|
||||
|
||||
return installedDependenciesVersions;
|
||||
}
|
||||
10
lib/cartodb/middleware/syntax-error.js
Normal file
10
lib/cartodb/middleware/syntax-error.js
Normal file
@ -0,0 +1,10 @@
|
||||
module.exports = function syntaxError () {
|
||||
return function syntaxErrorMiddleware (err, req, res, next) {
|
||||
if (err.name === 'SyntaxError') {
|
||||
err.http_status = 400;
|
||||
err.message = `${err.name}: ${err.message}`;
|
||||
}
|
||||
|
||||
next(err);
|
||||
};
|
||||
};
|
||||
@ -1,377 +1,55 @@
|
||||
var express = require('express');
|
||||
const express = require('express');
|
||||
const cors = require('./middleware/cors');
|
||||
const user = require('./middleware/user');
|
||||
var bodyParser = require('body-parser');
|
||||
var RedisPool = require('redis-mpool');
|
||||
var cartodbRedis = require('cartodb-redis');
|
||||
var _ = require('underscore');
|
||||
const bodyParser = require('body-parser');
|
||||
const _ = require('underscore');
|
||||
|
||||
const AnalysisLayergroupController = require('./controllers/layergroup/analysis');
|
||||
const AttributesLayergroupController = require('./controllers/layergroup/attributes');
|
||||
const DataviewLayergroupController = require('./controllers/layergroup/dataview');
|
||||
const PreviewLayergroupController = require('./controllers/layergroup/preview');
|
||||
const TileLayergroupController = require('./controllers/layergroup/tile');
|
||||
|
||||
const AnonymousMapController = require('./controllers/map/anonymous');
|
||||
const NamedMapController = require('./controllers/map/named');
|
||||
|
||||
const AdminTemplateController = require('./controllers/template/admin');
|
||||
const PreviewTemplateController = require('./controllers/template/preview');
|
||||
const TileTemplateController = require('./controllers/template/tile');
|
||||
|
||||
const AnalysesController = require('./controllers/analyses');
|
||||
|
||||
const ServerInfoController = require('./controllers/server-info');
|
||||
|
||||
var SurrogateKeysCache = require('./cache/surrogate_keys_cache');
|
||||
var NamedMapsCacheEntry = require('./cache/model/named_maps_entry');
|
||||
var VarnishHttpCacheBackend = require('./cache/backend/varnish_http');
|
||||
var FastlyCacheBackend = require('./cache/backend/fastly');
|
||||
|
||||
var StatsClient = require('./stats/client');
|
||||
const StatsClient = require('./stats/client');
|
||||
const stats = require('./middleware/stats');
|
||||
|
||||
var RendererStatsReporter = require('./stats/reporter/renderer');
|
||||
|
||||
var windshaft = require('windshaft');
|
||||
var mapnik = windshaft.mapnik;
|
||||
|
||||
var TemplateMaps = require('./backends/template_maps.js');
|
||||
var OverviewsMetadataApi = require('./api/overviews_metadata_api');
|
||||
var FilterStatsApi = require('./api/filter_stats_api');
|
||||
var UserLimitsApi = require('./api/user_limits_api');
|
||||
var AuthApi = require('./api/auth_api');
|
||||
var LayergroupAffectedTablesCache = require('./cache/layergroup_affected_tables');
|
||||
var NamedMapProviderCache = require('./cache/named_map_provider_cache');
|
||||
var PgQueryRunner = require('./backends/pg_query_runner');
|
||||
var PgConnection = require('./backends/pg_connection');
|
||||
|
||||
var AnalysisBackend = require('./backends/analysis');
|
||||
const AnalysisStatusBackend = require('./backends/analysis-status');
|
||||
const DataviewBackend = require('./backends/dataview');
|
||||
|
||||
var timeoutErrorTilePath = __dirname + '/../../assets/render-timeout-fallback.png';
|
||||
var timeoutErrorTile = require('fs').readFileSync(timeoutErrorTilePath, {encoding: null});
|
||||
|
||||
var SqlWrapMapConfigAdapter = require('./models/mapconfig/adapter/sql-wrap-mapconfig-adapter');
|
||||
var MapConfigNamedLayersAdapter = require('./models/mapconfig/adapter/mapconfig-named-layers-adapter');
|
||||
var MapConfigBufferSizeAdapter = require('./models/mapconfig/adapter/mapconfig-buffer-size-adapter');
|
||||
var AnalysisMapConfigAdapter = require('./models/mapconfig/adapter/analysis-mapconfig-adapter');
|
||||
var MapConfigOverviewsAdapter = require('./models/mapconfig/adapter/mapconfig-overviews-adapter');
|
||||
var TurboCartoAdapter = require('./models/mapconfig/adapter/turbo-carto-adapter');
|
||||
var DataviewsWidgetsAdapter = require('./models/mapconfig/adapter/dataviews-widgets-adapter');
|
||||
var AggregationMapConfigAdapter = require('./models/mapconfig/adapter/aggregation-mapconfig-adapter');
|
||||
var MapConfigAdapter = require('./models/mapconfig/adapter');
|
||||
|
||||
var StatsBackend = require('./backends/stats');
|
||||
|
||||
const ResourceLocator = require('./models/resource-locator');
|
||||
const LayergroupMetadata = require('./utils/layergroup-metadata');
|
||||
const { mapnik } = require('windshaft');
|
||||
|
||||
const lzmaMiddleware = require('./middleware/lzma');
|
||||
const errorMiddleware = require('./middleware/error-middleware');
|
||||
const syntaxError = require('./middleware/syntax-error');
|
||||
const servedByHostHeader = require('./middleware/served-by-host-header');
|
||||
|
||||
module.exports = function(serverOptions) {
|
||||
const ControllersFactory = require('./controllers/factory');
|
||||
|
||||
module.exports = function createServer (serverOptions) {
|
||||
validateOptions(serverOptions);
|
||||
|
||||
// Make stats client globally accessible
|
||||
global.statsClient = StatsClient.getInstance(serverOptions.statsd);
|
||||
|
||||
var redisPool = new RedisPool(_.defaults(global.environment.redis, {
|
||||
name: 'windshaft-server',
|
||||
unwatchOnRelease: false,
|
||||
noReadyCheck: true
|
||||
}));
|
||||
|
||||
redisPool.on('status', function(status) {
|
||||
var keyPrefix = 'windshaft.redis-pool.' + status.name + '.db' + status.db + '.';
|
||||
global.statsClient.gauge(keyPrefix + 'count', status.count);
|
||||
global.statsClient.gauge(keyPrefix + 'unused', status.unused);
|
||||
global.statsClient.gauge(keyPrefix + 'waiting', status.waiting);
|
||||
});
|
||||
|
||||
var metadataBackend = cartodbRedis({pool: redisPool});
|
||||
var pgConnection = new PgConnection(metadataBackend);
|
||||
var pgQueryRunner = new PgQueryRunner(pgConnection);
|
||||
var overviewsMetadataApi = new OverviewsMetadataApi(pgQueryRunner);
|
||||
var filterStatsApi = new FilterStatsApi(pgQueryRunner);
|
||||
var userLimitsApi = new UserLimitsApi(metadataBackend, {
|
||||
limits: {
|
||||
cacheOnTimeout: serverOptions.renderer.mapnik.limits.cacheOnTimeout || false,
|
||||
render: serverOptions.renderer.mapnik.limits.render || 0,
|
||||
rateLimitsEnabled: global.environment.enabledFeatures.rateLimitsEnabled
|
||||
}
|
||||
});
|
||||
|
||||
var templateMaps = new TemplateMaps(redisPool, {
|
||||
max_user_templates: global.environment.maxUserTemplates
|
||||
});
|
||||
|
||||
var surrogateKeysCache = new SurrogateKeysCache(surrogateKeysCacheBackends(serverOptions));
|
||||
|
||||
function invalidateNamedMap (owner, templateName) {
|
||||
var startTime = Date.now();
|
||||
surrogateKeysCache.invalidate(new NamedMapsCacheEntry(owner, templateName), function(err) {
|
||||
var logMessage = JSON.stringify({
|
||||
username: owner,
|
||||
type: 'named_map_invalidation',
|
||||
elapsed: Date.now() - startTime,
|
||||
error: !!err ? JSON.stringify(err.message) : undefined
|
||||
});
|
||||
if (err) {
|
||||
global.logger.warn(logMessage);
|
||||
} else {
|
||||
global.logger.info(logMessage);
|
||||
}
|
||||
});
|
||||
}
|
||||
['update', 'delete'].forEach(function(eventType) {
|
||||
templateMaps.on(eventType, invalidateNamedMap);
|
||||
});
|
||||
|
||||
serverOptions.grainstore.mapnik_version = mapnikVersion(serverOptions);
|
||||
|
||||
validateOptions(serverOptions);
|
||||
|
||||
bootstrapFonts(serverOptions);
|
||||
|
||||
// initialize express server
|
||||
var app = bootstrap(serverOptions);
|
||||
const app = bootstrap(serverOptions);
|
||||
|
||||
var mapStore = new windshaft.storage.MapStore({
|
||||
pool: redisPool,
|
||||
expire_time: serverOptions.grainstore.default_layergroup_ttl
|
||||
});
|
||||
app.use(bodyParser.json());
|
||||
|
||||
var onTileErrorStrategy;
|
||||
if (global.environment.enabledFeatures.onTileErrorStrategy !== false) {
|
||||
onTileErrorStrategy = function onTileErrorStrategy$TimeoutTile(err, tile, headers, stats, format, callback) {
|
||||
app.use(servedByHostHeader());
|
||||
|
||||
function isRenderTimeoutError (err) {
|
||||
return err.message === 'Render timed out';
|
||||
}
|
||||
app.use(stats({
|
||||
enabled: serverOptions.useProfiler,
|
||||
statsClient: global.statsClient
|
||||
}));
|
||||
|
||||
function isDatasourceTimeoutError (err) {
|
||||
return err.message && err.message.match(/canceling statement due to statement timeout/i);
|
||||
}
|
||||
app.use(lzmaMiddleware());
|
||||
|
||||
function isTimeoutError (err) {
|
||||
return isRenderTimeoutError(err) || isDatasourceTimeoutError(err);
|
||||
}
|
||||
|
||||
function isRasterFormat (format) {
|
||||
return format === 'png' || format === 'jpg';
|
||||
}
|
||||
setupLogger(app, serverOptions);
|
||||
|
||||
if (isTimeoutError(err) && isRasterFormat(format)) {
|
||||
return callback(null, timeoutErrorTile, {
|
||||
'Content-Type': 'image/png',
|
||||
}, {});
|
||||
} else {
|
||||
return callback(err, tile, headers, stats);
|
||||
}
|
||||
};
|
||||
}
|
||||
app.use(cors());
|
||||
app.use(user());
|
||||
|
||||
var rendererFactory = new windshaft.renderer.Factory({
|
||||
onTileErrorStrategy: onTileErrorStrategy,
|
||||
mapnik: {
|
||||
redisPool: redisPool,
|
||||
grainstore: serverOptions.grainstore,
|
||||
mapnik: serverOptions.renderer.mapnik
|
||||
},
|
||||
http: serverOptions.renderer.http,
|
||||
mvt: serverOptions.renderer.mvt
|
||||
});
|
||||
const controllers = new ControllersFactory({ serverOptions, environmentOptions: global.environment });
|
||||
|
||||
// initialize render cache
|
||||
var rendererCacheOpts = _.defaults(serverOptions.renderCache || {}, {
|
||||
ttl: 60000, // 60 seconds TTL by default
|
||||
statsInterval: 60000 // reports stats every milliseconds defined here
|
||||
});
|
||||
var rendererCache = new windshaft.cache.RendererCache(rendererFactory, rendererCacheOpts);
|
||||
var rendererStatsReporter = new RendererStatsReporter(rendererCache, rendererCacheOpts.statsInterval);
|
||||
rendererStatsReporter.start();
|
||||
|
||||
var attributesBackend = new windshaft.backend.Attributes();
|
||||
var previewBackend = new windshaft.backend.Preview(rendererCache);
|
||||
var tileBackend = new windshaft.backend.Tile(rendererCache);
|
||||
var mapValidatorBackend = new windshaft.backend.MapValidator(tileBackend, attributesBackend);
|
||||
var mapBackend = new windshaft.backend.Map(rendererCache, mapStore, mapValidatorBackend);
|
||||
|
||||
var analysisBackend = new AnalysisBackend(metadataBackend, serverOptions.analysis);
|
||||
const analysisStatusBackend = new AnalysisStatusBackend();
|
||||
|
||||
const dataviewBackend = new DataviewBackend(analysisBackend);
|
||||
|
||||
var statsBackend = new StatsBackend();
|
||||
|
||||
var layergroupAffectedTablesCache = new LayergroupAffectedTablesCache();
|
||||
app.layergroupAffectedTablesCache = layergroupAffectedTablesCache;
|
||||
|
||||
var mapConfigAdapter = new MapConfigAdapter(
|
||||
new MapConfigNamedLayersAdapter(templateMaps, pgConnection),
|
||||
new MapConfigBufferSizeAdapter(),
|
||||
new SqlWrapMapConfigAdapter(),
|
||||
new DataviewsWidgetsAdapter(),
|
||||
new AnalysisMapConfigAdapter(analysisBackend),
|
||||
new AggregationMapConfigAdapter(pgConnection),
|
||||
new MapConfigOverviewsAdapter(overviewsMetadataApi, filterStatsApi),
|
||||
new TurboCartoAdapter()
|
||||
);
|
||||
|
||||
var namedMapProviderCache = new NamedMapProviderCache(
|
||||
templateMaps,
|
||||
pgConnection,
|
||||
metadataBackend,
|
||||
userLimitsApi,
|
||||
mapConfigAdapter,
|
||||
layergroupAffectedTablesCache
|
||||
);
|
||||
|
||||
['update', 'delete'].forEach(function(eventType) {
|
||||
templateMaps.on(eventType, namedMapProviderCache.invalidate.bind(namedMapProviderCache));
|
||||
});
|
||||
|
||||
var authApi = new AuthApi(pgConnection, metadataBackend, mapStore, templateMaps);
|
||||
|
||||
var TablesExtentApi = require('./api/tables_extent_api');
|
||||
var tablesExtentApi = new TablesExtentApi(pgQueryRunner);
|
||||
|
||||
const resourceLocator = new ResourceLocator(global.environment);
|
||||
const layergroupMetadata = new LayergroupMetadata(resourceLocator);
|
||||
|
||||
var versions = getAndValidateVersions(serverOptions);
|
||||
|
||||
/*******************************************************************************************************************
|
||||
* Routing
|
||||
******************************************************************************************************************/
|
||||
|
||||
const mapRouter = express.Router();
|
||||
const templateRouter = express.Router();
|
||||
const monitorRouter = express.Router();
|
||||
|
||||
new AnalysisLayergroupController(
|
||||
analysisStatusBackend,
|
||||
pgConnection,
|
||||
mapStore,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
authApi,
|
||||
surrogateKeysCache
|
||||
).register(mapRouter);
|
||||
|
||||
new AttributesLayergroupController(
|
||||
attributesBackend,
|
||||
pgConnection,
|
||||
mapStore,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
authApi,
|
||||
surrogateKeysCache
|
||||
).register(mapRouter);
|
||||
|
||||
new DataviewLayergroupController(
|
||||
dataviewBackend,
|
||||
pgConnection,
|
||||
mapStore,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
authApi,
|
||||
surrogateKeysCache
|
||||
).register(mapRouter);
|
||||
|
||||
new PreviewLayergroupController(
|
||||
previewBackend,
|
||||
pgConnection,
|
||||
mapStore,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
authApi,
|
||||
surrogateKeysCache
|
||||
).register(mapRouter);
|
||||
|
||||
new TileLayergroupController(
|
||||
tileBackend,
|
||||
pgConnection,
|
||||
mapStore,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
authApi,
|
||||
surrogateKeysCache
|
||||
).register(mapRouter);
|
||||
|
||||
new AnonymousMapController(
|
||||
pgConnection,
|
||||
templateMaps,
|
||||
mapBackend,
|
||||
metadataBackend,
|
||||
surrogateKeysCache,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
mapConfigAdapter,
|
||||
statsBackend,
|
||||
authApi,
|
||||
layergroupMetadata
|
||||
).register(mapRouter);
|
||||
|
||||
new NamedMapController(
|
||||
pgConnection,
|
||||
templateMaps,
|
||||
mapBackend,
|
||||
metadataBackend,
|
||||
surrogateKeysCache,
|
||||
userLimitsApi,
|
||||
layergroupAffectedTablesCache,
|
||||
mapConfigAdapter,
|
||||
statsBackend,
|
||||
authApi,
|
||||
layergroupMetadata
|
||||
).register(templateRouter);
|
||||
|
||||
new TileTemplateController(
|
||||
namedMapProviderCache,
|
||||
tileBackend,
|
||||
surrogateKeysCache,
|
||||
pgConnection,
|
||||
authApi,
|
||||
userLimitsApi
|
||||
).register(templateRouter);
|
||||
|
||||
new PreviewTemplateController(
|
||||
namedMapProviderCache,
|
||||
previewBackend,
|
||||
surrogateKeysCache,
|
||||
tablesExtentApi,
|
||||
metadataBackend,
|
||||
pgConnection,
|
||||
authApi,
|
||||
userLimitsApi
|
||||
).register(mapRouter);
|
||||
|
||||
new AdminTemplateController(
|
||||
authApi,
|
||||
templateMaps,
|
||||
userLimitsApi
|
||||
).register(templateRouter);
|
||||
|
||||
new AnalysesController(
|
||||
pgConnection,
|
||||
authApi,
|
||||
userLimitsApi
|
||||
).register(mapRouter);
|
||||
|
||||
new ServerInfoController(versions).register(monitorRouter);
|
||||
|
||||
const { base_url_mapconfig: mapConfigBasePath, base_url_templated: templateBasePath } = serverOptions;
|
||||
|
||||
app.use(mapConfigBasePath, mapRouter);
|
||||
app.use(templateBasePath, templateRouter);
|
||||
app.use('/', monitorRouter);
|
||||
|
||||
/*******************************************************************************************************************
|
||||
* END Routing
|
||||
******************************************************************************************************************/
|
||||
controllers.regist(app);
|
||||
|
||||
app.use(syntaxError());
|
||||
app.use(errorMiddleware());
|
||||
|
||||
return app;
|
||||
@ -383,45 +61,6 @@ function validateOptions(opts) {
|
||||
}
|
||||
}
|
||||
|
||||
function getAndValidateVersions(options) {
|
||||
// jshint undef:false
|
||||
var warn = console.warn.bind(console);
|
||||
// jshint undef:true
|
||||
|
||||
var packageDefinition = require('../../package.json');
|
||||
|
||||
var declaredDependencies = packageDefinition.dependencies || {};
|
||||
var installedDependenciesVersions = {
|
||||
camshaft: require('camshaft').version,
|
||||
grainstore: windshaft.grainstore.version(),
|
||||
mapnik: windshaft.mapnik.versions.mapnik,
|
||||
node_mapnik: windshaft.mapnik.version,
|
||||
'turbo-carto': require('turbo-carto').version,
|
||||
windshaft: windshaft.version,
|
||||
windshaft_cartodb: packageDefinition.version
|
||||
};
|
||||
|
||||
var dependenciesToValidate = ['camshaft', 'turbo-carto', 'windshaft'];
|
||||
dependenciesToValidate.forEach(function(depName) {
|
||||
var declaredDependencyVersion = declaredDependencies[depName];
|
||||
var installedDependencyVersion = installedDependenciesVersions[depName];
|
||||
if (declaredDependencyVersion !== installedDependencyVersion) {
|
||||
warn(
|
||||
'Dependency="%s" installed version="%s" does not match declared version="%s". Check your installation.',
|
||||
depName, installedDependencyVersion, declaredDependencyVersion
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// Be nice and warn if configured mapnik version is != installed mapnik version
|
||||
if (mapnik.versions.mapnik !== options.grainstore.mapnik_version) {
|
||||
warn('WARNING: detected mapnik version (' + mapnik.versions.mapnik + ')' +
|
||||
' != configured mapnik version (' + options.grainstore.mapnik_version + ')');
|
||||
}
|
||||
|
||||
return installedDependenciesVersions;
|
||||
}
|
||||
|
||||
function bootstrapFonts(opts) {
|
||||
// Set carto renderer configuration for MMLStore
|
||||
opts.grainstore.carto_env = opts.grainstore.carto_env || {};
|
||||
@ -465,35 +104,6 @@ function bootstrap(opts) {
|
||||
return value;
|
||||
});
|
||||
|
||||
app.use(bodyParser.json());
|
||||
|
||||
app.use(servedByHostHeader());
|
||||
|
||||
app.use(stats({
|
||||
enabled: opts.useProfiler,
|
||||
statsClient: global.statsClient
|
||||
}));
|
||||
|
||||
app.use(lzmaMiddleware());
|
||||
|
||||
// temporary measure until we upgrade to newer version expressjs so we can check err.status
|
||||
app.use(function(err, req, res, next) {
|
||||
if (err) {
|
||||
if (err.name === 'SyntaxError') {
|
||||
res.status(400).json({ errors: [err.name + ': ' + err.message] });
|
||||
} else {
|
||||
next(err);
|
||||
}
|
||||
} else {
|
||||
next();
|
||||
}
|
||||
});
|
||||
|
||||
setupLogger(app, opts);
|
||||
|
||||
app.use(cors());
|
||||
app.use(user());
|
||||
|
||||
return app;
|
||||
}
|
||||
|
||||
@ -513,25 +123,6 @@ function setupLogger(app, opts) {
|
||||
}
|
||||
}
|
||||
|
||||
function surrogateKeysCacheBackends(serverOptions) {
|
||||
var cacheBackends = [];
|
||||
|
||||
if (serverOptions.varnish_purge_enabled) {
|
||||
cacheBackends.push(
|
||||
new VarnishHttpCacheBackend(serverOptions.varnish_host, serverOptions.varnish_http_port)
|
||||
);
|
||||
}
|
||||
|
||||
if (serverOptions.fastly &&
|
||||
!!serverOptions.fastly.enabled && !!serverOptions.fastly.apiKey && !!serverOptions.fastly.serviceId) {
|
||||
cacheBackends.push(
|
||||
new FastlyCacheBackend(serverOptions.fastly.apiKey, serverOptions.fastly.serviceId)
|
||||
);
|
||||
}
|
||||
|
||||
return cacheBackends;
|
||||
}
|
||||
|
||||
function mapnikVersion(opts) {
|
||||
return opts.grainstore.mapnik_version || mapnik.versions.mapnik;
|
||||
}
|
||||
|
||||
@ -230,7 +230,7 @@ describe('tests from old api translated to multilayer', function() {
|
||||
var parsed = JSON.parse(res.body);
|
||||
assert.ok(parsed.errors);
|
||||
assert.equal(parsed.errors.length, 1);
|
||||
assert.ok(parsed.errors[0].match(/^Unexpected token W/));
|
||||
assert.ok(parsed.errors[0].match(/Unexpected token W/));
|
||||
|
||||
done();
|
||||
}
|
||||
|
||||
@ -29,7 +29,7 @@ describe('windshaft', function() {
|
||||
function(){
|
||||
var ws = cartodbServer({unbuffered_logging:true});
|
||||
ws.listen();
|
||||
}, /Cannot read property 'mapnik' of undefined/
|
||||
}, /Must initialise server with/
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
Loading…
Reference in New Issue
Block a user