372 lines
12 KiB
JavaScript
372 lines
12 KiB
JavaScript
'use strict';
|
|
|
|
var queue = require('queue-async');
|
|
var debug = require('debug')('windshaft:analysis');
|
|
|
|
var camshaft = require('camshaft');
|
|
var dot = require('dot');
|
|
dot.templateSettings.strip = false;
|
|
|
|
function AnalysisMapConfigAdapter (analysisBackend) {
|
|
this.analysisBackend = analysisBackend;
|
|
}
|
|
|
|
module.exports = AnalysisMapConfigAdapter;
|
|
|
|
AnalysisMapConfigAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) {
|
|
// jshint maxcomplexity:7
|
|
var self = this;
|
|
|
|
if (!shouldAdaptLayers(requestMapConfig)) {
|
|
return callback(null, requestMapConfig);
|
|
}
|
|
|
|
var analysisConfiguration = context.analysisConfiguration;
|
|
|
|
var filters = {};
|
|
if (params.filters) {
|
|
try {
|
|
filters = JSON.parse(params.filters);
|
|
} catch (e) {
|
|
// ignore
|
|
}
|
|
}
|
|
|
|
var dataviewsFilters = filters.dataviews || {};
|
|
debug(dataviewsFilters);
|
|
var dataviews = requestMapConfig.dataviews || {};
|
|
|
|
var errors = getDataviewsErrors(dataviews);
|
|
if (errors.length > 0) {
|
|
return callback(errors);
|
|
}
|
|
|
|
var dataviewsFiltersBySourceId = Object.keys(dataviewsFilters).reduce(function (bySourceId, dataviewName) {
|
|
var dataview = dataviews[dataviewName];
|
|
if (dataview) {
|
|
var sourceId = dataview.source.id;
|
|
if (!Object.prototype.hasOwnProperty.call(bySourceId, sourceId)) {
|
|
bySourceId[sourceId] = {};
|
|
}
|
|
|
|
bySourceId[sourceId][dataviewName] = getFilter(dataview, dataviewsFilters[dataviewName]);
|
|
}
|
|
return bySourceId;
|
|
}, {});
|
|
|
|
debug(dataviewsFiltersBySourceId);
|
|
|
|
debug('mapconfig input', JSON.stringify(requestMapConfig, null, 4));
|
|
|
|
requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId);
|
|
|
|
// Expected format for analyses filters
|
|
// filters = {analyses: {
|
|
// a1: [{min, max}, {accept, reject}],
|
|
// b1: [{range, column, min, max}, {category, column, accept, reject}]
|
|
// }}
|
|
requestMapConfig = appendFiltersToNodes(requestMapConfig, filters.analyses);
|
|
|
|
function createAnalysis (analysisDefinition, done) {
|
|
self.analysisBackend.create(analysisConfiguration, analysisDefinition, function (err, analysis) {
|
|
if (err) {
|
|
var error = new Error(err.message);
|
|
error.type = 'analysis';
|
|
error.analysis = {
|
|
id: analysisDefinition.id,
|
|
node_id: err.node_id,
|
|
type: analysisDefinition.type
|
|
};
|
|
return done(error);
|
|
}
|
|
|
|
done(null, analysis);
|
|
});
|
|
}
|
|
|
|
var analysesQueue = queue(1);
|
|
requestMapConfig.analyses.forEach(function (analysis) {
|
|
analysesQueue.defer(createAnalysis, analysis);
|
|
});
|
|
|
|
analysesQueue.awaitAll(function (err, analysesResults) {
|
|
if (err) {
|
|
return callback(err);
|
|
}
|
|
|
|
var sourceId2Node = analysesResults.reduce(function (sourceId2Query, analysis) {
|
|
var rootNode = analysis.getRoot();
|
|
if (rootNode.params && rootNode.params.id) {
|
|
sourceId2Query[rootNode.params.id] = rootNode;
|
|
}
|
|
|
|
analysis.getNodes().forEach(function (node) {
|
|
if (node.params && node.params.id) {
|
|
sourceId2Query[node.params.id] = node;
|
|
}
|
|
});
|
|
|
|
return sourceId2Query;
|
|
}, {});
|
|
|
|
var analysesErrors = [];
|
|
|
|
requestMapConfig.layers = requestMapConfig.layers.map(function (layer, layerIndex) {
|
|
if (getLayerSourceId(layer)) {
|
|
var layerSourceId = getLayerSourceId(layer);
|
|
var layerNode = sourceId2Node[layerSourceId];
|
|
if (layerNode) {
|
|
try {
|
|
var analysisSql;
|
|
|
|
// it might throw error: Range filter from camshaft, for instance.
|
|
analysisSql = layerQuery(layerNode);
|
|
|
|
var sqlQueryWrap = layer.options.sql_wrap;
|
|
if (sqlQueryWrap) {
|
|
layer.options.sql_raw = analysisSql;
|
|
analysisSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, analysisSql);
|
|
}
|
|
layer.options.sql = analysisSql;
|
|
layer.options.columns = getDataviewsColumns(getLayerDataviews(layer, dataviews));
|
|
layer.options.affected_tables = getAllAffectedTablesFromSourceNodes(layerNode);
|
|
} catch (error) {
|
|
analysesErrors.push(error);
|
|
}
|
|
} else {
|
|
analysesErrors.push(
|
|
new Error('Missing analysis node.id="' + layerSourceId + '" for layer=' + layerIndex)
|
|
);
|
|
}
|
|
}
|
|
return layer;
|
|
});
|
|
|
|
var missingDataviewsNodesErrors = getMissingDataviewsSourceIds(dataviews, sourceId2Node);
|
|
if (analysesErrors.length > 0 || missingDataviewsNodesErrors.length > 0) {
|
|
return callback(analysesErrors.concat(missingDataviewsNodesErrors));
|
|
}
|
|
|
|
// Augment dataviews with sql from analyses
|
|
Object.keys(dataviews).forEach(function (dataviewName) {
|
|
var dataview = requestMapConfig.dataviews[dataviewName];
|
|
var dataviewSourceId = dataview.source.id;
|
|
var dataviewNode = sourceId2Node[dataviewSourceId];
|
|
dataview.node = {
|
|
type: dataviewNode.type,
|
|
filters: dataviewNode.getFilters()
|
|
};
|
|
dataview.sql = {
|
|
own_filter_on: dataviewQuery(dataviewNode, dataviewName, true),
|
|
own_filter_off: dataviewQuery(dataviewNode, dataviewName, false),
|
|
no_filters: dataviewNode.getQuery(Object.keys(dataviewNode.getFilters())
|
|
.reduce(function (applyFilters, filterId) {
|
|
applyFilters[filterId] = false;
|
|
return applyFilters;
|
|
}, {})
|
|
)
|
|
};
|
|
});
|
|
if (Object.keys(dataviews).length > 0) {
|
|
requestMapConfig.dataviews = dataviews;
|
|
}
|
|
|
|
debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4));
|
|
|
|
context.analysesResults = analysesResults;
|
|
|
|
return callback(null, requestMapConfig);
|
|
});
|
|
};
|
|
|
|
|
|
function stripQuotes(columnName) {
|
|
const quotedName = columnName.length > 2 && columnName[0] === '"' && columnName[columnName.length - 1] === '"';
|
|
if (quotedName)
|
|
{
|
|
return columnName.substring(1, columnName.length - 1);
|
|
}
|
|
return columnName;
|
|
}
|
|
|
|
var SKIP_COLUMNS = {
|
|
the_geom: true,
|
|
the_geom_webmercator: true
|
|
};
|
|
|
|
function skipColumns (columnNames) {
|
|
return columnNames
|
|
.filter(function (columnName) { return !SKIP_COLUMNS[stripQuotes(columnName)]; });
|
|
}
|
|
|
|
var wrappedQueryTpl = dot.template([
|
|
'SELECT {{=it._columns}}',
|
|
'FROM ({{=it._query}}) _cdb_analysis_query'
|
|
].join('\n'));
|
|
|
|
function layerQuery (node) {
|
|
if (node.type === 'source') {
|
|
return node.getQuery();
|
|
}
|
|
var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns()));
|
|
return wrappedQueryTpl({ _query: node.getQuery(), _columns: _columns.join(', ') });
|
|
}
|
|
|
|
function dataviewQuery (node, dataviewName, ownFilter) {
|
|
var applyFilters = {};
|
|
if (!ownFilter) {
|
|
applyFilters[dataviewName] = false;
|
|
}
|
|
|
|
if (node.type === 'source') {
|
|
return node.getQuery(applyFilters);
|
|
}
|
|
var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns()));
|
|
return wrappedQueryTpl({ _query: node.getQuery(applyFilters), _columns: _columns.join(', ') });
|
|
}
|
|
|
|
function appendFiltersToNodes (requestMapConfig, dataviewsFiltersBySourceId) {
|
|
var analyses = requestMapConfig.analyses || [];
|
|
dataviewsFiltersBySourceId = dataviewsFiltersBySourceId || {};
|
|
|
|
requestMapConfig.analyses = analyses.map(function (analysisDefinition) {
|
|
var analysisGraph = new camshaft.reference.AnalysisGraph(analysisDefinition);
|
|
var definition = analysisDefinition;
|
|
Object.keys(dataviewsFiltersBySourceId).forEach(function (sourceId) {
|
|
definition = analysisGraph.getDefinitionWith(sourceId, { filters: dataviewsFiltersBySourceId[sourceId] });
|
|
});
|
|
|
|
return definition;
|
|
});
|
|
|
|
return requestMapConfig;
|
|
}
|
|
|
|
function shouldAdaptLayers (requestMapConfig) {
|
|
return (Array.isArray(requestMapConfig.layers) && requestMapConfig.layers.some(getLayerSourceId)) ||
|
|
(Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0) ||
|
|
requestMapConfig.dataviews;
|
|
}
|
|
|
|
var DATAVIEW_TYPE_2_FILTER_TYPE = {
|
|
aggregation: 'category',
|
|
histogram: 'range'
|
|
};
|
|
function getFilter (dataview, params) {
|
|
var type = dataview.type;
|
|
|
|
return {
|
|
type: DATAVIEW_TYPE_2_FILTER_TYPE[type],
|
|
column: dataview.options.column,
|
|
params: params
|
|
};
|
|
}
|
|
|
|
function getLayerSourceId (layer) {
|
|
return layer.options.source && layer.options.source.id;
|
|
}
|
|
|
|
function getDataviewSourceId (dataview) {
|
|
return dataview.source && dataview.source.id;
|
|
}
|
|
|
|
function getLayerDataviews (layer, dataviews) {
|
|
var layerDataviews = [];
|
|
|
|
var layerSourceId = getLayerSourceId(layer);
|
|
if (layerSourceId) {
|
|
var dataviewsList = getDataviewsList(dataviews);
|
|
dataviewsList.forEach(function (dataview) {
|
|
if (getDataviewSourceId(dataview) === layerSourceId) {
|
|
layerDataviews.push(dataview);
|
|
}
|
|
});
|
|
}
|
|
|
|
return layerDataviews;
|
|
}
|
|
|
|
function getDataviewsColumns (dataviews) {
|
|
return Object.keys(dataviews.reduce(function (columnsDict, dataview) {
|
|
getDataviewColumns(dataview).forEach(function (columnName) {
|
|
if (columnName) {
|
|
columnsDict[columnName] = true;
|
|
}
|
|
});
|
|
return columnsDict;
|
|
}, {}));
|
|
}
|
|
|
|
function getDataviewColumns (dataview) {
|
|
var columns = [];
|
|
var options = dataview.options;
|
|
['column', 'aggregationColumn'].forEach(function (opt) {
|
|
if (Object.prototype.hasOwnProperty.call(options, opt) && !!options[opt]) {
|
|
columns.push(options[opt]);
|
|
}
|
|
});
|
|
return columns;
|
|
}
|
|
|
|
function getDataviewsList (dataviews) {
|
|
return Object.keys(dataviews).map(function (dataviewKey) { return dataviews[dataviewKey]; });
|
|
}
|
|
|
|
function getDataviewsErrors (dataviews) {
|
|
var dataviewType = typeof dataviews;
|
|
if (dataviewType !== 'object') {
|
|
return [new Error('"dataviews" must be a valid JSON object: "' + dataviewType + '" type found')];
|
|
}
|
|
|
|
if (Array.isArray(dataviews)) {
|
|
return [new Error('"dataviews" must be a valid JSON object: "array" type found')];
|
|
}
|
|
|
|
var errors = [];
|
|
|
|
Object.keys(dataviews).forEach(function (dataviewName) {
|
|
var dataview = dataviews[dataviewName];
|
|
if (!Object.prototype.hasOwnProperty.call(dataview, 'source') || !dataview.source.id) {
|
|
errors.push(new Error('Dataview "' + dataviewName + '" is missing `source.id` attribute'));
|
|
}
|
|
|
|
if (!dataview.type) {
|
|
errors.push(new Error('Dataview "' + dataviewName + '" is missing `type` attribute'));
|
|
}
|
|
});
|
|
|
|
return errors;
|
|
}
|
|
|
|
function getMissingDataviewsSourceIds (dataviews, sourceId2Node) {
|
|
var missingDataviewsSourceIds = [];
|
|
Object.keys(dataviews).forEach(function (dataviewName) {
|
|
var dataview = dataviews[dataviewName];
|
|
var dataviewSourceId = getDataviewSourceId(dataview);
|
|
if (!Object.prototype.hasOwnProperty.call(sourceId2Node, dataviewSourceId)) {
|
|
missingDataviewsSourceIds.push(new AnalysisError('Node with `source.id="' + dataviewSourceId + '"`' +
|
|
' not found in analyses for dataview "' + dataviewName + '"'));
|
|
}
|
|
});
|
|
|
|
return missingDataviewsSourceIds;
|
|
}
|
|
|
|
function AnalysisError (message) {
|
|
Error.captureStackTrace(this, this.constructor);
|
|
this.name = this.constructor.name;
|
|
this.type = 'analysis';
|
|
this.message = message;
|
|
}
|
|
|
|
function getAllAffectedTablesFromSourceNodes (node) {
|
|
var affectedTables = node.getAllInputNodes(function (node) {
|
|
return node.getType() === 'source';
|
|
}).reduce(function (list, node) {
|
|
return list.concat(node.getAffectedTables());
|
|
}, []);
|
|
return affectedTables;
|
|
}
|
|
|
|
require('util').inherits(AnalysisError, Error);
|