'use strict'; var queue = require('queue-async'); var debug = require('debug')('windshaft:analysis'); var camshaft = require('camshaft'); const queryUtils = require('../../../../lib/utils/query-utils'); var dot = require('dot'); dot.templateSettings.strip = false; function AnalysisMapConfigAdapter (analysisBackend) { this.analysisBackend = analysisBackend; } module.exports = AnalysisMapConfigAdapter; AnalysisMapConfigAdapter.prototype.getMapConfig = function (user, requestMapConfig, params, context, callback) { var self = this; if (!shouldAdaptLayers(requestMapConfig)) { return callback(null, requestMapConfig); } var analysisConfiguration = context.analysisConfiguration; var filters = {}; if (params.filters) { try { filters = JSON.parse(params.filters); } catch (e) { // ignore } } var dataviewsFilters = filters.dataviews || {}; debug(dataviewsFilters); var dataviews = requestMapConfig.dataviews || {}; var errors = getDataviewsErrors(dataviews); if (errors.length > 0) { return callback(errors); } var dataviewsFiltersBySourceId = Object.keys(dataviewsFilters).reduce(function (bySourceId, dataviewName) { var dataview = dataviews[dataviewName]; if (dataview) { var sourceId = dataview.source.id; if (!Object.prototype.hasOwnProperty.call(bySourceId, sourceId)) { bySourceId[sourceId] = {}; } bySourceId[sourceId][dataviewName] = getFilter(dataview, dataviewsFilters[dataviewName]); } return bySourceId; }, {}); debug(dataviewsFiltersBySourceId); debug('mapconfig input', JSON.stringify(requestMapConfig, null, 4)); requestMapConfig = appendFiltersToNodes(requestMapConfig, dataviewsFiltersBySourceId); // Expected format for analyses filters // filters = {analyses: { // a1: [{min, max}, {accept, reject}], // b1: [{range, column, min, max}, {category, column, accept, reject}] // }} requestMapConfig = appendFiltersToNodes(requestMapConfig, filters.analyses); function createAnalysis (analysisDefinition, done) { self.analysisBackend.create(analysisConfiguration, analysisDefinition, function (err, analysis) { if (err) { var error = new Error(err.message); error.type = 'analysis'; error.analysis = { id: analysisDefinition.id, node_id: err.node_id, type: analysisDefinition.type }; return done(error); } done(null, analysis); }); } var analysesQueue = queue(1); requestMapConfig.analyses.forEach(function (analysis) { analysesQueue.defer(createAnalysis, analysis); }); analysesQueue.awaitAll(function (err, analysesResults) { if (err) { return callback(err); } var sourceId2Node = analysesResults.reduce(function (sourceId2Query, analysis) { var rootNode = analysis.getRoot(); if (rootNode.params && rootNode.params.id) { sourceId2Query[rootNode.params.id] = rootNode; } analysis.getNodes().forEach(function (node) { if (node.params && node.params.id) { sourceId2Query[node.params.id] = node; } }); return sourceId2Query; }, {}); var analysesErrors = []; requestMapConfig.layers = requestMapConfig.layers.map(function (layer, layerIndex) { if (getLayerSourceId(layer)) { var layerSourceId = getLayerSourceId(layer); var layerNode = sourceId2Node[layerSourceId]; if (layerNode) { try { var analysisSql; // it might throw error: Range filter from camshaft, for instance. analysisSql = layerQuery(layerNode); var sqlQueryWrap = layer.options.sql_wrap; if (sqlQueryWrap) { layer.options.sql_raw = analysisSql; analysisSql = sqlQueryWrap.replace(/<%=\s*sql\s*%>/g, analysisSql); } layer.options.sql = analysisSql; layer.options.columns = getDataviewsColumns(getLayerDataviews(layer, dataviews)); layer.options.affected_tables = getAllAffectedTablesFromSourceNodes(layerNode); } catch (error) { analysesErrors.push(error); } } else { analysesErrors.push( new Error('Missing analysis node.id="' + layerSourceId + '" for layer=' + layerIndex) ); } } return layer; }); var missingDataviewsNodesErrors = getMissingDataviewsSourceIds(dataviews, sourceId2Node); if (analysesErrors.length > 0 || missingDataviewsNodesErrors.length > 0) { return callback(analysesErrors.concat(missingDataviewsNodesErrors)); } // Augment dataviews with sql from analyses Object.keys(dataviews).forEach(function (dataviewName) { var dataview = requestMapConfig.dataviews[dataviewName]; var dataviewSourceId = dataview.source.id; var dataviewNode = sourceId2Node[dataviewSourceId]; dataview.node = { type: dataviewNode.type, filters: dataviewNode.getFilters() }; dataview.sql = { own_filter_on: dataviewQuery(dataviewNode, dataviewName, true), own_filter_off: dataviewQuery(dataviewNode, dataviewName, false), no_filters: dataviewNode.getQuery(Object.keys(dataviewNode.getFilters()) .reduce(function (applyFilters, filterId) { applyFilters[filterId] = false; return applyFilters; }, {}) ) }; }); if (Object.keys(dataviews).length > 0) { requestMapConfig.dataviews = dataviews; } debug('mapconfig output', JSON.stringify(requestMapConfig, null, 4)); context.analysesResults = analysesResults; return callback(null, requestMapConfig); }); }; var SKIP_COLUMNS = { the_geom: true, the_geom_webmercator: true }; function skipColumns (columnNames) { return columnNames .filter(function (columnName) { return !SKIP_COLUMNS[queryUtils.stripQuotes(columnName)]; }); } var wrappedQueryTpl = dot.template([ 'SELECT {{=it._columns}}', 'FROM ({{=it._query}}) _cdb_analysis_query' ].join('\n')); function layerQuery (node) { if (node.type === 'source') { return node.getQuery(); } var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns())); return wrappedQueryTpl({ _query: node.getQuery(), _columns: _columns.join(', ') }); } function dataviewQuery (node, dataviewName, ownFilter) { var applyFilters = {}; if (!ownFilter) { applyFilters[dataviewName] = false; } if (node.type === 'source') { return node.getQuery(applyFilters); } var _columns = ['ST_Transform(the_geom, 3857) the_geom_webmercator'].concat(skipColumns(node.getColumns())); return wrappedQueryTpl({ _query: node.getQuery(applyFilters), _columns: _columns.join(', ') }); } function appendFiltersToNodes (requestMapConfig, dataviewsFiltersBySourceId) { var analyses = requestMapConfig.analyses || []; dataviewsFiltersBySourceId = dataviewsFiltersBySourceId || {}; requestMapConfig.analyses = analyses.map(function (analysisDefinition) { var analysisGraph = new camshaft.reference.AnalysisGraph(analysisDefinition); var definition = analysisDefinition; Object.keys(dataviewsFiltersBySourceId).forEach(function (sourceId) { definition = analysisGraph.getDefinitionWith(sourceId, { filters: dataviewsFiltersBySourceId[sourceId] }); }); return definition; }); return requestMapConfig; } function shouldAdaptLayers (requestMapConfig) { return (Array.isArray(requestMapConfig.layers) && requestMapConfig.layers.some(getLayerSourceId)) || (Array.isArray(requestMapConfig.analyses) && requestMapConfig.analyses.length > 0) || requestMapConfig.dataviews; } var DATAVIEW_TYPE_2_FILTER_TYPE = { aggregation: 'category', histogram: 'range' }; function getFilter (dataview, params) { var type = dataview.type; return { type: DATAVIEW_TYPE_2_FILTER_TYPE[type], column: dataview.options.column, params: params }; } function getLayerSourceId (layer) { return layer.options.source && layer.options.source.id; } function getDataviewSourceId (dataview) { return dataview.source && dataview.source.id; } function getLayerDataviews (layer, dataviews) { var layerDataviews = []; var layerSourceId = getLayerSourceId(layer); if (layerSourceId) { var dataviewsList = getDataviewsList(dataviews); dataviewsList.forEach(function (dataview) { if (getDataviewSourceId(dataview) === layerSourceId) { layerDataviews.push(dataview); } }); } return layerDataviews; } function getDataviewsColumns (dataviews) { return Object.keys(dataviews.reduce(function (columnsDict, dataview) { getDataviewColumns(dataview).forEach(function (columnName) { if (columnName) { columnsDict[columnName] = true; } }); return columnsDict; }, {})); } function getDataviewColumns (dataview) { var columns = []; var options = dataview.options; ['column', 'aggregationColumn'].forEach(function (opt) { if (Object.prototype.hasOwnProperty.call(options, opt) && !!options[opt]) { columns.push(options[opt]); } }); return columns; } function getDataviewsList (dataviews) { return Object.keys(dataviews).map(function (dataviewKey) { return dataviews[dataviewKey]; }); } function getDataviewsErrors (dataviews) { var dataviewType = typeof dataviews; if (dataviewType !== 'object') { return [new Error('"dataviews" must be a valid JSON object: "' + dataviewType + '" type found')]; } if (Array.isArray(dataviews)) { return [new Error('"dataviews" must be a valid JSON object: "array" type found')]; } var errors = []; Object.keys(dataviews).forEach(function (dataviewName) { var dataview = dataviews[dataviewName]; if (!Object.prototype.hasOwnProperty.call(dataview, 'source') || !dataview.source.id) { errors.push(new Error('Dataview "' + dataviewName + '" is missing `source.id` attribute')); } if (!dataview.type) { errors.push(new Error('Dataview "' + dataviewName + '" is missing `type` attribute')); } }); return errors; } function getMissingDataviewsSourceIds (dataviews, sourceId2Node) { var missingDataviewsSourceIds = []; Object.keys(dataviews).forEach(function (dataviewName) { var dataview = dataviews[dataviewName]; var dataviewSourceId = getDataviewSourceId(dataview); if (!Object.prototype.hasOwnProperty.call(sourceId2Node, dataviewSourceId)) { missingDataviewsSourceIds.push(new AnalysisError('Node with `source.id="' + dataviewSourceId + '"`' + ' not found in analyses for dataview "' + dataviewName + '"')); } }); return missingDataviewsSourceIds; } function AnalysisError (message) { Error.captureStackTrace(this, this.constructor); this.name = this.constructor.name; this.type = 'analysis'; this.message = message; } function getAllAffectedTablesFromSourceNodes (node) { var affectedTables = node.getAllInputNodes(function (node) { return node.getType() === 'source'; }).reduce(function (list, node) { return list.concat(node.getAffectedTables()); }, []); return affectedTables; } require('util').inherits(AnalysisError, Error);