const $ = require('jquery'); const Backbone = require('backbone'); const _ = require('underscore'); const TableDataCollection = require('dashboard/data/table/table-data-collection'); const RowModel = require('dashboard/data/table/row-model'); const checkAndBuildOpts = require('builder/helpers/required-opts'); const WKT = require('dashboard/common/wkt'); const safeTableNameQuoting = require('dashboard/helpers/safe-table-name-quoting'); const cartoMetadataStatic = require('dashboard/views/public-dataset/carto-table-metadata-static'); const REQUIRED_OPTS = [ 'configModel' ]; module.exports = TableDataCollection.extend({ _ADDED_ROW_TEXT: 'Row added correctly', _ADDING_ROW_TEXT: 'Adding a new row', _GEOMETRY_UPDATED: 'Table geometry updated', model: function (attrs, opts) { var configModel = opts.collection._configModel; return new RowModel(attrs, { configModel, // TODO: Check this collection: opts.collection }); }, initialize: function (models, options) { checkAndBuildOpts(options, REQUIRED_OPTS, this); this.table = options ? options.table : null; this.model.prototype.idAttribute = 'cartodb_id'; this.initOptions(); this.filter = null; this._fetching = false; this.pages = []; this.lastPage = false; this.bind('newPage', this.newPage, this); this.bind('reset', function () { var pages = Math.floor(this.size() / this.options.get('rows_per_page')); this.pages = []; for (var i = 0; i < pages; ++i) { this.pages.push(i); } }, this); if (this.table) { this.bind('add change:the_geom', function (row) { var gt = this.table.get('geometry_types'); if (gt && gt.length > 0) return; if (row.get('the_geom')) { // we set it to silent because a change in geometry_types // raises rendering and column feching this.table.addGeomColumnType(row.getGeomType()); } }, this); } TableDataCollection.prototype.initialize.call(this); }, initOptions: function () { this.options = new Backbone.Model({ rows_per_page: 40, page: 0, sort_order: 'asc', order_by: 'cartodb_id', filter_column: '', filter_value: '' }); this.options.bind('change', () => { if (this._fetching) { return; } this._fetching = true; var opt = {}; var previous = this.options.previous('page'); if (this.options.hasChanged('page')) { opt.add = true; opt.changingPage = true; // if user is going backwards insert new rows at the top if (previous > this.options.get('page')) { opt.at = 0; } } else { if (this.options.hasChanged('mode')) { this.options.set({ 'page': 0 }, { silent: true }); } } opt.success = (_coll, resp) => { this.trigger('loaded'); if (resp.rows && resp.rows.length !== 0) { if (opt.changingPage) { this.trigger('newPage', this.options.get('page'), opt.at === 0 ? 'up' : 'down'); } } else { // no data so do not change the page this.options.set({page: previous});//, { silent: true }); } this.trigger('endLoadingRows', this.options.get('page'), opt.at === 0 ? 'up' : 'down'); this._fetching = false; }; opt.error = () => { console.error('there was some problem fetching rows'); this.trigger('endLoadingRows'); this._fetching = false; }; this.trigger('loadingRows', opt.at === 0 ? 'up' : 'down'); this.fetch(opt); }, this); }, parse: function (d) { // when the query modifies the data modified flag is true // TODO: change this when SQL API was able to say if a // query modify some data // HACK, it will fail if using returning sql statement this.modify_rows = d.rows.length === 0 && _.size(d.fields) === 0; this.affected_rows = d.affected_rows; this.lastPage = false; if (d.rows.length < this.options.get('rows_per_page')) { this.lastPage = true; } return d.rows; }, // given fields array as they come from SQL create a map name -> type _schemaFromQueryFields: function (fields) { var sc = {}; for (var k in fields) { sc[k] = fields[k].type; } return sc; }, _createUrlOptions: function (filter) { var attr; if (filter) { var a = {}; for (var k in this.options.attributes) { if (filter(k)) { a[k] = this.options.attributes[k]; } } attr = _(a); } else { attr = _(this.options.attributes); } var params = attr.map(function (v, k) { return k + '=' + encodeURIComponent(v); }).join('&'); params += '&api_key=' + this._configModel.get('api_key'); return params; }, _geometryColumnSQL: function (c) { return [ 'CASE', 'WHEN GeometryType(' + c + ") = 'POINT' THEN", 'ST_AsGeoJSON(' + c + ',8)', 'WHEN (' + c + ' IS NULL) THEN', 'NULL', 'ELSE', 'GeometryType(' + c + ')', 'END ' + c ].join(' '); }, // return wrapped SQL removing the_geom and the_geom_webmercator // to avoid fetching those columns. // So for a sql like // select * from table the returned value is // select column1, column2, column3... from table wrappedSQL: function (schema, exclude, fetchGeometry) { exclude = exclude || ['the_geom_webmercator']; schema = _.clone(schema); var select_columns = _.chain(schema).omit(exclude).map((v, k) => { if (v === 'geometry') { if (fetchGeometry) { return 'st_astext("' + k + '") ' + 'as ' + k; } return this._geometryColumnSQL(k); } return '"' + k + '"'; }).value(); select_columns = select_columns.join(','); var mode = this.options.get('sort_order') === 'desc' ? 'desc' : 'asc'; var q = 'select ' + select_columns + ' from (' + this.getSQL() + ') __wrapped'; var order_by = this.options.get('order_by'); if (order_by && order_by.length > 0) { q += ' order by ' + order_by + ' ' + mode; } return q; }, url: function () { return this.sqlApiUrl(); }, /** * we need to override sync to avoid the sql request to be sent by GET. * For security reasons, we need them to be send as a PUT request. * @method sync * @param method {'save' || 'read' || 'delete' || 'create'} * @param model {Object} * @param options {Object} */ sync: function (method, model, options) { if (!options) { options = {}; } options.data = this._createUrlOptions(function (d) { return d !== 'sql'; }); if (cartoMetadataStatic.alterTableData(this.options.get('sql') || '')) { options.data += '&q=' + encodeURIComponent(this.options.get('sql')); options.type = 'POST'; } else { // when a geometry can be lazy fetched, don't fetch it var fetchGeometry = 'cartodb_id' in this.query_schema; options.data += '&q=' + encodeURIComponent(this.wrappedSQL(this.query_schema, [], !fetchGeometry)); if (options.data.length > 2048) { options.type = 'POST'; } } return Backbone.sync.call(this, method, this, options); }, sqlApiUrl: function () { return this._configModel.getSqlApiUrl(); }, setOptions: function (opt) { this.options.set(opt); }, // Refresh all table data refresh: function () { this.fetch(); }, isFetchingPage: function () { return this._fetching; }, loadPageAtTop: function () { if (!this._fetching) { var first = this.pages[0]; if (first > 0) { this.options.set('page', first - 1); } } }, loadPageAtBottom: function () { if (!this._fetching) { var last = this.pages[this.pages.length - 1]; if (!this.lastPage) { this.options.set('page', last + 1); } } }, /** * called when a new page is loaded * removes the models to max */ newPage: function (currentPage, direction) { if (this.pages.indexOf(currentPage) < 0) { this.pages.push(currentPage); } this.pages.sort(function (a, b) { return Number(a) > Number(b); }); // remove blocks if there are more rows than allowed var rowspp = this.options.get('rows_per_page'); var max_items = rowspp * 4; if (this.size() > max_items) { if (direction == 'up') { // eslint-disable-line eqeqeq // remove page from the bottom (the user is going up) this.pages.pop(); this.remove(this.models.slice(max_items, this.size())); } else { // remove page from the top (the user is going down) this.pages.shift(); this.remove(this.models.slice(0, rowspp)); } } }, addRow: function (opts) { this.table.notice(this._ADDING_ROW_TEXT, 'load', 0); opts = opts || {}; _.extend(opts, { wait: true, success: () => { this.table.notice(this._ADDED_ROW_TEXT); }, error: (e, resp) => { // TODO: notice user this.table.error(this._ADDING_ROW_TEXT, resp); } }); return this.create(null, opts); }, /** * creates a new row model in local, it is NOT serialized to the server */ newRow: function (attrs) { var r = new RowModel(attrs, { configModel: this._configModel }); r.table = this.table; r.bind('saved', function _saved () { if (r.table.data().length === 0) { r.table.data().fetch(); r.unbind('saved', _saved, r.table); } }, r.table); return r; }, /** * return a model row */ getRow: function (id, options) { options = options || {}; var r = this.get(id); if (!r) { r = new RowModel({cartodb_id: id}, { configModel: this._configModel }); } if (!options.no_add) { this.table._data.add(r); } r.table = this.table; return r; }, getRowAt: function (index) { var r = this.at(index); r.table = this.table; return r; }, deleteRow: function (row_id) { }, isReadOnly: function () { return false; }, quartiles: function (nslots, column, callback, error) { var tmpl = _.template('select quartile, max(<%= column %>) as maxamount from (select <%= column %>, ntile(<%= slots %>) over (order by <%= column %>) as quartile from (<%= sql %>) _table_sql where <%= column %> is not null) x group by quartile order by quartile'); this._sqlQuery(tmpl({ slots: nslots, sql: this.getSQL(), column: column }), function (data) { callback(_(data.rows).pluck('maxamount')); }, error); }, equalInterval: function (nslots, column, callback, error) { var tmpl = _.template(` with params as (select min(a), max(a) from ( select <%= column %> as a from (<%= sql %>) _table_sql where <%= column %> is not null ) as foo ) select (max-min)/<%= slots %> as s, min, max from params` ); this._sqlQuery(tmpl({ slots: nslots, sql: this.getSQL(), column: column }), function (data) { var min = data.rows[0].min; var max = data.rows[0].max; var range = data.rows[0].s; var values = []; for (var i = 1, l = nslots; i < l; i++) { values.push((range * i) + min); } // Add last value values.push(max); // Callback callback(values); }, error); }, _quantificationMethod: function (functionName, nslots, column, distinct, callback, error) { var tmpl = _.template('select unnest(<%= functionName %>(array_agg(<%= simplify_fn %>((<%= column %>::numeric))), <%= slots %>)) as buckets from (<%= sql %>) _table_sql where <%= column %> is not null'); this._sqlQuery(tmpl({ slots: nslots, sql: this.getSQL(), column: column, functionName: functionName, simplify_fn: 'distinct' }), function (data) { callback(_(data.rows).pluck('buckets')); }, error); }, discreteHistogram: function (nbuckets, column, callback, error) { var query = 'SELECT DISTINCT(<%= column %>) AS bucket, count(*) AS value FROM (<%= sql %>) _table_sql GROUP BY <%= column %> ORDER BY value DESC LIMIT <%= nbuckets %> + 1'; var sql = _.template(query, { column: column, nbuckets: nbuckets, sql: this.getSQL() }); this._sqlQuery(sql, function (data) { var count = data.rows.length; var reached_limit = false; if (count > nbuckets) { data.rows = data.rows.slice(0, nbuckets); reached_limit = true; } callback({ rows: data.rows, reached_limit: reached_limit }); // eslint-disable-line }); }, date_histogram: function (nbuckets, column, callback, error) { column = 'EXTRACT(EPOCH FROM ' + column + '::TIMESTAMP WITH TIME ZONE )'; var tmpl = _.template( 'with bounds as ( ' + 'SELECT ' + 'current_timestamp as tz, ' + 'min(<%= column %>) as lower, ' + 'max(<%= column %>) as upper, ' + '(max(<%= column %>) - min(<%= column %>)) as span, ' + 'CASE WHEN ABS((max(<%= column %>) - min(<%= column %>))/<%= nbuckets %>) <= 0 THEN 1 ELSE GREATEST(1.0, pow(10,ceil(log((max(<%= column %>) - min(<%= column %>))/<%= nbuckets %>)))) END as bucket_size ' + 'FROM (<%= sql %>) _table_sql ' + ') ' + 'select array_agg(v) val, array_agg(bucket) buckets, tz, bounds.upper, bounds.lower, bounds.span, bounds.bucket_size from ' + '( ' + 'select ' + 'count(<%= column %>) as v, ' + 'round((<%= column %> - bounds.lower)/bounds.bucket_size) as bucket ' + 'from (<%= sql %>) _table_sql, bounds ' + 'where <%= column %> is not null ' + 'group by bucket order by bucket ' + ') a, bounds ' + 'group by ' + 'bounds.upper, bounds.lower, bounds.span, bounds.bucket_size, bounds.tz '); // transform array_agg from postgres to a js array function agg_array (a) { return a.map(function (v) { return parseFloat(v); }); } this._sqlQuery(tmpl({ nbuckets: nbuckets, sql: this.getSQL(), column: column }), function (data) { if (!data.rows || data.rows.length === 0) { callback(null, null); return; } data = data.rows[0]; data.val = agg_array(data.val); data.buckets = agg_array(data.buckets); var hist = []; var bounds = {}; // create a sorted array and normalize var upper = data.upper; var lower = data.lower; var tz = data.tz; var bucket_size = data.bucket_size; var max; max = data.val[0]; for (var r = 0; r < data.buckets.length; ++r) { var b = data.buckets[r]; var v = hist[b] = data.val[r]; max = Math.max(max, v); } // var maxBucket = _.max(data.buckets) for (var i = 0; i < hist.length; ++i) { if (hist[i] === undefined) { hist[i] = 0; } else { hist[i] = hist[i] / max; } } bounds.upper = parseFloat(upper); bounds.lower = parseFloat(lower); bounds.bucket_size = parseFloat(bucket_size); bounds.tz = tz; callback(hist, bounds); }, error); }, histogram: function (nbuckets, column, callback, error) { var tmpl = _.template( 'with bounds as ( ' + 'SELECT ' + 'min(<%= column %>) as lower, ' + 'max(<%= column %>) as upper, ' + '(max(<%= column %>) - min(<%= column %>)) as span, ' + 'CASE WHEN ABS((max(<%= column %>) - min(<%= column %>))/<%= nbuckets %>) <= 0 THEN 1 ELSE GREATEST(1.0, pow(10,ceil(log((max(<%= column %>) - min(<%= column %>))/<%= nbuckets %>)))) END as bucket_size ' + 'FROM (<%= sql %>) _table_sql ' + ') ' + 'select array_agg(v) val, array_agg(bucket) buckets, bounds.upper, bounds.lower, bounds.span, bounds.bucket_size from ' + '( ' + 'select ' + 'count(<%= column %>) as v, ' + 'round((<%= column %> - bounds.lower)/bounds.bucket_size) as bucket ' + 'from (<%= sql %>) _table_sql, bounds ' + 'where <%= column %> is not null ' + 'group by bucket order by bucket ' + ') a, bounds ' + 'group by ' + 'bounds.upper, ' + 'bounds.lower, bounds.span, bounds.bucket_size '); // transform array_agg from postgres to a js array function agg_array (a) { return a.map(function (v) { return parseFloat(v); }); // return JSON.parse(a.replace('{', '[').replace('}', ']')) } this._sqlQuery(tmpl({ nbuckets: nbuckets, sql: this.getSQL(), column: column }), function (data) { if (!data.rows || data.rows.length === 0) { callback(null, null); return; } data = data.rows[0]; data.val = agg_array(data.val); data.buckets = agg_array(data.buckets); var hist = []; var bounds = {}; // create a sorted array and normalize var upper = data.upper; var lower = data.lower; var bucket_size = data.bucket_size; var max; max = data.val[0]; for (var r = 0; r < data.buckets.length; ++r) { var b = data.buckets[r]; var v = hist[b] = data.val[r]; max = Math.max(max, v); } // var maxBucket = _.max(data.buckets) for (var i = 0; i < hist.length; ++i) { if (hist[i] === undefined) { hist[i] = 0; } else { hist[i] = hist[i] / max; } } bounds.upper = parseFloat(upper); bounds.lower = parseFloat(lower); bounds.bucket_size = parseFloat(bucket_size); callback(hist, bounds); }, error); }, jenkBins: function (nslots, column, callback, error) { this._quantificationMethod('CDB_JenksBins', nslots, column, true, callback, error); }, headTails: function (nslots, column, callback, error) { this._quantificationMethod('CDB_HeadsTailsBins', nslots, column, false, callback, error); }, quantileBins: function (nslots, column, callback, error) { this._quantificationMethod('CDB_QuantileBins', nslots, column, false, callback, error); }, categoriesForColumn: function (max_values, column, callback, error) { var tmpl = _.template('SELECT <%= column %>, count(<%= column %>) FROM (<%= sql %>) _table_sql ' + 'GROUP BY <%= column %> ORDER BY count DESC LIMIT <%= max_values %> ' ); this._sqlQuery(tmpl({ sql: this.getSQL(), column: column, max_values: max_values + 1 }), function (data) { callback({// eslint-disable-line type: data.fields[column].type || 'string', categories: _(data.rows).pluck(column) }); }, error); }, /** * call callback with the geometry bounds */ geometryBounds: function (callback) { var tmpl = _.template('SELECT ST_XMin(ST_Extent(the_geom)) as minx,ST_YMin(ST_Extent(the_geom)) as miny, ST_XMax(ST_Extent(the_geom)) as maxx,ST_YMax(ST_Extent(the_geom)) as maxy from (<%= sql %>) _table_sql'); this._sqlQuery(tmpl({ sql: this.getSQL() }), function (result) { var coordinates = result.rows[0]; var lon0 = coordinates.maxx; var lat0 = coordinates.maxy; var lon1 = coordinates.minx; var lat1 = coordinates.miny; var minlat = -85.0511; var maxlat = 85.0511; var minlon = -179; var maxlon = 179; var clampNum = function (x, min, max) { return x < min ? min : x > max ? max : x; }; lon0 = clampNum(lon0, minlon, maxlon); lon1 = clampNum(lon1, minlon, maxlon); lat0 = clampNum(lat0, minlat, maxlat); lat1 = clampNum(lat1, minlat, maxlat); callback([ [lat0, lon0], [lat1, lon1]]); // eslint-disable-line } ); }, _sqlQuery: function (sql, callback, error, type) { var s = encodeURIComponent(sql); return $.ajax({ type: type || 'POST', data: 'q=' + s + '&api_key=' + this._configModel.get('api_key'), url: this.url(), success: callback, error: error }); }, getSQL: function () { // use table.id to fetch data because if always contains the real table name return 'select * from ' + safeTableNameQuoting(this.table.get('id')); }, fetch: function (opts) { opts = opts || {}; if (!opts || !opts.add) { this.options.attributes.page = 0; this.options._previousAttributes.page = 0; this.pages = []; } var error = opts.error; opts.error = (model, resp) => { this.fetched = true; this.trigger('error', model, resp); error && error(model, resp); }; var success = opts.success; opts.success = (model, resp) => { this.fetched = true; success && success.apply(this, arguments); }; this._fetch(opts); }, _fetch: function (opts) { var MAX_GET_LENGTH = 1024; this.trigger('loading', opts); var sql = this.getSQL(); // if the query changes the database just send it if (cartoMetadataStatic.alterTableData(sql)) { TableDataCollection.prototype.fetch.call(this, opts); return; } // use get to fetch the schema, probably cached this._sqlQuery(_.template('select * from (<%= sql %>) __wrapped limit 0')({ sql: sql }), (data) => { // get schema this.query_schema = this._schemaFromQueryFields(data.fields); if (!this.table.isInSQLView()) { if ('the_geom' in this.query_schema) { delete this.query_schema['the_geom_webmercator']; } } TableDataCollection.prototype.fetch.call(this, opts); }, (err) => { this.trigger('error', this, err); }, sql.length > MAX_GET_LENGTH ? 'POST' : 'GET'); }, /** * with the data from the rows fetch create an schema * if the schema from original table is passed the method * set the column types according to it * return an empty list if no data was fetch */ schemaFromData: function (originalTableSchema) { // build schema in format [ [field, type] , ...] return cartoMetadataStatic.sortSchema(_(this.query_schema).map(function (v, k) { return [k, v]; })); }, geometryTypeFromGeoJSON: function (geojson) { try { var geo = JSON.parse(geojson); return geo.type; } catch (e) { } }, geometryTypeFromWKT: function (wkt) { if (!wkt) return null; var types = WKT.types; wkt = wkt.toUpperCase(); for (var i = 0; i < types.length; ++i) { var t = types[i]; if (wkt.indexOf(t) !== -1) { return t; } } }, geometryTypeFromWKB: function (wkb) { if (!wkb) return null; var typeMap = { '0001': 'Point', '0002': 'LineString', '0003': 'Polygon', '0004': 'MultiPoint', '0005': 'MultiLineString', '0006': 'MultiPolygon' }; var bigendian = wkb[0] === '0' && wkb[1] === '0'; var type = wkb.substring(2, 6); if (!bigendian) { // swap '0100' => '0001' type = type[2] + type[3] + type[0] + type[1]; } return typeMap[type]; }, // // guesses from the first row the geometry types involved // returns an empty array where there is no rows // return postgist types, like st_GEOTYPE // getGeometryTypes: function () { var row = null; var i = this.size(); while (i-- && !(row && row.get('the_geom'))) { row = this.at(i); } if (!row) return []; var geom = row.get('the_geom') || row.get('the_geom_webmercator'); var geoType = this.geometryTypeFromWKB(geom) || this.geometryTypeFromWKT(geom); if (geoType) { return ['ST_' + geoType[0].toUpperCase() + geoType.substring(1).toLowerCase()]; } return []; } });