Merge branch 'bi_provider' of github.com:CartoDB/torque into bi_provider

specifying initial columns in the cartoons
updating code to use the new tile format
2015-11-06 17:10:25 +00:00 · 2015-11-06 17:06:45 +00:00 · 2015-11-06 17:06:21 +00:00 · 2015-11-06 17:35:49 +01:00 · 2015-11-03 18:36:10 -05:00 · 2015-11-03 18:35:56 -05:00
5 changed files with 806 additions and 23 deletions
--- a/examples/bi.html
+++ b/examples/bi.html
@ -0,0 +1,88 @@
+
+<html>
+  <link rel="stylesheet" href="vendor/leaflet.css" />
+  <style>
+    #map, html, body {
+      width: 100%; height: 100%; padding: 0; margin: 0;
+    }
+    #title {
+      position: absolute;
+      top: 100px;
+      left: 50px;
+      color: white;
+      font-size: 27px;
+      font-family: Helvetica, sans-serif;
+    }
+  </style>
+  <body>
+    <div id="map"></div>
+    <div id='graphs'></div>
+
+    <link rel="stylesheet" href="http://cdn.leafletjs.com/leaflet-0.7.3/leaflet.css" />
+    <script src="http://cdn.leafletjs.com/leaflet-0.7.3/leaflet.js"></script>
+    <script src="../dist/torque.full.uncompressed.js"></script>
+
+
+    <script>
+      // define the torque layer style using cartocss
+      // this creates a kind of density map
+      //color scale from http://colorbrewer2.org/
+      var CARTOCSS = [
+          'Map {',
+          '-torque-time-attribute: "transaction_date";',
+          '-torque-aggregation-function: "count(1);avg(confidence_score);merchant_name";',
+          '-torque-frame-count: 1;',
+          '-torque-animation-duration: 15;',
+          '-torque-resolution: 1',
+          '}',
+          '#layer {',
+          '  marker-width: 1;',
+          '  marker-fill-opacity: 1.0;',
+          '  marker-fill: #fff5eb; ',
+          '  marker-type: ellipse;',
+          '  [value > 1] { marker-fill: #fee6ce; }',
+          '  [value > 2] { marker-fill: #fdd0a2; }',
+          '  [value > 4] { marker-fill: #fdae6b; }',
+          '  [value > 10] { marker-fill: #fd8d3c; }',
+          '  [value > 15] { marker-fill: #f16913; }',
+          '  [value > 20] { marker-fill: #d94801; }',
+          '  [value > 25] { marker-fill: #8c2d04; }',
+          '}'
+      ].join('\n');
+
+
+      var map = new L.Map('map', {
+        zoomControl: true,
+        center: [40.76045572900912, -73.97601127624512],
+        zoom: 13
+      });
+
+      L.tileLayer('http://{s}.api.cartocdn.com/base-dark/{z}/{x}/{y}.png', {
+        attribution: 'CartoDB'
+      }).addTo(map);
+
+      var torqueLayer = new L.TorqueLayer({
+        user       : 'stuartlynn',
+        table      : 'yodlee_512',
+        cartocss: CARTOCSS,
+        provider: "filterable_sql_api"
+      });
+      torqueLayer.addTo(map);
+
+      var graphs=[]
+      // var ndx = crossfilter({});
+      //
+      // torqueLayer.onNewData(function(){
+      //   data = torqueLayer.getValues()
+      //   ndx  = crossfilter(data)
+      // })
+
+      function addGraph(variable, type){
+        var dataDim = ndx.dimension(function(d){return d[variable]})
+        var graphID =  "variable_graph"
+        $("graphs").append("<div id='"+graphID+"'></div>")
+      }
+
+    </script>
+  </body>
+</html>
--- a/lib/torque/leaflet/torque.js
+++ b/lib/torque/leaflet/torque.js
@ -10,6 +10,7 @@ L.TorqueLayer = L.CanvasLayer.extend({

  providers: {
    'sql_api': torque.providers.json,
+    'filterable_sql_api': torque.providers.filterableJson,
    'url_template': torque.providers.JsonArray,
    'windshaft': torque.providers.windshaft,
    'tileJSON': torque.providers.tileJSON
@ -28,10 +29,16 @@ L.TorqueLayer = L.CanvasLayer.extend({
    options.tileLoader = true;
    this.key = 0;
    this.prevRenderedKey = 0;
+    this._filters = {}
+
    if (options.cartocss) {
      torque.extend(options, torque.common.TorqueLayer.optionsFromCartoCSS(options.cartocss));
    }

+
+    options.columns = options.countby.split(";")
+
+
    options.resolution = options.resolution || 2;
    options.steps = options.steps || 100;
    options.visible = options.visible === undefined ? true: options.visible;
@ -98,11 +105,48 @@ L.TorqueLayer = L.CanvasLayer.extend({
          self.redraw();
        }
        self.fire('tileLoaded');
+        self.fire('dataUpdate')
      });
    }, this);

  },

+  setFilters: function() {
+    this.provider.setFilters(this._filters);
+    this._reloadTiles();
+    return this;
+  },
+
+  filterByRange: function(variableName, start, end) {
+    this._filters[variableName] = {type: 'range',  range: {start: start, end: end} }
+    this._filtersChanged()
+    this.fire('dataUpdate')
+    return this
+  },
+
+  filterByCat: function(variableName, categories) {
+    this._filters[variableName] = {type: 'cat',  categories: categories}
+    this._filtersChanged()
+    return this
+  },
+
+  clearFilter: function(name){
+    if(name) {
+      delete this._filters[name]
+    }
+    else {
+      this._filters = {}
+    }
+    this._filtersChanged()
+    return this
+  },
+
+  _filtersChanged:function(){
+    this.provider._filters = this._filters;
+    this._clearTileCaches()
+    this._render()
+  },
+
  _clearTileCaches: function() {
    var t, tile;
    for(t in this._tiles) {
@ -118,6 +162,96 @@ L.TorqueLayer = L.CanvasLayer.extend({
    this._clearTileCaches();
  },

+  valuesForRangeVariable:function(variable){
+    var t, tile;
+
+    var variable_id = this.provider.idForRange(variable)
+    var values  = [ ]
+
+
+    for(t in this._tiles){
+      tile = this._tiles[t]
+      var noPoints = tile.x.length;
+      for(var i=0; i < tile.x.length; i++){
+        if(tile.renderFlags[i] ){
+          value = tile.renderData[variable_id*noPoints + i]
+          values.push(value)
+        }
+      }
+    }
+    return values;
+  },
+
+  valuesForCatVariable:function(variable){
+    var t, tile;
+
+    var categories = this.provider.idsForCategory(variable)
+    var result  = [ ]
+
+
+    for(t in this._tiles){
+      tile = this._tiles[t]
+      var noPoints = tile.x.length;
+      for(var i=0; i < tile.x.length; i++){
+        if(tile.renderFlags[i] ){
+          var vals={}
+          Object.keys(categories).forEach(function(categoryName){
+            var variable_id = categories[categoryName]
+            value = tile.renderData[variable_id*noPoints + i]
+            vals[categoryName] = value
+          }.bind(this))
+          result.push(vals)
+        }
+      }
+    }
+    return result;
+
+  },
+  getValues:function(variable,callback){
+    var type= this.provider._mapping[variable].type
+    if(type=='float'){
+      callback(this.valuesForRangeVariable(variable))
+    }
+    else{
+      callback(this.valuesForCatVariable(variable))
+    }
+  },
+  getHistogram:function(variable,callback,noBins){
+    var type= this.provider._mapping[variable].type
+    if(type=='float'){
+      callback(this.histogramForRangeVariable(variable,noBins))
+    }
+    else if(type='cat'){
+      callback(this.histogramForCatVariable(variable))
+    }
+    return this
+  },
+  histogramForCatVariable:function(variable){
+    var result = {}
+    this.valuesForCatVariable(variable).forEach(function(point){
+      Object.keys(point).forEach(function(key){
+        result[key] = result[key] || 0
+        result[key] += point[key]
+      })
+    })
+    return result
+  },
+  histogramForRangeVariable:function(variable,noBins){
+    noBins = noBins || 10
+
+    var vals = this.valuesForRangeVariable(variable)
+
+    var min  = Math.min.apply(null, vals)
+    var max  = Math.max.apply(null, vals)
+    var binSize  = (max-min)/noBins
+    var result = []
+    vals.forEach(function(val){
+      var bin = (val -min)/binSize
+      result[bin]= result[bin] || 0
+      result[bin] +=  val
+    })
+    return result
+  },
  onAdd: function (map) {
    map.on({
      'zoomend': this._clearCaches,
@ -208,6 +342,40 @@ L.TorqueLayer = L.CanvasLayer.extend({
    canvas.width = canvas.width;
  },

+  /*
+  _filterTile:function(tile){
+    var noPoints = tile.x.length
+
+    var renderFlags = []
+    for(var i =0; i < noPoints; i++){
+      var includePoint = true
+      Object.keys(this._filters).forEach(function(key){
+        var filter = this._filters[key]
+        var variableId  = this.provider.idForRange(key)
+
+        var value  = tile.renderData[variableId*noPoints+i]
+        if(filter.type=='range'){
+          if(value < filter.range.start || value > filter.range.end){
+            includePoint = false;
+          }
+        }
+        else if (filter.type=='cat'){
+          var ids = this.provider.idsForCategory(key);
+          filter.categories.forEach(function(key){
+            var catId = ids[key]
+            var value  = tile.renderData[catId*noPoints + i]
+            if(value==0){
+              includePoint= false
+            }
+          }.bind(this))
+        }
+      }.bind(this))
+      renderFlags[i] = includePoint
+    }
+    return renderFlags
+  },
+  */
+
  /**
   * render the selectef key
   * don't call this function directly, it's called by
@ -236,10 +404,13 @@ L.TorqueLayer = L.CanvasLayer.extend({
          // all the points
          this.renderer._ctx.drawImage(tile._tileCache, 0, 0);
        } else {
+
+          //tile.renderFlags = this._filterTile(tile)
          this.renderer.renderTile(tile, this.key);
        }
      }
    }
+
    this.renderer.applyFilters();

    // prepare caches if the animation is not running
--- a/lib/torque/provider/filterableJson.js
+++ b/lib/torque/provider/filterableJson.js
@ -0,0 +1,521 @@
+var torque = require('../');
+var Profiler = require('../profiler');
+
+  var Uint8Array = torque.types.Uint8Array;
+  var Int32Array = torque.types.Int32Array;
+  var Uint32Array = torque.types.Uint32Array;
+
+  // format('hello, {0}', 'rambo') -> "hello, rambo"
+  function format(str) {
+    for(var i = 1; i < arguments.length; ++i) {
+      var attrs = arguments[i];
+      for(var attr in attrs) {
+        str = str.replace(RegExp('\\{' + attr + '\\}', 'g'), attrs[attr]);
+      }
+    }
+    return str;
+  }
+
+  var filterableJson = function (options) {
+    this._ready = false;
+    this._tileQueue = [];
+    this.options = options;
+    this._filters = {};
+    this._mapping = {
+      no:{
+        type: 'float',
+        col_id:6
+      },
+      passenger_count:{
+        type: 'float',
+        col_id: 0
+      },
+      tip_amount:{
+        type:'float',
+        col_id:1
+      },
+      payment_type:{
+        type:'cat',
+        col_ids:{
+          "Credit card": 2,
+          "Cash": 3,
+          "No charge": 4,
+          "Dispute": 5
+        }
+      },
+    };
+
+    this.options.is_time = this.options.is_time === undefined ? true: this.options.is_time;
+    this.options.tiler_protocol = options.tiler_protocol || 'http';
+    this.options.tiler_domain = options.tiler_domain || 'cartodb.com';
+    this.options.tiler_port = options.tiler_port || 80;
+
+    if (this.options.data_aggregation) {
+      this.options.cumulative = this.options.data_aggregation === 'cumulative';
+    }
+
+    // check options
+    if (options.resolution === undefined ) throw new Error("resolution should be provided");
+    if (options.steps === undefined ) throw new Error("steps should be provided");
+    if(options.start === undefined) {
+      this._fetchKeySpan();
+    } else {
+      this._setReady(true);
+    }
+  };
+
+
+
+  filterableJson.prototype = {
+
+    /**
+     * return the torque tile encoded in an efficient javascript
+     * structure:
+     * {
+     *   x:Uint8Array x coordinates in tile reference system, normally from 0-255
+     *   y:Uint8Array y coordinates in tile reference system
+     *   Index: Array index to the properties
+     * }
+     */
+    proccessTile: function(rows, coord, zoom) {
+      var r;
+      var x = new Uint8Array(rows.length);
+      var y = new Uint8Array(rows.length);
+
+      var prof_mem = Profiler.metric('ProviderJSON:mem');
+      var prof_point_count = Profiler.metric('ProviderJSON:point_count');
+      var prof_process_time = Profiler.metric('ProviderJSON:process_time').start()
+
+      // count number of steps
+      var steps = 0;
+      var maxDateSlots = -1;
+      for (r = 0; r < rows.length; ++r) {
+        var row = rows[r];
+        dates += row.steps.length;
+        for(var d = 0; d < row.steps.length; ++d) {
+          maxDateSlots = Math.max(maxDateSlots, row.steps[d]);
+        }
+      }
+
+      if(this.options.cumulative) {
+        steps = (1 + maxDateSlots) * rows.length;
+      }
+
+
+
+      // var type =  Uint8Array;
+
+
+      // reserve memory for all the dates
+      // var timeIndex = new Int32Array(maxDateSlots + 1); //index-size
+      // var timeCount = new Int32Array(maxDateSlots + 1);
+      var renderData = new Array()
+      // var renderDataPos = new Uint32Array(dates);
+
+      prof_mem.inc(
+        4 * maxDateSlots + // timeIndex
+        4 * maxDateSlots + // timeCount
+        steps + //renderData
+        steps * 4
+      ); //renderDataPos
+
+      prof_point_count.inc(rows.length);
+
+      var rowsPerSlot = {};
+
+      // precache pixel positions
+      for (var r = 0; r < rows.length; ++r) {
+        var row = rows[r];
+        x[r] = row.x //* this.options.resolution;
+        // fix value when it's in the tile EDGE
+        // TODO: this should be fixed in SQL query
+        if (row.y === -1) {
+          y[r] = 0;
+        } else {
+          y[r] = row.y //* this.options.resolution;
+        }
+
+        for(var i=0; i < row.steps.length; i++){
+          renderData[r*row.steps.length + i] = row.vals[i]
+        }
+          /*var lastDateSlot = dates[dates.length - 1];
+
+          y[r] = row.y * this.options.resolution;
+        }
+
+        var steps = row.steps;
+        var vals = row.vals;
+        if (!this.options.cumulative) {
+          for (var j = 0, len = steps.length; j < len; ++j) {
+              var rr = rowsPerSlot[steps[j]] || (rowsPerSlot[steps[j]] = []);
+              if(this.options.cumulative) {
+                  vals[j] += prev_val;
+              }
+              prev_val = vals[j];
+              rr.push([r, vals[j]]);
+          }
+        } else {
+          var valByDate = {}
+          for (var j = 0, len = steps.length; j < len; ++j) {
+            valByDate[steps[j]] = vals[j];
+          }
+          var accum = 0;
+
+          // extend the latest to the end
+          for (var j = steps[0]; j <= maxDateSlots; ++j) {
+              var rr = rowsPerSlot[j] || (rowsPerSlot[j] = []);
+              var v = valByDate[j];
+              if (v) {
+                accum += v;
+              }
+              rr.push([r, accum]);
+          }
+
+          /*var lastDateSlot = steps[steps.length - 1];
+>>>>>>> 4ad0dba547e2a57300064c484d42e196eb846356
+          for (var j = lastDateSlot + 1; j <= maxDateSlots; ++j) {
+            var rr = rowsPerSlot[j] || (rowsPerSlot[j] = []);
+            rr.push([r, prev_val]);
+          }
+          */
+
+      }
+
+      // for each timeslot search active buckets
+
+
+      prof_process_time.end();
+
+      return {
+        x: x,
+        y: y,
+        z: zoom,
+        coord: {
+          x: coord.x,
+          y: coord.y,
+          z: zoom
+        },
+        // timeCount: timeCount,
+        // timeIndex: timeIndex,
+        // renderDataPos: renderDataPos,
+        renderData: renderData,
+        // maxDate: maxDateSlots
+      };
+    },
+
+    _generateFilterSQLForCat:function(name,categories){
+      return name+" in "+categories;
+    },
+
+    _generateFilterSQLForRange:function(name,range){
+      var result = ""
+      if (range.start) {
+        result += " " + name + " > " + range.start;
+      }
+      if (range.end) {
+        if (range.start) {
+          result += " and "
+        }
+        result += " " + name + " < " + range.end;
+      }
+      return result
+    },
+
+    _setFilters:function(filters){
+      this.filters = filters
+    },
+
+    _generateFiltersSQL: function() {
+      var self = this;
+
+      return Object.keys(this._filters).map(function(filterName){
+        var filter = self._filters[filterName]
+        if (filter) {
+          if (filter.type == 'range') {
+            return self._generateFilterSQLForRange(filterName, filter.range)
+          }
+          else if (filter.type == 'cat') {
+            return self._generateFilterSQLForCat(filterName, filter.categories)
+          }
+          else {
+            return ""
+          }
+        }
+        else{
+          return ""
+        }
+      }).join(" and ")
+    },
+
+    _host: function() {
+      var opts = this.options;
+      var port = opts.sql_api_port;
+      var domain = ((opts.user_name || opts.user) + '.' + (opts.sql_api_domain || 'cartodb.com')) + (port ? ':' + port: '');
+      var protocol = opts.sql_api_protocol || 'http';
+      return this.options.url || protocol + '://' + domain + '/api/v2/sql';
+    },
+
+    url: function(subhost) {
+      var opts = this.options;
+      var protocol = opts.sql_api_protocol || 'http';
+      if (!this.options.cdn_url) {
+        return this._host();
+      }
+      var h = protocol+ "://";
+      if (subhost) {
+        h += subhost + ".";
+      }
+      var cdn_host = opts.cdn_url;
+      if(!cdn_host.http && !cdn_host.https) {
+        throw new Error("cdn_host should contain http and/or https entries");
+      }
+      h += cdn_host[protocol] + "/" + (opts.user_name || opts.user) + '/api/v2/sql';
+      return h;
+    },
+
+    _hash: function(str) {
+      var hash = 0;
+      if (!str || str.length == 0) return hash;
+      for (var i = 0, l = str.length; i < l; ++i) {
+          hash = (( (hash << 5 ) - hash ) + str.charCodeAt(i)) | 0;
+      }
+      return hash;
+    },
+
+    _extraParams: function() {
+      if (this.options.extra_params) {
+        var p = [];
+        for(var k in this.options.extra_params) {
+          var v = this.options.extra_params[k];
+          if (v) {
+            p.push(k + "=" + encodeURIComponent(v));
+          }
+        }
+        return p.join('&');
+      }
+      return null;
+    },
+
+    isHttps: function() {
+      return this.options.sql_api_protocol && this.options.sql_api_protocol === 'https';
+    },
+
+    // execute actual query
+    sql: function(sql, callback, options) {
+      options = options || {};
+      var subdomains = this.options.subdomains || '0123';
+      if(this.isHttps()) {
+        subdomains = [null]; // no subdomain
+      }
+
+
+      var url;
+      if (options.no_cdn) {
+        url = this._host();
+      } else {
+        url = this.url(subdomains[Math.abs(this._hash(sql))%subdomains.length]);
+      }
+      var extra = this._extraParams();
+      torque.net.get( url + "?q=" + encodeURIComponent(sql) + (extra ? "&" + extra: ''), function (data) {
+          if(options.parseJSON) {
+            data = JSON.parse(data && data.responseText);
+          }
+          callback && callback(data);
+      });
+    },
+
+    getTileData: function(coord, zoom, callback) {
+      if(!this._ready) {
+        this._tileQueue.push([coord, zoom, callback]);
+      } else {
+        this._getTileData(coord, zoom, callback);
+      }
+    },
+
+    _setReady: function(ready) {
+      this._ready = true;
+      this._processQueue();
+      this.options.ready && this.options.ready();
+    },
+
+    _processQueue: function() {
+      var item;
+      while (item = this._tileQueue.pop()) {
+        this._getTileData.apply(this, item);
+      }
+    },
+
+    /**
+     * `coord` object like {x : tilex, y: tiley }
+     * `zoom` quadtree zoom level
+     */
+
+    _getTileData: function(coord, zoom, callback) {
+      var prof_fetch_time = Profiler.metric('ProviderJSON:tile_fetch_time').start()
+      this.table = this.options.table;
+      var numTiles = 1 << zoom;
+
+      var column_conv = this.options.column;
+      //
+      // if(this.options.is_time) {
+      //   column_conv = format("date_part('epoch', {column})", this.options);
+      // }
+
+      // var sql = "" +
+      //   "WITH " +
+      //   "par AS (" +
+      //   "  SELECT CDB_XYZ_Resolution({zoom})*{resolution} as res" +
+      //   ",  256/{resolution} as tile_size" +
+      //   ", CDB_XYZ_Extent({x}, {y}, {zoom}) as ext "  +
+      //   ")," +
+      //   "cte AS ( "+
+      //   "  SELECT ST_SnapToGrid(i.the_geom_webmercator, p.res) g" +
+      //   ", avg(passenger_count) c1, avg(tip_amount) c2, sum( case when payment_type=1 then 1 else 0 end) c3, sum(case when payment_type=2 then 1 else 0 end ) c4, sum(case when payment_type=3 then 1 else 0 end ) c5,sum(case when payment_type=4 then 1 else 0 end ) c6, count(cartodb_id) c7  " +
+      //   "  FROM ({_sql}) i, par p " +
+      //   "  WHERE i.the_geom_webmercator && p.ext " +
+      //   "  GROUP BY g" +
+      //   ") " +
+      //   "" +
+      //   "SELECT (st_x(g)-st_xmin(p.ext))/p.res x__uint8, " +
+      //   "       (st_y(g)-st_ymin(p.ext))/p.res y__uint8," +
+      //   " Array[c1,c2,c3,c4,c5,c6,c7] vals__uint8," +
+      //   " Array[0,1,2,3,4,5,6] dates__uint16" +
+      //   // the tile_size where are needed because the overlaps query in cte subquery includes the points
+      //   // in the left and bottom borders of the tile
+      //   " FROM cte, par p where (st_y(g)-st_ymin(p.ext))/p.res < tile_size and (st_x(g)-st_xmin(p.ext))/p.res < tile_size ";
+      this.options.filters = this._generateFiltersSQL()
+      var sql = ""+
+      "SELECT * FROM torque_tile(x:={x}, y:={y}, z:={zoom}, aggr:=ARRAY['{columns}'], table_name:='{table}', where_clause:='{filters}');"
+
+
+      var query = format(sql, this.options, {
+        zoom: zoom,
+        x: coord.x,
+        y: coord.y,
+        column: column_conv,
+        table: this.options.table,
+        filters: this._generateFiltersSQL()
+      });
+
+
+      var self = this;
+      console.log("query is ", query)
+      this.sql(query, function (data) {
+
+        if (data) {
+          var rows = JSON.parse(data.responseText).rows;
+          callback(self.proccessTile(rows, coord, zoom));
+        } else {
+          callback(null);
+        }
+        prof_fetch_time.end();
+      });
+    },
+
+    getKeySpan: function() {
+      return {
+        start: this.options.start * 1000,
+        end: this.options.end * 1000,
+        step: this.options.step,
+        steps: this.options.steps,
+        columnType: this.options.is_time ? 'date': 'number'
+      };
+    },
+
+    setColumn: function(column, isTime) {
+      this.options.column = column;
+      this.options.is_time = isTime === undefined ? true: false;
+      this.reload();
+    },
+
+    setResolution: function(res) {
+      this.options.resolution = res;
+    },
+
+    // return true if tiles has been changed
+    setOptions: function(opt) {
+      var refresh = false;
+
+      if(opt.resolution !== undefined && opt.resolution !== this.options.resolution) {
+        this.options.resolution = opt.resolution;
+        refresh = true;
+      }
+
+      if(opt.steps !== undefined && opt.steps !== this.options.steps) {
+        this.setSteps(opt.steps, { silent: true });
+        refresh = true;
+      }
+
+      if(opt.column !== undefined && opt.column !== this.options.column) {
+        this.options.column = opt.column;
+        refresh = true;
+      }
+
+      if(opt.countby !== undefined && opt.countby !== this.options.countby) {
+        this.options.countby = opt.countby;
+        refresh = true;
+      }
+
+      if(opt.data_aggregation !== undefined) {
+        var c = opt.data_aggregation === 'cumulative';
+        if (this.options.cumulative !== c) {
+          this.options.cumulative = c;
+          refresh = true;
+        }
+      }
+
+      if (refresh) this.reload();
+      return refresh;
+
+    },
+
+    reload: function() {
+      this._ready = false;
+      this._fetchKeySpan();
+    },
+
+    setSQL: function(sql) {
+      if (this.options.sql != sql) {
+        this.options.sql = sql;
+        this.reload();
+      }
+    },
+
+    getSteps: function() {
+      return Math.min(this.options.steps, this.options.data_steps);
+    },
+
+    setSteps: function(steps, opt) {
+      opt = opt || {};
+      if (this.options.steps !== steps) {
+        this.options.steps = steps;
+        this.options.step = (this.options.end - this.options.start)/this.getSteps();
+        this.options.step = this.options.step || 1;
+        if (!opt.silent) this.reload();
+      }
+    },
+
+    getBounds: function() {
+      return this.options.bounds;
+    },
+
+    getSQL: function() {
+      return this.options.sql || "select * from " + this.options.table;
+    },
+
+    _tilerHost: function() {
+      var opts = this.options;
+      var user = (opts.user_name || opts.user);
+      return opts.tiler_protocol +
+           "://" + (user ? user + "." : "")  +
+           opts.tiler_domain +
+           ((opts.tiler_port != "") ? (":" + opts.tiler_port) : "");
+    },
+
+    _fetchKeySpan: function() {
+      this._setReady(true);
+    }
+
+  };
+
+module.exports = filterableJson;
--- a/lib/torque/provider/index.js
+++ b/lib/torque/provider/index.js
@ -1,5 +1,6 @@
 module.exports = {
    json: require('./json'),
+    filterableJson: require('./filterableJson'),
    JsonArray: require('./jsonarray'),
    windshaft: require('./windshaft'),
    tileJSON: require('./tilejson')
--- a/lib/torque/renderer/point.js
+++ b/lib/torque/renderer/point.js
@ -57,7 +57,7 @@ var Filters = require('./torque_filters');
    this.TILE_SIZE = 256;
    this._style = null;
    this._gradients = {};
-    
+
    this._forcePoints = false;
  }

@ -165,14 +165,14 @@ var Filters = require('./torque_filters');
        i.src = canvas.toDataURL();
        return i;
      }
-      
+
      return canvas;
    },

    //
    // renders all the layers (and frames for each layer) from cartocss
    //
-    renderTile: function(tile, key, callback) {
+    renderTile: function(tile, key, renderFlags, callback) {
      if (this._iconsToLoad > 0) {
          this.on('allIconsLoaded', function() {
              this.renderTile.apply(this, [tile, key, callback]);
@ -193,7 +193,7 @@ var Filters = require('./torque_filters');
          }
        }
      }
-      
+
      prof.end(true);

      return callback && callback(null);
@ -237,12 +237,12 @@ var Filters = require('./torque_filters');
    },

    //
-    // renders a tile in the canvas for key defined in 
+    // renders a tile in the canvas for key defined in
    // the torque tile
    //
    _renderTile: function(tile, key, frame_offset, sprites, shader, shaderVars) {
-      if (!this._canvas) return;

+      if (!this._canvas) return;
      var prof = Profiler.metric('torque.renderer.point.renderTile').start();
      var ctx = this._ctx;
      var blendMode = compop2canvas(shader.eval('comp-op')) || this.options.blendmode;
@ -254,27 +254,29 @@ var Filters = require('./torque_filters');
        key = tile.maxDate;
      }
      var tileMax = this.options.resolution * (this.TILE_SIZE/this.options.resolution - 1)
-      var activePixels = tile.timeCount[key];
+      var activePixels = tile.x.length;
      var anchor = this.options.resolution/2;
      if (activePixels) {
-        var pixelIndex = tile.timeIndex[key];
+        var pixelIndex = 0;//tile.timeIndex[key];
        for(var p = 0; p < activePixels; ++p) {
-          var posIdx = tile.renderDataPos[pixelIndex + p];
-          var c = tile.renderData[pixelIndex + p];
-          if (c) {
-           var sp = sprites[c];
-           if (sp === undefined) {
-             sp = sprites[c] = this.generateSprite(shader, c, torque.extend({ zoom: tile.z, 'frame-offset': frame_offset }, shaderVars));
-           }
-           if (sp) {
-             var x = tile.x[posIdx]- (sp.width >> 1) + anchor;
-             var y = tileMax - tile.y[posIdx] + anchor; // flip mercator
-             ctx.drawImage(sp, x, y - (sp.height >> 1));
-           }
-          }
+          if(tile.renderFlags[p]){
+            var posIdx = p// tile.renderDataPos[pixelIndex + p];
+
+            var posIdx = tile.renderDataPos[pixelIndex + p];
+
+            var c = tile.renderData[pixelIndex + p];
+            var sp = sprites[c];
+            if (sp === undefined) {
+               sp = sprites[c] = this.generateSprite(shader, c, torque.extend({ zoom: tile.z, 'frame-offset': frame_offset }, shaderVars));
+            }
+            if (sp) {
+               var x = tile.x[posIdx]- (sp.width >> 1) + anchor;
+               var y = tileMax - tile.y[posIdx] + anchor; // flip mercator
+               ctx.drawImage(sp, x, y - (sp.height >> 1));
+            }
        }
      }
-      
+

      prof.end(true);
    },
@ -442,7 +444,7 @@ var Filters = require('./torque_filters');
          }
          gradient = {};
          var colorize = this._style['image-filters'].args;
-          
+
          var increment = 1/colorize.length;
          for (var i = 0; i < colorize.length; i++){
            var key = increment * i + increment;
Author	SHA1	Message	Date
Stuart Lynn	8ce2a629f1	Merge branch 'bi_provider' of github.com:CartoDB/torque into bi_provider	2015-11-06 17:10:25 +00:00
Stuart Lynn	aafbf83d6c	specifying initial columns in the cartoons	2015-11-06 17:06:45 +00:00
Stuart Lynn	dea2f1f27f	updating code to use the new tile format	2015-11-06 17:06:21 +00:00
javi	4ad0dba547	removed client side filters temporally, integrated with server side queries	2015-11-06 17:35:49 +01:00
Stuart Lynn	71fc89a8d8	Examples of the filters and histogram api working	2015-11-03 18:36:10 -05:00
Stuart Lynn	fd5bc0f732	Adding the ability for leafletLayer to filter data on the client side and to calculate histograms and value arrays for the variables stored in the currently loaded tiles.	2015-11-03 18:35:56 -05:00
Stuart Lynn	130d72c872	adding a filterable json provider.	2015-11-03 18:35:01 -05:00