Compare commits

...

28 Commits

Author SHA1 Message Date
Stuart Lynn
880bf9870f adding methods to get the histogram for the visible region of the map. One using Pauls Grandfather tile idea and the other using quad tree ranges for each visible tile 2015-11-16 18:59:08 +00:00
Stuart Lynn
c9e87f4365 Adding method to get histogram by tiles 2015-11-16 18:58:16 +00:00
javi
ea2fe75fbc removed debugging messages 2015-11-16 12:38:42 +01:00
javi
e93c6169bd added method to know the point count and fixed cartoons setter 2015-11-15 11:16:57 +01:00
javi
33c79dad0c fixed xy position 2015-11-15 10:06:12 +01:00
javi
b46489d8eb utility methods for interactivity 2015-11-14 19:27:53 +01:00
javi
c4fa9b3351 fixed pixel calculation to fetch data for a point 2015-11-14 19:27:36 +01:00
javi
a822c369c7 using sql_api_template 2015-11-13 18:29:20 +01:00
javi
a61976727c merged 2015-11-13 17:21:14 +01:00
javi santana
1058d7ee8f Merge pull request #239 from CartoDB/bi_provider_get_point_data
Methods to get the data for a given torque point from the server
2015-11-13 16:54:14 +01:00
Stuart Lynn
b7f38d5996 Methods to get the data for a given torque tile from the server 2015-11-13 15:12:44 +00:00
javi
55847bda56 syntax 2015-11-13 11:41:53 +01:00
javi
164643d0f1 working with categories per tile 2015-11-12 12:27:08 +01:00
javi
e65367e8a6 fixed pixel rectangle 2015-11-11 18:08:00 +01:00
Stuart Lynn
1fa44a8ae1 Merge pull request #237 from CartoDB/bi_provider_web_workers
Adding web worker support for the BI provider
2015-11-11 15:47:15 +00:00
Stuart Lynn
87b93e6f9e whitespace for readability 2015-11-11 15:46:13 +00:00
javi
dbf48ea091 ose overview from options 2015-11-11 16:21:12 +01:00
Stuart Lynn
6dd0251be8 Allow worker pool to be configured by options 2015-11-11 15:09:24 +00:00
Stuart Lynn
19e35ffe2c Limiting the worker pool to a given size 2015-11-11 15:08:17 +00:00
Stuart Lynn
7a1f206d5e Adding web worker support for the BI provider 2015-11-11 13:30:47 +00:00
javi
1ec2324b83 fixed category count 2015-11-10 15:14:39 +01:00
javi
e0606ee295 fixed number of categories 2015-11-09 10:42:38 +01:00
javi
695cab290a added filters, variable mapping and support for new pixel type 2015-11-08 11:38:59 +01:00
javi
ff4809b08c histogram calculation 2015-11-06 19:47:37 +01:00
javi
4ad0dba547 removed client side filters temporally, integrated with server side queries 2015-11-06 17:35:49 +01:00
Stuart Lynn
71fc89a8d8 Examples of the filters and histogram api working 2015-11-03 18:36:10 -05:00
Stuart Lynn
fd5bc0f732 Adding the ability for leafletLayer to filter data on the client side and to calculate histograms and value arrays for the variables stored in the currently loaded tiles. 2015-11-03 18:35:56 -05:00
Stuart Lynn
130d72c872 adding a filterable json provider. 2015-11-03 18:35:01 -05:00
6 changed files with 1146 additions and 180 deletions

88
examples/bi.html Normal file
View File

@ -0,0 +1,88 @@
<html>
<link rel="stylesheet" href="vendor/leaflet.css" />
<style>
#map, html, body {
width: 100%; height: 100%; padding: 0; margin: 0;
}
#title {
position: absolute;
top: 100px;
left: 50px;
color: white;
font-size: 27px;
font-family: Helvetica, sans-serif;
}
</style>
<body>
<div id="map"></div>
<div id='graphs'></div>
<link rel="stylesheet" href="http://cdn.leafletjs.com/leaflet-0.7.3/leaflet.css" />
<script src="http://cdn.leafletjs.com/leaflet-0.7.3/leaflet.js"></script>
<script src="../dist/torque.full.uncompressed.js"></script>
<script>
// define the torque layer style using cartocss
// this creates a kind of density map
//color scale from http://colorbrewer2.org/
var CARTOCSS = [
'Map {',
'-torque-time-attribute: "tpep_dropoff_datetime";',
'-torque-aggregation-function: "avg(temp::float)";',
'-torque-frame-count: 1;',
'-torque-animation-duration: 15;',
'-torque-resolution: 1',
'}',
'#layer {',
' marker-width: 1;',
' marker-fill-opacity: 1.0;',
' marker-fill: #fff5eb; ',
' marker-type: ellipse;',
' [value > 1] { marker-fill: #fee6ce; }',
' [value > 2] { marker-fill: #fdd0a2; }',
' [value > 4] { marker-fill: #fdae6b; }',
' [value > 10] { marker-fill: #fd8d3c; }',
' [value > 15] { marker-fill: #f16913; }',
' [value > 20] { marker-fill: #d94801; }',
' [value > 25] { marker-fill: #8c2d04; }',
'}'
].join('\n');
var map = new L.Map('map', {
zoomControl: true,
center: [40.76045572900912, -73.97601127624512],
zoom: 13
});
L.tileLayer('http://{s}.api.cartocdn.com/base-dark/{z}/{x}/{y}.png', {
attribution: 'CartoDB'
}).addTo(map);
var torqueLayer = new L.TorqueLayer({
user : 'stuartlynn',
table : 'sampled_taxi_data',
cartocss: CARTOCSS,
provider: "filterable_sql_api"
});
torqueLayer.addTo(map);
var graphs=[]
// var ndx = crossfilter({});
//
// torqueLayer.onNewData(function(){
// data = torqueLayer.getValues()
// ndx = crossfilter(data)
// })
function addGraph(variable, type){
var dataDim = ndx.dimension(function(d){return d[variable]})
var graphID = "variable_graph"
$("graphs").append("<div id='"+graphID+"'></div>")
}
</script>
</body>
</html>

View File

@ -10,6 +10,7 @@ L.TorqueLayer = L.CanvasLayer.extend({
providers: {
'sql_api': torque.providers.json,
'filterable_sql_api': torque.providers.filterableJson,
'url_template': torque.providers.JsonArray,
'windshaft': torque.providers.windshaft,
'tileJSON': torque.providers.tileJSON
@ -28,6 +29,7 @@ L.TorqueLayer = L.CanvasLayer.extend({
options.tileLoader = true;
this.key = 0;
this.prevRenderedKey = 0;
this._filters = {}
if (options.cartocss) {
torque.extend(options, torque.common.TorqueLayer.optionsFromCartoCSS(options.cartocss));
}
@ -69,8 +71,8 @@ L.TorqueLayer = L.CanvasLayer.extend({
if (this.options.tileJSON) this.options.provider = 'tileJSON';
this.provider = new this.providers[this.options.provider](options);
this.renderer = new this.renderers[this.options.renderer](this.getCanvas(), options);
this.provider = new this.providers[this.options.provider](this.options);
this.renderer = new this.renderers[this.options.renderer](this.getCanvas(), this.options);
options.ready = function() {
self.fire("change:bounds", {
@ -98,11 +100,48 @@ L.TorqueLayer = L.CanvasLayer.extend({
self.redraw();
}
self.fire('tileLoaded');
self.fire('dataUpdate')
});
}, this);
},
setFilters: function() {
this.provider.setFilters(this._filters);
this._reloadTiles();
return this;
},
filterByRange: function(variableName, start, end) {
this._filters[variableName] = {type: 'range', range: {start: start, end: end} }
this._filtersChanged()
this.fire('dataUpdate')
return this
},
filterByCat: function(variableName, categories, exclusive) {
this._filters[variableName] = {type: 'cat', categories: categories, exclusive: !!exclusive };
this._filtersChanged()
return this
},
clearFilter: function(name){
if(name) {
delete this._filters[name]
}
else {
this._filters = {}
}
this._filtersChanged()
return this
},
_filtersChanged:function(){
this.provider._filters = this._filters;
this._clearTileCaches()
this._render()
},
_clearTileCaches: function() {
var t, tile;
for(t in this._tiles) {
@ -118,6 +157,119 @@ L.TorqueLayer = L.CanvasLayer.extend({
this._clearTileCaches();
},
valuesForRangeVariable:function(variable){
var t, tile;
var variable_id = this.provider.idForRange(variable)
var values = [ ]
for(t in this._tiles){
tile = this._tiles[t]
var noPoints = tile.x.length;
for(var i=0; i < tile.x.length; i++){
if(tile.renderFlags[i] ){
value = tile.renderData[variable_id*noPoints + i]
values.push(value)
}
}
}
return values;
},
valuesForCatVariable:function(variable){
var t, tile;
var categories = this.provider.idsForCategory(variable)
var result = [ ]
for(t in this._tiles){
tile = this._tiles[t]
var noPoints = tile.x.length;
for(var i=0; i < tile.x.length; i++){
if(tile.renderFlags[i] ){
var vals={}
Object.keys(categories).forEach(function(categoryName){
var variable_id = categories[categoryName]
value = tile.renderData[variable_id*noPoints + i]
vals[categoryName] = value
}.bind(this))
result.push(vals)
}
}
}
return result;
},
getValues:function(variable,callback){
var type= this.provider._mapping[variable].type
if(type=='float'){
callback(this.valuesForRangeVariable(variable))
}
else{
callback(this.valuesForCatVariable(variable))
}
},
getHistogramForVisibleRegion:function(varName,callback){
var center = this._map.getCenter()
var zoom = this._map.getZoom()
var xtile = parseInt(Math.floor( (center.lng + 180) / 360 * (1<<zoom) ));
var ytile = parseInt(Math.floor( (1 - Math.log(Math.tan(center.lat*Math.PI/180.0) + 1 / Math.cos(center.lat*Math.PI/180.0)) / Math.PI) / 2 * (1<<zoom) ));
var xx = xtile >> 2
var yy = ytile >> 2
var z = zoom - 2
this.provider.getHistogramForTiles(varName,[{x:xx,y:yy,z:z}],callback)
},
getHistogramForVisibleRegionBetter:function(varName,callback){
var tiles= []
for(var key in this._tiles){
tiles.push(this._tiles[key].coord)
}
this.provider.getHistogramForTiles(varName,tiles,callback)
},
getHistogram:function(variable,callback,noBins){
var type= this.provider._mapping[variable].type
if(type=='float'){
callback(this.histogramForRangeVariable(variable,noBins))
}
else if(type='cat'){
callback(this.histogramForCatVariable(variable))
}
return this
},
histogramForCatVariable:function(variable){
var result = {}
this.valuesForCatVariable(variable).forEach(function(point){
Object.keys(point).forEach(function(key){
result[key] = result[key] || 0
result[key] += point[key]
})
})
return result
},
histogramForRangeVariable:function(variable,noBins){
noBins = noBins || 10
var vals = this.valuesForRangeVariable(variable)
var min = Math.min.apply(null, vals)
var max = Math.max.apply(null, vals)
var binSize = (max-min)/noBins
var result = []
vals.forEach(function(val){
var bin = (val -min)/binSize
result[bin]= result[bin] || 0
result[bin] += val
})
return result
},
onAdd: function (map) {
map.on({
'zoomend': this._clearCaches,
@ -208,6 +360,62 @@ L.TorqueLayer = L.CanvasLayer.extend({
canvas.width = canvas.width;
},
getDataForPoint:function(x,y,callback, maxNo, tolerance){
var maxNo = maxNo || 10
var tolerance = tolerance || 10
for(var t in this._tiles ){
tile = this._tiles[t];
pos = this.getTilePos(tile.coord);
var tileWidth = 255.0/this.options.resolution
xx = x - pos.x;
yy = y - pos.y;
if(xx >= 0 && yy >= 0 && xx < 255 && yy <= 255) {
this.provider.getDataForTorquePixel(tile.coord,xx,yy,maxNo,tolerance,callback)
return this
}
}
callback(null)
return this
},
/*
_filterTile:function(tile){
var noPoints = tile.x.length
var renderFlags = []
for(var i =0; i < noPoints; i++){
var includePoint = true
Object.keys(this._filters).forEach(function(key){
var filter = this._filters[key]
var variableId = this.provider.idForRange(key)
var value = tile.renderData[variableId*noPoints+i]
if(filter.type=='range'){
if(value < filter.range.start || value > filter.range.end){
includePoint = false;
}
}
else if (filter.type=='cat'){
var ids = this.provider.idsForCategory(key);
filter.categories.forEach(function(key){
var catId = ids[key]
var value = tile.renderData[catId*noPoints + i]
if(value==0){
includePoint= false
}
}.bind(this))
}
}.bind(this))
renderFlags[i] = includePoint
}
return renderFlags
},
*/
/**
* render the selectef key
* don't call this function directly, it's called by
@ -236,10 +444,13 @@ L.TorqueLayer = L.CanvasLayer.extend({
// all the points
this.renderer._ctx.drawImage(tile._tileCache, 0, 0);
} else {
//tile.renderFlags = this._filterTile(tile)
this.renderer.renderTile(tile, this.key);
}
}
}
this.renderer.applyFilters();
// prepare caches if the animation is not running
@ -338,6 +549,7 @@ L.TorqueLayer = L.CanvasLayer.extend({
setCartoCSS: function(cartocss) {
if (this.provider.options.named_map) throw new Error("CartoCSS style on named maps is read-only");
if (!this.renderer) throw new Error('renderer is not valid');
this.options.cartocss = cartocss;
var shader = new carto.RendererJS().render(cartocss);
this.renderer.setShader(shader);
@ -378,11 +590,26 @@ L.TorqueLayer = L.CanvasLayer.extend({
*/
getValues: function(step) {
var values = [];
var idx = 0;
var mappedValues = [];
step = step === undefined ? this.key: step;
var t, tile;
for(t in this._tiles) {
tile = this._tiles[t];
this.renderer.getValues(tile, step, values);
if (tile) {
this.renderer.getValues(tile, step, values);
// map the categories
var mapping = tile.categories[step];
if (mapping) {
for (var i = idx; i <= values.length - idx; ++i) {
mappedValues.push(mapping[values[i]]);
}
idx = values.length;
}
}
}
if (mappedValues.length) {
return mappedValues;
}
return values;
},
@ -395,26 +622,77 @@ L.TorqueLayer = L.CanvasLayer.extend({
var t, tile, pos, value = null, xx, yy;
for(t in this._tiles) {
tile = this._tiles[t];
pos = this.getTilePos(tile.coord);
xx = x - pos.x;
yy = y - pos.y;
if (xx >= 0 && yy >= 0 && xx < this.renderer.TILE_SIZE && yy <= this.renderer.TILE_SIZE) {
value = this.renderer.getValueFor(tile, step, xx, yy);
}
if (value !== null) {
return value;
if (tile) {
pos = this.getTilePos(tile.coord);
xx = x - pos.x;
yy = y - pos.y;
if (xx >= 0 && yy >= 0 && xx < this.renderer.TILE_SIZE && yy <= this.renderer.TILE_SIZE) {
value = this.renderer.getValueFor(tile, step, xx, yy);
}
if (value !== null) {
return value;
}
}
}
return null;
},
/**
* return tile pos given screen x,y coordinates
*/
getTilePosForPixel: function(x, y) {
var t, tile, pos, xx, yy;
for(t in this._tiles) {
tile = this._tiles[t];
if (tile) {
pos = this.getTilePos(tile.coord);
xx = x - pos.x;
yy = y - pos.y;
if (xx >= 0 && yy >= 0 && xx < this.renderer.TILE_SIZE && yy <= this.renderer.TILE_SIZE) {
return {
x: xx,
y: yy,
tile: tile.coord
}
}
}
}
return null;
},
/**
* return a list of the closest values for a given coord.
* returned format is [{ x: .., y: .., value: ...}, .. ]
*/
getClosestValuesFor: function(x, y, dist, step) {
var xf = x + dist,
yf = y + dist,
_x = x;
var values = []
for(_y = y; _y < yf; _y += this.options.resolution){
for(_x = x; _x < xf; _x += this.options.resolution){
var thisValue = this.getValueForPos(_x,_y);
if (thisValue !== null) {
var bb = thisValue.bbox;
var xy = this._map.latLngToContainerPoint([bb[0].lat, bb[0].lon]);
values.push({
x: xy.x,
y: xy.y,
value: thisValue.value
})
}
}
}
return values;
},
getValueForBBox: function(x, y, w, h) {
var xf = x + w, yf = y + h, _x=x;
var sum = 0;
for(_y = y; _y<yf; _y+=this.options.resolution){
for(_x = x; _x<xf; _x+=this.options.resolution){
for(_y = y; _y < yf; _y += this.options.resolution){
for(_x = x; _x < xf; _x += this.options.resolution){
var thisValue = this.getValueForPos(_x,_y);
if (thisValue){
if (thisValue) {
var bb = thisValue.bbox;
var xy = this._map.latLngToContainerPoint([bb[1].lat, bb[1].lon]);
if(xy.x < xf && xy.y < yf){
@ -426,6 +704,20 @@ L.TorqueLayer = L.CanvasLayer.extend({
return sum;
},
/** return the number of points for a step */
pointCount: function(step) {
var t, tile;
step = step === undefined ? this.key: step;
var c = 0;
for(t in this._tiles) {
tile = this._tiles[t];
if (tile) {
c += tile.timeCount[step];
}
}
return c;
},
invalidate: function() {
this.provider.reload();
}

View File

@ -0,0 +1,687 @@
var torque = require('../');
var _ = require('underscore');
var Profiler = require('../profiler');
var Uint8Array = torque.types.Uint8Array;
var Int32Array = torque.types.Int32Array;
var Uint32Array = torque.types.Uint32Array;
// format('hello, {0}', 'rambo') -> "hello, rambo"
function format(str) {
for(var i = 1; i < arguments.length; ++i) {
var attrs = arguments[i];
for(var attr in attrs) {
str = str.replace(RegExp('\\{' + attr + '\\}', 'g'), attrs[attr]);
}
}
return str;
}
var filterableJson = function (options) {
this._ready = false;
this._tileQueue = [];
this.options = options;
this._filters = {};
this._tileProcessingQueue=[]
this._workers = [];
this._maxWorkerNo = this.options.maxWorkerNo || 4;
this.setupWorkerPool()
this.options.tiler_protocol = options.tiler_protocol || 'http';
this.options.tiler_domain = options.tiler_domain || 'cartodb.com';
this.options.tiler_port = options.tiler_port || 80;
// check options
if (options.resolution === undefined ) throw new Error("resolution should be provided");
if(options.start === undefined) {
this._fetchKeySpan();
} else {
this._setReady(true);
}
};
filterableJson.prototype = {
setupWorkerPool:function(){
for(var i=0; i< this._maxWorkerNo; i++){
this._workers.push(this.createProccessTileWorker())
}
},
getAvalaibleWorker:function(){
return this._workers.pop()
},
releaseWorker:function(worker){
this._workers.push(worker)
this.processNextTileRequestInQueue()
},
processNextTileRequestInQueue:function(){
if(this._tileProcessingQueue.length>0){
job = this._tileProcessingQueue.pop()
this.requestWorker(job.rows,job.coord,job.zoom, job.options, job.callback)
}
},
requestWorker:function(rows,coord,zoom,options,callback){
worker = this.getAvalaibleWorker()
self = this
if(worker){
worker.onmessage = function(e){
callback(e.data)
self.releaseWorker(this)
}
worker.postMessage(JSON.stringify({rows: rows, coord: {x:coord.x,y:coord.y}, zoom:zoom, options: options}))
}
else{
this.addToTileProcessingQueue(rows,coord,zoom,options,callback)
}
},
addToTileProcessingQueue:function(rows,coord,zoom, options, callback){
this._tileProcessingQueue.push({rows:rows, coord:coord, zoom:zoom, options: options, callback:callback})
},
/**
* Creates a worker to process the tile
*/
createProccessTileWorker: function(){
var workerFunction = "var proccessTile ="+ this.proccessTileSerial.toString()
var wrapper = "; self.onmessage = function(e){var data = JSON.parse(e.data); JSON.stringify(self.postMessage(proccessTile(data.rows,data.coord, data.zoom, data.options)))}"
var script = workerFunction + wrapper;
var blob = new Blob([script], {type: "text/javascript"})
var worker = new Worker(window.URL.createObjectURL(blob))
return worker
},
proccessTile: function(rows, coord, zoom, callback){
var self = this;
if(typeof(Worker) === "undefined"){
callback(this.proccessTileSerial(rows,coord,zoom, this.options))
}
else {
var workerSafeOptions = {
resolution: this.options.resolution,
fields: this.options.fields
}
this.requestWorker(rows, coord, zoom, workerSafeOptions, callback)
}
},
/**
* return the torque tile encoded in an efficient javascript
* structure:
* {
* x:Uint8Array x coordinates in tile reference system, normally from 0-255
* y:Uint8Array y coordinates in tile reference system
* Index: Array index to the properties
* }
*/
proccessTileSerial: function(rows, coord, zoom, options) {
// utility function for hashing categories
var r;
var x = new Uint8Array(rows.length);
var y = new Uint8Array(rows.length);
if(typeof(Profiler) != 'undefined') {
var prof_mem = Profiler.metric('ProviderJSON:mem');
var prof_point_count = Profiler.metric('ProviderJSON:point_count');
var prof_process_time = Profiler.metric('ProviderJSON:process_time').start()
}
var categoryMapping = {}
var categoryMappingSize = {}
var fields = options.fields;
for (var i = 0 ; i < fields.length; ++i) {
if (fields[i].type === 'cat') {
categoryMapping[i] = {};
categoryMappingSize[i] = 0;
}
}
// count number of steps
var maxDateSlots = Object.keys(rows[0].d).length;
var steps = maxDateSlots;
// reserve memory for all the steps
var timeIndex = new Int32Array(maxDateSlots + 1); //index-size
var timeCount = new Int32Array(maxDateSlots + 1);
var renderData = new Float32Array(rows.length * steps); //(this.options.valueDataType || type)(steps);
var renderDataPos = new Uint32Array(rows.length * steps);
if(typeof(Profiler) !='undefined'){
prof_mem.inc(
4 * maxDateSlots + // timeIndex
4 * maxDateSlots + // timeCount
steps + //renderData
steps * 4
); //renderDataPos
prof_point_count.inc(rows.length);
}
var rowsPerSlot = {};
// var steps = _.range(maxDateSlots);
var steps = []
for(var i=0 ; i< maxDateSlots; i++){
steps.push(i)
}
// precache pixel positions
for (var r = 0; r < rows.length; ++r) {
var row = rows[r];
x[r] = row.x * options.resolution;
// fix value when it's in the tile EDGE
// TODO: this should be fixed in SQL query
if (row.y === -1) {
y[r] = 0;
} else {
y[r] = row.y * options.resolution;
}
var vals = row.d;
for (var j = 0, len = steps.length; j < len; ++j) {
var rr = rowsPerSlot[steps[j]] || (rowsPerSlot[steps[j]] = []);
var k = 'f' + (j + 1)
var v = vals[k];
if (options.fields[j].type === 'cat') {
var mapping = categoryMapping[j];
var m = mapping[v]
if (!m) {
var count = ++categoryMappingSize[j];
v = mapping[v] = count;
} else {
v = m;
}
}
rr.push([r, v]);
}
}
// for each timeslot search active buckets
var renderDataIndex = 0;
var timeSlotIndex = 0;
var i = 0;
for(var i = 0; i <= maxDateSlots; ++i) {
var c = 0;
var slotRows = rowsPerSlot[i]
if(slotRows) {
for (var r = 0; r < slotRows.length; ++r) {
var rr = slotRows[r];
++c;
renderDataPos[renderDataIndex] = rr[0]
renderData[renderDataIndex] = rr[1];
++renderDataIndex;
}
}
timeIndex[i] = timeSlotIndex;
timeCount[i] = c;
timeSlotIndex += c;
}
if(typeof(Profiler) !='undefined'){
prof_process_time.end();
}
// invert the mapping
var invertedMapping = {}
for (var i = 0 ; i < fields.length; ++i) {
if (fields[i].type === 'cat') {
var cat = categoryMapping[i];
invertedMapping[i] = {}
for (var k in cat) {
invertedMapping[i][cat[k]] = k;
}
}
}
return {
x: x,
y: y,
z: zoom,
coord: {
x: coord.x,
y: coord.y,
z: zoom
},
timeCount: timeCount,
timeIndex: timeIndex,
renderDataPos: renderDataPos,
renderData: renderData,
maxDate: maxDateSlots,
categories: invertedMapping
};
},
_generateFilterSQLForCat: function(name, categories, exclusive) {
return name + (exclusive ? " not in ": " in") + '(' + categories.map(function(c) {
// escape ', double escape. one for javascript, another one for postgres
c = c.replace("'", "''''");
return "''" + c + "''";
}).join(',') + ')';
},
_generateFilterSQLForRange: function(name,range) {
var result = ""
if (range.start) {
result += " " + name + " > " + range.start;
}
if (range.end) {
if (range.start) {
result += " and "
}
result += " " + name + " < " + range.end;
}
return result
},
_setFilters:function(filters){
this.filters = filters
},
_generateFiltersSQL: function() {
var self = this;
return Object.keys(this._filters).map(function(filterName){
var filter = self._filters[filterName]
if (filter) {
if (filter.type == 'range') {
return self._generateFilterSQLForRange(filterName, filter.range)
}
else if (filter.type == 'cat' && filter.categories.length) {
return self._generateFilterSQLForCat(filterName, filter.categories, filter.exclusive)
}
else {
return ""
}
}
else{
return ""
}
}).filter(function(f) {
return f.length > 0;
}).map(function(f) {
return "(" + f + ")";
}).join(' and ')
},
_host: function() {
var opts = this.options;
var port = opts.sql_api_port;
var domain = ((opts.user_name || opts.user) + '.' + (opts.sql_api_domain || 'cartodb.com')) + (port ? ':' + port: '');
var protocol = opts.sql_api_protocol || 'http';
return this.options.url || protocol + '://' + domain + '/api/v2/sql';
},
url: function(subhost) {
var opts = this.options;
return opts.sql_api_template.replace('{user}', (opts.user_name || opts.user)).replace('{s}', subhost) + "/api/v1/sql";
},
_hash: function(str) {
var hash = 0;
if (!str || str.length == 0) return hash;
for (var i = 0, l = str.length; i < l; ++i) {
hash = (( (hash << 5 ) - hash ) + str.charCodeAt(i)) | 0;
}
return hash;
},
_extraParams: function() {
if (this.options.extra_params) {
var p = [];
for(var k in this.options.extra_params) {
var v = this.options.extra_params[k];
if (v) {
p.push(k + "=" + encodeURIComponent(v));
}
}
return p.join('&');
}
return null;
},
isHttps: function() {
return this.options.sql_api_protocol && this.options.sql_api_protocol === 'https';
},
// execute actual query
sql: function(sql, callback, options) {
options = options || {};
var subdomains = this.options.subdomains || 'abcd';
url = this.url(subdomains[Math.abs(this._hash(sql))%subdomains.length]);
var extra = this._extraParams();
torque.net.get( url + "?q=" + encodeURIComponent(sql) + (extra ? "&" + extra: ''), function (data) {
if(options.parseJSON) {
data = JSON.parse(data && data.responseText);
}
callback && callback(data);
});
},
getTileData: function(coord, zoom, callback) {
if(!this._ready) {
this._tileQueue.push([coord, zoom, callback]);
} else {
this._getTileData(coord, zoom, callback);
}
},
_setReady: function(ready) {
this._ready = true;
this._processQueue();
this.options.ready && this.options.ready();
},
_processQueue: function() {
var item;
while (item = this._tileQueue.pop()) {
this._getTileData.apply(this, item);
}
},
_getTableSQL: function(coord, zoom) {
return format("(WITH opt as ( SELECT table_name as tt FROM selectivity(x:={x}, y:={y}, z:={z}, tables:=ARRAY[{overview_tables}], where_clause:='{filters}') where nrows > 5000 order by nrows asc limit 1) select (CASE WHEN EXISTS(select 1 from opt) THEN (select tt from opt) else '{table}' END))", {
overview_tables: this.options.overview_tables.map(function(t) { return "'" + t + "'"; }).join(','),
table: this.options.table,
z: zoom,
x: coord.x,
y: coord.y,
filters: this._generateFiltersSQL()
});
},
/**
* `tile` the tile the point resides on
* `x` the x pixel coord on the tile
* 'y' the y pixel coord on the tile
* 'maxNo' the maximum number of points to return
* 'pixel tollerance around click' How many pixels to search around the click point
* 'callback' function(rows) returns an array of the data for that point
*/
getDataForTorquePixel:function(tile, x, y, maxNo, tolerance, callback){
shift = 23 - tile.z
tolerance = tolerance || 20
var sql = [
"select * from {table}",
"where (quadkey between (xyz2range({x},{y},{z})).min and (xyz2range({x},{y},{z})).max) ",
"and (((quadkey_x & (255 << {shift})) >> {shift}) - {torque_tile_x}) between -{tolerance} and {tolerance}",
"and (((quadkey_y & (255 << {shift})) >> {shift}) - {torque_tile_y}) between -{tolerance} and {tolerance} ",
"limit {maxNo}"
].join(' ')
var query = format(sql,{
x: tile.x,
y: tile.y,
z: tile.z,
table: this.options.table,
torque_tile_x: x,
torque_tile_y: y,
maxNo: maxNo,
shift: shift,
tolerance: tolerance
})
this.sql(query,function(data){
if(data) {
var rows = JSON.parse(data.responseText).rows;
callback(rows)
}
else{
callback(null)
}
})
},
/**
* `coord` object like {x : tilex, y: tiley }
* `zoom` quadtree zoom level
*/
_getTileData: function(coord, zoom, callback) {
var prof_fetch_time = Profiler.metric('ProviderJSON:tile_fetch_time').start()
this.table = this.options.table;
var numTiles = 1 << zoom;
var column_conv = this.options.column;
var sql = "select * from torque_tile_json({x}, {y}, {zoom}, ARRAY[{fields}], {table}, '{filters}')";
var query = format(sql, {
zoom: zoom,
x: coord.x,
y: coord.y,
fields: _.map(this.options.fields, function(f) {
if (f.type === 'cat') {
return "'mode() within group (order by " + f.name + ")'";
}
return "'avg(" + f.name + ")'";
}).join(','),
column: column_conv,
table: this._getTableSQL(coord, zoom),
filters: this._generateFiltersSQL()
});
var self = this;
this.sql(query, function (data) {
if (data) {
var rows = JSON.parse(data.responseText).rows;
if (rows.length !== 0) {
self.proccessTile(rows, coord, zoom,callback);
} else {
callback(null);
}
} else {
callback(null);
}
prof_fetch_time.end();
});
},
getKeySpan: function() {
return {
start: this.options.start * 1000,
end: this.options.end * 1000,
step: this.options.step,
steps: this.options.steps,
columnType: this.options.is_time ? 'date': 'number'
};
},
setColumn: function(column, isTime) {
this.options.column = column;
this.options.is_time = isTime === undefined ? true: false;
this.reload();
},
setResolution: function(res) {
this.options.resolution = res;
},
// return true if tiles has been changed
setOptions: function(opt) {
var refresh = false;
if(opt.resolution !== undefined && opt.resolution !== this.options.resolution) {
this.options.resolution = opt.resolution;
refresh = true;
}
if(opt.steps !== undefined && opt.steps !== this.options.steps) {
this.setSteps(opt.steps, { silent: true });
refresh = true;
}
if(opt.column !== undefined && opt.column !== this.options.column) {
this.options.column = opt.column;
refresh = true;
}
if(opt.countby !== undefined && opt.countby !== this.options.countby) {
this.options.countby = opt.countby;
refresh = true;
}
if(opt.data_aggregation !== undefined) {
var c = opt.data_aggregation === 'cumulative';
if (this.options.cumulative !== c) {
this.options.cumulative = c;
refresh = true;
}
}
if (refresh) this.reload();
return refresh;
},
reload: function() {
this._ready = false;
this._fetchKeySpan();
},
setSQL: function(sql) {
if (this.options.sql != sql) {
this.options.sql = sql;
this.reload();
}
},
getSteps: function() {
return Math.min(this.options.steps, this.options.data_steps);
},
setSteps: function(steps, opt) {
opt = opt || {};
if (this.options.steps !== steps) {
this.options.steps = steps;
this.options.step = (this.options.end - this.options.start)/this.getSteps();
this.options.step = this.options.step || 1;
if (!opt.silent) this.reload();
}
},
getBounds: function() {
return this.options.bounds;
},
getSQL: function() {
return this.options.sql || "select * from " + this.options.table;
},
_tilerHost: function() {
var opts = this.options;
var user = (opts.user_name || opts.user);
return opts.tiler_protocol +
"://" + (user ? user + "." : "") +
opts.tiler_domain +
((opts.tiler_port != "") ? (":" + opts.tiler_port) : "");
},
_fetchKeySpan: function() {
this._setReady(true);
},
_generateBoundsQuery:function(tiles){
return tiles.map( function(tile){
return "(quadkey between (xyz2range("+tile.x+","+tile.y+","+tile.z+")).min and (xyz2range("+tile.x+","+tile.y+","+tile.z+")).max)"
}).join(" or ")
},
getHistogramForTiles: function(varName,tiles,callback){
var sql = [
'with width as (',
'select min({varName}) as min,',
'max({varName}) as max,',
'{bins} as buckets',
'from {table}',
'),',
'_bw as ( select (max - min)/buckets as bw from width ),',
'histogram as (',
'select width_bucket({varName}, min, max, buckets) as bucket,',
'numrange(min({varName})::numeric, max({varName})::numeric, \'[]\') as range,',
'count(*) as freq',
'from {table}, width ',
'where {bounds}',
'{filters}',
//'where trip_time_in_secs between min and max',
'group by bucket',
'order by bucket',
')',
'select bucket*bw as start, (bucket+1)*bw as end, bucket as bin, lower(range) as min, upper(range) as max, freq from histogram, _bw;'
]
var filters = this._generateFiltersSQL() ? " and "+ this._generateFiltersSQL() : ""
var query = format(sql.join('\n'), this.options, {
varName: varName,
table: this.options.table,
filters: filters,
bounds: this._generateBoundsQuery(tiles),
bins: 20
});
var self = this;
this.sql(query, function (data) {
if (data) {
var rows = JSON.parse(data.responseText).rows;
callback(rows);
} else {
callback(null);
}
});
},
getHistogram: function(varName, callback) {
var sql = [
'with width as (',
'select min({column}) as min,',
'max({column}) as max,',
'20 as buckets',
'from {table}',
'),',
'_bw as ( select (max - min)/buckets as bw from width ),',
'histogram as (',
'select width_bucket({column}, min, max, buckets) as bucket,',
'numrange(min({column})::numeric, max({column})::numeric, \'[]\') as range,',
'count(*) as freq',
'from {table}, width ',
'where {filters}',
//'where trip_time_in_secs between min and max',
'group by bucket',
'order by bucket',
')',
'select bucket*bw as start, (bucket+1)*bw as end, bucket as bin, lower(range) as min, upper(range) as max, freq from histogram, _bw;'
]
var query = format(sql.join('\n'), this.options, {
column: varName,
table: this.options.table,
filters: this._generateFiltersSQL()
});
var self = this;
this.sql(query, function (data) {
if (data) {
var rows = JSON.parse(data.responseText).rows;
callback(rows);
} else {
callback(null);
}
});
}
};
module.exports = filterableJson;

View File

@ -1,5 +1,6 @@
module.exports = {
json: require('./json'),
filterableJson: require('./filterableJson'),
JsonArray: require('./jsonarray'),
windshaft: require('./windshaft'),
tileJSON: require('./tilejson')

View File

@ -57,7 +57,7 @@ var Filters = require('./torque_filters');
this.TILE_SIZE = 256;
this._style = null;
this._gradients = {};
this._forcePoints = false;
}
@ -165,14 +165,14 @@ var Filters = require('./torque_filters');
i.src = canvas.toDataURL();
return i;
}
return canvas;
},
//
// renders all the layers (and frames for each layer) from cartocss
//
renderTile: function(tile, key, callback) {
renderTile: function(tile, key, renderFlags, callback) {
if (this._iconsToLoad > 0) {
this.on('allIconsLoaded', function() {
this.renderTile.apply(this, [tile, key, callback]);
@ -193,7 +193,7 @@ var Filters = require('./torque_filters');
}
}
}
prof.end(true);
return callback && callback(null);
@ -237,12 +237,11 @@ var Filters = require('./torque_filters');
},
//
// renders a tile in the canvas for key defined in
// renders a tile in the canvas for key defined in
// the torque tile
//
_renderTile: function(tile, key, frame_offset, sprites, shader, shaderVars) {
if (!this._canvas) return;
var prof = Profiler.metric('torque.renderer.point.renderTile').start();
var ctx = this._ctx;
var blendMode = compop2canvas(shader.eval('comp-op')) || this.options.blendmode;
@ -254,27 +253,25 @@ var Filters = require('./torque_filters');
key = tile.maxDate;
}
var tileMax = this.options.resolution * (this.TILE_SIZE/this.options.resolution - 1)
var activePixels = tile.timeCount[key];
var activePixels = tile.x.length;
var anchor = this.options.resolution/2;
if (activePixels) {
var pixelIndex = tile.timeIndex[key];
var pixelIndex = 0;//tile.timeIndex[key];
for(var p = 0; p < activePixels; ++p) {
var posIdx = tile.renderDataPos[pixelIndex + p];
var c = tile.renderData[pixelIndex + p];
if (c) {
var sp = sprites[c];
if (sp === undefined) {
sp = sprites[c] = this.generateSprite(shader, c, torque.extend({ zoom: tile.z, 'frame-offset': frame_offset }, shaderVars));
}
if (sp) {
var x = tile.x[posIdx]- (sp.width >> 1) + anchor;
var y = tileMax - tile.y[posIdx] + anchor; // flip mercator
ctx.drawImage(sp, x, y - (sp.height >> 1));
}
}
var posIdx = tile.renderDataPos[pixelIndex + p];
var c = tile.renderData[pixelIndex + p];
var sp = sprites[c];
if (sp === undefined) {
sp = sprites[c] = this.generateSprite(shader, c, torque.extend({ zoom: tile.z, 'frame-offset': frame_offset }, shaderVars));
}
if (sp) {
var x = tile.x[posIdx]- (sp.width >> 1) + anchor;
var y = tileMax - tile.y[posIdx] + anchor; // flip mercator
ctx.drawImage(sp, x, y - (sp.height >> 1));
}
}
}
prof.end(true);
},
@ -442,7 +439,7 @@ var Filters = require('./torque_filters');
}
gradient = {};
var colorize = this._style['image-filters'].args;
var increment = 1/colorize.length;
for (var i = 0; i < colorize.length; i++){
var key = increment * i + increment;

View File

@ -1,160 +1,61 @@
var torque = require('../');
var cartocss = require('./cartocss_render');
var Profiler = require('../profiler');
var carto = global.carto || require('carto');
var Filters = require('./torque_filters');
var PointRenderer = require('./point')
var DEFAULT_CARTOCSS = [
'#layer {',
' polygon-fill: #FFFF00;',
' [value > 10] { polygon-fill: #FFFF00; }',
' [value > 100] { polygon-fill: #FFCC00; }',
' [value > 1000] { polygon-fill: #FE9929; }',
' [value > 10000] { polygon-fill: #FF6600; }',
' [value > 100000] { polygon-fill: #FF3300; }',
'}'
].join('\n');
var PixelRenderer = function(canvas, options) {
PointRenderer.call(this, canvas, options);
}
var TAU = Math.PI * 2;
torque.extend(PixelRenderer.prototype, PointRenderer.prototype, {
//
// this renderer just render points depending of the value
//
function RectanbleRenderer(canvas, options) {
this.options = options;
carto.tree.Reference.set(torque['torque-reference']);
this.setCanvas(canvas);
this.setCartoCSS(this.options.cartocss || DEFAULT_CARTOCSS);
}
RectanbleRenderer.prototype = {
//
// sets the cartocss style to render stuff
//
setCartoCSS: function(cartocss) {
this._cartoCssStyle = new carto.RendererJS().render(cartocss);
if(this._cartoCssStyle.getLayers().length > 1) {
throw new Error("only one CartoCSS layer is supported");
}
this._shader = this._cartoCssStyle.getLayers()[0].shader;
},
setCanvas: function(canvas) {
if(!canvas) return;
this._canvas = canvas;
this._ctx = canvas.getContext('2d');
},
accumulate: function(tile, keys) {
var prof = Profiler.metric('RectangleRender:accumulate').start();
var x, y, posIdx, p, k, key, activePixels, pixelIndex;
var res = this.options.resolution;
var s = 256/res;
var accum = new Float32Array(s*s);
if(typeof(keys) !== 'object') {
keys = [keys];
generateSprite: function(shader, value, shaderVars) {
var self = this;
var prof = Profiler.metric('torque.renderer.point.generateSprite').start();
var st = shader.getStyle({
value: value
}, shaderVars);
if(this._style === null || this._style !== st){
this._style = st;
}
for(k = 0; k < keys.length; ++k) {
key = keys[k];
activePixels = tile.timeCount[key];
if(activePixels) {
pixelIndex = tile.timeIndex[key];
for(p = 0; p < activePixels; ++p) {
posIdx = tile.renderDataPos[pixelIndex + p];
x = tile.x[posIdx]/res;
y = tile.y[posIdx]/res;
accum[x*s + y] += tile.renderData[pixelIndex + p];
}
}
return {
width: st['marker-width'],
color: st['marker-fill']
}
prof.end();
return accum;
},
renderTileAccum: function(accum, px, py) {
var prof = Profiler.metric('RectangleRender:renderTileAccum').start();
var color, x, y, alpha;
var res = this.options.resolution;
_renderTile: function(tile, key, frame_offset, sprites, shader, shaderVars) {
if (!this._canvas) return;
var prof = Profiler.metric('torque.renderer.point.renderTile').start();
var ctx = this._ctx;
var s = (256/res) | 0;
var s2 = s*s;
var colors = this._colors;
if(this.options.blendmode) {
ctx.globalCompositeOperation = this.options.blendmode;
if (this.options.cumulative && key > tile.maxDate) {
//TODO: precache because this tile is not going to change
key = tile.maxDate;
}
var polygon_alpha = this._shader['polygon-opacity'] || function() { return 1.0; };
for(var i = 0; i < s2; ++i) {
var xy = i;
var value = accum[i];
if(value) {
x = (xy/s) | 0;
y = xy % s;
// by-pass the style generation for improving performance
color = this._shader['polygon-fill']({ value: value }, { zoom: 0 });
ctx.fillStyle = color;
//TODO: each function should have a default value for each
//property defined in the cartocss
alpha = polygon_alpha({ value: value }, { zoom: 0 });
if(alpha === null) {
alpha = 1.0;
}
ctx.globalAlpha = alpha;
ctx.fillRect(x * res, 256 - res - y * res, res, res);
}
}
prof.end();
},
//
// renders a tile in the canvas for key defined in
// the torque tile
//
renderTile: function(tile, key, callback) {
if(!this._canvas) return;
var res = this.options.resolution;
//var prof = Profiler.get('render').start();
var ctx = this._ctx;
var colors = this._colors;
var activepixels = tile.timeCount[key];
if(activepixels) {
var w = this._canvas.width;
var h = this._canvas.height;
//var imageData = ctx.getImageData(0, 0, w, h);
//var pixels = imageData.data;
var pixelIndex = tile.timeIndex[key];
var tileMax = this.options.resolution * (this.TILE_SIZE/this.options.resolution - 1)
var activePixels = tile.x.length;
var anchor = this.options.resolution/2;
if (activePixels) {
var pixelIndex = 0;//tile.timeIndex[key];
for(var p = 0; p < activePixels; ++p) {
var posIdx = tile.renderDataPos[pixelIndex + p];
var c = tile.renderData[pixelIndex + p];
if(c) {
var color = colors[Math.min(c, colors.length - 1)];
var x = tile.x[posIdx];// + px;
var y = tile.y[posIdx]; //+ py;
ctx.fillStyle = color;
ctx.fillRect(x, y, res, res);
/*
for(var xx = 0; xx < res; ++xx) {
for(var yy = 0; yy < res; ++yy) {
var idx = 4*((x+xx) + w*(y + yy));
pixels[idx + 0] = color[0];
pixels[idx + 1] = color[1];
pixels[idx + 2] = color[2];
pixels[idx + 3] = color[3];
var posIdx = tile.renderDataPos[pixelIndex + p];
var c = tile.renderData[pixelIndex + p];
var sp = sprites[c];
if (sp === undefined) {
sp = sprites[c] = this.generateSprite(shader, c, torque.extend({ zoom: tile.z, 'frame-offset': frame_offset }, shaderVars));
}
if (sp) {
var x = tile.x[posIdx]- (sp.width >> 1) + anchor;
var y = tileMax - tile.y[posIdx] + anchor; // flip mercator
ctx.fillStyle = sp.color;
ctx.fillRect(x, y, sp.width, sp.width);
}
}
*/
}
}
//ctx.putImageData(imageData, 0, 0);
}
//prof.end();
return callback && callback(null);
}
};
});
// exports public api
module.exports = RectanbleRenderer;
module.exports = PixelRenderer;