cartodb/torque
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

use splunk's PostProcessManager to request tile data instead of

Browse Source 
hardcoded job requests
This commit is contained in:
Dani Carrion 2015-08-31 18:22:52 +02:00
parent b69a8e5354
commit d56b403cfe
1 changed files with 51 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

122
lib/torque/provider/splunk.js
View File

@ -58,6 +58,12 @@
splunk.prototype = { splunk.prototype = {
setManager: function (PostProcessManager, managerId) {
this.searchManagerId = managerId;
this.managerClass = PostProcessManager;
this.managers = [];
},
/** /**
* return the torque tile encoded in an efficient javascript * return the torque tile encoded in an efficient javascript
* structure: * structure:
@ -198,7 +204,7 @@
this.options.cartocss = c; this.options.cartocss = c;
},*/ },*/
setSteps: function(steps, opt) { setSteps: function(steps, opt) {
opt = opt || {}; opt = opt || {};
if (this.options.steps !== steps) { if (this.options.steps !== steps) {
this.options.steps = steps; this.options.steps = steps;
@ -266,10 +272,10 @@
getTileData: function(coord, zoom, callback) { getTileData: function(coord, zoom, callback) {
if(!this._ready) { if(!this._ready) {
this._tileQueue.push([coord, zoom, callback]); this._tileQueue.push([coord, zoom, callback]);
} else { } else {
this._getTileData(coord, zoom, callback); this._getTileData(coord, zoom, callback);
} }
}, },
@ -293,7 +299,7 @@
*/ */
_getTileData: function(coord, zoom, callback) { _getTileData: function(coord, zoom, callback) {
console.log("Fetching tile " + coord.zoom + '/' + coord.x + '/' + coord.y); console.log("Fetching tile " + coord.zoom + '/' + coord.x + '/' + coord.y);
var self = this; var self = this;
@ -301,11 +307,11 @@
getSplunkData(function(data){ getSplunkData(function(data){
//parseData converts splunk results into a valid torque tile //parseData converts splunk results into a valid torque tile
var tile = parseData(data); var tile = parseData(data);
if (tile) { if (tile) {
var rows = tile; var rows = tile;
callback(self.proccessTile(rows, coord, zoom)); callback(self.proccessTile(rows, coord, zoom));
} else { } else {
@ -316,7 +322,7 @@
function getSplunkData(callback) { function getSplunkData(callback) {
var tileBounds = boundsFromTile(coord.zoom,coord.x,coord.y); var tileBounds = boundsFromTile(coord.zoom,coord.x,coord.y);
@ -330,40 +336,19 @@
tileBounds.maxLng <= 180 tileBounds.maxLng <= 180
) { ) {
var options = { if (!self.managers[coord.zoom + "_" + coord.x + "_" + coord.y]) {
host: 'localhost', self.managers[coord.zoom + "_" + coord.x + "_" + coord.y] = new self.managerClass({
port: '8000', id: coord.zoom + "_" + coord.x + "_" + coord.y,
sid: '1439866488.13', managerid: self.searchManagerId,
bounds: tileBounds search: "geofilter south=" + tileBounds.minLat + " west=" + tileBounds.minLng + " north=" + tileBounds.maxLat + " east=" + tileBounds.maxLng + " maxclusters=65536"
}
var template = 'http://{{host}}:{{port}}/en-US/splunkd/__raw/services/search/jobs/{{sid}}/results_preview?output_mode=json_rows&count=65536&show_metadata=false&search=geofilter+south={{bounds.minLat}}+west={{bounds.minLng}}+north={{bounds.maxLat}}+east={{bounds.maxLng}}+maxclusters%3D65536';
var queryString = Mustache.render(template,options);
console.log(queryString);
$.getJSON(queryString,function(data) {
console.log("Got " + (data.rows.length-8) + " bins for this tile");
callback(data);
}); });
// $.ajax({ self.managers[coord.zoom + "_" + coord.x + "_" + coord.y].data("results", {count: 0, output_mode: 'json_rows'}).on("data", function (results) {
// type: "POST", console.log("Got " + (results.data().rows.length - 8) + " bins for this tile");
// crossDomain: true, callback(results.data());
// url: "https://localhost:8089/servicesNS/admin/search/search/jobs/export", });
// username: 'admin', }
// password: 'cartodb',
// dataType: 'text', self.managers[coord.zoom + "_" + coord.x + "_" + coord.y].startSearch();
// data: queryString,
// success: callback,
// beforeSend: function(xhr) {
// xhr.setRequestHeader(
// 'Authorization',
// 'Basic ' + window.btoa(unescape(encodeURIComponent('admin' + ':' + 'cartodb')))
// )
// }
// });
} }
} }
@ -373,7 +358,7 @@
var bounds = tileBounds(z,x,y); var bounds = tileBounds(z,x,y);
mins = metersToLatLng(bounds[0]); mins = metersToLatLng(bounds[0]);
maxs = metersToLatLng(bounds[1]); maxs = metersToLatLng(bounds[1]);
bounds={ bounds={
minLat:mins[1], minLat:mins[1],
maxLat:maxs[1], maxLat:maxs[1],
@ -386,17 +371,17 @@
function metersToLatLng(coord) { function metersToLatLng(coord) {
lng = (coord[0] / (2 * Math.PI * 6378137 / 2.0)) * 180.0 lng = (coord[0] / (2 * Math.PI * 6378137 / 2.0)) * 180.0
lat = (coord[1] / (2 * Math.PI * 6378137 / 2.0)) * 180.0 lat = (coord[1] / (2 * Math.PI * 6378137 / 2.0)) * 180.0
lat = 180 / Math.PI * (2 * Math.atan( Math.exp( lat * Math.PI / 180.0)) - Math.PI / 2.0) lat = 180 / Math.PI * (2 * Math.atan( Math.exp( lat * Math.PI / 180.0)) - Math.PI / 2.0)
return [lng,lat] return [lng,lat]
} }
function tileBounds(z,x,y) { function tileBounds(z,x,y) {
var mins = pixelsToMeters( z, x*256, (y+1)*256 ) var mins = pixelsToMeters( z, x*256, (y+1)*256 )
var maxs = pixelsToMeters( z, (x+1)*256, y*256 ) var maxs = pixelsToMeters( z, (x+1)*256, y*256 )
return [mins,maxs]; return [mins,maxs];
} }
@ -416,7 +401,7 @@
// data = '[' + data.split('}}').join('}},').replace(/,\s*$/, "") + ']'; // data = '[' + data.split('}}').join('}},').replace(/,\s*$/, "") + ']';
// data = JSON.parse(data); // data = JSON.parse(data);
var torqueTile = []; var torqueTile = [];
@ -426,10 +411,10 @@
//console.log(bin); //console.log(bin);
torqueTile.push(torqueTransform(bin)); torqueTile.push(torqueTransform(bin));
function torqueTransform(bin){ function torqueTransform(bin){
//var binSpan = 1010384; //hard coded to figure out step from unixtimestamp, TODO figure out how we will deal with the time range and bin span //var binSpan = 1010384; //hard coded to figure out step from unixtimestamp, TODO figure out how we will deal with the time range and bin span
@ -451,11 +436,11 @@
var point = latLngToTileXY(parseInt(lat),parseInt(lng),zoom) var point = latLngToTileXY(parseInt(lat),parseInt(lng),zoom)
//console.log('tileXY', point) //console.log('tileXY', point)
torqueBin.x__uint8 = point.x; torqueBin.x__uint8 = point.x;
torqueBin.y__uint8 = point.y; torqueBin.y__uint8 = point.y;
//iterate over remaining rows minus the last 6 //iterate over remaining rows minus the last 6
var dates = [], var dates = [],
@ -464,7 +449,7 @@
if(bin[i] !== null) { if(bin[i] !== null) {
dates.push(i); dates.push(i);
vals.push(bin[i]); vals.push(bin[i]);
} }
} }
torqueBin.vals__uint8 = vals; torqueBin.vals__uint8 = vals;
@ -482,7 +467,7 @@
MaxLongitude = 180, MaxLongitude = 180,
mapSize = Math.pow(2, zoom) * 256; mapSize = Math.pow(2, zoom) * 256;
latitude = clip(lat, MinLatitude, MaxLatitude) latitude = clip(lat, MinLatitude, MaxLatitude)
longitude = clip(lng, MinLongitude, MaxLongitude) longitude = clip(lng, MinLongitude, MaxLongitude)
@ -491,12 +476,12 @@
p.x = (longitude + 180.0) / 360.0 * (1 << zoom) p.x = (longitude + 180.0) / 360.0 * (1 << zoom)
p.y = (1.0 - Math.log(Math.tan(latitude * Math.PI / 180.0) + 1.0 / Math.cos(lat.toRad())) / Math.PI) / 2.0 * (1 << zoom) p.y = (1.0 - Math.log(Math.tan(latitude * Math.PI / 180.0) + 1.0 / Math.cos(lat.toRad())) / Math.PI) / 2.0 * (1 << zoom)
var tilex = parseInt(Math.trunc(p.x)); var tilex = parseInt(Math.trunc(p.x));
var tiley = parseInt(Math.trunc(p.y)); var tiley = parseInt(Math.trunc(p.y));
var pixelX = clipByRange((tilex * 256) + ((p.x - tilex) * 256), mapSize - 1)/2 //<-- divide by 2 because we only have 128 bins var pixelX = clipByRange((tilex * 256) + ((p.x - tilex) * 256), mapSize - 1)/2 //<-- divide by 2 because we only have 128 bins
var pixelY = (256 - clipByRange((tiley * 256) + ((p.y - tiley) * 256), mapSize - 1))/2 var pixelY = (256 - clipByRange((tiley * 256) + ((p.y - tiley) * 256), mapSize - 1))/2
@ -511,7 +496,7 @@
} }
function clipByRange(n,range) { function clipByRange(n,range) {
return n % range; return n % range;
} }
} }
@ -522,8 +507,8 @@
return torqueTile; return torqueTile;
} }
}, },
@ -634,7 +619,7 @@
_fetchMap: function(callback) { _fetchMap: function(callback) {
var self = this; var self = this;
var layergroup = {}; var layergroup = {};
var host = this.options.dynamic_cdn ? this.url().replace('{s}', '0'): this._tilerHost(); var host = this.options.dynamic_cdn ? this.url().replace('{s}', '0'): this._tilerHost();
@ -662,7 +647,7 @@
}] }]
}; };
} }
if(this.options.stat_tag){ if(this.options.stat_tag){
allParams["stat_tag"] = this.options.stat_tag; allParams["stat_tag"] = this.options.stat_tag;
} }
@ -680,22 +665,17 @@
"&callback=?" + (extra ? "&" + extra: ''); "&callback=?" + (extra ? "&" + extra: '');
var map_instance_time = Profiler.metric('torque.provider.windshaft.layergroup.time').start(); var map_instance_time = Profiler.metric('torque.provider.windshaft.layergroup.time').start();
// var opt = {start: 1000, end: 5000, data_steps: 5, column_type: "number"};
var opt = {start: 1262311701000, // var opt = {start: 1000, end: 5000, data_steps: 5, column_type: "number"};
end: 1391640787000,
data_steps: 8247, var opt = {start: 1262311701000,
end: 1391640787000,
data_steps: 8247,
column_type: "date"}; column_type: "date"};
for(var k in opt) { for(var k in opt) {
self.options[k] = opt[k]; self.options[k] = opt[k];
} }
//in Windshaft, this was happening after an ajax call. Calling self._setReady() too early causes the viz to fail, so it is delayed here. We should fix this
setTimeout(function(){
self._setReady(true);
},500)
} }
}; };