Add arraybuffer format

NEWS.md

@@ -1,3 +1,7 @@
+1.4.0
+-----
+* Add arraybuffer format
+
 1.3.10
 ------
 * Fixed problem identifying OAuth request protocol

app/controllers/app.js

@@ -91,7 +91,7 @@ function window_sql (sql, limit, offset) {
 // request handlers
 function handleQuery(req, res) {
 
-    var supportedFormats = ['json', 'geojson', 'topojson', 'csv', 'svg', 'shp', 'kml'];
+    //var supportedFormats = ['json', 'geojson', 'topojson', 'csv', 'svg', 'shp', 'kml', 'arraybuffer'];
 
     // extract input
     var body      = (req.body) ? req.body : {};
@@ -136,7 +136,8 @@ function handleQuery(req, res) {
           skipfields = [];
         }
 
-        if ( -1 === supportedFormats.indexOf(format) )
+        //if ( -1 === supportedFormats.indexOf(format) )
+        if ( ! formats.hasOwnProperty(format) ) 
           throw new Error("Invalid format: " + format);
 
         if (!_.isString(sql)) throw new Error("You must indicate a sql query");
@@ -235,7 +236,6 @@ function handleQuery(req, res) {
                 }
 
 
-                if ( ! formats.hasOwnProperty(format) ) throw new Error("Unknown format " + format);
                 var fClass = formats[format]
                 formatter = new fClass();
 

app/models/bin_encoder.js

@@ -0,0 +1,148 @@
+
+function ArrayBufferSer(type, data, options) {
+  if(type === undefined) throw "ArrayBufferSer should be created with a type";
+  this.options = options || {}
+  this._initFunctions();
+  this.headerSize = 8;
+  this.data = data;
+  this.type = type = Math.min(type, ArrayBufferSer.BUFFER);
+  var size = this._sizeFor(this.headerSize, data);
+  this.buffer = new Buffer(this.headerSize + size);
+  this.buffer.writeUInt32BE(type, 0); // this could be one byte but for byte padding is better to be 4 bytes
+  this.buffer.writeUInt32BE(size, 4);
+  this.offset = this.headerSize;
+
+  var w = this.writeFn[type];
+
+  if(!this.options.delta) {
+    for(var i = 0; i < data.length; ++i) {
+      this[w](data[i]);
+    }
+  } else {
+    this[w](data[0]);
+    for(var i = 1; i < data.length; ++i) {
+      this[w](data[i] - data[i - 1]);
+    }
+  }
+}
+
+//
+// constants
+//
+ArrayBufferSer.INT8 = 1;
+ArrayBufferSer.UINT8 = 2;
+ArrayBufferSer.UINT8_CLAMP = 3;
+ArrayBufferSer.INT16 = 4;
+ArrayBufferSer.UINT16 = 5;
+ArrayBufferSer.INT32 = 6;
+ArrayBufferSer.UINT32 = 7;
+ArrayBufferSer.FLOAT32 = 8;
+//ArrayBufferSer.FLOAT64 = 9; not supported
+ArrayBufferSer.STRING = 10;
+ArrayBufferSer.BUFFER = 11;
+
+ArrayBufferSer.MAX_PADDING = ArrayBufferSer.INT32;
+
+
+ArrayBufferSer.typeNames = {
+  'int8': ArrayBufferSer.INT8,
+  'uint8': ArrayBufferSer.UINT8,
+  'uintclamp': ArrayBufferSer.UINT8_CLAMP,
+  'int16': ArrayBufferSer.INT16,
+  'uint16': ArrayBufferSer.UINT16,
+  'int32': ArrayBufferSer.INT32,
+  'uint32': ArrayBufferSer.UINT32,
+  'float32': ArrayBufferSer.FLOAT32,
+  'string': ArrayBufferSer.STRING,
+  'buffer': ArrayBufferSer.BUFFER
+};
+
+ArrayBufferSer.prototype = {
+
+  // 0 not used
+  sizes: [NaN, 1, 1, 1, 2, 2, 4, 4, 4, 8],
+
+  _paddingFor: function(off, type) {
+    var s = this.sizes[type]
+    if(s) {
+      var r = off % s;
+      return r == 0 ? 0 : s - r;
+    }
+    return 0;
+  },
+
+  _sizeFor: function(offset, t) {
+    var self = this;
+    var s = this.sizes[this.type]
+    if(s) {
+      return s*t.length;
+    }
+    s = 0;
+    if(this.type == ArrayBufferSer.STRING) {
+      // calculate size with padding
+      t.forEach(function(arr) { 
+        var pad = self._paddingFor(offset, ArrayBufferSer.MAX_PADDING);
+        s += pad;
+        offset += pad;
+        var len = (self.headerSize + arr.length*2) 
+        s += len;
+        offset += len;
+      });
+    } else {
+      t.forEach(function(arr) {
+        var pad = self._paddingFor(offset, ArrayBufferSer.MAX_PADDING);
+        s += pad;
+        offset += pad;
+        s += arr.getSize()
+        offset += arr.getSize();
+      });
+    }
+    return s;
+  },
+
+  getDataSize: function() {
+    return this._sizeFor(0, this.data);
+  },
+
+  getSize: function() {
+    return this.headerSize + this._sizeFor(this.headerSize, this.data);
+  },
+
+  writeFn: ['', 'writeInt8', 'writeUInt8','writeUInt8Clamp', 'writeInt16LE', 'writeUInt16LE', 'writeUInt32LE', 'writeUInt32LE', 'writeFloatLE', 'writeDoubleLE', 'writeString', 'writteBuffer'],
+
+  _initFunctions: function() {
+    var self = this;
+    this.writeFn.forEach(function(fn) {
+      if(self[fn] === undefined)
+        self[fn] = function(d) {
+          self.buffer[fn](d, self.offset);
+          self.offset += self.sizes[self.type];
+        }
+    });
+  },
+
+  writeUInt8Clamp: function(c) {
+    this.buffer.writeUInt8(Math.min(255, c), this.offset);
+    this.offset += 1;
+  },
+
+  writeString: function(s) {
+    var arr = [];
+    for(var i = 0, len = s.length; i < len; ++i) {
+      arr.push(s.charCodeAt(i));
+    }
+    var str = new ArrayBufferSer(ArrayBufferSer.UINT16, arr);
+    this.writteBuffer(str);
+  },
+
+  writteBuffer: function(b) {
+    this.offset += this._paddingFor(this.offset, ArrayBufferSer.MAX_PADDING);
+    // copy header
+    b.buffer.copy(this.buffer, this.offset)
+    this.offset += b.buffer.length;
+  }
+
+};
+
+
+module.exports = ArrayBufferSer;

app/models/formats/arraybuffer.js

@@ -0,0 +1,82 @@
+var pg  = require('./pg');
+var ArrayBufferSer = require("../bin_encoder");
+var _ = require('underscore');
+
+function binary() {}
+
+binary.prototype = new pg('arraybuffer');
+
+binary.prototype._contentType = "application/octet-stream";
+
+binary.prototype._extractTypeFromName = function(name) {
+  var g = name.match(new RegExp(/.*__(uintclamp|uint|int|float)(8|16|32)/i))
+  if(g && g.length == 3) {
+    var typeName = g[1] + g[2];
+    return ArrayBufferSer.typeNames[typeName];
+  }
+};
+
+binary.prototype.transform = function(result, options, callback) {
+  var total_rows = result.rowCount;
+  var rows = result.rows
+
+  // get headers 
+  if(!total_rows) {
+    callback(null, new Buffer(0));
+    return;
+  }
+
+  var headersNames = Object.keys(rows[0]);
+  var headerTypes = [];
+
+  if(_.contains(headersNames, 'the_geom')) {
+    callback(new Error("geometry types are not supported"), null);
+    return;
+  }
+
+  try {
+
+    // get header types (and guess from name)
+    for(var i = 0; i < headersNames.length; ++i) {
+      var r = rows[0];
+      var n = headersNames[i];
+      if(typeof(r[n]) == 'string') {
+        headerTypes.push(ArrayBufferSer.STRING);
+      } else if(typeof(r[n]) == 'object') {
+        var t = this._extractTypeFromName(n);
+        t = t == undefined ? ArrayBufferSer.FLOAT32: t;
+        headerTypes.push(ArrayBufferSer.BUFFER + t);
+      } else {
+        var t = this._extractTypeFromName(n);
+        headerTypes.push(t == undefined ? ArrayBufferSer.FLOAT32: t);
+      }
+    }
+
+    // pack the data
+    var header = new ArrayBufferSer(ArrayBufferSer.STRING, headersNames);
+    var data = [header];
+    for(var i = 0; i < headersNames.length; ++i) {
+      var d = [];
+      var n = headersNames[i];
+      for(var r = 0; r < total_rows; ++r) {
+        var row = rows[r][n]; 
+        if(headerTypes[i] > ArrayBufferSer.BUFFER) {
+          row = new ArrayBufferSer(headerTypes[i] - ArrayBufferSer.BUFFER, row);
+        }
+        d.push(row);
+      };
+      var b = new ArrayBufferSer(headerTypes[i], d);
+      data.push(b);
+    }
+
+    // create the final buffer
+    var all = new ArrayBufferSer(ArrayBufferSer.BUFFER, data);
+
+    callback(null, all.buffer);
+
+  } catch(e) {
+    callback(e, null);
+  }
+};
+
+module.exports = binary;

client/bin_decoder.js

@@ -0,0 +1,104 @@
+
+
+function ArrayBufferSer(arrayBuffer) {
+  this.buffer = arrayBuffer;
+  this.offset = 0;
+  this.sections = this.readArray();
+  this.headers = this.sections[0];
+  this._headersIndex = {};
+  for(var i = 0; i < this.headers.length; ++i) {
+    this._headersIndex[this.headers[i]] = i;
+  }
+  if(this.sections.length > 1) {
+    this.length = this.sections[1].length;
+  }
+}
+
+ArrayBufferSer.INT8 = 1;
+ArrayBufferSer.UINT8 = 2;
+ArrayBufferSer.UINT8_CLAMP = 3;
+ArrayBufferSer.INT16 = 4;
+ArrayBufferSer.UINT16 = 5;
+ArrayBufferSer.INT32 = 6;
+ArrayBufferSer.UINT32 = 7;
+ArrayBufferSer.FLOAT32 = 8;
+//ArrayBufferSer.FLOAT64 = 9; not supported
+ArrayBufferSer.STRING = 10;
+ArrayBufferSer.BUFFER = 11;
+
+
+ArrayBufferSer.prototype = {
+
+   sizes: [NaN, 1, 1, 1, 2, 2, 4, 4, 4, 8],
+
+   types: [
+    null,
+    Int8Array,
+    Uint8Array,
+    Uint8ClampedArray,
+    Int16Array,
+    Uint16Array,
+    Int32Array,
+    Uint32Array,
+    Float32Array,
+    Float64Array
+   ],
+
+  get: function(columnName) {
+    var i = this._headersIndex[columnName]
+    if(i != undefined) {
+      return this.sections[i + 1]
+    }
+    return;
+  },
+
+  _paddingFor: function(offset, type) {
+    var s = this.sizes[type]
+    if(s) {
+      var r = offset % s;
+      return r == 0 ? 0 : s - r;
+    }
+    return 0;
+  },
+
+  readUInt32: function() {
+    var i = new DataView(this.buffer).getUint32(this.offset);
+    this.offset += 4;
+    return i
+  },
+
+  readArray: function() {
+    var type = this.readUInt32();
+    var size = this.readUInt32();
+    if(type < ArrayBufferSer.STRING) {
+      var a = new this.types[type](this.buffer, this.offset, size/this.sizes[type]);
+      this.offset += size;
+      return a;
+    } else if(type == ArrayBufferSer.STRING) {
+      var target = this.offset + size;
+      var b = [];
+      while(this.offset < target) {
+        this.offset += this._paddingFor(this.offset, ArrayBufferSer.INT32);
+        var arr = this.readArray();
+        if(arr) {
+          var str = '';
+          for(var i = 0; i < arr.length; ++i) {
+            str += String.fromCharCode(arr[i]);
+          }
+          b.push(str);
+        }
+        // parse sttring
+      }
+      return b;
+    } else if(type == ArrayBufferSer.BUFFER) {
+      var b = [];
+      var target = this.offset + size;
+      while(this.offset < target) {
+        this.offset += this._paddingFor(this.offset, ArrayBufferSer.INT32);
+        b.push(this.readArray());
+      }
+      return b;
+    }
+  }
+
+};

client/test.html

@@ -0,0 +1,32 @@
+
+<html>
+  <body>
+    <script src="bin_decoder.js"></script>
+    <script>
+      function get(url, callback) {
+        var req = new XMLHttpRequest();
+        req.onreadystatechange = function() {
+          if (req.readyState == 4){
+            if (req.status == 200){
+              callback(req);
+            } else {
+              callback(null);
+            }
+          }
+        };
+        req.open("GET", url, true)
+        req.responseType = 'arraybuffer';
+        req.send(null)
+        return req;
+      }
+
+      var q = 'select%20owner,%20array_agg(lat::integer)%20as%20arr_lat__int8,%20array_agg(long)%20as%20arr_long%20%20%20from%20antenas%20%20group%20by%20owner,%20lat,%20long%20limit%2020';
+
+      get('http://development.localhost.lan:8080/api/v1/sql?q=' + q+ '&format=bin', function(req) {
+        var b = new ArrayBufferSer(req.response);
+        console.log(b.length);
+        window.b = b;
+      });
+    </script>
+  </body>
+</html>

package.json

@@ -2,7 +2,7 @@
     "private": true,
     "name": "cartodb_api",
     "description": "high speed SQL api for cartodb",
-    "version": "1.3.10",
+    "version": "1.4.0",
     "author": {
         "name": "Simon Tokumine, Sandro Santilli, Vizzuality",
         "url": "http://vizzuality.com",
@@ -30,7 +30,7 @@
         "libxmljs": "~0.6.1"
     },
     "scripts": {
-        "test": "test/run_tests.sh ${RUNTESTFLAGS} test/unit/*.js test/acceptance/*.js  test/acceptance/export/*.js"
+        "test": "test/run_tests.sh ${RUNTESTFLAGS} test/unit/*.js test/unit/model/*.js test/acceptance/*.js  test/acceptance/export/*.js"
     },
     "engines": { "node": ">= 0.4.1 < 0.9" }
 }

test/acceptance/export/arraybuffer.js

@@ -0,0 +1,64 @@
+require('../../helper');
+require('../../support/assert');
+
+
+var app    = require(global.settings.app_root + '/app/controllers/app')
+    , assert = require('assert')
+    , querystring = require('querystring')
+    , _ = require('underscore')
+    , zipfile = require('zipfile')
+    , fs      = require('fs')
+    , libxmljs = require('libxmljs')
+    , Step = require('step')
+    ;
+
+// allow lots of emitters to be set to silence warning
+app.setMaxListeners(0);
+
+suite('export.arraybuffer', function() {
+
+var expected_cache_control = 'no-cache,max-age=3600,must-revalidate,public';
+var expected_cache_control_persist = 'public,max-age=31536000';
+
+// use dec_sep for internationalization
+var checkDecimals = function(x, dec_sep){
+    var tmp='' + x;
+    if (tmp.indexOf(dec_sep)>-1)
+        return tmp.length-tmp.indexOf(dec_sep)-1;
+    else
+        return 0;
+}
+
+test('GET /api/v1/sql as arraybuffer ', function(done){
+    assert.response(app, {
+        url: '/api/v1/sql?' + querystring.stringify({
+          q: 'SELECT cartodb_id,name,1::integer,187.9 FROM untitle_table_4',
+          format: 'arraybuffer'
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(res){
+        assert.equal(res.statusCode, 200, res.body);
+        assert.equal(res.headers['content-type'], "application/octet-stream")
+        done();
+    });
+});
+
+test('GET /api/v1/sql as arraybuffer does not support geometry types ', function(done){
+    assert.response(app, {
+        url: '/api/v1/sql?' + querystring.stringify({
+          q: 'SELECT cartodb_id, the_geom FROM untitle_table_4',
+          format: 'arraybuffer'
+        }),
+        headers: {host: 'vizzuality.cartodb.com'},
+        method: 'GET'
+    },{ }, function(res){
+        assert.equal(res.statusCode, 400, res.body);
+        var result = JSON.parse(res.body);
+        assert.equal(result.error[0], "geometry types are not supported");
+
+        done();
+    });
+});
+
+});

test/unit/model/bin_encoder.js

@@ -0,0 +1,74 @@
+require('../../helper');
+var assert = require('assert');
+
+var ArrayBufferSer = require('../../../app/models/bin_encoder')
+
+suite('ArrayBufferSer', function() {
+
+  test('calculate size for basic types', function() {
+    var b = new ArrayBufferSer(ArrayBufferSer.INT16, [1,2,3,4])
+    assert.equal(4*2, b.getDataSize());
+
+    b = new ArrayBufferSer(ArrayBufferSer.INT8, [1,2,3,4])
+    assert.equal(4*1, b.getDataSize());
+
+    b = new ArrayBufferSer(ArrayBufferSer.INT32, [1,2,3,4])
+    assert.equal(4*4, b.getDataSize());
+  });
+
+
+  test('calculate size for arrays', function() {
+    var b = new ArrayBufferSer(ArrayBufferSer.STRING, ["test","kease"])
+    assert.equal((b.headerSize + 4 + 5)*2, b.getDataSize());
+
+    var ba = new ArrayBufferSer(ArrayBufferSer.INT16, [1,2,3,4])
+    var bc = new ArrayBufferSer(ArrayBufferSer.INT16, [1,4])
+
+    b = new ArrayBufferSer(ArrayBufferSer.BUFFER, [ba, bc])
+    assert.equal((b.headerSize + 4 + 2)*2, b.getDataSize());
+    assert.equal(b.type, ArrayBufferSer.BUFFER);
+  });
+
+  function assert_buffer_equals(a, b) {
+    assert.equal(a.length, b.length);
+    for(var i = 0; i < a.length; ++i) {
+      assert.equal(a[i], b[i], "byte i " + i + " is different: " + a[i] + " != " + b[i]);
+    }
+  }
+
+  test('binary data is ok', function() {
+    var b = new ArrayBufferSer(ArrayBufferSer.INT16, [1,2,3,4])
+    var bf = new Buffer([0, 0, 0, ArrayBufferSer.INT16, 0, 0, 0, 8, 1, 0, 2, 0, 3, 0, 4, 0]);
+    assert_buffer_equals(bf, b.buffer);
+  });
+
+  test('binary data is ok with arrays', function() {
+    var ba = new ArrayBufferSer(ArrayBufferSer.INT16, [1,2, 3, 4])
+    var bc = new ArrayBufferSer(ArrayBufferSer.INT16, [1,4])
+
+    var b = new ArrayBufferSer(ArrayBufferSer.BUFFER, [ba, bc])
+    var bf = new Buffer([
+      0, 0, 0, ArrayBufferSer.BUFFER, // type
+      0, 0, 0, 28,
+      0, 0, 0, ArrayBufferSer.INT16, 0, 0, 0, 8, 1, 0, 2, 0, 3, 0, 4, 0,
+      0, 0, 0, ArrayBufferSer.INT16, 0, 0, 0, 4, 1, 0, 4, 0])
+    assert_buffer_equals(bf, b.buffer);
+  });
+
+  test('binary data is ok with strings', function() {
+    var s = 'test'
+    var b = new ArrayBufferSer(ArrayBufferSer.STRING, [s])
+    var bf = new Buffer([
+      0, 0, 0, ArrayBufferSer.STRING, // type
+      0, 0, 0, 16,
+      0, 0, 0, ArrayBufferSer.UINT16,
+      0, 0, 0, 8,
+      s.charCodeAt(0), 0,
+      s.charCodeAt(1), 0,
+      s.charCodeAt(2), 0,
+      s.charCodeAt(3), 0
+    ]);
+    assert_buffer_equals(bf, b.buffer);
+  });
+
+});