From ed38139c0b3bd835de13621102bd8539b5dfcae7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20K=C3=A4fer?= Date: Mon, 24 Jan 2011 17:23:18 -0500 Subject: [PATCH] refactor external to make tests working --- lib/mess/external.js | 445 +++++++++---------------- lib/mess/renderer.js | 45 ++- test/rendering.js | 37 ++ test/rendering/complex_cascades.mml | 31 ++ test/rendering/complex_cascades.mss | 31 ++ test/rendering/complex_cascades.result | 89 +++++ 6 files changed, 366 insertions(+), 312 deletions(-) create mode 100644 test/rendering.js create mode 100644 test/rendering/complex_cascades.mml create mode 100644 test/rendering/complex_cascades.mss create mode 100644 test/rendering/complex_cascades.result diff --git a/lib/mess/external.js b/lib/mess/external.js index 1acb123..644c7af 100644 --- a/lib/mess/external.js +++ b/lib/mess/external.js @@ -2,307 +2,176 @@ var fs = require('fs'), netlib = require('./netlib'), url = require('url'), path = require('path'), - Step = require('step'), - _ = require('underscore')._, - spawn = require('child_process').spawn; + crypto = require('crypto'), + assert = require('assert'), + zip = require('zipfile'), + Step = require('step'); // node compatibility for mkdirs below var constants = {}; -if (!process.EEXIST >= 1) +if (!process.EEXIST >= 1) constants = require('constants'); else constants.EEXIST = process.EEXIST; -/** - * TODO: use node-minizip instead of shelling - * TODO: include or don't use underscore - */ -var External = function External(env) { - var env = env; - - return { - env: env, +function External(env, uri, callback) { + assert.ok(env.data_dir); + assert.ok(env.local_data_dir); - /** - * Download an external, process it, and return the usable filepath for - * Mapnik - * @param {String} resource_url the URI of the datasource from a mapfile. - * @param {Function} callback passed into processor function after - * localizing. - */ - process: function(resource_url, callback) { - var file_format = path.extname(resource_url).toLowerCase(), - that = this; + this.env = env; + this.callback = callback; + this.uri = uri; + this.format = path.extname(uri).toLowerCase(); - if (url.parse(resource_url).protocol == 'http:') { - fs.stat(this.tmppos(resource_url), function(err, stats) { - if (stats && stats.isFile()) { - callback(null, [ - resource_url, - that.destinations(file_format)(resource_url, that) - ]); - } else { - netlib.download( - resource_url, - that.tmppos(resource_url), - that.encodings[file_format], - function(err, url, filename) { - if (that.processors(file_format)) { - that.processors(file_format)( - filename, - resource_url, - callback, - that); - } else { - console.log('no processor found for %s', file_format); - } - }); - } - }); - } else { - // is this a fully-qualified URL? - fs.stat(resource_url, function(err, stat) { - if (!err && stat.isFile()) { - if (that.processors(file_format)) { - that.processors(file_format)( - resource_url, - resource_url, - callback, - that); - } else { - console.log('no processor found for %s', file_format); - } - } - }); - } - }, + External.mkdirp(path.join(this.env.data_dir, 'cache'), 0755); + External.mkdirp(path.join(this.env.data_dir, 'temp'), 0755); - /** - * Get a processor, given a file's extension - * @param {String} extension the file's extension. - * @return {Function} processor function. - */ - processors: function(extension) { - return { - '.zip': this.unzip, - '.mss': this.plainfile, - '.shp': this.inplace, - '.geojson': this.plainfile, - '.kml': this.plainfile - }[extension]; - }, + if (/^https?:\/\//i.test(uri)) { + this.downloadFile(); + } else { + this.localFile(); + } +} - destinations: function(extension) { - return { - '.zip': this.unzip_dest, - '.mss': this.plainfile_dest, - '.shp': this.inplace_dest, - '.geojson': this.plainfile_dest, - '.kml': this.plainfile_dest - }[extension]; - }, +External.prototype.localFile = function() { + // Only treat files as local that don't have to be processed. + this.isLocal = !External.processors[this.format]; - encodings: function(extension) { - return { - '.zip': 'binary', - '.mss': 'utf-8', - '.shp': 'binary', - '.geojson': 'utf-8', - '.kml': 'utf-8' - }[extension]; - }, - - /** - * Get the final resting position of an external's directory - * @param {String} ext name of the external. - * @return {String} file path. - */ - pos: function(ext) { - return path.join(this.env.data_dir, netlib.safe64(ext)); - }, - - /** - * Get the temporary path of an external before processing - * @param {String} ext filename of the external. - * @return {String} file path. - */ - tmppos: function(ext) { - return path.join(this.env.data_dir, require('crypto') - .createHash('md5').update(ext).digest('hex')); - }, - - plainname: function(resource_url) { - return require('crypto') - .createHash('md5').update(resource_url).digest('hex') + - path.extname(resource_url); - - }, - - unzip_dest: function(resource_url, that) { - return that.locateShp(that.pos(resource_url)); - }, - - plainfile_dest: function(resource_url, that) { - return path.join(that.pos(resource_url), - that.plainname(resource_url)); - }, - - inplace_dest: function(resource_url, that) { - console.log(url.parse(resource_url)); - return resource_url; - }, - - /** - * Deal with a plain file, which is likely to be - * GeoJSON, KML, or one of the other OGR-supported formats, - * returning a Mapnik-usable filename - * - * @param {String} filename the place of the file on your system. - * @param {String} resource_url - * @param {Function} callback - */ - plainfile: function(filename, resource_url, callback, that) { - // TODO: possibly decide upon default extension - var extension = path.extname(resource_url); - if (extension !== '') { - // TODO: make sure dir doesn't exist - var destination = path.join(that.pos(resource_url), - that.plainname(resource_url)); - fs.mkdir(that.pos(resource_url), 0777, function(err) { - err && callback(err); - fs.rename( - filename, - destination, - function(err) { - err && callback(err); - callback(null, [resource_url, destination]); - } - ) - }); - } else { - throw Exception('Non-extended files cannot be processed'); - } - }, - - /** - * Deal with an inplace local file - * - * @param {String} filename the place of the file on your system. - * @param {String} resource_url - * @param {Function} callback - */ - inplace: function(filename, resource_url, callback, that) { - - callback(null, [resource_url, filename]); - }, - - locateShp: function(dir) { - try { - var unzipped = fs.readdirSync(dir); - var shp = _.detect(unzipped, - function(f) { - return path.extname(f).toLowerCase() == '.shp'; - } - ); - if (!shp) { - var dirs = _.select(unzipped, - function(f) { - return fs.statSync(path.join(dir, f)).isDirectory(); - } - ); - if (dirs) { - for (var i = 0, l = dirs.length; i < l; i++) { - var located = locateShp(path.join(dir, dirs[i])); - if (located) { - return located; - } - } - } - } else { - return path.join(dir, shp); - } - } catch (e) { - return false; - } - }, - - /** - * Recursively make directory tree - * - * @param {String} directory path - * @param {Number} mode - * @param {Function} callback - */ - mkdirs: function (p, mode, f) { - var that = this; - var cb = f || function () {}; - // if relative - if (p.charAt(0) != '/') { - // TODO if >= node 0.3.0 use path.resolve() ? - p = path.join(__dirname,p); - } - var ps = path.normalize(p).split('/'); - path.exists(p, function (exists) { - if (exists) cb(null); - else that.mkdirs(ps.slice(0,-1).join('/'), mode, function (err) { - if (err && err.errno != constants.EEXIST) cb(err) - else fs.mkdir(p, mode, cb); - }); - }); - }, - - /** - * Unzip a file and return a shapefile contained within it - * - * TODO: handle other files than shapefiles - * @param {String} filename the place of the shapefile on your system. - * @param {String} resource_url - * @param {Function} callback - */ - unzip: function(filename, resource_url, callback, that) { - console.log('unzipping download from ' + filename); - // https://github.com/springmeyer/node-zipfile - var zip = require('zipfile'); - try { - var zf = new zip.ZipFile(filename); - } catch (e) { - callback(e); - } - var dirname = that.pos(resource_url); - Step( - function() { - var group = this.group(); - zf.names.forEach(function(name) { - var uncompressed = path.join(dirname,name); - var g = group(); - that.mkdirs(dirname, 0755 , function(err) { - if (err && err.errno != constants.EEXIST) { - console.log('unzip failed'); - callback(err, [resource_url, false]); - } - if (path.extname(name)) { - var buffer = zf.readFileSync(name); - console.log('saving to: ' + uncompressed); - fd = fs.openSync(uncompressed,'w'); - fs.writeSync(fd, buffer, 0, buffer.length, null); - fs.closeSync(fd); - } - g(null, uncompressed); - }) - }); - }, - function(err, results) { - // TODO: simplify locateShp but also - // allow non-redownloading of shapefiles. - err && callback(err); - callback(null, [ - resource_url, - that.locateShp(dirname)]); - } - ); - } - }; + this.tempPath = path.join(this.env.local_data_dir, this.uri); + fs.stat(this.tempPath, this.processFile.bind(this)); }; +External.prototype.downloadFile = function() { + this.tempPath = path.join( + this.env.data_dir, 'temp', + crypto.createHash('md5').update(this.uri).digest('hex') + path.extname(this.uri)); + + fs.stat(this.path(), function(err, stats) { + if (err) { + // This file does not yet exist. Download it! + netlib.download( + this.uri, + this.tempPath, + External.encodings[this.format] || External.encodings['default'], + this.processFile.bind(this) + ); + } else { + this.callback(null, this); + } + }.bind(this)); +}; + +External.prototype.processFile = function(err) { + if (err) { + this.callback(err); + } else { + if (this.isLocal) { + this.callback(null, this); + } else { + (External.processors[this.format] || External.processors['default'])( + this.tempPath, + this.path(), + function(err) { + if (err) { + this.callback(err); + } else { + this.callback(null, this); + } + }.bind(this) + ); + } + } +}; + + +External.prototype.path = function() { + if (this.isLocal) { + return this.tempPath; + } + else { + return path.join( + this.env.data_dir, + 'cache', + (External.destinations[this.format] || External.destinations['default'])(this.uri) + ); + } +}; + +// https://gist.github.com/707661 +External.mkdirp = function mkdirP (p, mode, f) { + var cb = f || function () {}; + if (p.charAt(0) != '/') { cb('Relative path: ' + p); return } + + var ps = path.normalize(p).split('/'); + path.exists(p, function (exists) { + if (exists) cb(null); + else mkdirP(ps.slice(0,-1).join('/'), mode, function (err) { + if (err && err.errno != process.EEXIST) cb(err) + else { + fs.mkdir(p, mode, cb); + } + }); + }); +}; + + + +External.encodings = { + '.zip': 'binary', + '.shp': 'binary', + 'default': 'utf-8' +}; + +// Destinations are names in the data_dir/cache directory. +External.destinations = {}; +External.destinations['default'] = function(uri) { + return crypto.createHash('md5').update(uri).digest('hex') + path.extname(uri); +}; +External.destinations['.zip'] = function(uri) { + return crypto.createHash('md5').update(uri).digest('hex'); +}; + + +External.processors = {}; +External.processors['default'] = function(tempPath, destPath, callback) { + if (tempPath === destPath) { + callback(null); + } else { + fs.rename(tempPath, destPath, callback); + } +}; +External.processors['.zip'] = function(tempPath, destPath, callback) { + try { + var zf = new zip.ZipFile(tempPath); + // This is blocking, sequential and synchronous. + zf.names.forEach(function(name) { + var uncompressed = path.join(destPath, name); + External.mkdirp(path.dirname(uncompressed), 0755, function(err) { + if (err && err.errno != constants.EEXIST) { + throw "Couldn't create directory " + path.dirname(name); + } + else { + if (path.extname(name)) { + var buffer = zf.readFileSync(name); + console.log('saving to: ' + uncompressed); + fd = fs.openSync(uncompressed, 'w'); + fs.writeSync(fd, buffer, 0, buffer.length, null); + fs.closeSync(fd); + } + else { + throw 'UNIMPLEMENTED'; + } + } + }); + }); + callback(null); + } catch (err) { + console.log(err); + callback(err); + } +}; + + + module.exports = External; diff --git a/lib/mess/renderer.js b/lib/mess/renderer.js index eec59fe..2c9dccb 100644 --- a/lib/mess/renderer.js +++ b/lib/mess/renderer.js @@ -48,7 +48,7 @@ mess.Renderer = function Renderer(env) { * @param {Function} callback */ grab: function(uri, callback) { - new External(this.env).process(uri, callback); + new External(this.env, uri, callback); }, /** @@ -119,23 +119,21 @@ mess.Renderer = function Renderer(env) { var group = this.group(); m.Layer.forEach(function(l) { if (l.Datasource.file) { - that.grab(l.Datasource.file, group()); + var next = group(); + that.grab(l.Datasource.file, function(err, external) { + if (err) throw err; + // Extract the shapefile. + l.Datasource.file = external.path(); + next(); + }); + } else { + throw 'Layer does not have a datasource'; } }); - if (m.Layer.length == 0) group()(); + // Continue even if we don't have any layers. + group()(); }, - function(err, results) { - var result_map = to(results); - m.Layer = _.map(_.filter(m.Layer, - function(l) { - return l.Datasource.file && - result_map[l.Datasource.file]; - }), - function(l) { - l.Datasource.file = result_map[l.Datasource.file]; - return l; - } - ); + function(err) { callback(err, m); } ); @@ -152,20 +150,19 @@ mess.Renderer = function Renderer(env) { Step( function() { var group = this.group(); - m.Stylesheet.forEach(function(s) { + m.Stylesheet.forEach(function(s, k) { if (!s.id) { - that.grab(s, group()); + var next = group(); + that.grab(s, function(err, external) { + if (err) throw err; + m.Stylesheet[k] = external.path(); + next(); + }); } }); group()(); }, function(err, results) { - var result_map = to(results); - for (s in m.Stylesheet) { - if (!m.Stylesheet[s].id) { - m.Stylesheet[s] = result_map[m.Stylesheet[s]]; - } - } callback(err, m); } ); @@ -201,7 +198,7 @@ mess.Renderer = function Renderer(env) { group = this.group(); for (var i = 0, l = results.length; i < l; i++) { new mess.Parser(_.extend(_.extend({ - filename: s + filename: results[i][0] }, that.env), this.env)).parse(results[i][1], function(err, tree) { if (err) { diff --git a/test/rendering.js b/test/rendering.js new file mode 100644 index 0000000..f674eda --- /dev/null +++ b/test/rendering.js @@ -0,0 +1,37 @@ +var path = require('path'), + sys = require('sys'), + assert = require('assert'), + fs = require('fs'); + +var mess = require('mess'); +var tree = require('mess/tree'); +var helper = require('./support/helper'); + +helper.files('rendering', 'mml', function(file) { + exports['test rendering ' + file] = function(beforeExit) { + var success = false; + + helper.file(file, function(mml) { + new mess.Renderer({ + paths: [ path.dirname(file) ], + data_dir: path.join(__dirname, '../data'), + local_data_dir: path.join(__dirname, 'rendering'), + filename: file + }).render(mml, function (err, output) { + if (err) { + throw err; + } else { + var result = helper.resultFile(file); + helper.file(result, function(result) { + assert.equal(output, result); + success = true; + }); + } + }); + }); + + beforeExit(function() { + assert.ok(success, 'Rendering finished.'); + }); + } +}); diff --git a/test/rendering/complex_cascades.mml b/test/rendering/complex_cascades.mml new file mode 100644 index 0000000..c5e8dca --- /dev/null +++ b/test/rendering/complex_cascades.mml @@ -0,0 +1,31 @@ +{ + "srs": "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs", + "Stylesheet": [ + "complex_cascades.mss" + ], + + "Layer": [{ + "id": "world", + "name": "world", + "srs": "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs", + "Datasource": { + "file": "http://localhost/map.zip", + "type": "shape" + } + }, + { + "id": "countries", + "class": "new", + "name": "world", + "srs": "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs", + "Datasource": { + "file": "http://localhost/map.zip", + "type": "shape" + } + }], + "center": { + "lat": 47.687579167, + "lon": 5.6799316403973, + "zoom": 6 + } +} \ No newline at end of file diff --git a/test/rendering/complex_cascades.mss b/test/rendering/complex_cascades.mss new file mode 100644 index 0000000..13f6fa3 --- /dev/null +++ b/test/rendering/complex_cascades.mss @@ -0,0 +1,31 @@ +#world { + polygon-fill: #FFF; + line-color:#F00; + line-width: 0.5; +} + +#world[NAME='United States'] { + polygon-fill:#CCC; + [zoom > 6] { polygon-fill:#DDD; } + [zoom > 7] { polygon-fill:#999; } + [zoom > 5] { polygon-fill:#666; } +} + +#world[NAME='Canada'], +#countries { + polygon-fill: #eee; + line-color: #ccc; + line-width: 1; + + .new { + polygon-fill: #CCC; + } + + .new[zoom > 5] { + line-width:0.5; + + [NAME='United States'] { + polygon-fill:#AFC; + } + } +} diff --git a/test/rendering/complex_cascades.result b/test/rendering/complex_cascades.result new file mode 100644 index 0000000..d375e5c --- /dev/null +++ b/test/rendering/complex_cascades.result @@ -0,0 +1,89 @@ + + + + + + + + + world-line/__default__ + world-polygon/__default__ + + /Users/kkaefer/Code/devseed/mess.js/data/cache/af08a2b537079e720a9293424a319a1e + shape + + + + + + + countries-line/__default__ + countries-polygon/__default__ + + /Users/kkaefer/Code/devseed/mess.js/data/cache/af08a2b537079e720a9293424a319a1e + shape + + + + \ No newline at end of file