refactor external to make tests working

lib/mess/external.js

@@ -2,9 +2,10 @@ var fs = require('fs'),
     netlib = require('./netlib'),
     url = require('url'),
     path = require('path'),
-    Step = require('step'),
+    crypto = require('crypto'),
-    _ = require('underscore')._,
+    assert = require('assert'),
-    spawn = require('child_process').spawn;
+    zip = require('zipfile'),
+    Step = require('step');
 
 // node compatibility for mkdirs below
 var constants = {};
@@ -13,274 +14,144 @@ if (!process.EEXIST >= 1)
 else
     constants.EEXIST = process.EEXIST;
 
-/**
- * TODO: use node-minizip instead of shelling
- * TODO: include or don't use underscore
- */
 
-var External = function External(env) {
+function External(env, uri, callback) {
-    var env = env;
+    assert.ok(env.data_dir);
+    assert.ok(env.local_data_dir);
 
-    return {
+    this.env = env;
-        env: env,
+    this.callback = callback;
+    this.uri = uri;
+    this.format = path.extname(uri).toLowerCase();
 
-        /**
+    External.mkdirp(path.join(this.env.data_dir, 'cache'), 0755);
-         * Download an external, process it, and return the usable filepath for
+    External.mkdirp(path.join(this.env.data_dir, 'temp'), 0755);
-         * Mapnik
-         * @param {String} resource_url the URI of the datasource from a mapfile.
-         * @param {Function} callback passed into processor function after
-         *     localizing.
-         */
-        process: function(resource_url, callback) {
-            var file_format = path.extname(resource_url).toLowerCase(),
-                that = this;
 
-            if (url.parse(resource_url).protocol == 'http:') {
+    if (/^https?:\/\//i.test(uri)) {
-                fs.stat(this.tmppos(resource_url), function(err, stats) {
+        this.downloadFile();
-                    if (stats && stats.isFile()) {
-                        callback(null, [
-                            resource_url,
-                            that.destinations(file_format)(resource_url, that)
-                            ]);
     } else {
+        this.localFile();
+    }
+}
+
+External.prototype.localFile = function() {
+    // Only treat files as local that don't have to be processed.
+    this.isLocal = !External.processors[this.format];
+
+    this.tempPath = path.join(this.env.local_data_dir, this.uri);
+    fs.stat(this.tempPath, this.processFile.bind(this));
+};
+
+External.prototype.downloadFile = function() {
+    this.tempPath = path.join(
+        this.env.data_dir, 'temp',
+        crypto.createHash('md5').update(this.uri).digest('hex') + path.extname(this.uri));
+
+    fs.stat(this.path(), function(err, stats) {
+        if (err) {
+            // This file does not yet exist. Download it!
             netlib.download(
-                            resource_url,
+                this.uri,
-                            that.tmppos(resource_url),
+                this.tempPath,
-                            that.encodings[file_format],
+                External.encodings[this.format] || External.encodings['default'],
-                            function(err, url, filename) {
+                this.processFile.bind(this)
-                                if (that.processors(file_format)) {
+            );
-                                    that.processors(file_format)(
-                                        filename,
-                                        resource_url,
-                                        callback,
-                                        that);
         } else {
-                                    console.log('no processor found for %s', file_format);
+            this.callback(null, this);
         }
-                            });
+    }.bind(this));
-                    }
+};
-                });
+
+External.prototype.processFile = function(err) {
+    if (err) {
+        this.callback(err);
     } else {
-                // is this a fully-qualified URL?
+        if (this.isLocal) {
-                fs.stat(resource_url, function(err, stat) {
+            this.callback(null, this);
-                    if (!err && stat.isFile()) {
-                        if (that.processors(file_format)) {
-                            that.processors(file_format)(
-                                resource_url,
-                                resource_url,
-                                callback,
-                                that);
         } else {
-                            console.log('no processor found for %s', file_format);
+            (External.processors[this.format] || External.processors['default'])(
-                        }
+                this.tempPath,
-                    }
+                this.path(),
-                });
-            }
-        },
-
-        /**
-         * Get a processor, given a file's extension
-         * @param {String} extension the file's extension.
-         * @return {Function} processor function.
-         */
-        processors: function(extension) {
-            return {
-                '.zip': this.unzip,
-                '.mss': this.plainfile,
-                '.shp': this.inplace,
-                '.geojson': this.plainfile,
-                '.kml': this.plainfile
-            }[extension];
-        },
-
-        destinations: function(extension) {
-            return {
-                '.zip': this.unzip_dest,
-                '.mss': this.plainfile_dest,
-                '.shp': this.inplace_dest,
-                '.geojson': this.plainfile_dest,
-                '.kml': this.plainfile_dest
-            }[extension];
-        },
-
-        encodings: function(extension) {
-            return {
-                '.zip': 'binary',
-                '.mss': 'utf-8',
-                '.shp': 'binary',
-                '.geojson': 'utf-8',
-                '.kml': 'utf-8'
-            }[extension];
-        },
-
-        /**
-         * Get the final resting position of an external's directory
-         * @param {String} ext name of the external.
-         * @return {String} file path.
-         */
-        pos: function(ext) {
-            return path.join(this.env.data_dir, netlib.safe64(ext));
-        },
-
-        /**
-         * Get the temporary path of an external before processing
-         * @param {String} ext filename of the external.
-         * @return {String} file path.
-         */
-        tmppos: function(ext) {
-            return path.join(this.env.data_dir, require('crypto')
-                .createHash('md5').update(ext).digest('hex'));
-        },
-
-        plainname: function(resource_url) {
-            return require('crypto')
-                .createHash('md5').update(resource_url).digest('hex') +
-                path.extname(resource_url);
-
-        },
-
-        unzip_dest: function(resource_url, that) {
-            return that.locateShp(that.pos(resource_url));
-        },
-
-        plainfile_dest: function(resource_url, that) {
-            return path.join(that.pos(resource_url),
-                that.plainname(resource_url));
-        },
-
-        inplace_dest: function(resource_url, that) {
-            console.log(url.parse(resource_url));
-            return resource_url;
-        },
-
-        /**
-         * Deal with a plain file, which is likely to be
-         * GeoJSON, KML, or one of the other OGR-supported formats,
-         * returning a Mapnik-usable filename
-         *
-         * @param {String} filename the place of the file on your system.
-         * @param {String} resource_url
-         * @param {Function} callback
-         */
-        plainfile: function(filename, resource_url, callback, that) {
-            // TODO: possibly decide upon default extension
-            var extension = path.extname(resource_url);
-            if (extension !== '') {
-                // TODO: make sure dir doesn't exist
-                var destination = path.join(that.pos(resource_url),
-                    that.plainname(resource_url));
-                fs.mkdir(that.pos(resource_url), 0777, function(err) {
-                    err && callback(err);
-                    fs.rename(
-                        filename,
-                        destination,
                 function(err) {
-                            err && callback(err);
+                    if (err) {
-                            callback(null, [resource_url, destination]);
+                        this.callback(err);
-                        }
-                    )
-                });
                     } else {
-                throw Exception('Non-extended files cannot be processed');
+                        this.callback(null, this);
-            }
-        },
-
-        /**
-         * Deal with an inplace local file
-         *
-         * @param {String} filename the place of the file on your system.
-         * @param {String} resource_url
-         * @param {Function} callback
-         */
-        inplace: function(filename, resource_url, callback, that) {
-
-            callback(null, [resource_url, filename]);
-        },
-
-        locateShp: function(dir) {
-            try {
-                var unzipped = fs.readdirSync(dir);
-                var shp = _.detect(unzipped,
-                    function(f) {
-                        return path.extname(f).toLowerCase() == '.shp';
                     }
+                }.bind(this)
             );
-                if (!shp) {
-                    var dirs = _.select(unzipped,
-                        function(f) {
-                            return fs.statSync(path.join(dir, f)).isDirectory();
-                        }
-                    );
-                    if (dirs) {
-                        for (var i = 0, l = dirs.length; i < l; i++) {
-                            var located = locateShp(path.join(dir, dirs[i]));
-                            if (located) {
-                                return located;
         }
     }
-                    }
+};
-                } else {
-                    return path.join(dir, shp);
-                }
-            } catch (e) {
-                return false;
-            }
-        },
 
-        /**
+
-         * Recursively make directory tree
+External.prototype.path = function() {
-         *
+    if (this.isLocal) {
-         * @param {String} directory path
+        return this.tempPath;
-         * @param {Number} mode
+    }
-         * @param {Function} callback
+    else {
-         */
+        return path.join(
-        mkdirs: function (p, mode, f) {
+            this.env.data_dir,
-            var that = this;
+            'cache',
+            (External.destinations[this.format] || External.destinations['default'])(this.uri)
+        );
+    }
+};
+
+// https://gist.github.com/707661
+External.mkdirp = function mkdirP (p, mode, f) {
     var cb = f || function () {};
-            // if relative
+    if (p.charAt(0) != '/') { cb('Relative path: ' + p); return }
-            if (p.charAt(0) != '/') {
+
-                // TODO if >= node 0.3.0 use path.resolve() ?
-                p = path.join(__dirname,p);
-            }
     var ps = path.normalize(p).split('/');
     path.exists(p, function (exists) {
         if (exists) cb(null);
-                else that.mkdirs(ps.slice(0,-1).join('/'), mode, function (err) {
+        else mkdirP(ps.slice(0,-1).join('/'), mode, function (err) {
-                    if (err && err.errno != constants.EEXIST) cb(err)
+            if (err && err.errno != process.EEXIST) cb(err)
-                    else fs.mkdir(p, mode, cb);
+            else {
+                fs.mkdir(p, mode, cb);
+            }
         });
     });
-        },
+};
 
-        /**
+
-         * Unzip a file and return a shapefile contained within it
+
-         *
+External.encodings = {
-         * TODO: handle other files than shapefiles
+    '.zip': 'binary',
-         * @param {String} filename the place of the shapefile on your system.
+    '.shp': 'binary',
-         * @param {String} resource_url
+    'default': 'utf-8'
-         * @param {Function} callback
+};
-         */
+
-        unzip: function(filename, resource_url, callback, that) {
+// Destinations are names in the data_dir/cache directory.
-            console.log('unzipping download from ' + filename);
+External.destinations = {};
-            // https://github.com/springmeyer/node-zipfile
+External.destinations['default'] = function(uri) {
-            var zip = require('zipfile');
+    return crypto.createHash('md5').update(uri).digest('hex') + path.extname(uri);
+};
+External.destinations['.zip'] = function(uri) {
+    return crypto.createHash('md5').update(uri).digest('hex');
+};
+
+
+External.processors = {};
+External.processors['default'] = function(tempPath, destPath, callback) {
+    if (tempPath === destPath) {
+        callback(null);
+    } else {
+        fs.rename(tempPath, destPath, callback);
+    }
+};
+External.processors['.zip'] = function(tempPath, destPath, callback) {
     try {
-                var zf = new zip.ZipFile(filename);
+        var zf = new zip.ZipFile(tempPath);
-            } catch (e) {
+        // This is blocking, sequential and synchronous.
-                callback(e);
-            }
-            var dirname = that.pos(resource_url);
-            Step(
-                function() {
-                    var group = this.group();
         zf.names.forEach(function(name) {
-                        var uncompressed = path.join(dirname,name);   
+            var uncompressed = path.join(destPath, name);
-                        var g = group();             
+            External.mkdirp(path.dirname(uncompressed), 0755, function(err) {
-                        that.mkdirs(dirname, 0755 , function(err) {
                 if (err && err.errno != constants.EEXIST) {
-                                console.log('unzip failed');
+                    throw "Couldn't create directory " + path.dirname(name);
-                                callback(err, [resource_url, false]);
                 }
+                else {
                     if (path.extname(name)) {
                         var buffer = zf.readFileSync(name);
                         console.log('saving to: ' + uncompressed);
@@ -288,21 +159,19 @@ var External = function External(env) {
                         fs.writeSync(fd, buffer, 0, buffer.length, null);
                         fs.closeSync(fd);
                     }
-                            g(null, uncompressed);
+                    else {
-                        })
+                        throw 'UNIMPLEMENTED';
+                    }
+                }
             });
-                },
+        });
-                function(err, results) {
+        callback(null);
-                    // TODO: simplify locateShp but also
+    } catch (err) {
-                    // allow non-redownloading of shapefiles.
+        console.log(err);
-                    err && callback(err);
+        callback(err);
-                    callback(null, [
-                        resource_url,
-                        that.locateShp(dirname)]);
     }
-            );
-        }
-    };
 };
 
+
+
 module.exports = External;

lib/mess/renderer.js

@@ -48,7 +48,7 @@ mess.Renderer = function Renderer(env) {
          * @param {Function} callback
          */
         grab: function(uri, callback) {
-            new External(this.env).process(uri, callback);
+            new External(this.env, uri, callback);
         },
 
         /**
@@ -119,23 +119,21 @@ mess.Renderer = function Renderer(env) {
                     var group = this.group();
                     m.Layer.forEach(function(l) {
                         if (l.Datasource.file) {
-                            that.grab(l.Datasource.file, group());
+                            var next = group();
+                            that.grab(l.Datasource.file, function(err, external) {
+                                if (err) throw err;
+                                // Extract the shapefile.
+                                l.Datasource.file = external.path();
+                                next();
+                            });
+                        } else {
+                            throw 'Layer does not have a datasource';
                         }
                     });
-                    if (m.Layer.length == 0) group()();
+                    // Continue even if we don't have any layers.
+                    group()();
                 },
-                function(err, results) {
+                function(err) {
-                    var result_map = to(results);
-                    m.Layer = _.map(_.filter(m.Layer,
-                        function(l) {
-                            return l.Datasource.file &&
-                                result_map[l.Datasource.file];
-                        }),
-                        function(l) {
-                            l.Datasource.file = result_map[l.Datasource.file];
-                            return l;
-                        }
-                    );
                     callback(err, m);
                 }
             );
@@ -152,20 +150,19 @@ mess.Renderer = function Renderer(env) {
             Step(
                 function() {
                     var group = this.group();
-                    m.Stylesheet.forEach(function(s) {
+                    m.Stylesheet.forEach(function(s, k) {
                         if (!s.id) {
-                            that.grab(s, group());
+                            var next = group();
+                            that.grab(s, function(err, external) {
+                                if (err) throw err;
+                                m.Stylesheet[k] = external.path();
+                                next();
+                            });
                         }
                     });
                     group()();
                 },
                 function(err, results) {
-                    var result_map = to(results);
-                    for (s in m.Stylesheet) {
-                        if (!m.Stylesheet[s].id) {
-                            m.Stylesheet[s] = result_map[m.Stylesheet[s]];
-                        }
-                    }
                     callback(err, m);
                 }
             );
@@ -201,7 +198,7 @@ mess.Renderer = function Renderer(env) {
                         group = this.group();
                     for (var i = 0, l = results.length; i < l; i++) {
                         new mess.Parser(_.extend(_.extend({
-                            filename: s
+                            filename: results[i][0]
                         }, that.env), this.env)).parse(results[i][1],
                             function(err, tree) {
                             if (err) {

test/rendering.js

@@ -0,0 +1,37 @@
+var path = require('path'),
+    sys = require('sys'),
+    assert = require('assert'),
+    fs = require('fs');
+
+var mess = require('mess');
+var tree = require('mess/tree');
+var helper = require('./support/helper');
+
+helper.files('rendering', 'mml', function(file) {
+    exports['test rendering ' + file] = function(beforeExit) {
+        var success = false;
+
+        helper.file(file, function(mml) {
+            new mess.Renderer({
+                paths: [ path.dirname(file) ],
+                data_dir: path.join(__dirname, '../data'),
+                local_data_dir: path.join(__dirname, 'rendering'),
+                filename: file
+            }).render(mml, function (err, output) {
+                if (err) {
+                    throw err;
+                } else {
+                    var result = helper.resultFile(file);
+                    helper.file(result, function(result) {
+                        assert.equal(output, result);
+                        success = true;
+                    });
+                }
+            });
+        });
+
+        beforeExit(function() {
+            assert.ok(success, 'Rendering finished.');
+        });
+    }
+});

test/rendering/complex_cascades.mml

@@ -0,0 +1,31 @@
+{
+    "srs": "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs",
+    "Stylesheet": [
+        "complex_cascades.mss"
+    ],
+    
+    "Layer": [{
+        "id": "world",
+        "name": "world",
+        "srs": "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs",
+        "Datasource": {
+            "file": "http://localhost/map.zip",
+            "type": "shape"
+        }
+    },
+    {
+        "id": "countries",
+        "class": "new",
+        "name": "world",
+        "srs": "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs",
+        "Datasource": {
+            "file": "http://localhost/map.zip",
+            "type": "shape"
+        }
+    }],
+    "center": {
+        "lat": 47.687579167,
+        "lon": 5.6799316403973,
+        "zoom": 6
+    }
+}

test/rendering/complex_cascades.mss

@@ -0,0 +1,31 @@
+#world {
+  polygon-fill: #FFF;
+  line-color:#F00;
+  line-width: 0.5;
+}
+
+#world[NAME='United States'] {
+  polygon-fill:#CCC;
+  [zoom > 6] { polygon-fill:#DDD; }
+  [zoom > 7] { polygon-fill:#999; }
+  [zoom > 5] { polygon-fill:#666; }
+}
+
+#world[NAME='Canada'],
+#countries {
+  polygon-fill: #eee;
+  line-color: #ccc;
+  line-width: 1;
+
+  .new {
+    polygon-fill: #CCC;
+  }
+
+  .new[zoom > 5] {
+    line-width:0.5;
+
+    [NAME='United States'] {
+      polygon-fill:#AFC;
+    }
+  }
+}

test/rendering/complex_cascades.result

@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE Map[]>
+<Map  srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs">
+
+
+<Style name="world-polygon/__default__">
+  <Rule>
+    <MaxScaleDenominator>12500000</MaxScaleDenominator>
+    <Filter>([NAME] = 'United States')</Filter>
+    <PolygonSymbolizer fill="#666666"/>
+  </Rule>
+  <Rule>
+    <MinScaleDenominator>12500000</MinScaleDenominator>
+    <Filter>([NAME] = 'Canada') and ([NAME] != 'United States')</Filter>
+    <PolygonSymbolizer fill="#eeeeee"/>
+  </Rule>
+  <Rule>
+    <MinScaleDenominator>12500000</MinScaleDenominator>
+    <Filter>([NAME] = 'United States') and ([NAME] != 'United States') and ([NAME] != 'Canada')</Filter>
+    <PolygonSymbolizer fill="#cccccc"/>
+  </Rule>
+  <Rule>
+    <MinScaleDenominator>12500000</MinScaleDenominator>
+    <Filter>([NAME] != 'United States') and ([NAME] != 'Canada') and ([NAME] != 'United States')</Filter>
+    <PolygonSymbolizer fill="#ffffff"/>
+  </Rule>
+</Style>
+<Style name="world-line/__default__">
+  <Rule>
+    
+    <Filter>([NAME] = 'Canada')</Filter>
+    <LineSymbolizer stroke-width="1"
+      stroke="#cccccc"/>
+  </Rule>
+  <Rule>
+    
+    <Filter>([NAME] != 'Canada')</Filter>
+    <LineSymbolizer stroke-width="0.5"
+      stroke="#ff0000"/>
+  </Rule>
+</Style>
+<Layer
+   id="world"
+   name="world"
+   srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs">
+    <StyleName>world-line/__default__</StyleName>
+    <StyleName>world-polygon/__default__</StyleName>
+    <Datasource>
+       <Parameter name="file">/Users/kkaefer/Code/devseed/mess.js/data/cache/af08a2b537079e720a9293424a319a1e</Parameter>
+       <Parameter name="type">shape</Parameter>
+    </Datasource>
+  </Layer>
+
+<Style name="countries-polygon/__default__">
+  <Rule>
+    <MaxScaleDenominator>12500000</MaxScaleDenominator>
+    <Filter>([NAME] = 'United States')</Filter>
+    <PolygonSymbolizer fill="#aaffcc"/>
+  </Rule>
+  <Rule>
+    <MinScaleDenominator>12500000</MinScaleDenominator>
+    <Filter>([NAME] != 'United States')</Filter>
+    <PolygonSymbolizer fill="#cccccc"/>
+  </Rule>
+</Style>
+<Style name="countries-line/__default__">
+  <Rule>
+    <MaxScaleDenominator>12500000</MaxScaleDenominator>
+    <LineSymbolizer stroke-width="0.5"/>
+  </Rule>
+  <Rule>
+    <MinScaleDenominator>12500000</MinScaleDenominator>
+    <LineSymbolizer stroke-width="1"
+      stroke="#cccccc"/>
+  </Rule>
+</Style>
+<Layer
+   id="countries"
+   name="world"
+   srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs">
+    <StyleName>countries-line/__default__</StyleName>
+    <StyleName>countries-polygon/__default__</StyleName>
+    <Datasource>
+       <Parameter name="file">/Users/kkaefer/Code/devseed/mess.js/data/cache/af08a2b537079e720a9293424a319a1e</Parameter>
+       <Parameter name="type">shape</Parameter>
+    </Datasource>
+  </Layer>
+
+</Map>