Adding rendering machinery

bin/messc

@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+
+var path = require('path'),
+    fs = require('fs'),
+    sys = require('sys');
+
+require.paths.unshift(path.join(__dirname, '..', 'lib'));
+
+var less = require('less');
+var args = process.argv.slice(1);
+var options = {
+    compress: false,
+    optimization: 1,
+    silent: false
+};
+
+args = args.filter(function (arg) {
+    var match;
+
+    if (match = arg.match(/^--?([a-z][0-9a-z-]*)$/i)) { arg = match[1] }
+    else { return arg }
+
+    switch (arg) {
+        case 'v':
+        case 'version':
+            sys.puts("lessc " + less.version.join('.') + " (LESS Compiler) [JavaScript]");
+            process.exit(0);
+        case 'verbose':
+            options.verbose = true;
+            break;
+        case 's':
+        case 'silent':
+            options.silent = true;
+            break;
+        case 'h':
+        case 'help':
+            sys.puts("usage: lessc source [destination]");
+            process.exit(0);
+        case 'x':
+        case 'compress':
+            options.compress = true;
+            break;
+        case 'O0': options.optimization = 0; break;
+        case 'O1': options.optimization = 1; break;
+        case 'O2': options.optimization = 2; break;
+    }
+});
+
+var input = args[1];
+if (input && input[0] != '/') {
+    input = path.join(process.cwd(), input);
+}
+var output = args[2];
+if (output && output[0] != '/') {
+    output = path.join(process.cwd(), output);
+}
+
+var css, fd, tree;
+
+if (! input) {
+    sys.puts("lessc: no input files");
+    process.exit(1);
+}
+
+fs.readFile(input, 'utf-8', function (e, data) {
+    if (e) {
+        sys.puts("lessc: " + e.message);
+        process.exit(1);
+    }
+
+    new(less.Renderer)({
+        paths: [path.dirname(input)],
+        optimization: options.optimization,
+        filename: input
+    }).render(data, function (err, tree) {
+        if (err) {
+            less.writeError(err, options);
+            process.exit(1);
+        } else {
+            try {
+                css = tree.toCSS({ compress: options.compress });
+                if (output) {
+                    fd = fs.openSync(output, "w");
+                    fs.writeSync(fd, css, 0, "utf8");
+                } else {
+                    sys.print(css);
+                }
+            } catch (e) {
+                less.writeError(e, options);
+                process.exit(2);
+            }
+        }
+    });
+});

lib/less/external.js

@@ -0,0 +1,160 @@
+var fs = require('fs'),
+    netlib = require('./netlib'),
+    url = require('url'),
+    path = require('path'),
+    _ = require('underscore')._,
+    spawn = require('child_process').spawn;
+
+/**
+ * TODO: use node-minizip instead of shelling
+ * TODO: include or don't use underscore
+ */
+
+var External = function External(env) {
+    this.env = env;
+
+    return {
+        /**
+         * Get a processor, given a file's extension
+         * @param {String} extension the file's extension.
+         * @return {Function} processor function.
+         */
+        processors: function(extension) {
+            return {
+                '.zip': this.unzip,
+                '.mss': this.plainfile,
+                '.geojson': this.plainfile,
+                '.kml': this.plainfile
+            }[extension];
+        },
+
+        /**
+         * Get the final resting position of an external's directory
+         * @param {String} ext name of the external.
+         * @return {String} file path.
+         */
+        pos: function(ext) {
+            return path.join(env.data_dir, netlib.safe64(ext));
+        },
+
+        /**
+         * Get the temporary path of an external before processing
+         * @param {String} ext filename of the external.
+         * @return {String} file path.
+         */
+        tmppos: function(ext) {
+            return path.join(env.data_dir, require('crypto')
+                .createHash('md5').update(ext).digest('hex'));
+        },
+
+        plainname: function(resource_url) {
+            return require('crypto')
+                .createHash('md5').update(resource_url).digest('hex') +
+                path.extname(resource_url);
+
+        },
+
+        /**
+         * Download an external, process it, and return the usable filepath for
+         * Mapnik
+         * @param {String} resource_url the URI of the datasource from a mapfile.
+         * @param {Function} callback passed into processor function after
+         *     localizing.
+         */
+        process: function(resource_url, callback) {
+            var file_format = path.extname(resource_url),
+                that = this;
+            netlib.download(resource_url, this.tmppos(resource_url),
+                function(err, url, filename) {
+                if (that.processors(file_format)) {
+                    that.processors(file_format)(
+                        filename,
+                        resource_url,
+                        callback,
+                        that);
+                } else {
+                    console.log('no processor found for %s', file_format);
+                }
+            });
+
+        },
+
+        /**
+         * Deal with a plain file, which is likely to be
+         * GeoJSON, KML, or one of the other OGR-supported formats,
+         * returning a Mapnik-usable filename
+         *
+         * @param {String} filename the place of the file on your system.
+         * @param {String} resource_url
+         * @param {Function} callback
+         */
+        plainfile: function(filename, resource_url, callback, that) {
+            // TODO: possibly decide upon default extension
+            var extension = path.extname(resource_url);
+            if (extension !== '') {
+                // TODO: make sure dir doesn't exist
+                var destination = path.join(that.pos(resource_url),
+                    that.plainname(resource_url));
+                fs.mkdirSync(that.pos(resource_url), 0777);
+                fs.renameSync(
+                    filename,
+                    destination);
+                return destination;
+            } else {
+                throw Exception('Non-extended files cannot be processed');
+            }
+        },
+
+        /**
+         * Unzip a file and return a shapefile contained within it
+         *
+         * TODO: handle other files than shapefiles
+         * @param {String} filename the place of the shapefile on your system.
+         * @param {String} resource_url
+         * @param {Function} callback
+         */
+        unzip: function(filename, resource_url, callback, that) {
+            // regrettably complex because zip library isn't written for
+            // node yet.
+            var locateShp = function(dir) {
+                var unzipped = fs.readdirSync(dir);
+                var shp = _.detect(unzipped,
+                    function(f) {
+                        return path.extname(f) == '.shp';
+                    }
+                );
+                if (!shp) {
+                    var dirs = _.select(unzipped,
+                        function(f) {
+                            return fs.statSync(path.join(dir, f)).isDirectory();
+                        }
+                    );
+                    if (dirs) {
+                        for (var i = 0, l = dirs.length; i < l; i++) {
+                            var located = locateShp(path.join(dir, dirs[i]));
+                            if (located) {
+                                return located;
+                            }
+                        }
+                    }
+                }
+                else {
+                    return dir + '/' + shp;
+                }
+            };
+
+            spawn('unzip', [filename, '-d', that.pos(resource_url)])
+                .on('exit', function(code) {
+                if (code > 0) {
+                    console.log('Unzip returned a code of %d', code);
+                } else {
+                    // TODO; eliminate locality of reference
+                    var shpfile = locateShp(that.pos(resource_url));
+                    callback(null, [resource_url, shpfile]);
+                }
+            });
+        }
+    }
+};
+
+module.exports = External;

lib/less/index.js

@@ -7,6 +7,7 @@ require.paths.unshift(path.join(__dirname, '..'));
 var less = {
     version: [1, 0, 40],
     Parser: require('less/parser').Parser,
+    Renderer: require('less/renderer').Renderer,
     importer: require('less/parser').importer,
     tree: require('less/tree'),
     reference: JSON.parse(fs.readFileSync(

lib/less/netlib.js

@@ -0,0 +1,126 @@
+var fs = require('fs'),
+    http = require('http'),
+    url = require('url');
+
+/**
+ * A library of net-interaction functions - this could be simplified
+ */
+
+module.exports = {
+    /**
+     * Download a file to the disk and return the downloaded
+     * filename and its data
+     *
+     * @param {String} file_url the URI of the file.
+     * @param {String} filename the filename on the system.
+     * @param {Function} callback to call after finishing the download and
+     *   run with arguments [err, filename, data].
+     */
+    downloadAndGet: function(file_url, filename, callback) {
+        var file_url = url.parse(file_url);
+        var c = http.createClient(file_url.port || 80, file_url.hostname);
+        var request = c.request('GET', file_url.pathname + '?' + (file_url.query || ''), {
+            host: file_url.hostname
+        });
+        request.end();
+
+        var data = '';
+        var f = fs.createWriteStream(filename);
+        request.on('response', function(response) {
+            response.on('data', function(chunk) {
+                data += chunk;
+                f.write(chunk);
+            });
+            response.on('end', function() {
+                f.destroy();
+                callback(null, filename, data);
+            });
+            response.on('error', function(err) {
+                console.log('error downloading file');
+                callback(err, null);
+            });
+        });
+    },
+
+    /**
+     * Download a file and return data
+     *
+     * @param {String} file_url the URI of the file.
+     * @param {String} filename the filename on the system.
+     * @param {Function} callback to call after finishing the download and
+     *   run with arguments [err, filename, data].
+     */
+    get: function(file_url, filename, callback) {
+        var file_url = url.parse(file_url);
+        var c = http.createClient(file_url.port || 80, file_url.hostname);
+        // TODO: more robust method for getting things with params?
+        console.log(file_url.pathname + '?' + file_url.query);
+        var request = c.request('GET', file_url.pathname + '?' + file_url.query, {
+            host: file_url.host,
+            query: file_url.query
+        });
+        request.end();
+
+        var data = '';
+        request.on('response', function(response) {
+            response.on('data', function(chunk) {
+                data += chunk;
+            });
+            response.on('end', function() {
+                callback(null, filename, data);
+            });
+            response.on('error', function(err) {
+                console.log('error downloading file');
+                callback(err, null);
+            });
+        });
+    },
+
+    /**
+     * Download a file
+     *
+     * @param {String} file_url the URI of the file.
+     * @param {String} filename the filename on the system.
+     * @param {Function} callback to call after finishing the download and
+     *   run with arguments [err, filename, data].
+     */
+    download: function(file_url_raw, filename, callback) {
+        var file_url = url.parse(file_url_raw);
+        var c = http.createClient(file_url.port || 80, file_url.hostname);
+        var request = c.request('GET', file_url.pathname + '?' + file_url.query, {
+            host: file_url.hostname
+        });
+        request.end();
+
+        console.log('Downloading\n\t%s', file_url.hostname);
+        var f = fs.createWriteStream(filename);
+        request.on('response', function(response) {
+            response.on('data', function(chunk) {
+                f.write(chunk);
+            });
+            response.on('end', function() {
+                f.destroy();
+                console.log('Download finished');
+                callback(null, file_url_raw, filename);
+            });
+            response.on('error', function(err) {
+                console.log('Error downloading file');
+                callback(err, null);
+            });
+        });
+    },
+
+    /**
+     * Encode a string as base64
+     *
+     * TODO: actually make safe64 by chunking
+     *
+     * @param {String} s the string to be encoded.
+     * @return {String} base64 encoded string.
+     */
+    safe64: function(s) {
+        var b = new Buffer(s, 'utf-8');
+        return b.toString('base64');
+    }
+};
+

lib/less/step.js

@@ -0,0 +1,154 @@
+/*
+Copyright (c) 2010 Tim Caswell <tim@creationix.com>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+*/
+
+// Inspired by http://github.com/willconant/flow-js, but reimplemented and
+// modified to fit my taste and the node.JS error handling system.
+function Step() {
+  var steps = Array.prototype.slice.call(arguments),
+      counter, results, lock;
+
+  // Define the main callback that's given as `this` to the steps.
+  function next() {
+
+    // Check if there are no steps left
+    if (steps.length === 0) {
+      // Throw uncaught errors
+      if (arguments[0]) {
+        throw arguments[0];
+      }
+      return;
+    }
+
+    // Get the next step to execute
+    var fn = steps.shift();
+    counter = 0;
+    results = [];
+
+    // Run the step in a try..catch block so exceptions don't get out of hand.
+    try {
+      lock = true;
+      var result = fn.apply(next, arguments);
+    } catch (e) {
+      // Pass any exceptions on through the next callback
+      next(e);
+    }
+
+
+    // If a syncronous return is used, pass it to the callback
+    if (result !== undefined) {
+      next(undefined, result);
+    }
+    lock = false;
+  }
+
+  // Add a special callback generator `this.parallel()` that groups stuff.
+  next.parallel = function () {
+    var i = counter;
+    counter++;
+    function check() {
+      counter--;
+      if (counter === 0) {
+        // When they're all done, call the callback
+        next.apply(null, results);
+      }
+    }
+    return function () {
+      // Compress the error from any result to the first argument
+      if (arguments[0]) {
+        results[0] = arguments[0];
+      }
+      // Send the other results as arguments
+      results[i + 1] = arguments[1];
+      if (lock) {
+        process.nextTick(check);
+        return
+      }
+      check();
+    };
+  };
+
+  // Generates a callback generator for grouped results
+  next.group = function () {
+    var localCallback = next.parallel();
+    var counter = 0;
+    var result = [];
+    var error = undefined;
+    // Generates a callback for the group
+    return function () {
+      var i = counter;
+      counter++;
+      function check() {
+        counter--;
+        if (counter === 0) {
+          // When they're all done, call the callback
+          localCallback(error, result);
+        }
+      }
+      return function () {
+        // Compress the error from any result to the first argument
+        if (arguments[0]) {
+          error = arguments[0];
+        }
+        // Send the other results as arguments
+        result[i] = arguments[1];
+        if (lock) {
+          process.nextTick(check);
+          return
+        }
+        check();
+      }
+
+    }
+  };
+
+  // Start the engine an pass nothing to the first step.
+  next([]);
+}
+
+// Tack on leading and tailing steps for input and output and return
+// the whole thing as a function.  Basically turns step calls into function
+// factories.
+Step.fn = function StepFn() {
+  var steps = Array.prototype.slice.call(arguments);
+  return function () {
+    var args = Array.prototype.slice.call(arguments);
+
+    // Insert a first step that primes the data stream
+    var toRun = [function () {
+      this.apply(null, args);
+    }].concat(steps);
+
+    // If the last arg is a function add it as a last step
+    if (typeof args[args.length-1] === 'function') {
+      toRun.push(args.pop());
+    }
+
+
+    Step.apply(null, toRun);
+  }
+}
+
+
+// Hook into commonJS module systems
+if (typeof module !== 'undefined' && "exports" in module) {
+  module.exports = Step;
+}