var fs = require('fs'), netlib = require('./netlib'), url = require('url'), sys = require('sys'), EventEmitter = require('events').EventEmitter, path = require('path'), crypto = require('crypto'), assert = require('assert'), zip = require('zipfile'), Step = require('step'); // node compatibility for mkdirs below var constants = {}; if (!process.EEXIST >= 1) constants = require('constants'); else constants.EEXIST = process.EEXIST; function External(env, uri) { var local = !(/^https?:\/\//i.test(uri)) if (local) { uri = path.join(env.local_data_dir, uri); } if (External.instances[uri]) return External.instances[uri]; this.uri = uri; this.env = env; this.format = path.extname(uri).toLowerCase(); this._callbacks = []; this.done = false; External.mkdirp(this.env.data_dir, 0755); if (local) { this.localFile(); } else { this.downloadFile(); } External.instances[this.uri] = this; } sys.inherits(External, EventEmitter); External.instances = {}; External.prototype.invokeCallbacks = function(err) { delete External.instances[this.uri]; if (err) { this.emit('error', err); } else { this.emit('complete', this); } }; External.prototype.localFile = function() { // Only treat files as local that don't have to be processed. this.isLocal = !External.processors[this.format]; this.tempPath = this.uri; fs.stat(this.tempPath, this.processFile.bind(this)); }; External.prototype.downloadFile = function() { this.tempPath = path.join( this.env.data_dir, crypto.createHash('md5').update(this.uri).digest('hex') + path.extname(this.uri)); fs.stat(this.path(), function(err, stats) { if (err) { // This file does not yet exist. Download it! netlib.download( this.uri, this.tempPath, External.encodings[this.format] || External.encodings['default'], this.processFile.bind(this) ); } else { this.invokeCallbacks(null); } }.bind(this)); }; External.prototype.processFile = function(err) { if (err) { this.invokeCallbacks(err); } else { if (this.isLocal) { this.invokeCallbacks(null); } else { (External.processors[this.format] || External.processors['default'])( this.tempPath, this.path(), function(err) { if (err) { this.invokeCallbacks(err); } else { this.invokeCallbacks(null); } }.bind(this) ); } } }; External.prototype.path = function() { if (this.isLocal) { return this.tempPath; } else { return path.join( this.env.data_dir, (External.destinations[this.format] || External.destinations['default'])(this.uri) ); } }; External.prototype.findByExtension = function(ext, callback) { var running = 0; var found = []; (function find(dir) { running++; fs.readdir(dir, function(err, files) { if (err) { running = 0; callback(err); return; i } files.forEach(function(file) { running++; file = path.join(dir, file); fs.stat(file, function(err, stats) { if (err) { running = 0; callback(err); return; } if (stats.isDirectory()) { find(file); } else if (stats.isFile() && path.extname(file) === ext) { found.push(file); } if (running && !--running) callback(null, found); }); }); if (running && !--running) callback(null, found); }) })(this.path()); }; // https://gist.github.com/707661 External.mkdirp = function mkdirP (p, mode, f) { var cb = f || function () {}; if (p.charAt(0) != '/') { cb('Relative path: ' + p); return } var ps = path.normalize(p).split('/'); path.exists(p, function (exists) { if (exists) cb(null); else mkdirP(ps.slice(0,-1).join('/'), mode, function (err) { if (err && err.errno != process.EEXIST) cb(err) else { fs.mkdir(p, mode, cb); } }); }); }; External.encodings = { '.zip': 'binary', '.shp': 'binary', '.png': 'binary', '.jpg': 'binary', 'default': 'utf-8' }; // Destinations are names in the data_dir/cache directory. External.destinations = {}; External.destinations['default'] = function(uri) { return crypto.createHash('md5').update(uri).digest('hex') + path.extname(uri); }; External.destinations['.zip'] = function(uri) { return crypto.createHash('md5').update(uri).digest('hex'); }; External.processors = {}; External.processors['default'] = function(tempPath, destPath, callback) { if (tempPath === destPath) { callback(null); } else { fs.rename(tempPath, destPath, callback); } }; External.processors['.zip'] = function(tempPath, destPath, callback) { try { var zf = new zip.ZipFile(tempPath); } catch (err) { callback(err); } Step( function() { var group = this.group(); zf.names.forEach(function(name) { var next = group(); var uncompressed = path.join(destPath, name); External.mkdirp(path.dirname(uncompressed), 0755, function(err) { if (err && err.errno != constants.EEXIST) { callback("Couldn't create directory " + path.dirname(name)); } else { if (path.extname(name)) { var buffer = zf.readFile(name, function(err, buffer) { fd = fs.open(uncompressed, 'w', 0755, function(err, fd) { sys.debug('saving to: ' + uncompressed); fs.write(fd, buffer, 0, buffer.length, null, function(err, written) { fs.close(fd, function(err) { next(); }); }); }); }); } else { throw 'UNIMPLEMENTED'; } } }); }); }, callback ); }; module.exports = External;