303 lines
11 KiB
JavaScript
303 lines
11 KiB
JavaScript
var fs = require('fs'),
|
|
netlib = require('./netlib'),
|
|
url = require('url'),
|
|
path = require('path'),
|
|
Step = require('step'),
|
|
_ = require('underscore')._,
|
|
spawn = require('child_process').spawn;
|
|
|
|
// node compatibility for mkdirs below
|
|
var constants = {};
|
|
if (!process.EEXIST >= 1)
|
|
constants = require('constants');
|
|
else
|
|
constants.EEXIST = process.EEXIST;
|
|
|
|
/**
|
|
* TODO: use node-minizip instead of shelling
|
|
* TODO: include or don't use underscore
|
|
*/
|
|
|
|
var External = function External(env) {
|
|
var env = env;
|
|
|
|
return {
|
|
env: env,
|
|
|
|
/**
|
|
* Download an external, process it, and return the usable filepath for
|
|
* Mapnik
|
|
* @param {String} resource_url the URI of the datasource from a mapfile.
|
|
* @param {Function} callback passed into processor function after
|
|
* localizing.
|
|
*/
|
|
process: function(resource_url, callback) {
|
|
var file_format = path.extname(resource_url).toLowerCase(),
|
|
that = this;
|
|
|
|
if (url.parse(resource_url).protocol == 'http:') {
|
|
fs.stat(this.tmppos(resource_url), function(err, stats) {
|
|
if (stats && stats.isFile()) {
|
|
callback(null, [
|
|
resource_url,
|
|
that.destinations(file_format)(resource_url, that)
|
|
]);
|
|
} else {
|
|
netlib.download(
|
|
resource_url,
|
|
that.tmppos(resource_url),
|
|
that.encodings[file_format],
|
|
function(err, url, filename) {
|
|
if (that.processors(file_format)) {
|
|
that.processors(file_format)(
|
|
filename,
|
|
resource_url,
|
|
callback,
|
|
that);
|
|
} else {
|
|
console.log('no processor found for %s', file_format);
|
|
}
|
|
});
|
|
}
|
|
});
|
|
} else {
|
|
// is this a fully-qualified URL?
|
|
fs.stat(resource_url, function(err, stat) {
|
|
if (!err && stat.isFile()) {
|
|
if (that.processors(file_format)) {
|
|
that.processors(file_format)(
|
|
resource_url,
|
|
resource_url,
|
|
callback,
|
|
that);
|
|
} else {
|
|
console.log('no processor found for %s', file_format);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Get a processor, given a file's extension
|
|
* @param {String} extension the file's extension.
|
|
* @return {Function} processor function.
|
|
*/
|
|
processors: function(extension) {
|
|
return {
|
|
'.zip': this.unzip,
|
|
'.mss': this.plainfile,
|
|
'.shp': this.inplace,
|
|
'.geojson': this.plainfile,
|
|
'.kml': this.plainfile
|
|
}[extension];
|
|
},
|
|
|
|
destinations: function(extension) {
|
|
return {
|
|
'.zip': this.unzip_dest,
|
|
'.mss': this.plainfile_dest,
|
|
'.shp': this.inplace_dest,
|
|
'.geojson': this.plainfile_dest,
|
|
'.kml': this.plainfile_dest
|
|
}[extension];
|
|
},
|
|
|
|
encodings: function(extension) {
|
|
return {
|
|
'.zip': 'binary',
|
|
'.mss': 'utf-8',
|
|
'.shp': 'binary',
|
|
'.geojson': 'utf-8',
|
|
'.kml': 'utf-8'
|
|
}[extension];
|
|
},
|
|
|
|
/**
|
|
* Get the final resting position of an external's directory
|
|
* @param {String} ext name of the external.
|
|
* @return {String} file path.
|
|
*/
|
|
pos: function(ext) {
|
|
return path.join(this.env.data_dir, netlib.safe64(ext));
|
|
},
|
|
|
|
/**
|
|
* Get the temporary path of an external before processing
|
|
* @param {String} ext filename of the external.
|
|
* @return {String} file path.
|
|
*/
|
|
tmppos: function(ext) {
|
|
return path.join(this.env.data_dir, require('crypto')
|
|
.createHash('md5').update(ext).digest('hex'));
|
|
},
|
|
|
|
plainname: function(resource_url) {
|
|
return require('crypto')
|
|
.createHash('md5').update(resource_url).digest('hex') +
|
|
path.extname(resource_url);
|
|
|
|
},
|
|
|
|
unzip_dest: function(resource_url, that) {
|
|
return that.locateShp(that.pos(resource_url));
|
|
},
|
|
|
|
plainfile_dest: function(resource_url, that) {
|
|
return path.join(that.pos(resource_url),
|
|
that.plainname(resource_url));
|
|
},
|
|
|
|
inplace_dest: function(resource_url, that) {
|
|
console.log(url.parse(resource_url));
|
|
return resource_url;
|
|
},
|
|
|
|
/**
|
|
* Deal with a plain file, which is likely to be
|
|
* GeoJSON, KML, or one of the other OGR-supported formats,
|
|
* returning a Mapnik-usable filename
|
|
*
|
|
* @param {String} filename the place of the file on your system.
|
|
* @param {String} resource_url
|
|
* @param {Function} callback
|
|
*/
|
|
plainfile: function(filename, resource_url, callback, that) {
|
|
// TODO: possibly decide upon default extension
|
|
var extension = path.extname(resource_url);
|
|
if (extension !== '') {
|
|
// TODO: make sure dir doesn't exist
|
|
var destination = path.join(that.pos(resource_url),
|
|
that.plainname(resource_url));
|
|
fs.mkdir(that.pos(resource_url), 0777, function(err) {
|
|
err && callback(err);
|
|
fs.rename(
|
|
filename,
|
|
destination,
|
|
function(err) {
|
|
err && callback(err);
|
|
callback(null, [resource_url, destination]);
|
|
}
|
|
)
|
|
});
|
|
} else {
|
|
throw Exception('Non-extended files cannot be processed');
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Deal with an inplace local file
|
|
*
|
|
* @param {String} filename the place of the file on your system.
|
|
* @param {String} resource_url
|
|
* @param {Function} callback
|
|
*/
|
|
inplace: function(filename, resource_url, callback, that) {
|
|
|
|
callback(null, [resource_url, filename]);
|
|
},
|
|
|
|
locateShp: function(dir) {
|
|
try {
|
|
var unzipped = fs.readdirSync(dir);
|
|
var shp = _.detect(unzipped,
|
|
function(f) {
|
|
return path.extname(f).toLowerCase() == '.shp';
|
|
}
|
|
);
|
|
if (!shp) {
|
|
var dirs = _.select(unzipped,
|
|
function(f) {
|
|
return fs.statSync(path.join(dir, f)).isDirectory();
|
|
}
|
|
);
|
|
if (dirs) {
|
|
for (var i = 0, l = dirs.length; i < l; i++) {
|
|
var located = locateShp(path.join(dir, dirs[i]));
|
|
if (located) {
|
|
return located;
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
return path.join(dir, shp);
|
|
}
|
|
} catch (e) {
|
|
return false;
|
|
}
|
|
},
|
|
|
|
/**
|
|
* Recursively make directory tree
|
|
*
|
|
* @param {String} directory path
|
|
* @param {Number} mode
|
|
* @param {Function} callback
|
|
*/
|
|
mkdirs: function (p, mode, f) {
|
|
var that = this;
|
|
var cb = f || function () {};
|
|
// if relative
|
|
if (p.charAt(0) != '/') {
|
|
// TODO if >= node 0.3.0 use path.resolve() ?
|
|
p = path.join(__dirname,p);
|
|
}
|
|
var ps = path.normalize(p).split('/');
|
|
path.exists(p, function (exists) {
|
|
if (exists) cb(null);
|
|
else that.mkdirs(ps.slice(0,-1).join('/'), mode, function (err) {
|
|
if (err && err.errno != constants.EEXIST) cb(err)
|
|
else fs.mkdir(p, mode, cb);
|
|
});
|
|
});
|
|
},
|
|
|
|
/**
|
|
* Unzip a file and return a shapefile contained within it
|
|
*
|
|
* TODO: handle other files than shapefiles
|
|
* @param {String} filename the place of the shapefile on your system.
|
|
* @param {String} resource_url
|
|
* @param {Function} callback
|
|
*/
|
|
unzip: function(filename, resource_url, callback, that) {
|
|
console.log('unzipping download from ' + filename);
|
|
// https://github.com/springmeyer/node-zipfile
|
|
var zip = require('zipfile');
|
|
var zf = new zip.ZipFile(filename);
|
|
var dirname = that.pos(resource_url);
|
|
Step(
|
|
function uncompress_all() {
|
|
var group = this.group();
|
|
zf.names.forEach(function(name) {
|
|
var uncompressed = path.join(dirname,name);
|
|
var g = group();
|
|
that.mkdirs(dirname, 0755 , function(err) {
|
|
if (err && err.errno != constants.EEXIST) {
|
|
console.log('unzip failed');
|
|
callback(err, [resource_url, false]);
|
|
}
|
|
if (path.extname(name)) {
|
|
var buffer = zf.readFileSync(name);
|
|
console.log('saving to: ' + uncompressed);
|
|
fd = fs.openSync(uncompressed,'w');
|
|
fs.writeSync(fd, buffer, 0, buffer.length, null);
|
|
fs.closeSync(fd);
|
|
}
|
|
g();
|
|
})
|
|
});
|
|
},
|
|
function finish_up(err) {
|
|
if (err) throw err;
|
|
callback(null, [
|
|
resource_url,
|
|
that.locateShp(dirname)]);
|
|
}
|
|
);
|
|
}
|
|
};
|
|
};
|
|
|
|
module.exports = External;
|