cartodb/carto
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor external to make tests working

Browse Source
This commit is contained in:
Konstantin Käfer 2011-01-24 17:23:18 -05:00
parent 32b81b142d
commit ed38139c0b
6 changed files with 366 additions and 312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

383
lib/mess/external.js
View File

@ -2,9 +2,10 @@ var fs = require('fs'),
netlib = require('./netlib'),
url = require('url'),
path = require('path'),
Step = require('step'),
_ = require('underscore')._,
spawn = require('child_process').spawn;
crypto = require('crypto'),
assert = require('assert'),
zip = require('zipfile'),
Step = require('step');
// node compatibility for mkdirs below
var constants = {};
@ -13,296 +14,164 @@ if (!process.EEXIST >= 1)
else
constants.EEXIST = process.EEXIST;
/**
* TODO: use node-minizip instead of shelling
* TODO: include or don't use underscore
*/
var External = function External(env) {
var env = env;
function External(env, uri, callback) {
assert.ok(env.data_dir);
assert.ok(env.local_data_dir);
return {
env: env,
this.env = env;
this.callback = callback;
this.uri = uri;
this.format = path.extname(uri).toLowerCase();
/**
* Download an external, process it, and return the usable filepath for
* Mapnik
* @param {String} resource_url the URI of the datasource from a mapfile.
* @param {Function} callback passed into processor function after
* localizing.
*/
process: function(resource_url, callback) {
var file_format = path.extname(resource_url).toLowerCase(),
that = this;
External.mkdirp(path.join(this.env.data_dir, 'cache'), 0755);
External.mkdirp(path.join(this.env.data_dir, 'temp'), 0755);
if (url.parse(resource_url).protocol == 'http:') {
fs.stat(this.tmppos(resource_url), function(err, stats) {
if (stats && stats.isFile()) {
callback(null, [
resource_url,
that.destinations(file_format)(resource_url, that)
]);
if (/^https?:\/\//i.test(uri)) {
this.downloadFile();
} else {
this.localFile();
}
}
External.prototype.localFile = function() {
// Only treat files as local that don't have to be processed.
this.isLocal = !External.processors[this.format];
this.tempPath = path.join(this.env.local_data_dir, this.uri);
fs.stat(this.tempPath, this.processFile.bind(this));
};
External.prototype.downloadFile = function() {
this.tempPath = path.join(
this.env.data_dir, 'temp',
crypto.createHash('md5').update(this.uri).digest('hex') + path.extname(this.uri));
fs.stat(this.path(), function(err, stats) {
if (err) {
// This file does not yet exist. Download it!
netlib.download(
resource_url,
that.tmppos(resource_url),
that.encodings[file_format],
function(err, url, filename) {
if (that.processors(file_format)) {
that.processors(file_format)(
filename,
resource_url,
callback,
that);
this.uri,
this.tempPath,
External.encodings[this.format] || External.encodings['default'],
this.processFile.bind(this)
);
} else {
console.log('no processor found for %s', file_format);
this.callback(null, this);
}
});
}
});
}.bind(this));
};
External.prototype.processFile = function(err) {
if (err) {
this.callback(err);
} else {
// is this a fully-qualified URL?
fs.stat(resource_url, function(err, stat) {
if (!err && stat.isFile()) {
if (that.processors(file_format)) {
that.processors(file_format)(
resource_url,
resource_url,
callback,
that);
if (this.isLocal) {
this.callback(null, this);
} else {
console.log('no processor found for %s', file_format);
}
}
});
}
},
/**
* Get a processor, given a file's extension
* @param {String} extension the file's extension.
* @return {Function} processor function.
*/
processors: function(extension) {
return {
'.zip': this.unzip,
'.mss': this.plainfile,
'.shp': this.inplace,
'.geojson': this.plainfile,
'.kml': this.plainfile
}[extension];
},
destinations: function(extension) {
return {
'.zip': this.unzip_dest,
'.mss': this.plainfile_dest,
'.shp': this.inplace_dest,
'.geojson': this.plainfile_dest,
'.kml': this.plainfile_dest
}[extension];
},
encodings: function(extension) {
return {
'.zip': 'binary',
'.mss': 'utf-8',
'.shp': 'binary',
'.geojson': 'utf-8',
'.kml': 'utf-8'
}[extension];
},
/**
* Get the final resting position of an external's directory
* @param {String} ext name of the external.
* @return {String} file path.
*/
pos: function(ext) {
return path.join(this.env.data_dir, netlib.safe64(ext));
},
/**
* Get the temporary path of an external before processing
* @param {String} ext filename of the external.
* @return {String} file path.
*/
tmppos: function(ext) {
return path.join(this.env.data_dir, require('crypto')
.createHash('md5').update(ext).digest('hex'));
},
plainname: function(resource_url) {
return require('crypto')
.createHash('md5').update(resource_url).digest('hex') +
path.extname(resource_url);
},
unzip_dest: function(resource_url, that) {
return that.locateShp(that.pos(resource_url));
},
plainfile_dest: function(resource_url, that) {
return path.join(that.pos(resource_url),
that.plainname(resource_url));
},
inplace_dest: function(resource_url, that) {
console.log(url.parse(resource_url));
return resource_url;
},
/**
* Deal with a plain file, which is likely to be
* GeoJSON, KML, or one of the other OGR-supported formats,
* returning a Mapnik-usable filename
*
* @param {String} filename the place of the file on your system.
* @param {String} resource_url
* @param {Function} callback
*/
plainfile: function(filename, resource_url, callback, that) {
// TODO: possibly decide upon default extension
var extension = path.extname(resource_url);
if (extension !== '') {
// TODO: make sure dir doesn't exist
var destination = path.join(that.pos(resource_url),
that.plainname(resource_url));
fs.mkdir(that.pos(resource_url), 0777, function(err) {
err && callback(err);
fs.rename(
filename,
destination,
(External.processors[this.format] || External.processors['default'])(
this.tempPath,
this.path(),
function(err) {
err && callback(err);
callback(null, [resource_url, destination]);
}
)
});
if (err) {
this.callback(err);
} else {
throw Exception('Non-extended files cannot be processed');
}
},
/**
* Deal with an inplace local file
*
* @param {String} filename the place of the file on your system.
* @param {String} resource_url
* @param {Function} callback
*/
inplace: function(filename, resource_url, callback, that) {
callback(null, [resource_url, filename]);
},
locateShp: function(dir) {
try {
var unzipped = fs.readdirSync(dir);
var shp = _.detect(unzipped,
function(f) {
return path.extname(f).toLowerCase() == '.shp';
this.callback(null, this);
}
}.bind(this)
);
if (!shp) {
var dirs = _.select(unzipped,
function(f) {
return fs.statSync(path.join(dir, f)).isDirectory();
}
);
if (dirs) {
for (var i = 0, l = dirs.length; i < l; i++) {
var located = locateShp(path.join(dir, dirs[i]));
if (located) {
return located;
}
}
}
} else {
return path.join(dir, shp);
}
} catch (e) {
return false;
}
},
};
/**
* Recursively make directory tree
*
* @param {String} directory path
* @param {Number} mode
* @param {Function} callback
*/
mkdirs: function (p, mode, f) {
var that = this;
External.prototype.path = function() {
if (this.isLocal) {
return this.tempPath;
}
else {
return path.join(
this.env.data_dir,
'cache',
(External.destinations[this.format] || External.destinations['default'])(this.uri)
);
}
};
// https://gist.github.com/707661
External.mkdirp = function mkdirP (p, mode, f) {
var cb = f || function () {};
// if relative
if (p.charAt(0) != '/') {
// TODO if >= node 0.3.0 use path.resolve() ?
p = path.join(__dirname,p);
}
if (p.charAt(0) != '/') { cb('Relative path: ' + p); return }
var ps = path.normalize(p).split('/');
path.exists(p, function (exists) {
if (exists) cb(null);
else that.mkdirs(ps.slice(0,-1).join('/'), mode, function (err) {
if (err && err.errno != constants.EEXIST) cb(err)
else fs.mkdir(p, mode, cb);
else mkdirP(ps.slice(0,-1).join('/'), mode, function (err) {
if (err && err.errno != process.EEXIST) cb(err)
else {
fs.mkdir(p, mode, cb);
}
});
});
},
};
/**
* Unzip a file and return a shapefile contained within it
*
* TODO: handle other files than shapefiles
* @param {String} filename the place of the shapefile on your system.
* @param {String} resource_url
* @param {Function} callback
*/
unzip: function(filename, resource_url, callback, that) {
console.log('unzipping download from ' + filename);
// https://github.com/springmeyer/node-zipfile
var zip = require('zipfile');
External.encodings = {
'.zip': 'binary',
'.shp': 'binary',
'default': 'utf-8'
};
// Destinations are names in the data_dir/cache directory.
External.destinations = {};
External.destinations['default'] = function(uri) {
return crypto.createHash('md5').update(uri).digest('hex') + path.extname(uri);
};
External.destinations['.zip'] = function(uri) {
return crypto.createHash('md5').update(uri).digest('hex');
};
External.processors = {};
External.processors['default'] = function(tempPath, destPath, callback) {
if (tempPath === destPath) {
callback(null);
} else {
fs.rename(tempPath, destPath, callback);
}
};
External.processors['.zip'] = function(tempPath, destPath, callback) {
try {
var zf = new zip.ZipFile(filename);
} catch (e) {
callback(e);
}
var dirname = that.pos(resource_url);
Step(
function() {
var group = this.group();
var zf = new zip.ZipFile(tempPath);
// This is blocking, sequential and synchronous.
zf.names.forEach(function(name) {
var uncompressed = path.join(dirname,name);
var g = group();
that.mkdirs(dirname, 0755 , function(err) {
var uncompressed = path.join(destPath, name);
External.mkdirp(path.dirname(uncompressed), 0755, function(err) {
if (err && err.errno != constants.EEXIST) {
console.log('unzip failed');
callback(err, [resource_url, false]);
throw "Couldn't create directory " + path.dirname(name);
}
else {
if (path.extname(name)) {
var buffer = zf.readFileSync(name);
console.log('saving to: ' + uncompressed);
fd = fs.openSync(uncompressed,'w');
fd = fs.openSync(uncompressed, 'w');
fs.writeSync(fd, buffer, 0, buffer.length, null);
fs.closeSync(fd);
}
g(null, uncompressed);
})
else {
throw 'UNIMPLEMENTED';
}
}
});
},
function(err, results) {
// TODO: simplify locateShp but also
// allow non-redownloading of shapefiles.
err && callback(err);
callback(null, [
resource_url,
that.locateShp(dirname)]);
});
callback(null);
} catch (err) {
console.log(err);
callback(err);
}
);
}
};
};
module.exports = External;

45
lib/mess/renderer.js
View File

@ -48,7 +48,7 @@ mess.Renderer = function Renderer(env) {
* @param {Function} callback
*/
grab: function(uri, callback) {
new External(this.env).process(uri, callback);
new External(this.env, uri, callback);
},
/**
@ -119,23 +119,21 @@ mess.Renderer = function Renderer(env) {
var group = this.group();
m.Layer.forEach(function(l) {
if (l.Datasource.file) {
that.grab(l.Datasource.file, group());
var next = group();
that.grab(l.Datasource.file, function(err, external) {
if (err) throw err;
// Extract the shapefile.
l.Datasource.file = external.path();
next();
});
} else {
throw 'Layer does not have a datasource';
}
});
if (m.Layer.length == 0) group()();
// Continue even if we don't have any layers.
group()();
},
function(err, results) {
var result_map = to(results);
m.Layer = _.map(_.filter(m.Layer,
function(l) {
return l.Datasource.file &&
result_map[l.Datasource.file];
}),
function(l) {
l.Datasource.file = result_map[l.Datasource.file];
return l;
}
);
function(err) {
callback(err, m);
}
);
@ -152,20 +150,19 @@ mess.Renderer = function Renderer(env) {
Step(
function() {
var group = this.group();
m.Stylesheet.forEach(function(s) {
m.Stylesheet.forEach(function(s, k) {
if (!s.id) {
that.grab(s, group());
var next = group();
that.grab(s, function(err, external) {
if (err) throw err;
m.Stylesheet[k] = external.path();
next();
});
}
});
group()();
},
function(err, results) {
var result_map = to(results);
for (s in m.Stylesheet) {
if (!m.Stylesheet[s].id) {
m.Stylesheet[s] = result_map[m.Stylesheet[s]];
}
}
callback(err, m);
}
);
@ -201,7 +198,7 @@ mess.Renderer = function Renderer(env) {
group = this.group();
for (var i = 0, l = results.length; i < l; i++) {
new mess.Parser(_.extend(_.extend({
filename: s
filename: results[i][0]
}, that.env), this.env)).parse(results[i][1],
function(err, tree) {
if (err) {

37
test/rendering.js Normal file
View File

@ -0,0 +1,37 @@
var path = require('path'),
sys = require('sys'),
assert = require('assert'),
fs = require('fs');
var mess = require('mess');
var tree = require('mess/tree');
var helper = require('./support/helper');
helper.files('rendering', 'mml', function(file) {
exports['test rendering ' + file] = function(beforeExit) {
var success = false;
helper.file(file, function(mml) {
new mess.Renderer({
paths: [ path.dirname(file) ],
data_dir: path.join(__dirname, '../data'),
local_data_dir: path.join(__dirname, 'rendering'),
filename: file
}).render(mml, function (err, output) {
if (err) {
throw err;
} else {
var result = helper.resultFile(file);
helper.file(result, function(result) {
assert.equal(output, result);
success = true;
});
}
});
});
beforeExit(function() {
assert.ok(success, 'Rendering finished.');
});
}
});

31
test/rendering/complex_cascades.mml Normal file
View File

@ -0,0 +1,31 @@
{
"srs": "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs",
"Stylesheet": [
"complex_cascades.mss"
],
"Layer": [{
"id": "world",
"name": "world",
"srs": "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs",
"Datasource": {
"file": "http://localhost/map.zip",
"type": "shape"
}
},
{
"id": "countries",
"class": "new",
"name": "world",
"srs": "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs",
"Datasource": {
"file": "http://localhost/map.zip",
"type": "shape"
}
}],
"center": {
"lat": 47.687579167,
"lon": 5.6799316403973,
"zoom": 6
}
}

31
test/rendering/complex_cascades.mss Normal file
View File

@ -0,0 +1,31 @@
#world {
polygon-fill: #FFF;
line-color:#F00;
line-width: 0.5;
}
#world[NAME='United States'] {
polygon-fill:#CCC;
[zoom > 6] { polygon-fill:#DDD; }
[zoom > 7] { polygon-fill:#999; }
[zoom > 5] { polygon-fill:#666; }
}
#world[NAME='Canada'],
#countries {
polygon-fill: #eee;
line-color: #ccc;
line-width: 1;
.new {
polygon-fill: #CCC;
}
.new[zoom > 5] {
line-width:0.5;
[NAME='United States'] {
polygon-fill:#AFC;
}
}
}

89
test/rendering/complex_cascades.result Normal file
View File

@ -0,0 +1,89 @@
<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE Map[]>
<Map srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs">
<Style name="world-polygon/__default__">
<Rule>
<MaxScaleDenominator>12500000</MaxScaleDenominator>
<Filter>([NAME] = 'United States')</Filter>
<PolygonSymbolizer fill="#666666"/>
</Rule>
<Rule>
<MinScaleDenominator>12500000</MinScaleDenominator>
<Filter>([NAME] = 'Canada') and ([NAME] != 'United States')</Filter>
<PolygonSymbolizer fill="#eeeeee"/>
</Rule>
<Rule>
<MinScaleDenominator>12500000</MinScaleDenominator>
<Filter>([NAME] = 'United States') and ([NAME] != 'United States') and ([NAME] != 'Canada')</Filter>
<PolygonSymbolizer fill="#cccccc"/>
</Rule>
<Rule>
<MinScaleDenominator>12500000</MinScaleDenominator>
<Filter>([NAME] != 'United States') and ([NAME] != 'Canada') and ([NAME] != 'United States')</Filter>
<PolygonSymbolizer fill="#ffffff"/>
</Rule>
</Style>
<Style name="world-line/__default__">
<Rule>
<Filter>([NAME] = 'Canada')</Filter>
<LineSymbolizer stroke-width="1"
stroke="#cccccc"/>
</Rule>
<Rule>
<Filter>([NAME] != 'Canada')</Filter>
<LineSymbolizer stroke-width="0.5"
stroke="#ff0000"/>
</Rule>
</Style>
<Layer
id="world"
name="world"
srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs">
<StyleName>world-line/__default__</StyleName>
<StyleName>world-polygon/__default__</StyleName>
<Datasource>
<Parameter name="file">/Users/kkaefer/Code/devseed/mess.js/data/cache/af08a2b537079e720a9293424a319a1e</Parameter>
<Parameter name="type">shape</Parameter>
</Datasource>
</Layer>
<Style name="countries-polygon/__default__">
<Rule>
<MaxScaleDenominator>12500000</MaxScaleDenominator>
<Filter>([NAME] = 'United States')</Filter>
<PolygonSymbolizer fill="#aaffcc"/>
</Rule>
<Rule>
<MinScaleDenominator>12500000</MinScaleDenominator>
<Filter>([NAME] != 'United States')</Filter>
<PolygonSymbolizer fill="#cccccc"/>
</Rule>
</Style>
<Style name="countries-line/__default__">
<Rule>
<MaxScaleDenominator>12500000</MaxScaleDenominator>
<LineSymbolizer stroke-width="0.5"/>
</Rule>
<Rule>
<MinScaleDenominator>12500000</MinScaleDenominator>
<LineSymbolizer stroke-width="1"
stroke="#cccccc"/>
</Rule>
</Style>
<Layer
id="countries"
name="world"
srs="+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +no_defs">
<StyleName>countries-line/__default__</StyleName>
<StyleName>countries-polygon/__default__</StyleName>
<Datasource>
<Parameter name="file">/Users/kkaefer/Code/devseed/mess.js/data/cache/af08a2b537079e720a9293424a319a1e</Parameter>
<Parameter name="type">shape</Parameter>
</Datasource>
</Layer>
</Map>