cartodb4/log4js-node
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

moved streams code around, added stub for DateRollingFileStream

Browse Source
This commit is contained in:
Gareth Jones 2012-09-05 10:58:28 +10:00
parent cbc1dd32f9
commit be1272cd7c
9 changed files with 349 additions and 271 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

268
lib/streams.js
View File

@ -1,268 +0,0 @@
var util = require('util'),
fs = require('fs'),
path = require('path'),
events = require('events'),
async = require('async');
function debug(message) {
// util.debug(message);
// console.log(message);
}
function BufferedWriteStream(stream) {
var that = this;
this.stream = stream;
this.buffer = [];
this.canWrite = false;
this.bytes = 0;
this.stream.on("open", function() {
that.canWrite = true;
that.flushBuffer();
});
this.stream.on("error", function (err) {
that.emit("error", err);
});
this.stream.on("drain", function() {
that.canWrite = true;
that.flushBuffer();
});
}
util.inherits(BufferedWriteStream, events.EventEmitter);
Object.defineProperty(
BufferedWriteStream.prototype,
"fd",
{
get: function() { return this.stream.fd; },
set: function(newFd) {
this.stream.fd = newFd;
this.bytes = 0;
}
}
);
Object.defineProperty(
BufferedWriteStream.prototype,
"bytesWritten",
{
get: function() { return this.bytes; }
}
);
BufferedWriteStream.prototype.write = function(data, encoding) {
this.buffer.push({ data: data, encoding: encoding });
this.flushBuffer();
};
BufferedWriteStream.prototype.end = function(data, encoding) {
if (data) {
this.buffer.push({ data: data, encoding: encoding });
}
this.flushBufferEvenIfCannotWrite();
};
BufferedWriteStream.prototype.writeToStream = function(toWrite) {
this.bytes += toWrite.data.length;
this.canWrite = this.stream.write(toWrite.data, toWrite.encoding);
};
BufferedWriteStream.prototype.flushBufferEvenIfCannotWrite = function() {
while (this.buffer.length > 0) {
this.writeToStream(this.buffer.shift());
}
};
BufferedWriteStream.prototype.flushBuffer = function() {
while (this.buffer.length > 0 && this.canWrite) {
this.writeToStream(this.buffer.shift());
}
};
function BaseRollingFileStream(filename, options) {
this.filename = filename;
this.options = options || { encoding: 'utf8', mode: 0644, flags: 'a' };
this.rolling = false;
this.writesWhileRolling = [];
this.currentSize = 0;
function currentFileSize(file) {
var fileSize = 0;
try {
fileSize = fs.statSync(file).size;
} catch (e) {
// file does not exist
}
return fileSize;
}
function throwErrorIfArgumentsAreNotValid() {
if (!filename) {
throw new Error("You must specify a filename");
}
}
throwErrorIfArgumentsAreNotValid();
BaseRollingFileStream.super_.call(this, this.filename, this.options);
this.currentSize = currentFileSize(this.filename);
}
util.inherits(BaseRollingFileStream, fs.FileWriteStream);
BaseRollingFileStream.prototype.initRolling = function() {
var that = this;
function emptyRollingQueue() {
debug("emptying the rolling queue");
var toWrite;
while ((toWrite = that.writesWhileRolling.shift())) {
BaseRollingFileStream.super_.prototype.write.call(that, toWrite.data, toWrite.encoding);
that.currentSize += toWrite.data.length;
if (that.shouldRoll()) {
that.flush();
return true;
}
}
that.flush();
return false;
}
this.rolling = true;
this.roll(this.filename, function() {
that.currentSize = 0;
that.rolling = emptyRollingQueue();
if (that.rolling) {
process.nextTick(function() { that.initRolling(); });
}
});
};
BaseRollingFileStream.prototype.write = function(data, encoding) {
if (this.rolling) {
this.writesWhileRolling.push({ data: data, encoding: encoding });
return false;
} else {
var canWrite = BaseRollingFileStream.super_.prototype.write.call(this, data, encoding);
this.currentSize += data.length;
debug('current size = ' + this.currentSize);
if (this.shouldRoll()) {
this.initRolling();
}
return canWrite;
}
};
BaseRollingFileStream.prototype.shouldRoll = function() {
return false; // default behaviour is never to roll
};
BaseRollingFileStream.prototype.roll = function(filename, callback) {
callback(); // default behaviour is not to do anything
};
function RollingFileStream (filename, size, backups, options) {
this.size = size;
this.backups = backups || 1;
function throwErrorIfArgumentsAreNotValid() {
if (!filename || !size || size <= 0) {
throw new Error("You must specify a filename and file size");
}
}
throwErrorIfArgumentsAreNotValid();
RollingFileStream.super_.call(this, filename, options);
}
util.inherits(RollingFileStream, BaseRollingFileStream);
RollingFileStream.prototype.shouldRoll = function() {
return this.currentSize >= this.size;
};
RollingFileStream.prototype.roll = function(filename, callback) {
var that = this,
nameMatcher = new RegExp('^' + path.basename(filename));
function justTheseFiles (item) {
return nameMatcher.test(item);
}
function index(filename_) {
return parseInt(filename_.substring((path.basename(filename) + '.').length), 10) || 0;
}
function byIndex(a, b) {
if (index(a) > index(b)) {
return 1;
} else if (index(a) < index(b) ) {
return -1;
} else {
return 0;
}
}
function increaseFileIndex (fileToRename, cb) {
var idx = index(fileToRename);
debug('Index of ' + fileToRename + ' is ' + idx);
if (idx < that.backups) {
//on windows, you can get a EEXIST error if you rename a file to an existing file
//so, we'll try to delete the file we're renaming to first
fs.unlink(filename + '.' + (idx+1), function (err) {
//ignore err: if we could not delete, it's most likely that it doesn't exist
debug('Renaming ' + fileToRename + ' -> ' + filename + '.' + (idx+1));
fs.rename(path.join(path.dirname(filename), fileToRename), filename + '.' + (idx + 1), cb);
});
} else {
cb();
}
}
function renameTheFiles(cb) {
//roll the backups (rename file.n to file.n+1, where n <= numBackups)
debug("Renaming the old files");
fs.readdir(path.dirname(filename), function (err, files) {
async.forEachSeries(
files.filter(justTheseFiles).sort(byIndex).reverse(),
increaseFileIndex,
cb
);
});
}
function openANewFile(cb) {
debug("Opening a new file");
fs.open(
filename,
that.options.flags,
that.options.mode,
function (err, fd) {
debug("opened new file");
var oldLogFileFD = that.fd;
that.fd = fd;
that.writable = true;
fs.close(oldLogFileFD, cb);
}
);
}
debug("Starting roll");
debug("Queueing up data until we've finished rolling");
debug("Flushing underlying stream");
this.flush();
async.series([
renameTheFiles,
openANewFile
], callback);
};
exports.BaseRollingFileStream = BaseRollingFileStream;
exports.RollingFileStream = RollingFileStream;
exports.BufferedWriteStream = BufferedWriteStream;

91
lib/streams/BaseRollingFileStream.js Normal file
View File

@ -0,0 +1,91 @@
var fs = require('fs'),
util = require('util');
function debug(message) {
// util.debug(message);
// console.log(message);
}
module.exports = BaseRollingFileStream;
function BaseRollingFileStream(filename, options) {
this.filename = filename;
this.options = options || { encoding: 'utf8', mode: 0644, flags: 'a' };
this.rolling = false;
this.writesWhileRolling = [];
this.currentSize = 0;
function currentFileSize(file) {
var fileSize = 0;
try {
fileSize = fs.statSync(file).size;
} catch (e) {
// file does not exist
}
return fileSize;
}
function throwErrorIfArgumentsAreNotValid() {
if (!filename) {
throw new Error("You must specify a filename");
}
}
throwErrorIfArgumentsAreNotValid();
BaseRollingFileStream.super_.call(this, this.filename, this.options);
this.currentSize = currentFileSize(this.filename);
}
util.inherits(BaseRollingFileStream, fs.FileWriteStream);
BaseRollingFileStream.prototype.initRolling = function() {
var that = this;
function emptyRollingQueue() {
debug("emptying the rolling queue");
var toWrite;
while ((toWrite = that.writesWhileRolling.shift())) {
BaseRollingFileStream.super_.prototype.write.call(that, toWrite.data, toWrite.encoding);
that.currentSize += toWrite.data.length;
if (that.shouldRoll()) {
that.flush();
return true;
}
}
that.flush();
return false;
}
this.rolling = true;
this.roll(this.filename, function() {
that.currentSize = 0;
that.rolling = emptyRollingQueue();
if (that.rolling) {
process.nextTick(function() { that.initRolling(); });
}
});
};
BaseRollingFileStream.prototype.write = function(data, encoding) {
if (this.rolling) {
this.writesWhileRolling.push({ data: data, encoding: encoding });
return false;
} else {
var canWrite = BaseRollingFileStream.super_.prototype.write.call(this, data, encoding);
this.currentSize += data.length;
debug('current size = ' + this.currentSize);
if (this.shouldRoll()) {
this.initRolling();
}
return canWrite;
}
};
BaseRollingFileStream.prototype.shouldRoll = function() {
return false; // default behaviour is never to roll
};
BaseRollingFileStream.prototype.roll = function(filename, callback) {
callback(); // default behaviour is not to do anything
};

77
lib/streams/BufferedWriteStream.js Normal file
View File

@ -0,0 +1,77 @@
var events = require('events'),
util = require('util');
module.exports = BufferedWriteStream;
function BufferedWriteStream(stream) {
var that = this;
this.stream = stream;
this.buffer = [];
this.canWrite = false;
this.bytes = 0;
this.stream.on("open", function() {
that.canWrite = true;
that.flushBuffer();
});
this.stream.on("error", function (err) {
that.emit("error", err);
});
this.stream.on("drain", function() {
that.canWrite = true;
that.flushBuffer();
});
}
util.inherits(BufferedWriteStream, events.EventEmitter);
Object.defineProperty(
BufferedWriteStream.prototype,
"fd",
{
get: function() { return this.stream.fd; },
set: function(newFd) {
this.stream.fd = newFd;
this.bytes = 0;
}
}
);
Object.defineProperty(
BufferedWriteStream.prototype,
"bytesWritten",
{
get: function() { return this.bytes; }
}
);
BufferedWriteStream.prototype.write = function(data, encoding) {
this.buffer.push({ data: data, encoding: encoding });
this.flushBuffer();
};
BufferedWriteStream.prototype.end = function(data, encoding) {
if (data) {
this.buffer.push({ data: data, encoding: encoding });
}
this.flushBufferEvenIfCannotWrite();
};
BufferedWriteStream.prototype.writeToStream = function(toWrite) {
this.bytes += toWrite.data.length;
this.canWrite = this.stream.write(toWrite.data, toWrite.encoding);
};
BufferedWriteStream.prototype.flushBufferEvenIfCannotWrite = function() {
while (this.buffer.length > 0) {
this.writeToStream(this.buffer.shift());
}
};
BufferedWriteStream.prototype.flushBuffer = function() {
while (this.buffer.length > 0 && this.canWrite) {
this.writeToStream(this.buffer.shift());
}
};

16
lib/streams/DateRollingFileStream.js Normal file
View File

@ -0,0 +1,16 @@
var BaseRollingFileStream = require('./BaseRollingFileStream'),
util = require('util');
module.exports = DateRollingFileStream;
function DateRollingFileStream(filename, pattern, options) {
if (typeof(pattern) === 'object') {
options = pattern;
pattern = null;
}
this.pattern = pattern || 'yyyy-mm-dd';
DateRollingFileStream.super_.call(this, filename, options);
}
util.inherits(DateRollingFileStream, BaseRollingFileStream);

110
lib/streams/RollingFileStream.js Normal file
View File

@ -0,0 +1,110 @@
var BaseRollingFileStream = require('./BaseRollingFileStream'),
util = require('util'),
path = require('path'),
fs = require('fs'),
async = require('async');
function debug(message) {
// util.debug(message);
// console.log(message);
}
module.exports = RollingFileStream;
function RollingFileStream (filename, size, backups, options) {
this.size = size;
this.backups = backups || 1;
function throwErrorIfArgumentsAreNotValid() {
if (!filename || !size || size <= 0) {
throw new Error("You must specify a filename and file size");
}
}
throwErrorIfArgumentsAreNotValid();
RollingFileStream.super_.call(this, filename, options);
}
util.inherits(RollingFileStream, BaseRollingFileStream);
RollingFileStream.prototype.shouldRoll = function() {
return this.currentSize >= this.size;
};
RollingFileStream.prototype.roll = function(filename, callback) {
var that = this,
nameMatcher = new RegExp('^' + path.basename(filename));
function justTheseFiles (item) {
return nameMatcher.test(item);
}
function index(filename_) {
return parseInt(filename_.substring((path.basename(filename) + '.').length), 10) || 0;
}
function byIndex(a, b) {
if (index(a) > index(b)) {
return 1;
} else if (index(a) < index(b) ) {
return -1;
} else {
return 0;
}
}
function increaseFileIndex (fileToRename, cb) {
var idx = index(fileToRename);
debug('Index of ' + fileToRename + ' is ' + idx);
if (idx < that.backups) {
//on windows, you can get a EEXIST error if you rename a file to an existing file
//so, we'll try to delete the file we're renaming to first
fs.unlink(filename + '.' + (idx+1), function (err) {
//ignore err: if we could not delete, it's most likely that it doesn't exist
debug('Renaming ' + fileToRename + ' -> ' + filename + '.' + (idx+1));
fs.rename(path.join(path.dirname(filename), fileToRename), filename + '.' + (idx + 1), cb);
});
} else {
cb();
}
}
function renameTheFiles(cb) {
//roll the backups (rename file.n to file.n+1, where n <= numBackups)
debug("Renaming the old files");
fs.readdir(path.dirname(filename), function (err, files) {
async.forEachSeries(
files.filter(justTheseFiles).sort(byIndex).reverse(),
increaseFileIndex,
cb
);
});
}
function openANewFile(cb) {
debug("Opening a new file");
fs.open(
filename,
that.options.flags,
that.options.mode,
function (err, fd) {
debug("opened new file");
var oldLogFileFD = that.fd;
that.fd = fd;
that.writable = true;
fs.close(oldLogFileFD, cb);
}
);
}
debug("Starting roll");
debug("Queueing up data until we've finished rolling");
debug("Flushing underlying stream");
this.flush();
async.series([
renameTheFiles,
openANewFile
], callback);
};

3
lib/streams/index.js Normal file
View File

@ -0,0 +1,3 @@
exports.BufferedWriteStream = require('./BufferedWriteStream');
exports.RollingFileStream = require('./RollingFileStream');
exports.DateRollingFileStream = require('./DateRollingFileStream');

49
test/streams/DateRollingFileStream-test.js Normal file
View File

@ -0,0 +1,49 @@
var vows = require('vows'),
assert = require('assert'),
fs = require('fs'),
DateRollingFileStream = require('../../lib/streams').DateRollingFileStream;
vows.describe('DateRollingFileStream').addBatch({
'arguments': {
topic: new DateRollingFileStream('test-date-rolling-file-stream', 'yyyy-mm-dd.hh'),
'should take a filename and a pattern and return a FileWriteStream': function(stream) {
assert.equal(stream.filename, 'test-date-rolling-file-stream');
assert.equal(stream.pattern, 'yyyy-mm-dd.hh');
assert.instanceOf(stream, fs.FileWriteStream);
},
'with default settings for the underlying stream': function(stream) {
assert.equal(stream.mode, 420);
assert.equal(stream.flags, 'a');
assert.equal(stream.encoding, 'utf8');
}
},
'default arguments': {
topic: new DateRollingFileStream('test-date-rolling-file-stream'),
'pattern should be yyyy-mm-dd': function(stream) {
assert.equal(stream.pattern, 'yyyy-mm-dd');
}
},
'with stream arguments': {
topic: new DateRollingFileStream('test-rolling-file-stream', 'yyyy-mm-dd', { mode: 0666 }),
'should pass them to the underlying stream': function(stream) {
assert.equal(stream.mode, 0666);
}
},
'with stream arguments but no pattern': {
topic: new DateRollingFileStream('test-rolling-file-stream', { mode: 0666 }),
'should pass them to the underlying stream': function(stream) {
assert.equal(stream.mode, 0666);
},
'should use default pattern': function(stream) {
assert.equal(stream.pattern, 'yyyy-mm-dd');
}
}
}).exportTo(module);

4
test/bufferedStream-test.js → test/streams/bufferedStream-test.js
View File

@ -1,7 +1,7 @@
var vows = require('vows')
, assert = require('assert')
, events = require('events')
, BufferedWriteStream = require('../lib/streams').BufferedWriteStream;
, BufferedWriteStream = require('../../lib/streams').BufferedWriteStream;
function FakeStream() {
this.writes = [];
@ -127,4 +127,4 @@ vows.describe('BufferedWriteStream').addBatch({
}
}
}).exportTo(module);
}).exportTo(module);

2
test/rollingFileStream-test.js → test/streams/rollingFileStream-test.js
View File

@ -2,7 +2,7 @@ var vows = require('vows')
, assert = require('assert')
, events = require('events')
, fs = require('fs')
, RollingFileStream = require('../lib/streams').RollingFileStream;
, RollingFileStream = require('../../lib/streams').RollingFileStream;
function remove(filename) {
try {