From f01c2786151fc3087d633de56d1a0d0250d57923 Mon Sep 17 00:00:00 2001 From: jeromew Date: Thu, 14 Mar 2019 18:24:15 +0000 Subject: [PATCH] Improve copy-to overall performance by keeping the in-out chunk balance (ported from CartoDB's fork) --- copy-to.js | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/copy-to.js b/copy-to.js index f078713..5947bdd 100644 --- a/copy-to.js +++ b/copy-to.js @@ -44,6 +44,16 @@ CopyStreamQuery.prototype._transform = function(chunk, enc, cb) { var messageCode; var needPush = false; + var buffer = Buffer.alloc(chunk.length); + var buffer_offset = 0; + + this.pushBufferIfneeded = function() { + if (needPush && buffer_offset > 0) { + this.push(buffer.slice(0, buffer_offset)) + buffer_offset = 0; + } + } + while((chunk.length - offset) >= (Byte1Len + Int32Len)) { var messageCode = chunk[offset] @@ -69,9 +79,10 @@ CopyStreamQuery.prototype._transform = function(chunk, enc, cb) { case code.NoticeResponse: case code.NotificationResponse: break; - + case code.ErrorResponse: case code.CopyDone: + this.pushBufferIfneeded(); this._detach() this.push(null) return cb(); @@ -86,7 +97,8 @@ CopyStreamQuery.prototype._transform = function(chunk, enc, cb) { if (needPush) { var row = chunk.slice(offset, offset + length - Int32Len) this.rowCount++ - this.push(row) + row.copy(buffer, buffer_offset); + buffer_offset += row.length; } offset += (length - Int32Len) } else { @@ -95,6 +107,8 @@ CopyStreamQuery.prototype._transform = function(chunk, enc, cb) { } } + this.pushBufferIfneeded(); + if(chunk.length - offset) { var slice = chunk.slice(offset) this._remainder = slice