string_decoder: reimplement in C++

Implement string decoder in C++. The perks are a decent speed boost (for decoding, whereas creation show some performance degradation), that this can now be used more easily to add native decoding support to C++ streams and (arguably) more readable variable names. $ ./node benchmark/compare.js --new ./node --old ./node-before --set n=25e4 string_decoder | Rscript benchmark/compare.R [03:07:14|% 100| 2/2 files | 60/60 runs | 80/80 configs]: Done confidence improvement accuracy (*) (**) (***) string_decoder/string-decoder-create.js n=250000 encoding="ascii" *** -77.32 % ±4.32% ±5.80% ±7.68% string_decoder/string-decoder-create.js n=250000 encoding="AscII" *** -74.15 % ±2.55% ±3.43% ±4.53% string_decoder/string-decoder-create.js n=250000 encoding="base64" *** -56.88 % ±4.75% ±6.36% ±8.37% string_decoder/string-decoder-create.js n=250000 encoding="ucs2" *** -64.00 % ±3.53% ±4.72% ±6.21% string_decoder/string-decoder-create.js n=250000 encoding="UTF-16LE" *** -54.64 % ±3.82% ±5.12% ±6.74% string_decoder/string-decoder-create.js n=250000 encoding="utf-8" *** -66.98 % ±4.92% ±6.62% ±8.75% string_decoder/string-decoder-create.js n=250000 encoding="UTF-8" *** -59.94 % ±3.45% ±4.61% ±6.03% string_decoder/string-decoder-create.js n=250000 encoding="utf8" *** -66.55 % ±4.65% ±6.26% ±8.28% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=1024 encoding="ascii" 12.89 % ±14.67% ±19.54% ±25.46% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=1024 encoding="base64-ascii" ** 12.83 % ±9.20% ±12.24% ±15.93% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=1024 encoding="base64-utf8" ** 12.19 % ±8.97% ±11.93% ±15.53% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=1024 encoding="utf16le" 7.69 % ±11.02% ±14.67% ±19.12% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=1024 encoding="utf8" 0.55 % ±8.17% ±10.88% ±14.17% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=128 encoding="ascii" * 23.55 % ±17.71% ±23.57% ±30.69% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=128 encoding="base64-ascii" *** 45.47 % ±13.60% ±18.13% ±23.66% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=128 encoding="base64-utf8" *** 39.64 % ±14.04% ±18.70% ±24.37% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=128 encoding="utf16le" * 13.59 % ±13.43% ±17.86% ±23.26% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=128 encoding="utf8" 11.52 % ±12.00% ±15.98% ±20.81% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=32 encoding="ascii" * 14.10 % ±13.51% ±18.01% ±23.50% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=32 encoding="base64-ascii" *** 67.77 % ±17.03% ±22.72% ±29.69% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=32 encoding="base64-utf8" *** 54.15 % ±16.54% ±22.08% ±28.87% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=32 encoding="utf16le" 15.89 % ±16.84% ±22.44% ±29.26% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=32 encoding="utf8" 9.28 % ±12.42% ±16.52% ±21.51% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=4096 encoding="ascii" 10.09 % ±10.59% ±14.09% ±18.34% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=4096 encoding="base64-ascii" 6.93 % ±8.86% ±11.79% ±15.36% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=4096 encoding="base64-utf8" * 9.24 % ±7.98% ±10.62% ±13.83% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=4096 encoding="utf16le" -0.67 % ±9.40% ±12.51% ±16.31% string_decoder/string-decoder.js n=250000 chunkLen=1024 inLen=4096 encoding="utf8" 2.27 % ±8.52% ±11.34% ±14.76% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=1024 encoding="ascii" 8.53 % ±10.65% ±14.18% ±18.48% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=1024 encoding="base64-ascii" *** 54.82 % ±10.50% ±14.00% ±18.29% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=1024 encoding="base64-utf8" *** 51.75 % ±10.83% ±14.43% ±18.85% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=1024 encoding="utf16le" -5.01 % ±9.15% ±12.18% ±15.85% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=1024 encoding="utf8" 4.55 % ±8.20% ±10.92% ±14.21% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=128 encoding="ascii" * 13.88 % ±12.62% ±16.81% ±21.91% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=128 encoding="base64-ascii" *** 52.79 % ±10.56% ±14.08% ±18.39% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=128 encoding="base64-utf8" *** 52.56 % ±11.39% ±15.21% ±19.90% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=128 encoding="utf16le" 0.43 % ±10.06% ±13.38% ±17.42% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=128 encoding="utf8" 3.33 % ±10.17% ±13.53% ±17.63% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=32 encoding="ascii" *** 31.81 % ±14.71% ±19.58% ±25.52% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=32 encoding="base64-ascii" *** 51.38 % ±13.23% ±17.65% ±23.09% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=32 encoding="base64-utf8" *** 48.89 % ±14.06% ±18.82% ±24.71% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=32 encoding="utf16le" 8.70 % ±12.70% ±16.90% ±22.01% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=32 encoding="utf8" * 17.94 % ±14.58% ±19.40% ±25.26% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=4096 encoding="ascii" ** 13.42 % ±9.78% ±13.03% ±17.00% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=4096 encoding="base64-ascii" *** 49.02 % ±10.60% ±14.13% ±18.45% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=4096 encoding="base64-utf8" *** 52.44 % ±11.03% ±14.72% ±19.23% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=4096 encoding="utf16le" -5.55 % ±9.18% ±12.22% ±15.92% string_decoder/string-decoder.js n=250000 chunkLen=16 inLen=4096 encoding="utf8" 6.32 % ±8.12% ±10.80% ±14.07% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=1024 encoding="ascii" 8.21 % ±12.47% ±16.61% ±21.65% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=1024 encoding="base64-ascii" *** 28.39 % ±9.32% ±12.42% ±16.19% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=1024 encoding="base64-utf8" *** 21.83 % ±9.94% ±13.24% ±17.26% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=1024 encoding="utf16le" 5.92 % ±10.21% ±13.60% ±17.71% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=1024 encoding="utf8" 2.83 % ±8.47% ±11.27% ±14.67% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=128 encoding="ascii" * 16.92 % ±16.02% ±21.34% ±27.81% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=128 encoding="base64-ascii" *** 36.24 % ±14.01% ±18.68% ±24.37% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=128 encoding="base64-utf8" *** 24.74 % ±13.22% ±17.65% ±23.10% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=128 encoding="utf16le" 6.26 % ±12.01% ±15.98% ±20.81% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=128 encoding="utf8" 1.51 % ±10.89% ±14.50% ±18.88% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=32 encoding="ascii" ** 25.58 % ±19.19% ±25.54% ±33.28% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=32 encoding="base64-ascii" *** 67.36 % ±15.88% ±21.21% ±27.76% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=32 encoding="base64-utf8" *** 55.56 % ±17.89% ±23.92% ±31.36% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=32 encoding="utf16le" ** 22.91 % ±15.06% ±20.03% ±26.07% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=32 encoding="utf8" 11.45 % ±12.80% ±17.03% ±22.18% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=4096 encoding="ascii" * 10.71 % ±10.00% ±13.32% ±17.35% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=4096 encoding="base64-ascii" *** 21.56 % ±9.11% ±12.12% ±15.77% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=4096 encoding="base64-utf8" *** 25.72 % ±9.04% ±12.03% ±15.68% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=4096 encoding="utf16le" 0.64 % ±9.49% ±12.64% ±16.48% string_decoder/string-decoder.js n=250000 chunkLen=256 inLen=4096 encoding="utf8" 1.64 % ±8.54% ±11.37% ±14.80% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=1024 encoding="ascii" 11.38 % ±11.58% ±15.44% ±20.15% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=1024 encoding="base64-ascii" *** 44.38 % ±10.72% ±14.31% ±18.70% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=1024 encoding="base64-utf8" *** 40.75 % ±10.92% ±14.57% ±19.03% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=1024 encoding="utf16le" 0.01 % ±9.09% ±12.09% ±15.73% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=1024 encoding="utf8" 4.25 % ±8.23% ±10.95% ±14.26% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=128 encoding="ascii" * 21.15 % ±16.30% ±21.71% ±28.31% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=128 encoding="base64-ascii" *** 50.31 % ±11.71% ±15.64% ±20.46% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=128 encoding="base64-utf8" *** 42.86 % ±11.50% ±15.34% ±20.07% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=128 encoding="utf16le" 3.08 % ±14.43% ±19.21% ±25.01% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=128 encoding="utf8" 5.19 % ±10.15% ±13.51% ±17.59% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=32 encoding="ascii" 12.04 % ±16.01% ±21.32% ±27.79% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=32 encoding="base64-ascii" *** 52.38 % ±15.90% ±21.27% ±27.92% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=32 encoding="base64-utf8" *** 38.86 % ±16.88% ±22.57% ±29.60% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=32 encoding="utf16le" ** 19.55 % ±13.93% ±18.54% ±24.15% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=32 encoding="utf8" ** 19.12 % ±12.42% ±16.52% ±21.51% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=4096 encoding="ascii" ** 14.49 % ±9.70% ±12.91% ±16.81% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=4096 encoding="base64-ascii" *** 37.39 % ±10.67% ±14.21% ±18.52% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=4096 encoding="base64-utf8" *** 46.33 % ±10.02% ±13.34% ±17.40% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=4096 encoding="utf16le" -1.83 % ±9.42% ±12.54% ±16.32% string_decoder/string-decoder.js n=250000 chunkLen=64 inLen=4096 encoding="utf8" 2.81 % ±8.96% ±11.93% ±15.52%
nodejs · Feb 2, 2018 · 7eae61d · 7eae61d
1 parent d2a6110
commit 7eae61d
Show file tree

Hide file tree

Showing 7 changed files with 476 additions and 238 deletions.
diff --git a/lib/string_decoder.js b/lib/string_decoder.js
@@ -22,10 +22,23 @@
 'use strict';
 
 const { Buffer } = require('buffer');
+const {
+  kIncompleteCharactersStart,
+  kIncompleteCharactersEnd,
+  kMissingBytes,
+  kBufferedBytes,
+  kEncodingField,
+  kSize,
+  decode,
+  flush,
+  encodings
+} = internalBinding('string_decoder');
 const internalUtil = require('internal/util');
 const errors = require('internal/errors');
 const isEncoding = Buffer[internalUtil.kIsEncodingSymbol];
 
+const kNativeDecoder = Symbol('kNativeDecoder');
+
 // Do not cache `Buffer.isEncoding` when checking encoding names as some
 // modules monkey-patch it to support additional encodings
 function normalizeEncoding(enc) {
@@ -36,258 +49,52 @@ function normalizeEncoding(enc) {
   return nenc || enc;
 }
 
+const encodingsMap = {};
+for (var i = 0; i < encodings.length; ++i)
+  encodingsMap[encodings[i]] = i;
+
 // StringDecoder provides an interface for efficiently splitting a series of
 // buffers into a series of JS strings without breaking apart multi-byte
 // characters.
-exports.StringDecoder = StringDecoder;
-function StringDecoder(encoding) {
-  this.encoding = normalizeEncoding(encoding);
-  var nb;
-  switch (this.encoding) {
-    case 'utf16le':
-      this.text = utf16Text;
-      this.end = utf16End;
-      nb = 4;
-      break;
-    case 'utf8':
-      this.fillLast = utf8FillLast;
-      nb = 4;
-      break;
-    case 'base64':
-      this.text = base64Text;
-      this.end = base64End;
-      nb = 3;
-      break;
-    default:
-      this.write = simpleWrite;
-      this.end = simpleEnd;
-      return;
-  }
-  this.lastNeed = 0;
-  this.lastTotal = 0;
-  this.lastChar = Buffer.allocUnsafe(nb);
-}
-
-StringDecoder.prototype.write = function(buf) {
-  if (buf.length === 0)
-    return '';
-  var r;
-  var i;
-  if (this.lastNeed) {
-    r = this.fillLast(buf);
-    if (r === undefined)
-      return '';
-    i = this.lastNeed;
-    this.lastNeed = 0;
-  } else {
-    i = 0;
-  }
-  if (i < buf.length)
-    return (r ? r + this.text(buf, i) : this.text(buf, i));
-  return r || '';
-};
-
-StringDecoder.prototype.end = utf8End;
-
-// Returns only complete characters in a Buffer
-StringDecoder.prototype.text = utf8Text;
-
-// Attempts to complete a partial non-UTF-8 character using bytes from a Buffer
-StringDecoder.prototype.fillLast = function(buf) {
-  if (this.lastNeed <= buf.length) {
-    buf.copy(this.lastChar, this.lastTotal - this.lastNeed, 0, this.lastNeed);
-    return this.lastChar.toString(this.encoding, 0, this.lastTotal);
-  }
-  buf.copy(this.lastChar, this.lastTotal - this.lastNeed, 0, buf.length);
-  this.lastNeed -= buf.length;
-};
-
-// Checks the type of a UTF-8 byte, whether it's ASCII, a leading byte, or a
-// continuation byte. If an invalid byte is detected, -2 is returned.
-function utf8CheckByte(byte) {
-  if (byte <= 0x7F)
-    return 0;
-  else if (byte >> 5 === 0x06)
-    return 2;
-  else if (byte >> 4 === 0x0E)
-    return 3;
-  else if (byte >> 3 === 0x1E)
-    return 4;
-  return (byte >> 6 === 0x02 ? -1 : -2);
-}
-
-// Checks at most 3 bytes at the end of a Buffer in order to detect an
-// incomplete multi-byte UTF-8 character. The total number of bytes (2, 3, or 4)
-// needed to complete the UTF-8 character (if applicable) are returned.
-function utf8CheckIncomplete(self, buf, i) {
-  var j = buf.length - 1;
-  if (j < i)
-    return 0;
-  var nb = utf8CheckByte(buf[j]);
-  if (nb >= 0) {
-    if (nb > 0)
-      self.lastNeed = nb - 1;
-    return nb;
-  }
-  if (--j < i || nb === -2)
-    return 0;
-  nb = utf8CheckByte(buf[j]);
-  if (nb >= 0) {
-    if (nb > 0)
-      self.lastNeed = nb - 2;
-    return nb;
-  }
-  if (--j < i || nb === -2)
-    return 0;
-  nb = utf8CheckByte(buf[j]);
-  if (nb >= 0) {
-    if (nb > 0) {
-      if (nb === 2)
-        nb = 0;
-      else
-        self.lastNeed = nb - 3;
-    }
-    return nb;
-  }
-  return 0;
-}
-
-// Validates as many continuation bytes for a multi-byte UTF-8 character as
-// needed or are available. If we see a non-continuation byte where we expect
-// one, we "replace" the validated continuation bytes we've seen so far with
-// a single UTF-8 replacement character ('\ufffd'), to match v8's UTF-8 decoding
-// behavior. The continuation byte check is included three times in the case
-// where all of the continuation bytes for a character exist in the same buffer.
-// It is also done this way as a slight performance increase instead of using a
-// loop.
-function utf8CheckExtraBytes(self, buf, p) {
-  if ((buf[0] & 0xC0) !== 0x80) {
-    self.lastNeed = 0;
-    return '\ufffd';
-  }
-  if (self.lastNeed > 1 && buf.length > 1) {
-    if ((buf[1] & 0xC0) !== 0x80) {
-      self.lastNeed = 1;
-      return '\ufffd';
-    }
-    if (self.lastNeed > 2 && buf.length > 2) {
-      if ((buf[2] & 0xC0) !== 0x80) {
-        self.lastNeed = 2;
-        return '\ufffd';
-      }
-    }
+class StringDecoder {
+  constructor(encoding) {
+    this.encoding = normalizeEncoding(encoding);
+    this[kNativeDecoder] = Buffer.alloc(kSize);
+    this[kNativeDecoder][kEncodingField] = encodingsMap[this.encoding];
   }
-}
 
-// Attempts to complete a multi-byte UTF-8 character using bytes from a Buffer.
-function utf8FillLast(buf) {
-  const p = this.lastTotal - this.lastNeed;
-  var r = utf8CheckExtraBytes(this, buf, p);
-  if (r !== undefined)
-    return r;
-  if (this.lastNeed <= buf.length) {
-    buf.copy(this.lastChar, p, 0, this.lastNeed);
-    return this.lastChar.toString(this.encoding, 0, this.lastTotal);
+  write(buf) {
+    if (typeof buf === 'string')
+      return buf;
+    if (!ArrayBuffer.isView(buf))
+      throw new errors.TypeError('ERR_INVALID_ARG_TYPE', 'buf',
+                                 ['Buffer', 'Uint8Array', 'ArrayBufferView']);
+    return decode(this[kNativeDecoder], buf);
   }
-  buf.copy(this.lastChar, p, 0, buf.length);
-  this.lastNeed -= buf.length;
-}
 
-// Returns all complete UTF-8 characters in a Buffer. If the Buffer ended on a
-// partial character, the character's bytes are buffered until the required
-// number of bytes are available.
-function utf8Text(buf, i) {
-  const total = utf8CheckIncomplete(this, buf, i);
-  if (!this.lastNeed)
-    return buf.toString('utf8', i);
-  this.lastTotal = total;
-  const end = buf.length - (total - this.lastNeed);
-  buf.copy(this.lastChar, 0, end);
-  return buf.toString('utf8', i, end);
-}
-
-// For UTF-8, a replacement character is added when ending on a partial
-// character.
-function utf8End(buf) {
-  const r = (buf && buf.length ? this.write(buf) : '');
-  if (this.lastNeed) {
-    this.lastNeed = 0;
-    this.lastTotal = 0;
-    return r + '\ufffd';
+  end(buf) {
+    let ret = '';
+    if (buf !== undefined)
+      ret = this.write(buf);
+    return ret + flush(this[kNativeDecoder]);
   }
-  return r;
-}
 
-// UTF-16LE typically needs two bytes per character, but even if we have an even
-// number of bytes available, we need to check if we end on a leading/high
-// surrogate. In that case, we need to wait for the next two bytes in order to
-// decode the last character properly.
-function utf16Text(buf, i) {
-  if ((buf.length - i) % 2 === 0) {
-    const r = buf.toString('utf16le', i);
-    if (r) {
-      const c = r.charCodeAt(r.length - 1);
-      if (c >= 0xD800 && c <= 0xDBFF) {
-        this.lastNeed = 2;
-        this.lastTotal = 4;
-        this.lastChar[0] = buf[buf.length - 2];
-        this.lastChar[1] = buf[buf.length - 1];
-        return r.slice(0, -1);
-      }
-    }
-    return r;
-  }
-  this.lastNeed = 1;
-  this.lastTotal = 2;
-  this.lastChar[0] = buf[buf.length - 1];
-  return buf.toString('utf16le', i, buf.length - 1);
-}
+  /* Everything below this line is undocumented legacy stuff. */
 
-// For UTF-16LE we do not explicitly append special replacement characters if we
-// end on a partial character, we simply let v8 handle that.
-function utf16End(buf) {
-  const r = (buf && buf.length ? this.write(buf) : '');
-  if (this.lastNeed) {
-    const end = this.lastTotal - this.lastNeed;
-    this.lastNeed = 0;
-    this.lastTotal = 0;
-    return r + this.lastChar.toString('utf16le', 0, end);
+  text(buf, offset) {
+    this[kNativeDecoder][kMissingBytes] = 0;
+    this[kNativeDecoder][kBufferedBytes] = 0;
+    return this.write(buf.slice(offset));
   }
-  return r;
-}
 
-function base64Text(buf, i) {
-  const n = (buf.length - i) % 3;
-  if (n === 0)
-    return buf.toString('base64', i);
-  this.lastNeed = 3 - n;
-  this.lastTotal = 3;
-  if (n === 1) {
-    this.lastChar[0] = buf[buf.length - 1];
-  } else {
-    this.lastChar[0] = buf[buf.length - 2];
-    this.lastChar[1] = buf[buf.length - 1];
+  get lastTotal() {
+    return this[kNativeDecoder][kBufferedBytes] + this.lastNeed;
   }
-  return buf.toString('base64', i, buf.length - n);
-}
-
 
-function base64End(buf) {
-  const r = (buf && buf.length ? this.write(buf) : '');
-  if (this.lastNeed) {
-    const end = 3 - this.lastNeed;
-    this.lastNeed = 0;
-    this.lastTotal = 0;
-    return r + this.lastChar.toString('base64', 0, end);
+  get lastChar() {
+    return this[kNativeDecoder].subarray(kIncompleteCharactersStart,
+                                         kIncompleteCharactersEnd);
   }
-  return r;
 }
 
-// Pass bytes on through for single-byte encodings (e.g. ascii, latin1, hex)
-function simpleWrite(buf) {
-  return buf.toString(this.encoding);
-}
-
-function simpleEnd(buf) {
-  return (buf && buf.length ? this.write(buf) : '');
-}
+exports.StringDecoder = StringDecoder;
diff --git a/node.gyp b/node.gyp
@@ -325,6 +325,7 @@
         'src/signal_wrap.cc',
         'src/spawn_sync.cc',
         'src/string_bytes.cc',
+        'src/string_decoder.cc',
         'src/string_search.cc',
         'src/stream_base.cc',
         'src/stream_wrap.cc',
@@ -378,6 +379,8 @@
         'src/req_wrap.h',
         'src/req_wrap-inl.h',
         'src/string_bytes.h',
+        'src/string_decoder.h',
+        'src/string_decoder-inl.h',
         'src/stream_base.h',
         'src/stream_base-inl.h',
         'src/stream_wrap.h',
@@ -988,6 +991,7 @@
         '<(obj_path)<(obj_separator)node_url.<(obj_suffix)',
         '<(obj_path)<(obj_separator)util.<(obj_suffix)',
         '<(obj_path)<(obj_separator)string_bytes.<(obj_suffix)',
+        '<(obj_path)<(obj_separator)string_decoder.<(obj_suffix)',
         '<(obj_path)<(obj_separator)string_search.<(obj_suffix)',
         '<(obj_path)<(obj_separator)stream_base.<(obj_suffix)',
         '<(obj_path)<(obj_separator)node_constants.<(obj_suffix)',

diff --git a/src/node_internals.h b/src/node_internals.h
@@ -120,6 +120,7 @@ struct sockaddr;
     V(signal_wrap)                                                            \
     V(spawn_sync)                                                             \
     V(stream_wrap)                                                            \
+    V(string_decoder)                                                         \
     V(tcp_wrap)                                                               \
     V(timer_wrap)                                                             \
     V(trace_events)                                                           \

diff --git a/src/string_decoder-inl.h b/src/string_decoder-inl.h
@@ -0,0 +1,38 @@
+#ifndef SRC_STRING_DECODER_INL_H_
+#define SRC_STRING_DECODER_INL_H_
+
+#if defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS
+
+#include "string_decoder.h"
+#include "util.h"
+
+namespace node {
+
+inline void StringDecoder::SetEncoding(enum encoding encoding) {
+  state_[kBufferedBytes] = 0;
+  state_[kMissingBytes] = 0;
+  state_[kEncodingField] = encoding;
+}
+
+inline enum encoding StringDecoder::Encoding() const {
+  return static_cast<enum encoding>(state_[kEncodingField]);
+}
+
+inline unsigned StringDecoder::BufferedBytes() const {
+  return state_[kBufferedBytes];
+}
+
+inline unsigned StringDecoder::MissingBytes() const {
+  return state_[kMissingBytes];
+}
+
+inline char* StringDecoder::IncompleteCharacterBuffer() {
+  return reinterpret_cast<char*>(state_ + kIncompleteCharactersStart);
+}
+
+
+}  // namespace node
+
+#endif  // defined(NODE_WANT_INTERNALS) && NODE_WANT_INTERNALS
+
+#endif   // SRC_STRING_DECODER_INL_H_