Update Acorn.js to v0.1.01

lib/acorn.js

@@ -12,13 +12,20 @@
 // Please use the [github bug tracker][ghbt] to report issues.
 //
 // [ghbt]: https://github.com/marijnh/acorn/issues
+//
+// This file defines the main parser interface. The library also comes
+// with a [error-tolerant parser][dammit] and an
+// [abstract syntax tree walker][walk], defined in other files.
+//
+// [dammit]: acorn_loose.js
+// [walk]: util/walk.js
 
 (function(exports) {
   "use strict";
 
-  exports.version = "0.0.1";
+  exports.version = "0.1.01";
 
-  // The main exported interface (under `window.acorn` when in the
+  // The main exported interface (under `self.acorn` when in the
   // browser) is a `parse` function that takes a code string and
   // returns an abstract syntax tree as specified by [Mozilla parser
   // API][api], with the caveat that the SpiderMonkey-specific syntax
@@ -30,10 +37,8 @@
 
   exports.parse = function(inpt, opts) {
     input = String(inpt); inputLen = input.length;
-    options = opts || {};
-    for (var opt in defaultOptions) if (!options.hasOwnProperty(opt))
-      options[opt] = defaultOptions[opt];
-    sourceFile = options.sourceFile || null;
+    setOptions(opts);
+    initTokenState();
     return parseTopLevel(options.program);
   };
 
@@ -55,18 +60,21 @@
     // By default, reserved words are not enforced. Enable
     // `forbidReserved` to enforce them.
     forbidReserved: false,
-    // When `trackComments` is turned on, the parser will attach
-    // `commentsBefore` and `commentsAfter` properties to AST nodes
-    // holding arrays of strings. A single comment may appear in both
-    // a `commentsBefore` and `commentsAfter` array (of the nodes
-    // after and before it), but never twice in the before (or after)
-    // array of different nodes.
-    trackComments: false,
     // When `locations` is on, `loc` properties holding objects with
     // `start` and `end` properties in `{line, column}` form (with
     // line being 1-based and column 0-based) will be attached to the
     // nodes.
     locations: false,
+    // A function can be passed as `onComment` option, which will
+    // cause Acorn to call that function with `(block, text, start,
+    // end)` parameters whenever a comment is skipped. `block` is a
+    // boolean indicating whether this is a block (`/* */`) comment,
+    // `text` is the content of the comment, and `start` and `end` are
+    // character offsets that denote the start and end of the comment.
+    // When the `locations` option is on, two more parameters are
+    // passed, the full `{line, column}` locations of the start and
+    // end of the comments.
+    onComment: null,
     // Nodes have their start and end characters offsets recorded in
     // `start` and `end` properties (directly on the node, rather than
     // the `loc` object, which holds line/column data. To also add a
@@ -87,6 +95,13 @@
     sourceFile: null
   };
 
+  function setOptions(opts) {
+    options = opts || {};
+    for (var opt in defaultOptions) if (!options.hasOwnProperty(opt))
+      options[opt] = defaultOptions[opt];
+    sourceFile = options.sourceFile || null;
+  }
+
   // The `getLineInfo` function is mostly useful when the
   // `locations` option is off (for performance reasons) and you
   // want to find the line/column position for a given character
@@ -106,9 +121,44 @@
   };
 
   // Acorn is organized as a tokenizer and a recursive-descent parser.
-  // Both use (closure-)global variables to keep their state and
-  // communicate. We already saw the `options`, `input`, and
-  // `inputLen` variables above (set in `parse`).
+  // The `tokenize` export provides an interface to the tokenizer.
+  // Because the tokenizer is optimized for being efficiently used by
+  // the Acorn parser itself, this interface is somewhat crude and not
+  // very modular. Performing another parse or call to `tokenize` will
+  // reset the internal state, and invalidate existing tokenizers.
+
+  exports.tokenize = function(inpt, opts) {
+    input = String(inpt); inputLen = input.length;
+    setOptions(opts);
+    initTokenState();
+
+    var t = {};
+    function getToken(forceRegexp) {
+      readToken(forceRegexp);
+      t.start = tokStart; t.end = tokEnd;
+      t.startLoc = tokStartLoc; t.endLoc = tokEndLoc;
+      t.type = tokType; t.value = tokVal;
+      return t;
+    }
+    getToken.jumpTo = function(pos, reAllowed) {
+      tokPos = pos;
+      if (options.locations) {
+        tokCurLine = tokLineStart = lineBreak.lastIndex = 0;
+        var match;
+        while ((match = lineBreak.exec(input)) && match.index < pos) {
+          ++tokCurLine;
+          tokLineStart = match.index + match[0].length;
+        }
+      }
+      var ch = input.charAt(pos - 1);
+      tokRegexpAllowed = reAllowed;
+      skipSpace();
+    };
+    return getToken;
+  };
+
+  // State is kept in (closure-)global variables. We already saw the
+  // `options`, `input`, and `inputLen` variables above.
 
   // The current position of the tokenizer in the input.
 
@@ -133,11 +183,6 @@
 
   var tokType, tokVal;
 
-  // These are used to hold arrays of comments when
-  // `options.trackComments` is true.
-
-  var tokCommentsBefore, tokCommentsAfter;
-
   // Interal state for the tokenizer. To distinguish between division
   // operators and regular expressions, it remembers whether the last
   // token was one that is allowed to be followed by an expression.
@@ -145,13 +190,13 @@
   // division operator. See the `parseStatement` function for a
   // caveat.)
 
-  var tokRegexpAllowed, tokComments;
+  var tokRegexpAllowed;
 
   // When `options.locations` is true, these are used to keep
   // track of the current line, and know when a new line has been
-  // entered. See the `curLineLoc` function.
+  // entered.
 
-  var tokCurLine, tokLineStart, tokLineStartNext;
+  var tokCurLine, tokLineStart;
 
   // These store the position of the previous token, which is useful
   // when finishing a node and assigning its `end` position.
@@ -166,15 +211,17 @@
   var inFunction, labels, strict;
 
   // This function is used to raise exceptions on parse errors. It
-  // takes either a `{line, column}` object or an offset integer (into
-  // the current `input`) as `pos` argument. It attaches the position
-  // to the end of the error message, and then raises a `SyntaxError`
-  // with that message.
+  // takes an offset integer (into the current `input`) to indicate
+  // the location of the error, attaches the position to the end
+  // of the error message, and then raises a `SyntaxError` with that
+  // message.
 
   function raise(pos, message) {
-    if (typeof pos == "number") pos = getLineInfo(input, pos);
-    message += " (" + pos.line + ":" + pos.column + ")";
-    throw new SyntaxError(message);
+    var loc = getLineInfo(input, pos);
+    message += " (" + loc.line + ":" + loc.column + ")";
+    var err = new SyntaxError(message);
+    err.pos = pos; err.loc = loc; err.raisedAt = tokPos;
+    throw err;
   }
 
   // ## Token types
@@ -233,10 +280,10 @@
                       "function": _function, "if": _if, "return": _return, "switch": _switch,
                       "throw": _throw, "try": _try, "var": _var, "while": _while, "with": _with,
                       "null": _null, "true": _true, "false": _false, "new": _new, "in": _in,
-                      "instanceof": {keyword: "instanceof", binop: 7}, "this": _this,
-                      "typeof": {keyword: "typeof", prefix: true},
-                      "void": {keyword: "void", prefix: true},
-                      "delete": {keyword: "delete", prefix: true}};
+                      "instanceof": {keyword: "instanceof", binop: 7, beforeExpr: true}, "this": _this,
+                      "typeof": {keyword: "typeof", prefix: true, beforeExpr: true},
+                      "void": {keyword: "void", prefix: true, beforeExpr: true},
+                      "delete": {keyword: "delete", prefix: true, beforeExpr: true}};
 
   // Punctuation token types. Again, the `type` property is purely for debugging.
 
@@ -270,6 +317,15 @@
   var _bin7 = {binop: 7, beforeExpr: true}, _bin8 = {binop: 8, beforeExpr: true};
   var _bin10 = {binop: 10, beforeExpr: true};
 
+  // Provide access to the token types for external users of the
+  // tokenizer.
+
+  exports.tokTypes = {bracketL: _bracketL, bracketR: _bracketR, braceL: _braceL, braceR: _braceR,
+                      parenL: _parenL, parenR: _parenR, comma: _comma, semi: _semi, colon: _colon,
+                      dot: _dot, question: _question, slash: _slash, eq: _eq, name: _name, eof: _eof,
+                      num: _num, regexp: _regexp, string: _string};
+  for (var kw in keywordTypes) exports.tokTypes[kw] = keywordTypes[kw];
+
   // This is a trick taken from Esprima. It turns out that, on
   // non-Chrome browsers, to check whether a string is in a set, a
   // predicate containing a big ugly `switch` statement is faster than
@@ -384,23 +440,12 @@
 
   // ## Tokenizer
 
-  // These are used when `options.locations` is on, in order to track
-  // the current line number and start of line offset, in order to set
-  // `tokStartLoc` and `tokEndLoc`.
+  // These are used when `options.locations` is on, for the
+  // `tokStartLoc` and `tokEndLoc` properties.
 
-  function nextLineStart() {
-    lineBreak.lastIndex = tokLineStart;
-    var match = lineBreak.exec(input);
-    return match ? match.index + match[0].length : input.length + 1;
-  }
-
-  function curLineLoc() {
-    while (tokLineStartNext <= tokPos) {
-      ++tokCurLine;
-      tokLineStart = tokLineStartNext;
-      tokLineStartNext = nextLineStart();
-    }
-    return {line: tokCurLine, column: tokPos - tokLineStart};
+  function line_loc_t() {
+    this.line = tokCurLine;
+    this.column = tokPos - tokLineStart;
   }
 
   // Reset the token state. Used at the start of a parse.
@@ -408,64 +453,86 @@
   function initTokenState() {
     tokCurLine = 1;
     tokPos = tokLineStart = 0;
-    tokLineStartNext = nextLineStart();
     tokRegexpAllowed = true;
-    tokComments = null;
     skipSpace();
   }
 
-  // Called at the end of every token. Sets `tokEnd`, `tokVal`,
-  // `tokCommentsAfter`, and `tokRegexpAllowed`, and skips the space
-  // after the token, so that the next one's `tokStart` will point at
-  // the right position.
+  // Called at the end of every token. Sets `tokEnd`, `tokVal`, and
+  // `tokRegexpAllowed`, and skips the space after the token, so that
+  // the next one's `tokStart` will point at the right position.
 
   function finishToken(type, val) {
     tokEnd = tokPos;
-    if (options.locations) tokEndLoc = curLineLoc();
+    if (options.locations) tokEndLoc = new line_loc_t;
     tokType = type;
     skipSpace();
     tokVal = val;
-    tokCommentsAfter = tokComments;
     tokRegexpAllowed = type.beforeExpr;
   }
 
   function skipBlockComment() {
-    var end = input.indexOf("*/", tokPos += 2);
+    var startLoc = options.onComment && options.locations && new line_loc_t;
+    var start = tokPos, end = input.indexOf("*/", tokPos += 2);
     if (end === -1) raise(tokPos - 2, "Unterminated comment");
-    if (options.trackComments)
-      (tokComments || (tokComments = [])).push(input.slice(tokPos, end));
     tokPos = end + 2;
+    if (options.locations) {
+      lineBreak.lastIndex = start;
+      var match;
+      while ((match = lineBreak.exec(input)) && match.index < tokPos) {
+        ++tokCurLine;
+        tokLineStart = match.index + match[0].length;
+      }
+    }
+    if (options.onComment)
+      options.onComment(true, input.slice(start + 2, end), start, tokPos,
+                        startLoc, options.locations && new line_loc_t);
   }
 
   function skipLineComment() {
     var start = tokPos;
+    var startLoc = options.onComment && options.locations && new line_loc_t;
     var ch = input.charCodeAt(tokPos+=2);
     while (tokPos < inputLen && ch !== 10 && ch !== 13 && ch !== 8232 && ch !== 8329) {
       ++tokPos;
       ch = input.charCodeAt(tokPos);
     }
-    if (options.trackComments)
-      (tokComments || (tokComments = [])).push(input.slice(start, tokPos));
+    if (options.onComment)
+      options.onComment(false, input.slice(start + 2, tokPos), start, tokPos,
+                        startLoc, options.locations && new line_loc_t);
   }
 
   // Called at the start of the parse and after every token. Skips
-  // whitespace and comments, and, if `options.trackComments` is on,
-  // will store all skipped comments in `tokComments`.
+  // whitespace and comments, and.
 
   function skipSpace() {
-    tokComments = null;
     while (tokPos < inputLen) {
       var ch = input.charCodeAt(tokPos);
-      if (ch === 47) { // '/'
+      if (ch === 32) { // ' '
+        ++tokPos;
+      } else if(ch === 13) {
+        ++tokPos;
+        var next = input.charCodeAt(tokPos);
+        if(next === 10) {
+          ++tokPos;
+        }
+        if(options.locations) {
+          ++tokCurLine;
+          tokLineStart = tokPos;
+        }
+      } else if (ch === 10) {
+        ++tokPos;
+        ++tokCurLine;
+        tokLineStart = tokPos;
+      } else if(ch < 14 && ch > 8) {
+        ++tokPos;
+      } else if (ch === 47) { // '/'
         var next = input.charCodeAt(tokPos+1);
         if (next === 42) { // '*'
           skipBlockComment();
         } else if (next === 47) { // '/'
           skipLineComment();
         } else break;
-      } else if (ch < 14 && ch > 8) {
-        ++tokPos;
-      } else if (ch === 32 || ch === 160) { // ' ', '\xa0'
+      } else if ((ch < 14 && ch > 8) || ch === 32 || ch === 160) { // ' ', '\xa0'
         ++tokPos;
       } else if (ch >= 5760 && nonASCIIwhitespace.test(String.fromCharCode(ch))) {
         ++tokPos;
@@ -487,26 +554,71 @@
   // The `forceRegexp` parameter is used in the one case where the
   // `tokRegexpAllowed` trick does not work. See `parseStatement`.
 
-  function readToken(forceRegexp) {
-    tokStart = tokPos;
-    if (options.locations) tokStartLoc = curLineLoc();
-    tokCommentsBefore = tokComments;
-    if (forceRegexp) return readRegexp();
-    if (tokPos >= inputLen) return finishToken(_eof);
-
-    var code = input.charCodeAt(tokPos);
-    // Identifier or keyword. '\uXXXX' sequences are allowed in
-    // identifiers, so '\' also dispatches to that.
-    if (isIdentifierStart(code) || code === 92 /* '\' */) return readWord();
+  function readToken_dot() {
     var next = input.charCodeAt(tokPos+1);
+    if (next >= 48 && next <= 57) return readNumber(true);
+    ++tokPos;
+    return finishToken(_dot);
+  }
 
+  function readToken_slash() { // '/'
+    var next = input.charCodeAt(tokPos+1);
+    if (tokRegexpAllowed) {++tokPos; return readRegexp();}
+    if (next === 61) return finishOp(_assign, 2);
+    return finishOp(_slash, 1);
+  }
+
+  function readToken_mult_modulo() { // '%*'
+    var next = input.charCodeAt(tokPos+1);
+    if (next === 61) return finishOp(_assign, 2);
+    return finishOp(_bin10, 1);
+  }
+
+  function readToken_pipe_amp(code) { // '|&'
+    var next = input.charCodeAt(tokPos+1);
+    if (next === code) return finishOp(code === 124 ? _bin1 : _bin2, 2);
+    if (next === 61) return finishOp(_assign, 2);
+    return finishOp(code === 124 ? _bin3 : _bin5, 1);
+  }
+
+  function readToken_caret() { // '^'
+    var next = input.charCodeAt(tokPos+1);
+    if (next === 61) return finishOp(_assign, 2);
+    return finishOp(_bin4, 1);    
+  }
+
+  function readToken_plus_min(code) { // '+-'
+    var next = input.charCodeAt(tokPos+1);
+    if (next === code) return finishOp(_incdec, 2);
+    if (next === 61) return finishOp(_assign, 2);
+    return finishOp(_plusmin, 1);    
+  }
+
+  function readToken_lt_gt(code) { // '<>'
+    var next = input.charCodeAt(tokPos+1);
+    var size = 1;
+    if (next === code) {
+      size = code === 62 && input.charCodeAt(tokPos+2) === 62 ? 3 : 2;
+      if (input.charCodeAt(tokPos + size) === 61) return finishOp(_assign, size + 1);
+      return finishOp(_bin8, size);
+    }
+    if (next === 61)
+      size = input.charCodeAt(tokPos+2) === 61 ? 3 : 2;
+    return finishOp(_bin7, size);
+  }
+  
+  function readToken_eq_excl(code) { // '=!'
+    var next = input.charCodeAt(tokPos+1);
+    if (next === 61) return finishOp(_bin6, input.charCodeAt(tokPos+2) === 61 ? 3 : 2);
+    return finishOp(code === 61 ? _eq : _prefix, 1);
+  }
+
+  function getTokenFromCode(code) {
     switch(code) {
       // The interpretation of a dot depends on whether it is followed
       // by a digit.
     case 46: // '.'
-      if (next >= 48 && next <= 57) return readNumber(String.fromCharCode(code));
-      ++tokPos;
-      return finishToken(_dot);
+      return readToken_dot();
 
       // Punctuation tokens.
     case 40: ++tokPos; return finishToken(_parenL);
@@ -522,11 +634,12 @@
 
       // '0x' is a hexadecimal number.
     case 48: // '0'
+      var next = input.charCodeAt(tokPos+1);
       if (next === 120 || next === 88) return readHexNumber();
       // Anything else beginning with a digit is an integer, octal
       // number, or float.
     case 49: case 50: case 51: case 52: case 53: case 54: case 55: case 56: case 57: // 1-9
-      return readNumber(String.fromCharCode(code));
+      return readNumber(false);
 
       // Quotes produce strings.
     case 34: case 39: // '"', "'"
@@ -538,52 +651,54 @@
     // of the type given by its first argument.
 
     case 47: // '/'
-      if (tokRegexpAllowed) {++tokPos; return readRegexp();}
-      if (next === 61) return finishOp(_assign, 2);
-      return finishOp(_slash, 1);
+      return readToken_slash(code);
 
     case 37: case 42: // '%*'
-      if (next === 61) return finishOp(_assign, 2);
-      return finishOp(_bin10, 1);
+      return readToken_mult_modulo();
 
     case 124: case 38: // '|&'
-      if (next === code) return finishOp(code === 124 ? _bin1 : _bin2, 2);
-      if (next === 61) return finishOp(_assign, 2);
-      return finishOp(code === 124 ? _bin3 : _bin5, 1);
+      return readToken_pipe_amp(code);
 
     case 94: // '^'
-      if (next === 61) return finishOp(_assign, 2);
-      return finishOp(_bin4, 1);
+      return readToken_caret();
 
     case 43: case 45: // '+-'
-      if (next === code) return finishOp(_incdec, 2);
-      if (next === 61) return finishOp(_assign, 2);
-      return finishOp(_plusmin, 1);
+      return readToken_plus_min(code);
 
     case 60: case 62: // '<>'
-      var size = 1;
-      if (next === code) {
-        size = code === 62 && input.charCodeAt(tokPos+2) === 62 ? 3 : 2;
-        if (input.charCodeAt(tokPos + size) === 61) return finishOp(_assign, size + 1);
-        return finishOp(_bin8, size);
-      }
-      if (next === 61)
-        size = input.charCodeAt(tokPos+2) === 61 ? 3 : 2;
-      return finishOp(_bin7, size);
+      return readToken_lt_gt(code);
 
     case 61: case 33: // '=!'
-      if (next === 61) return finishOp(_bin6, input.charCodeAt(tokPos+2) === 61 ? 3 : 2);
-      return finishOp(code === 61 ? _eq : _prefix, 1);
+      return readToken_eq_excl(code);
 
     case 126: // '~'
       return finishOp(_prefix, 1);
     }
 
-    // If we are here, we either found a non-ASCII identifier
-    // character, or something that's entirely disallowed.
-    var ch = String.fromCharCode(code);
-    if (ch === "\\" || nonASCIIidentifierStart.test(ch)) return readWord();
-    raise(tokPos, "Unexpected character '" + ch + "'");
+    return false;
+  }
+
+  function readToken(forceRegexp) {
+    tokStart = tokPos;
+    if (options.locations) tokStartLoc = new line_loc_t;
+    if (forceRegexp) return readRegexp();
+    if (tokPos >= inputLen) return finishToken(_eof);
+
+    var code = input.charCodeAt(tokPos);
+    // Identifier or keyword. '\uXXXX' sequences are allowed in
+    // identifiers, so '\' also dispatches to that.
+    if (isIdentifierStart(code) || code === 92 /* '\' */) return readWord();
+    
+    var tok = getTokenFromCode(code);
+
+    if (tok === false) {
+      // If we are here, we either found a non-ASCII identifier
+      // character, or something that's entirely disallowed.
+      var ch = String.fromCharCode(code);
+      if (ch === "\\" || nonASCIIidentifierStart.test(ch)) return readWord();
+      raise(tokPos, "Unexpected character '" + ch + "'");
+    } 
+    return tok;
   }
 
   function finishOp(type, size) {
@@ -624,7 +739,7 @@
 
   function readInt(radix, len) {
     var start = tokPos, total = 0;
-    for (;;) {
+    for (var i = 0, e = len == null ? Infinity : len; i < e; ++i) {
       var code = input.charCodeAt(tokPos), val;
       if (code >= 97) val = code - 97 + 10; // a
       else if (code >= 65) val = code - 65 + 10; // A
@@ -649,18 +764,18 @@
 
   // Read an integer, octal integer, or floating-point number.
   
-  function readNumber(ch) {
-    var start = tokPos, isFloat = ch === ".";
-    if (!isFloat && readInt(10) == null) raise(start, "Invalid number");
-    if (isFloat || input.charAt(tokPos) === ".") {
-      var next = input.charAt(++tokPos);
-      if (next === "-" || next === "+") ++tokPos;
-      if (readInt(10) === null && ch === ".") raise(start, "Invalid number");
+  function readNumber(startsWithDot) {
+    var start = tokPos, isFloat = false, octal = input.charCodeAt(tokPos) === 48;
+    if (!startsWithDot && readInt(10) === null) raise(start, "Invalid number");
+    if (input.charCodeAt(tokPos) === 46) {
+      ++tokPos;
+      readInt(10);
       isFloat = true;
     }
-    if (/e/i.test(input.charAt(tokPos))) {
-      var next = input.charAt(++tokPos);
-      if (next === "-" || next === "+") ++tokPos;
+    var next = input.charCodeAt(tokPos);
+    if (next === 69 || next === 101) { // 'eE'
+      next = input.charCodeAt(++tokPos);
+      if (next === 43 || next === 45) ++tokPos; // '+-'
       if (readInt(10) === null) raise(start, "Invalid number")
       isFloat = true;
     }
@@ -668,7 +783,7 @@
 
     var str = input.slice(start, tokPos), val;
     if (isFloat) val = parseFloat(str);
-    else if (ch !== "0" || str.length === 1) val = parseInt(str, 10);
+    else if (!octal || str.length === 1) val = parseInt(str, 10);
     else if (/[89]/.test(str) || strict) raise(start, "Invalid number");
     else val = parseInt(str, 8);
     return finishToken(_num, val);
@@ -678,13 +793,13 @@
 
   function readString(quote) {
     tokPos++;
-    var str = [];
+    var out = "";
     for (;;) {
       if (tokPos >= inputLen) raise(tokStart, "Unterminated string constant");
       var ch = input.charCodeAt(tokPos);
       if (ch === quote) {
         ++tokPos;
-        return finishToken(_string, String.fromCharCode.apply(null, str));
+        return finishToken(_string, out);
       }
       if (ch === 92) { // '\'
         ch = input.charCodeAt(++tokPos);
@@ -695,28 +810,30 @@
         ++tokPos;
         if (octal) {
           if (strict) raise(tokPos - 2, "Octal literal in strict mode");
-          str.push(parseInt(octal, 8));
+          out += String.fromCharCode(parseInt(octal, 8));
           tokPos += octal.length - 1;
         } else {
           switch (ch) {
-          case 110: str.push(10); break; // 'n' -> '\n'
-          case 114: str.push(13); break; // 'r' -> '\r'
-          case 120: str.push(readHexChar(2)); break; // 'x'
-          case 117: str.push(readHexChar(4)); break; // 'u'
-          case 85: str.push(readHexChar(8)); break; // 'U'
-          case 116: str.push(9); break; // 't' -> '\t'
-          case 98: str.push(8); break; // 'b' -> '\b'
-          case 118: str.push(11); break; // 'v' -> '\u000b'
-          case 102: str.push(12); break; // 'f' -> '\f'
-          case 48: str.push(0); break; // 0 -> '\0'
+          case 110: out += "\n"; break; // 'n' -> '\n'
+          case 114: out += "\r"; break; // 'r' -> '\r'
+          case 120: out += String.fromCharCode(readHexChar(2)); break; // 'x'
+          case 117: out += String.fromCharCode(readHexChar(4)); break; // 'u'
+          case 85: out += String.fromCharCode(readHexChar(8)); break; // 'U'
+          case 116: out += "\t"; break; // 't' -> '\t'
+          case 98: out += "\b"; break; // 'b' -> '\b'
+          case 118: out += "\v"; break; // 'v' -> '\u000b'
+          case 102: out += "\f"; break; // 'f' -> '\f'
+          case 48: out += "\0"; break; // 0 -> '\0'
           case 13: if (input.charCodeAt(tokPos) === 10) ++tokPos; // '\r\n'
-          case 10: break; // ' \n'
-          default: str.push(ch); break;
+          case 10: // ' \n'
+            if (options.locations) { tokLineStart = tokPos; ++tokCurLine; }
+            break;
+          default: out += String.fromCharCode(ch); break;
           }
         }
       } else {
         if (ch === 13 || ch === 10 || ch === 8232 || ch === 8329) raise(tokStart, "Unterminated string constant");
-        if (ch !== 92) str.push(ch); // '\'
+        out += String.fromCharCode(ch); // '\'
         ++tokPos;
       }
     }
@@ -827,65 +944,51 @@
     readToken();
   }
 
-  // Start an AST node, attaching a start offset and optionally a
-  // `commentsBefore` property to it.
+  // Start an AST node, attaching a start offset.
+
+  function node_t() {
+    this.type = null;
+    this.start = tokStart;
+    this.end = null;
+  }
+
+  function node_loc_t() {
+    this.start = tokStartLoc;
+    this.end = null;
+    if (sourceFile !== null) this.source = sourceFile;
+  }
 
   function startNode() {
-    var node = {type: null, start: tokStart, end: null};
-    if (options.trackComments && tokCommentsBefore) {
-      node.commentsBefore = tokCommentsBefore;
-      tokCommentsBefore = null;
-    }
+    var node = new node_t();
     if (options.locations)
-      node.loc = {start: tokStartLoc, end: null, source: sourceFile};
+      node.loc = new node_loc_t();
     if (options.ranges)
       node.range = [tokStart, 0];
     return node;
   }
 
-  // Start a node whose start offset/comments information should be
-  // based on the start of another node. For example, a binary
-  // operator node is only started after its left-hand side has
-  // already been parsed.
+  // Start a node whose start offset information should be based on
+  // the start of another node. For example, a binary operator node is
+  // only started after its left-hand side has already been parsed.
 
   function startNodeFrom(other) {
-    var node = {type: null, start: other.start};
-    if (other.commentsBefore) {
-      node.commentsBefore = other.commentsBefore;
-      other.commentsBefore = null;
+    var node = new node_t();
+    node.start = other.start;
+    if (options.locations) {
+      node.loc = new node_loc_t();
+      node.loc.start = other.loc.start;
     }
-    if (options.locations)
-      node.loc = {start: other.loc.start, end: null, source: other.loc.source};
     if (options.ranges)
       node.range = [other.range[0], 0];
 
     return node;
   }
 
-  // Finish an AST node, adding `type`, `end`, and `commentsAfter`
-  // properties.
-  //
-  // We keep track of the last node that we finished, in order
-  // 'bubble' `commentsAfter` properties up to the biggest node. I.e.
-  // in '`1 + 1 // foo', the comment should be attached to the binary
-  // operator node, not the second literal node.
-
-  var lastFinishedNode;
+  // Finish an AST node, adding `type` and `end` properties.
 
   function finishNode(node, type) {
     node.type = type;
     node.end = lastEnd;
-    if (options.trackComments) {
-      if (tokCommentsAfter) {
-        node.commentsAfter = tokCommentsAfter;
-        tokCommentsAfter = null;
-      } else if (lastFinishedNode && lastFinishedNode.end === lastEnd &&
-                 lastFinishedNode.commentsAfter) {
-        node.commentsAfter = lastFinishedNode.commentsAfter;
-        lastFinishedNode.commentsAfter = null;
-      }
-      lastFinishedNode = node;
-    }
     if (options.locations)
       node.loc.end = lastEndLoc;
     if (options.ranges)
@@ -956,9 +1059,8 @@
   // to its body instead of creating a new node.
 
   function parseTopLevel(program) {
-    initTokenState();
     lastStart = lastEnd = tokPos;
-    if (options.locations) lastEndLoc = curLineLoc();
+    if (options.locations) lastEndLoc = new line_loc_t;
     inFunction = strict = null;
     labels = [];
     readToken();
@@ -972,7 +1074,7 @@
       first = false;
     }
     return finishNode(node, "Program");
-  };
+  }
 
   var loopLabel = {kind: "loop"}, switchLabel = {kind: "switch"};
 
@@ -1018,6 +1120,7 @@
 
     case _debugger:
       next();
+      semicolon();
       return finishNode(node, "DebuggerStatement");
 
     case _do:
@@ -1117,6 +1220,7 @@
       if (newline.test(input.slice(lastEnd, tokStart)))
         raise(lastEnd, "Illegal newline after throw");
       node.argument = parseExpression();
+      semicolon();
       return finishNode(node, "ThrowStatement");
 
     case _try:
@@ -1182,6 +1286,7 @@
         var kind = tokType.isLoop ? "loop" : tokType === _switch ? "switch" : null;
         labels.push({name: maybeName, kind: kind});
         node.body = parseStatement();
+        labels.pop();
         node.label = expr;
         return finishNode(node, "LabeledStatement");
       } else {
@@ -1429,6 +1534,7 @@
     case _null: case _true: case _false:
       var node = startNode();
       node.value = tokType.atomValue;
+      node.raw = tokType.keyword
       next();
       return finishNode(node, "Literal");
 
@@ -1476,7 +1582,7 @@
   function parseNew() {
     var node = startNode();
     next();
-    node.callee = parseSubscripts(parseExprAtom(false), true);
+    node.callee = parseSubscripts(parseExprAtom(), true);
     if (eat(_parenL)) node.arguments = parseExprList(_parenR, false);
     else node.arguments = [];
     return finishNode(node, "NewExpression");
@@ -1503,7 +1609,7 @@
         isGetSet = sawGetSet = true;
         kind = prop.kind = prop.key.name;
         prop.key = parsePropertyName();
-        if (!tokType === _parenL) unexpected();
+        if (tokType !== _parenL) unexpected();
         prop.value = parseFunction(startNode(), false);
       } else unexpected();
 
@@ -1601,4 +1707,4 @@
     return finishNode(node, "Identifier");
   }
 
-})(typeof exports === "undefined" ? (window.acorn = {}) : exports);
+})(typeof exports === "undefined" ? (self.acorn = {}) : exports);