X-Git-Url: https://git.saurik.com/apple/javascriptcore.git/blobdiff_plain/4e4e5a6f2694187498445a6ac6f1634ce8141119..4be4e30906bcb8ee30b4d189205cb70bad6707ce:/parser/Lexer.cpp?ds=sidebyside

diff --git a/parser/Lexer.cpp b/parser/Lexer.cpp
index bdeb049..4925656 100644
--- a/parser/Lexer.cpp
+++ b/parser/Lexer.cpp
@@ -1,7 +1,9 @@
 /*
  *  Copyright (C) 1999-2000 Harri Porten (porten@kde.org)
- *  Copyright (C) 2006, 2007, 2008, 2009 Apple Inc. All Rights Reserved.
+ *  Copyright (C) 2006, 2007, 2008, 2009, 2011, 2012, 2013 Apple Inc. All Rights Reserved.
  *  Copyright (C) 2007 Cameron Zwarich (cwzwarich@uwaterloo.ca)
+ *  Copyright (C) 2010 Zoltan Herczeg (zherczeg@inf.u-szeged.hu)
+ *  Copyright (C) 2012 Mathias Bynens (mathias@qiwi.be)
  *
  *  This library is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU Library General Public
@@ -24,10 +26,12 @@
 #include "Lexer.h"
 
 #include "JSFunction.h"
+
 #include "JSGlobalObjectFunctions.h"
+#include "Identifier.h"
 #include "NodeInfo.h"
 #include "Nodes.h"
-#include "dtoa.h"
+#include <wtf/dtoa.h>
 #include <ctype.h>
 #include <limits.h>
 #include <string.h>
@@ -36,879 +40,1698 @@
 using namespace WTF;
 using namespace Unicode;
 
-// We can't specify the namespace in yacc's C output, so do it here instead.
-using namespace JSC;
-
-#include "Grammar.h"
-#include "Lookup.h"
+#include "KeywordLookup.h"
 #include "Lexer.lut.h"
+#include "Parser.h"
 
 namespace JSC {
 
-static const UChar byteOrderMark = 0xFEFF;
+Keywords::Keywords(VM* vm)
+    : m_vm(vm)
+    , m_keywordTable(JSC::mainTable)
+{
+}
 
-Lexer::Lexer(JSGlobalData* globalData)
+enum CharacterType {
+    // Types for the main switch
+
+    // The first three types are fixed, and also used for identifying
+    // ASCII alpha and alphanumeric characters (see isIdentStart and isIdentPart).
+    CharacterIdentifierStart,
+    CharacterZero,
+    CharacterNumber,
+
+    CharacterInvalid,
+    CharacterLineTerminator,
+    CharacterExclamationMark,
+    CharacterOpenParen,
+    CharacterCloseParen,
+    CharacterOpenBracket,
+    CharacterCloseBracket,
+    CharacterComma,
+    CharacterColon,
+    CharacterQuestion,
+    CharacterTilde,
+    CharacterQuote,
+    CharacterDot,
+    CharacterSlash,
+    CharacterBackSlash,
+    CharacterSemicolon,
+    CharacterOpenBrace,
+    CharacterCloseBrace,
+
+    CharacterAdd,
+    CharacterSub,
+    CharacterMultiply,
+    CharacterModulo,
+    CharacterAnd,
+    CharacterXor,
+    CharacterOr,
+    CharacterLess,
+    CharacterGreater,
+    CharacterEqual,
+
+    // Other types (only one so far)
+    CharacterWhiteSpace,
+};
+
+// 256 Latin-1 codes
+static const unsigned short typesOfLatin1Characters[256] = {
+/*   0 - Null               */ CharacterInvalid,
+/*   1 - Start of Heading   */ CharacterInvalid,
+/*   2 - Start of Text      */ CharacterInvalid,
+/*   3 - End of Text        */ CharacterInvalid,
+/*   4 - End of Transm.     */ CharacterInvalid,
+/*   5 - Enquiry            */ CharacterInvalid,
+/*   6 - Acknowledgment     */ CharacterInvalid,
+/*   7 - Bell               */ CharacterInvalid,
+/*   8 - Back Space         */ CharacterInvalid,
+/*   9 - Horizontal Tab     */ CharacterWhiteSpace,
+/*  10 - Line Feed          */ CharacterLineTerminator,
+/*  11 - Vertical Tab       */ CharacterWhiteSpace,
+/*  12 - Form Feed          */ CharacterWhiteSpace,
+/*  13 - Carriage Return    */ CharacterLineTerminator,
+/*  14 - Shift Out          */ CharacterInvalid,
+/*  15 - Shift In           */ CharacterInvalid,
+/*  16 - Data Line Escape   */ CharacterInvalid,
+/*  17 - Device Control 1   */ CharacterInvalid,
+/*  18 - Device Control 2   */ CharacterInvalid,
+/*  19 - Device Control 3   */ CharacterInvalid,
+/*  20 - Device Control 4   */ CharacterInvalid,
+/*  21 - Negative Ack.      */ CharacterInvalid,
+/*  22 - Synchronous Idle   */ CharacterInvalid,
+/*  23 - End of Transmit    */ CharacterInvalid,
+/*  24 - Cancel             */ CharacterInvalid,
+/*  25 - End of Medium      */ CharacterInvalid,
+/*  26 - Substitute         */ CharacterInvalid,
+/*  27 - Escape             */ CharacterInvalid,
+/*  28 - File Separator     */ CharacterInvalid,
+/*  29 - Group Separator    */ CharacterInvalid,
+/*  30 - Record Separator   */ CharacterInvalid,
+/*  31 - Unit Separator     */ CharacterInvalid,
+/*  32 - Space              */ CharacterWhiteSpace,
+/*  33 - !                  */ CharacterExclamationMark,
+/*  34 - "                  */ CharacterQuote,
+/*  35 - #                  */ CharacterInvalid,
+/*  36 - $                  */ CharacterIdentifierStart,
+/*  37 - %                  */ CharacterModulo,
+/*  38 - &                  */ CharacterAnd,
+/*  39 - '                  */ CharacterQuote,
+/*  40 - (                  */ CharacterOpenParen,
+/*  41 - )                  */ CharacterCloseParen,
+/*  42 - *                  */ CharacterMultiply,
+/*  43 - +                  */ CharacterAdd,
+/*  44 - ,                  */ CharacterComma,
+/*  45 - -                  */ CharacterSub,
+/*  46 - .                  */ CharacterDot,
+/*  47 - /                  */ CharacterSlash,
+/*  48 - 0                  */ CharacterZero,
+/*  49 - 1                  */ CharacterNumber,
+/*  50 - 2                  */ CharacterNumber,
+/*  51 - 3                  */ CharacterNumber,
+/*  52 - 4                  */ CharacterNumber,
+/*  53 - 5                  */ CharacterNumber,
+/*  54 - 6                  */ CharacterNumber,
+/*  55 - 7                  */ CharacterNumber,
+/*  56 - 8                  */ CharacterNumber,
+/*  57 - 9                  */ CharacterNumber,
+/*  58 - :                  */ CharacterColon,
+/*  59 - ;                  */ CharacterSemicolon,
+/*  60 - <                  */ CharacterLess,
+/*  61 - =                  */ CharacterEqual,
+/*  62 - >                  */ CharacterGreater,
+/*  63 - ?                  */ CharacterQuestion,
+/*  64 - @                  */ CharacterInvalid,
+/*  65 - A                  */ CharacterIdentifierStart,
+/*  66 - B                  */ CharacterIdentifierStart,
+/*  67 - C                  */ CharacterIdentifierStart,
+/*  68 - D                  */ CharacterIdentifierStart,
+/*  69 - E                  */ CharacterIdentifierStart,
+/*  70 - F                  */ CharacterIdentifierStart,
+/*  71 - G                  */ CharacterIdentifierStart,
+/*  72 - H                  */ CharacterIdentifierStart,
+/*  73 - I                  */ CharacterIdentifierStart,
+/*  74 - J                  */ CharacterIdentifierStart,
+/*  75 - K                  */ CharacterIdentifierStart,
+/*  76 - L                  */ CharacterIdentifierStart,
+/*  77 - M                  */ CharacterIdentifierStart,
+/*  78 - N                  */ CharacterIdentifierStart,
+/*  79 - O                  */ CharacterIdentifierStart,
+/*  80 - P                  */ CharacterIdentifierStart,
+/*  81 - Q                  */ CharacterIdentifierStart,
+/*  82 - R                  */ CharacterIdentifierStart,
+/*  83 - S                  */ CharacterIdentifierStart,
+/*  84 - T                  */ CharacterIdentifierStart,
+/*  85 - U                  */ CharacterIdentifierStart,
+/*  86 - V                  */ CharacterIdentifierStart,
+/*  87 - W                  */ CharacterIdentifierStart,
+/*  88 - X                  */ CharacterIdentifierStart,
+/*  89 - Y                  */ CharacterIdentifierStart,
+/*  90 - Z                  */ CharacterIdentifierStart,
+/*  91 - [                  */ CharacterOpenBracket,
+/*  92 - \                  */ CharacterBackSlash,
+/*  93 - ]                  */ CharacterCloseBracket,
+/*  94 - ^                  */ CharacterXor,
+/*  95 - _                  */ CharacterIdentifierStart,
+/*  96 - `                  */ CharacterInvalid,
+/*  97 - a                  */ CharacterIdentifierStart,
+/*  98 - b                  */ CharacterIdentifierStart,
+/*  99 - c                  */ CharacterIdentifierStart,
+/* 100 - d                  */ CharacterIdentifierStart,
+/* 101 - e                  */ CharacterIdentifierStart,
+/* 102 - f                  */ CharacterIdentifierStart,
+/* 103 - g                  */ CharacterIdentifierStart,
+/* 104 - h                  */ CharacterIdentifierStart,
+/* 105 - i                  */ CharacterIdentifierStart,
+/* 106 - j                  */ CharacterIdentifierStart,
+/* 107 - k                  */ CharacterIdentifierStart,
+/* 108 - l                  */ CharacterIdentifierStart,
+/* 109 - m                  */ CharacterIdentifierStart,
+/* 110 - n                  */ CharacterIdentifierStart,
+/* 111 - o                  */ CharacterIdentifierStart,
+/* 112 - p                  */ CharacterIdentifierStart,
+/* 113 - q                  */ CharacterIdentifierStart,
+/* 114 - r                  */ CharacterIdentifierStart,
+/* 115 - s                  */ CharacterIdentifierStart,
+/* 116 - t                  */ CharacterIdentifierStart,
+/* 117 - u                  */ CharacterIdentifierStart,
+/* 118 - v                  */ CharacterIdentifierStart,
+/* 119 - w                  */ CharacterIdentifierStart,
+/* 120 - x                  */ CharacterIdentifierStart,
+/* 121 - y                  */ CharacterIdentifierStart,
+/* 122 - z                  */ CharacterIdentifierStart,
+/* 123 - {                  */ CharacterOpenBrace,
+/* 124 - |                  */ CharacterOr,
+/* 125 - }                  */ CharacterCloseBrace,
+/* 126 - ~                  */ CharacterTilde,
+/* 127 - Delete             */ CharacterInvalid,
+/* 128 - Cc category        */ CharacterInvalid,
+/* 129 - Cc category        */ CharacterInvalid,
+/* 130 - Cc category        */ CharacterInvalid,
+/* 131 - Cc category        */ CharacterInvalid,
+/* 132 - Cc category        */ CharacterInvalid,
+/* 133 - Cc category        */ CharacterInvalid,
+/* 134 - Cc category        */ CharacterInvalid,
+/* 135 - Cc category        */ CharacterInvalid,
+/* 136 - Cc category        */ CharacterInvalid,
+/* 137 - Cc category        */ CharacterInvalid,
+/* 138 - Cc category        */ CharacterInvalid,
+/* 139 - Cc category        */ CharacterInvalid,
+/* 140 - Cc category        */ CharacterInvalid,
+/* 141 - Cc category        */ CharacterInvalid,
+/* 142 - Cc category        */ CharacterInvalid,
+/* 143 - Cc category        */ CharacterInvalid,
+/* 144 - Cc category        */ CharacterInvalid,
+/* 145 - Cc category        */ CharacterInvalid,
+/* 146 - Cc category        */ CharacterInvalid,
+/* 147 - Cc category        */ CharacterInvalid,
+/* 148 - Cc category        */ CharacterInvalid,
+/* 149 - Cc category        */ CharacterInvalid,
+/* 150 - Cc category        */ CharacterInvalid,
+/* 151 - Cc category        */ CharacterInvalid,
+/* 152 - Cc category        */ CharacterInvalid,
+/* 153 - Cc category        */ CharacterInvalid,
+/* 154 - Cc category        */ CharacterInvalid,
+/* 155 - Cc category        */ CharacterInvalid,
+/* 156 - Cc category        */ CharacterInvalid,
+/* 157 - Cc category        */ CharacterInvalid,
+/* 158 - Cc category        */ CharacterInvalid,
+/* 159 - Cc category        */ CharacterInvalid,
+/* 160 - Zs category (nbsp) */ CharacterWhiteSpace,
+/* 161 - Po category        */ CharacterInvalid,
+/* 162 - Sc category        */ CharacterInvalid,
+/* 163 - Sc category        */ CharacterInvalid,
+/* 164 - Sc category        */ CharacterInvalid,
+/* 165 - Sc category        */ CharacterInvalid,
+/* 166 - So category        */ CharacterInvalid,
+/* 167 - So category        */ CharacterInvalid,
+/* 168 - Sk category        */ CharacterInvalid,
+/* 169 - So category        */ CharacterInvalid,
+/* 170 - Ll category        */ CharacterIdentifierStart,
+/* 171 - Pi category        */ CharacterInvalid,
+/* 172 - Sm category        */ CharacterInvalid,
+/* 173 - Cf category        */ CharacterInvalid,
+/* 174 - So category        */ CharacterInvalid,
+/* 175 - Sk category        */ CharacterInvalid,
+/* 176 - So category        */ CharacterInvalid,
+/* 177 - Sm category        */ CharacterInvalid,
+/* 178 - No category        */ CharacterInvalid,
+/* 179 - No category        */ CharacterInvalid,
+/* 180 - Sk category        */ CharacterInvalid,
+/* 181 - Ll category        */ CharacterIdentifierStart,
+/* 182 - So category        */ CharacterInvalid,
+/* 183 - Po category        */ CharacterInvalid,
+/* 184 - Sk category        */ CharacterInvalid,
+/* 185 - No category        */ CharacterInvalid,
+/* 186 - Ll category        */ CharacterIdentifierStart,
+/* 187 - Pf category        */ CharacterInvalid,
+/* 188 - No category        */ CharacterInvalid,
+/* 189 - No category        */ CharacterInvalid,
+/* 190 - No category        */ CharacterInvalid,
+/* 191 - Po category        */ CharacterInvalid,
+/* 192 - Lu category        */ CharacterIdentifierStart,
+/* 193 - Lu category        */ CharacterIdentifierStart,
+/* 194 - Lu category        */ CharacterIdentifierStart,
+/* 195 - Lu category        */ CharacterIdentifierStart,
+/* 196 - Lu category        */ CharacterIdentifierStart,
+/* 197 - Lu category        */ CharacterIdentifierStart,
+/* 198 - Lu category        */ CharacterIdentifierStart,
+/* 199 - Lu category        */ CharacterIdentifierStart,
+/* 200 - Lu category        */ CharacterIdentifierStart,
+/* 201 - Lu category        */ CharacterIdentifierStart,
+/* 202 - Lu category        */ CharacterIdentifierStart,
+/* 203 - Lu category        */ CharacterIdentifierStart,
+/* 204 - Lu category        */ CharacterIdentifierStart,
+/* 205 - Lu category        */ CharacterIdentifierStart,
+/* 206 - Lu category        */ CharacterIdentifierStart,
+/* 207 - Lu category        */ CharacterIdentifierStart,
+/* 208 - Lu category        */ CharacterIdentifierStart,
+/* 209 - Lu category        */ CharacterIdentifierStart,
+/* 210 - Lu category        */ CharacterIdentifierStart,
+/* 211 - Lu category        */ CharacterIdentifierStart,
+/* 212 - Lu category        */ CharacterIdentifierStart,
+/* 213 - Lu category        */ CharacterIdentifierStart,
+/* 214 - Lu category        */ CharacterIdentifierStart,
+/* 215 - Sm category        */ CharacterInvalid,
+/* 216 - Lu category        */ CharacterIdentifierStart,
+/* 217 - Lu category        */ CharacterIdentifierStart,
+/* 218 - Lu category        */ CharacterIdentifierStart,
+/* 219 - Lu category        */ CharacterIdentifierStart,
+/* 220 - Lu category        */ CharacterIdentifierStart,
+/* 221 - Lu category        */ CharacterIdentifierStart,
+/* 222 - Lu category        */ CharacterIdentifierStart,
+/* 223 - Ll category        */ CharacterIdentifierStart,
+/* 224 - Ll category        */ CharacterIdentifierStart,
+/* 225 - Ll category        */ CharacterIdentifierStart,
+/* 226 - Ll category        */ CharacterIdentifierStart,
+/* 227 - Ll category        */ CharacterIdentifierStart,
+/* 228 - Ll category        */ CharacterIdentifierStart,
+/* 229 - Ll category        */ CharacterIdentifierStart,
+/* 230 - Ll category        */ CharacterIdentifierStart,
+/* 231 - Ll category        */ CharacterIdentifierStart,
+/* 232 - Ll category        */ CharacterIdentifierStart,
+/* 233 - Ll category        */ CharacterIdentifierStart,
+/* 234 - Ll category        */ CharacterIdentifierStart,
+/* 235 - Ll category        */ CharacterIdentifierStart,
+/* 236 - Ll category        */ CharacterIdentifierStart,
+/* 237 - Ll category        */ CharacterIdentifierStart,
+/* 238 - Ll category        */ CharacterIdentifierStart,
+/* 239 - Ll category        */ CharacterIdentifierStart,
+/* 240 - Ll category        */ CharacterIdentifierStart,
+/* 241 - Ll category        */ CharacterIdentifierStart,
+/* 242 - Ll category        */ CharacterIdentifierStart,
+/* 243 - Ll category        */ CharacterIdentifierStart,
+/* 244 - Ll category        */ CharacterIdentifierStart,
+/* 245 - Ll category        */ CharacterIdentifierStart,
+/* 246 - Ll category        */ CharacterIdentifierStart,
+/* 247 - Sm category        */ CharacterInvalid,
+/* 248 - Ll category        */ CharacterIdentifierStart,
+/* 249 - Ll category        */ CharacterIdentifierStart,
+/* 250 - Ll category        */ CharacterIdentifierStart,
+/* 251 - Ll category        */ CharacterIdentifierStart,
+/* 252 - Ll category        */ CharacterIdentifierStart,
+/* 253 - Ll category        */ CharacterIdentifierStart,
+/* 254 - Ll category        */ CharacterIdentifierStart,
+/* 255 - Ll category        */ CharacterIdentifierStart
+};
+
+// This table provides the character that results from \X where X is the index in the table beginning
+// with SPACE. A table value of 0 means that more processing needs to be done.
+static const LChar singleCharacterEscapeValuesForASCII[128] = {
+/*   0 - Null               */ 0,
+/*   1 - Start of Heading   */ 0,
+/*   2 - Start of Text      */ 0,
+/*   3 - End of Text        */ 0,
+/*   4 - End of Transm.     */ 0,
+/*   5 - Enquiry            */ 0,
+/*   6 - Acknowledgment     */ 0,
+/*   7 - Bell               */ 0,
+/*   8 - Back Space         */ 0,
+/*   9 - Horizontal Tab     */ 0,
+/*  10 - Line Feed          */ 0,
+/*  11 - Vertical Tab       */ 0,
+/*  12 - Form Feed          */ 0,
+/*  13 - Carriage Return    */ 0,
+/*  14 - Shift Out          */ 0,
+/*  15 - Shift In           */ 0,
+/*  16 - Data Line Escape   */ 0,
+/*  17 - Device Control 1   */ 0,
+/*  18 - Device Control 2   */ 0,
+/*  19 - Device Control 3   */ 0,
+/*  20 - Device Control 4   */ 0,
+/*  21 - Negative Ack.      */ 0,
+/*  22 - Synchronous Idle   */ 0,
+/*  23 - End of Transmit    */ 0,
+/*  24 - Cancel             */ 0,
+/*  25 - End of Medium      */ 0,
+/*  26 - Substitute         */ 0,
+/*  27 - Escape             */ 0,
+/*  28 - File Separator     */ 0,
+/*  29 - Group Separator    */ 0,
+/*  30 - Record Separator   */ 0,
+/*  31 - Unit Separator     */ 0,
+/*  32 - Space              */ ' ',
+/*  33 - !                  */ '!',
+/*  34 - "                  */ '"',
+/*  35 - #                  */ '#',
+/*  36 - $                  */ '$',
+/*  37 - %                  */ '%',
+/*  38 - &                  */ '&',
+/*  39 - '                  */ '\'',
+/*  40 - (                  */ '(',
+/*  41 - )                  */ ')',
+/*  42 - *                  */ '*',
+/*  43 - +                  */ '+',
+/*  44 - ,                  */ ',',
+/*  45 - -                  */ '-',
+/*  46 - .                  */ '.',
+/*  47 - /                  */ '/',
+/*  48 - 0                  */ 0,
+/*  49 - 1                  */ 0,
+/*  50 - 2                  */ 0,
+/*  51 - 3                  */ 0,
+/*  52 - 4                  */ 0,
+/*  53 - 5                  */ 0,
+/*  54 - 6                  */ 0,
+/*  55 - 7                  */ 0,
+/*  56 - 8                  */ 0,
+/*  57 - 9                  */ 0,
+/*  58 - :                  */ ':',
+/*  59 - ;                  */ ';',
+/*  60 - <                  */ '<',
+/*  61 - =                  */ '=',
+/*  62 - >                  */ '>',
+/*  63 - ?                  */ '?',
+/*  64 - @                  */ '@',
+/*  65 - A                  */ 'A',
+/*  66 - B                  */ 'B',
+/*  67 - C                  */ 'C',
+/*  68 - D                  */ 'D',
+/*  69 - E                  */ 'E',
+/*  70 - F                  */ 'F',
+/*  71 - G                  */ 'G',
+/*  72 - H                  */ 'H',
+/*  73 - I                  */ 'I',
+/*  74 - J                  */ 'J',
+/*  75 - K                  */ 'K',
+/*  76 - L                  */ 'L',
+/*  77 - M                  */ 'M',
+/*  78 - N                  */ 'N',
+/*  79 - O                  */ 'O',
+/*  80 - P                  */ 'P',
+/*  81 - Q                  */ 'Q',
+/*  82 - R                  */ 'R',
+/*  83 - S                  */ 'S',
+/*  84 - T                  */ 'T',
+/*  85 - U                  */ 'U',
+/*  86 - V                  */ 'V',
+/*  87 - W                  */ 'W',
+/*  88 - X                  */ 'X',
+/*  89 - Y                  */ 'Y',
+/*  90 - Z                  */ 'Z',
+/*  91 - [                  */ '[',
+/*  92 - \                  */ '\\',
+/*  93 - ]                  */ ']',
+/*  94 - ^                  */ '^',
+/*  95 - _                  */ '_',
+/*  96 - `                  */ '`',
+/*  97 - a                  */ 'a',
+/*  98 - b                  */ 0x08,
+/*  99 - c                  */ 'c',
+/* 100 - d                  */ 'd',
+/* 101 - e                  */ 'e',
+/* 102 - f                  */ 0x0C,
+/* 103 - g                  */ 'g',
+/* 104 - h                  */ 'h',
+/* 105 - i                  */ 'i',
+/* 106 - j                  */ 'j',
+/* 107 - k                  */ 'k',
+/* 108 - l                  */ 'l',
+/* 109 - m                  */ 'm',
+/* 110 - n                  */ 0x0A,
+/* 111 - o                  */ 'o',
+/* 112 - p                  */ 'p',
+/* 113 - q                  */ 'q',
+/* 114 - r                  */ 0x0D,
+/* 115 - s                  */ 's',
+/* 116 - t                  */ 0x09,
+/* 117 - u                  */ 0,
+/* 118 - v                  */ 0x0B,
+/* 119 - w                  */ 'w',
+/* 120 - x                  */ 0,
+/* 121 - y                  */ 'y',
+/* 122 - z                  */ 'z',
+/* 123 - {                  */ '{',
+/* 124 - |                  */ '|',
+/* 125 - }                  */ '}',
+/* 126 - ~                  */ '~',
+/* 127 - Delete             */ 0
+};
+
+template <typename T>
+Lexer<T>::Lexer(VM* vm)
     : m_isReparsing(false)
-    , m_globalData(globalData)
-    , m_keywordTable(JSC::mainTable)
+    , m_vm(vm)
 {
 }
 
-Lexer::~Lexer()
+template <typename T>
+Lexer<T>::~Lexer()
 {
-    m_keywordTable.deleteTable();
 }
 
-inline const UChar* Lexer::currentCharacter() const
+template <typename T>
+String Lexer<T>::invalidCharacterMessage() const
 {
-    return m_code - 4;
+    switch (m_current) {
+    case 0:
+        return "Invalid character: '\\0'";
+    case 10:
+        return "Invalid character: '\\n'";
+    case 11:
+        return "Invalid character: '\\v'";
+    case 13:
+        return "Invalid character: '\\r'";
+    case 35:
+        return "Invalid character: '#'";
+    case 64:
+        return "Invalid character: '@'";
+    case 96:
+        return "Invalid character: '`'";
+    default:
+        return String::format("Invalid character '\\u%04u'", static_cast<unsigned>(m_current)).impl();
+    }
 }
 
-inline int Lexer::currentOffset() const
+template <typename T>
+ALWAYS_INLINE const T* Lexer<T>::currentSourcePtr() const
 {
-    return currentCharacter() - m_codeStart;
+    ASSERT(m_code <= m_codeEnd);
+    return m_code;
 }
 
-ALWAYS_INLINE void Lexer::shift1()
+template <typename T>
+void Lexer<T>::setCode(const SourceCode& source, ParserArena* arena)
 {
-    m_current = m_next1;
-    m_next1 = m_next2;
-    m_next2 = m_next3;
-    if (LIKELY(m_code < m_codeEnd))
-        m_next3 = m_code[0];
+    m_arena = &arena->identifierArena();
+    
+    m_lineNumber = source.firstLine();
+    m_lastToken = -1;
+    
+    const String& sourceString = source.provider()->source();
+
+    if (!sourceString.isNull())
+        setCodeStart(sourceString.impl());
     else
-        m_next3 = -1;
+        m_codeStart = 0;
 
-    ++m_code;
+    m_source = &source;
+    m_sourceOffset = source.startOffset();
+    m_codeStartPlusOffset = m_codeStart + source.startOffset();
+    m_code = m_codeStartPlusOffset;
+    m_codeEnd = m_codeStart + source.endOffset();
+    m_error = false;
+    m_atLineStart = true;
+    m_lineStart = m_code;
+    m_lexErrorMessage = String();
+    
+    m_buffer8.reserveInitialCapacity(initialReadBufferCapacity);
+    m_buffer16.reserveInitialCapacity((m_codeEnd - m_code) / 2);
+    
+    if (LIKELY(m_code < m_codeEnd))
+        m_current = *m_code;
+    else
+        m_current = 0;
+    ASSERT(currentOffset() == source.startOffset());
 }
 
-ALWAYS_INLINE void Lexer::shift2()
+template <typename T>
+template <int shiftAmount> ALWAYS_INLINE void Lexer<T>::internalShift()
 {
-    m_current = m_next2;
-    m_next1 = m_next3;
-    if (LIKELY(m_code + 1 < m_codeEnd)) {
-        m_next2 = m_code[0];
-        m_next3 = m_code[1];
-    } else {
-        m_next2 = m_code < m_codeEnd ? m_code[0] : -1;
-        m_next3 = -1;
-    }
-
-    m_code += 2;
+    m_code += shiftAmount;
+    ASSERT(currentOffset() >= currentLineStartOffset());
+    m_current = *m_code;
 }
 
-ALWAYS_INLINE void Lexer::shift3()
+template <typename T>
+ALWAYS_INLINE void Lexer<T>::shift()
 {
-    m_current = m_next3;
-    if (LIKELY(m_code + 2 < m_codeEnd)) {
-        m_next1 = m_code[0];
-        m_next2 = m_code[1];
-        m_next3 = m_code[2];
-    } else {
-        m_next1 = m_code < m_codeEnd ? m_code[0] : -1;
-        m_next2 = m_code + 1 < m_codeEnd ? m_code[1] : -1;
-        m_next3 = -1;
-    }
+    // At one point timing showed that setting m_current to 0 unconditionally was faster than an if-else sequence.
+    m_current = 0;
+    ++m_code;
+    if (LIKELY(m_code < m_codeEnd))
+        m_current = *m_code;
+}
 
-    m_code += 3;
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::atEnd() const
+{
+    ASSERT(!m_current || m_code < m_codeEnd);
+    return UNLIKELY(UNLIKELY(!m_current) && m_code == m_codeEnd);
 }
 
-ALWAYS_INLINE void Lexer::shift4()
+template <typename T>
+ALWAYS_INLINE T Lexer<T>::peek(int offset) const
 {
-    if (LIKELY(m_code + 3 < m_codeEnd)) {
-        m_current = m_code[0];
-        m_next1 = m_code[1];
-        m_next2 = m_code[2];
-        m_next3 = m_code[3];
-    } else {
-        m_current = m_code < m_codeEnd ? m_code[0] : -1;
-        m_next1 = m_code + 1 < m_codeEnd ? m_code[1] : -1;
-        m_next2 = m_code + 2 < m_codeEnd ? m_code[2] : -1;
-        m_next3 = -1;
-    }
+    ASSERT(offset > 0 && offset < 5);
+    const T* code = m_code + offset;
+    return (code < m_codeEnd) ? *code : 0;
+}
 
-    m_code += 4;
+template <typename T>
+typename Lexer<T>::UnicodeHexValue Lexer<T>::parseFourDigitUnicodeHex()
+{
+    T char1 = peek(1);
+    T char2 = peek(2);
+    T char3 = peek(3);
+
+    if (UNLIKELY(!isASCIIHexDigit(m_current) || !isASCIIHexDigit(char1) || !isASCIIHexDigit(char2) || !isASCIIHexDigit(char3)))
+        return UnicodeHexValue((m_code + 4) >= m_codeEnd ? UnicodeHexValue::IncompleteHex : UnicodeHexValue::InvalidHex);
+
+    int result = convertUnicode(m_current, char1, char2, char3);
+    shift();
+    shift();
+    shift();
+    shift();
+    return UnicodeHexValue(result);
 }
 
-void Lexer::setCode(const SourceCode& source, ParserArena& arena)
+template <typename T>
+void Lexer<T>::shiftLineTerminator()
 {
-    m_arena = &arena.identifierArena();
+    ASSERT(isLineTerminator(m_current));
 
-    m_lineNumber = source.firstLine();
-    m_delimited = false;
-    m_lastToken = -1;
+    T prev = m_current;
+    shift();
 
-    const UChar* data = source.provider()->data();
+    // Allow both CRLF and LFCR.
+    if (prev + m_current == '\n' + '\r')
+        shift();
 
-    m_source = &source;
-    m_codeStart = data;
-    m_code = data + source.startOffset();
-    m_codeEnd = data + source.endOffset();
-    m_error = false;
-    m_atLineStart = true;
+    ++m_lineNumber;
+}
 
-    m_buffer8.reserveInitialCapacity(initialReadBufferCapacity);
-    m_buffer16.reserveInitialCapacity((m_codeEnd - m_code) / 2);
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::lastTokenWasRestrKeyword() const
+{
+    return m_lastToken == CONTINUE || m_lastToken == BREAK || m_lastToken == RETURN || m_lastToken == THROW;
+}
 
-    // ECMA-262 calls for stripping all Cf characters, but we only strip BOM characters.
-    // See <https://bugs.webkit.org/show_bug.cgi?id=4931> for details.
-    if (source.provider()->hasBOMs()) {
-        for (const UChar* p = m_codeStart; p < m_codeEnd; ++p) {
-            if (UNLIKELY(*p == byteOrderMark)) {
-                copyCodeWithoutBOMs();
-                break;
-            }
-        }
-    }
+static NEVER_INLINE bool isNonLatin1IdentStart(int c)
+{
+    return category(c) & (Letter_Uppercase | Letter_Lowercase | Letter_Titlecase | Letter_Modifier | Letter_Other);
+}
 
-    // Read the first characters into the 4-character buffer.
-    shift4();
-    ASSERT(currentOffset() == source.startOffset());
+static ALWAYS_INLINE bool isLatin1(LChar)
+{
+    return true;
 }
 
-void Lexer::copyCodeWithoutBOMs()
+static ALWAYS_INLINE bool isLatin1(UChar c)
 {
-    // Note: In this case, the character offset data for debugging will be incorrect.
-    // If it's important to correctly debug code with extraneous BOMs, then the caller
-    // should strip the BOMs when creating the SourceProvider object and do its own
-    // mapping of offsets within the stripped text to original text offset.
-
-    m_codeWithoutBOMs.reserveCapacity(m_codeEnd - m_code);
-    for (const UChar* p = m_code; p < m_codeEnd; ++p) {
-        UChar c = *p;
-        if (c != byteOrderMark)
-            m_codeWithoutBOMs.append(c);
-    }
-    ptrdiff_t startDelta = m_codeStart - m_code;
-    m_code = m_codeWithoutBOMs.data();
-    m_codeStart = m_code + startDelta;
-    m_codeEnd = m_codeWithoutBOMs.data() + m_codeWithoutBOMs.size();
+    return c < 256;
 }
 
-void Lexer::shiftLineTerminator()
+static inline bool isIdentStart(LChar c)
 {
-    ASSERT(isLineTerminator(m_current));
+    return typesOfLatin1Characters[c] == CharacterIdentifierStart;
+}
 
-    // Allow both CRLF and LFCR.
-    if (m_current + m_next1 == '\n' + '\r')
-        shift2();
-    else
-        shift1();
+static inline bool isIdentStart(UChar c)
+{
+    return isLatin1(c) ? isIdentStart(static_cast<LChar>(c)) : isNonLatin1IdentStart(c);
+}
 
-    ++m_lineNumber;
+static NEVER_INLINE bool isNonLatin1IdentPart(int c)
+{
+    return (category(c) & (Letter_Uppercase | Letter_Lowercase | Letter_Titlecase | Letter_Modifier | Letter_Other
+        | Mark_NonSpacing | Mark_SpacingCombining | Number_DecimalDigit | Punctuation_Connector)) || c == 0x200C || c == 0x200D;
 }
 
-ALWAYS_INLINE const Identifier* Lexer::makeIdentifier(const UChar* characters, size_t length)
+static ALWAYS_INLINE bool isIdentPart(LChar c)
 {
-    return &m_arena->makeIdentifier(m_globalData, characters, length);
+    // Character types are divided into two groups depending on whether they can be part of an
+    // identifier or not. Those whose type value is less or equal than CharacterNumber can be
+    // part of an identifier. (See the CharacterType definition for more details.)
+    return typesOfLatin1Characters[c] <= CharacterNumber;
 }
 
-inline bool Lexer::lastTokenWasRestrKeyword() const
+static ALWAYS_INLINE bool isIdentPart(UChar c)
 {
-    return m_lastToken == CONTINUE || m_lastToken == BREAK || m_lastToken == RETURN || m_lastToken == THROW;
+    return isLatin1(c) ? isIdentPart(static_cast<LChar>(c)) : isNonLatin1IdentPart(c);
 }
 
-static NEVER_INLINE bool isNonASCIIIdentStart(int c)
+static inline LChar singleEscape(int c)
 {
-    return category(c) & (Letter_Uppercase | Letter_Lowercase | Letter_Titlecase | Letter_Modifier | Letter_Other);
+    if (c < 128) {
+        ASSERT(static_cast<size_t>(c) < ARRAY_SIZE(singleCharacterEscapeValuesForASCII));
+        return singleCharacterEscapeValuesForASCII[c];
+    }
+    return 0;
 }
 
-static inline bool isIdentStart(int c)
+template <typename T>
+inline void Lexer<T>::record8(int c)
 {
-    return isASCII(c) ? isASCIIAlpha(c) || c == '$' || c == '_' : isNonASCIIIdentStart(c);
+    ASSERT(c >= 0);
+    ASSERT(c <= 0xFF);
+    m_buffer8.append(static_cast<LChar>(c));
 }
 
-static NEVER_INLINE bool isNonASCIIIdentPart(int c)
+template <typename T>
+inline void assertCharIsIn8BitRange(T c)
 {
-    return category(c) & (Letter_Uppercase | Letter_Lowercase | Letter_Titlecase | Letter_Modifier | Letter_Other
-        | Mark_NonSpacing | Mark_SpacingCombining | Number_DecimalDigit | Punctuation_Connector);
+    UNUSED_PARAM(c);
+    ASSERT(c >= 0);
+    ASSERT(c <= 0xFF);
 }
 
-static inline bool isIdentPart(int c)
+template <>
+inline void assertCharIsIn8BitRange(UChar c)
 {
-    return isASCII(c) ? isASCIIAlphanumeric(c) || c == '$' || c == '_' : isNonASCIIIdentPart(c);
+    UNUSED_PARAM(c);
+    ASSERT(c <= 0xFF);
 }
 
-static inline int singleEscape(int c)
+template <>
+inline void assertCharIsIn8BitRange(LChar)
 {
-    switch (c) {
-        case 'b':
-            return 0x08;
-        case 't':
-            return 0x09;
-        case 'n':
-            return 0x0A;
-        case 'v':
-            return 0x0B;
-        case 'f':
-            return 0x0C;
-        case 'r':
-            return 0x0D;
-        default:
-            return c;
+}
+
+template <typename T>
+inline void Lexer<T>::append8(const T* p, size_t length)
+{
+    size_t currentSize = m_buffer8.size();
+    m_buffer8.grow(currentSize + length);
+    LChar* rawBuffer = m_buffer8.data() + currentSize;
+
+    for (size_t i = 0; i < length; i++) {
+        T c = p[i];
+        assertCharIsIn8BitRange(c);
+        rawBuffer[i] = c;
     }
 }
 
-inline void Lexer::record8(int c)
+template <typename T>
+inline void Lexer<T>::append16(const LChar* p, size_t length)
 {
-    ASSERT(c >= 0);
-    ASSERT(c <= 0xFF);
-    m_buffer8.append(static_cast<char>(c));
+    size_t currentSize = m_buffer16.size();
+    m_buffer16.grow(currentSize + length);
+    UChar* rawBuffer = m_buffer16.data() + currentSize;
+
+    for (size_t i = 0; i < length; i++)
+        rawBuffer[i] = p[i];
 }
 
-inline void Lexer::record16(UChar c)
+template <typename T>
+inline void Lexer<T>::record16(T c)
 {
     m_buffer16.append(c);
 }
 
-inline void Lexer::record16(int c)
+template <typename T>
+inline void Lexer<T>::record16(int c)
 {
     ASSERT(c >= 0);
-    ASSERT(c <= USHRT_MAX);
-    record16(UChar(static_cast<unsigned short>(c)));
+    ASSERT(c <= static_cast<int>(USHRT_MAX));
+    m_buffer16.append(static_cast<UChar>(c));
 }
 
-int Lexer::lex(void* p1, void* p2)
+template <>
+template <bool shouldCreateIdentifier> ALWAYS_INLINE JSTokenType Lexer<LChar>::parseIdentifier(JSTokenData* tokenData, unsigned lexerFlags, bool strictMode)
 {
-    ASSERT(!m_error);
-    ASSERT(m_buffer8.isEmpty());
-    ASSERT(m_buffer16.isEmpty());
+    const ptrdiff_t remaining = m_codeEnd - m_code;
+    if ((remaining >= maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords)) {
+        JSTokenType keyword = parseKeyword<shouldCreateIdentifier>(tokenData);
+        if (keyword != IDENT) {
+            ASSERT((!shouldCreateIdentifier) || tokenData->ident);
+            return keyword == RESERVED_IF_STRICT && !strictMode ? IDENT : keyword;
+        }
+    }
 
-    YYSTYPE* lvalp = static_cast<YYSTYPE*>(p1);
-    YYLTYPE* llocp = static_cast<YYLTYPE*>(p2);
-    int token = 0;
-    m_terminator = false;
+    const LChar* identifierStart = currentSourcePtr();
+    unsigned identifierLineStart = currentLineStartOffset();
+    
+    while (isIdentPart(m_current))
+        shift();
+    
+    if (UNLIKELY(m_current == '\\')) {
+        setOffsetFromSourcePtr(identifierStart, identifierLineStart);
+        return parseIdentifierSlowCase<shouldCreateIdentifier>(tokenData, lexerFlags, strictMode);
+    }
 
-start:
-    while (isWhiteSpace(m_current))
-        shift1();
+    const Identifier* ident = 0;
+    
+    if (shouldCreateIdentifier) {
+        int identifierLength = currentSourcePtr() - identifierStart;
+        ident = makeIdentifier(identifierStart, identifierLength);
+
+        tokenData->ident = ident;
+    } else
+        tokenData->ident = 0;
+
+    if (UNLIKELY((remaining < maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords))) {
+        ASSERT(shouldCreateIdentifier);
+        if (remaining < maxTokenLength) {
+            const HashEntry* entry = m_vm->keywords->getKeyword(*ident);
+            ASSERT((remaining < maxTokenLength) || !entry);
+            if (!entry)
+                return IDENT;
+            JSTokenType token = static_cast<JSTokenType>(entry->lexerValue());
+            return (token != RESERVED_IF_STRICT) || strictMode ? token : IDENT;
+        }
+        return IDENT;
+    }
 
-    int startOffset = currentOffset();
+    return IDENT;
+}
 
-    if (m_current == -1) {
-        if (!m_terminator && !m_delimited && !m_isReparsing) {
-            // automatic semicolon insertion if program incomplete
-            token = ';';
-            goto doneSemicolon;
+template <>
+template <bool shouldCreateIdentifier> ALWAYS_INLINE JSTokenType Lexer<UChar>::parseIdentifier(JSTokenData* tokenData, unsigned lexerFlags, bool strictMode)
+{
+    const ptrdiff_t remaining = m_codeEnd - m_code;
+    if ((remaining >= maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords)) {
+        JSTokenType keyword = parseKeyword<shouldCreateIdentifier>(tokenData);
+        if (keyword != IDENT) {
+            ASSERT((!shouldCreateIdentifier) || tokenData->ident);
+            return keyword == RESERVED_IF_STRICT && !strictMode ? IDENT : keyword;
         }
-        return 0;
     }
 
-    m_delimited = false;
-    switch (m_current) {
-        case '>':
-            if (m_next1 == '>' && m_next2 == '>') {
-                if (m_next3 == '=') {
-                    shift4();
-                    token = URSHIFTEQUAL;
-                    break;
-                }
-                shift3();
-                token = URSHIFT;
-                break;
-            }
-            if (m_next1 == '>') {
-                if (m_next2 == '=') {
-                    shift3();
-                    token = RSHIFTEQUAL;
-                    break;
-                }
-                shift2();
-                token = RSHIFT;
-                break;
-            }
-            if (m_next1 == '=') {
-                shift2();
-                token = GE;
-                break;
-            }
-            shift1();
-            token = '>';
-            break;
-        case '=':
-            if (m_next1 == '=') {
-                if (m_next2 == '=') {
-                    shift3();
-                    token = STREQ;
-                    break;
-                }
-                shift2();
-                token = EQEQ;
-                break;
-            }
-            shift1();
-            token = '=';
-            break;
-        case '!':
-            if (m_next1 == '=') {
-                if (m_next2 == '=') {
-                    shift3();
-                    token = STRNEQ;
-                    break;
-                }
-                shift2();
-                token = NE;
-                break;
-            }
-            shift1();
-            token = '!';
-            break;
-        case '<':
-            if (m_next1 == '!' && m_next2 == '-' && m_next3 == '-') {
-                // <!-- marks the beginning of a line comment (for www usage)
-                shift4();
-                goto inSingleLineComment;
-            }
-            if (m_next1 == '<') {
-                if (m_next2 == '=') {
-                    shift3();
-                    token = LSHIFTEQUAL;
-                    break;
-                }
-                shift2();
-                token = LSHIFT;
-                break;
-            }
-            if (m_next1 == '=') {
-                shift2();
-                token = LE;
-                break;
-            }
-            shift1();
-            token = '<';
-            break;
-        case '+':
-            if (m_next1 == '+') {
-                shift2();
-                if (m_terminator) {
-                    token = AUTOPLUSPLUS;
-                    break;
-                }
-                token = PLUSPLUS;
-                break;
-            }
-            if (m_next1 == '=') {
-                shift2();
-                token = PLUSEQUAL;
-                break;
-            }
-            shift1();
-            token = '+';
-            break;
-        case '-':
-            if (m_next1 == '-') {
-                if (m_atLineStart && m_next2 == '>') {
-                    shift3();
-                    goto inSingleLineComment;
-                }
-                shift2();
-                if (m_terminator) {
-                    token = AUTOMINUSMINUS;
-                    break;
-                }
-                token = MINUSMINUS;
-                break;
-            }
-            if (m_next1 == '=') {
-                shift2();
-                token = MINUSEQUAL;
-                break;
-            }
-            shift1();
-            token = '-';
-            break;
-        case '*':
-            if (m_next1 == '=') {
-                shift2();
-                token = MULTEQUAL;
-                break;
-            }
-            shift1();
-            token = '*';
-            break;
-        case '/':
-            if (m_next1 == '/') {
-                shift2();
-                goto inSingleLineComment;
-            }
-            if (m_next1 == '*')
-                goto inMultiLineComment;
-            if (m_next1 == '=') {
-                shift2();
-                token = DIVEQUAL;
-                break;
-            }
-            shift1();
-            token = '/';
-            break;
-        case '&':
-            if (m_next1 == '&') {
-                shift2();
-                token = AND;
-                break;
-            }
-            if (m_next1 == '=') {
-                shift2();
-                token = ANDEQUAL;
-                break;
-            }
-            shift1();
-            token = '&';
-            break;
-        case '^':
-            if (m_next1 == '=') {
-                shift2();
-                token = XOREQUAL;
-                break;
-            }
-            shift1();
-            token = '^';
-            break;
-        case '%':
-            if (m_next1 == '=') {
-                shift2();
-                token = MODEQUAL;
-                break;
-            }
-            shift1();
-            token = '%';
-            break;
-        case '|':
-            if (m_next1 == '=') {
-                shift2();
-                token = OREQUAL;
-                break;
-            }
-            if (m_next1 == '|') {
-                shift2();
-                token = OR;
-                break;
-            }
-            shift1();
-            token = '|';
-            break;
-        case '.':
-            if (isASCIIDigit(m_next1)) {
-                record8('.');
-                shift1();
-                goto inNumberAfterDecimalPoint;
-            }
-            token = '.';
-            shift1();
-            break;
-        case ',':
-        case '~':
-        case '?':
-        case ':':
-        case '(':
-        case ')':
-        case '[':
-        case ']':
-            token = m_current;
-            shift1();
-            break;
-        case ';':
-            shift1();
-            m_delimited = true;
-            token = ';';
-            break;
-        case '{':
-            lvalp->intValue = currentOffset();
-            shift1();
-            token = OPENBRACE;
-            break;
-        case '}':
-            lvalp->intValue = currentOffset();
-            shift1();
-            m_delimited = true;
-            token = CLOSEBRACE;
-            break;
-        case '\\':
-            goto startIdentifierWithBackslash;
-        case '0':
-            goto startNumberWithZeroDigit;
-        case '1':
-        case '2':
-        case '3':
-        case '4':
-        case '5':
-        case '6':
-        case '7':
-        case '8':
-        case '9':
-            goto startNumber;
-        case '"':
-        case '\'':
-            goto startString;
-        default:
-            if (isIdentStart(m_current))
-                goto startIdentifierOrKeyword;
-            if (isLineTerminator(m_current)) {
-                shiftLineTerminator();
-                m_atLineStart = true;
-                m_terminator = true;
-                if (lastTokenWasRestrKeyword()) {
-                    token = ';';
-                    goto doneSemicolon;
-                }
-                goto start;
-            }
-            goto returnError;
+    const UChar* identifierStart = currentSourcePtr();
+    int identifierLineStart = currentLineStartOffset();
+
+    UChar orAllChars = 0;
+    
+    while (isIdentPart(m_current)) {
+        orAllChars |= m_current;
+        shift();
+    }
+    
+    if (UNLIKELY(m_current == '\\')) {
+        setOffsetFromSourcePtr(identifierStart, identifierLineStart);
+        return parseIdentifierSlowCase<shouldCreateIdentifier>(tokenData, lexerFlags, strictMode);
     }
 
-    m_atLineStart = false;
-    goto returnToken;
+    bool isAll8Bit = false;
 
-startString: {
-    int stringQuoteCharacter = m_current;
-    shift1();
+    if (!(orAllChars & ~0xff))
+        isAll8Bit = true;
 
-    const UChar* stringStart = currentCharacter();
-    while (m_current != stringQuoteCharacter) {
-        // Fast check for characters that require special handling.
-        // Catches -1, \n, \r, \, 0x2028, and 0x2029 as efficiently
-        // as possible, and lets through all common ASCII characters.
-        if (UNLIKELY(m_current == '\\') || UNLIKELY(((static_cast<unsigned>(m_current) - 0xE) & 0x2000))) {
-            m_buffer16.append(stringStart, currentCharacter() - stringStart);
-            goto inString;
+    const Identifier* ident = 0;
+    
+    if (shouldCreateIdentifier) {
+        int identifierLength = currentSourcePtr() - identifierStart;
+        if (isAll8Bit)
+            ident = makeIdentifierLCharFromUChar(identifierStart, identifierLength);
+        else
+            ident = makeIdentifier(identifierStart, identifierLength);
+        
+        tokenData->ident = ident;
+    } else
+        tokenData->ident = 0;
+    
+    if (UNLIKELY((remaining < maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords))) {
+        ASSERT(shouldCreateIdentifier);
+        if (remaining < maxTokenLength) {
+            const HashEntry* entry = m_vm->keywords->getKeyword(*ident);
+            ASSERT((remaining < maxTokenLength) || !entry);
+            if (!entry)
+                return IDENT;
+            JSTokenType token = static_cast<JSTokenType>(entry->lexerValue());
+            return (token != RESERVED_IF_STRICT) || strictMode ? token : IDENT;
         }
-        shift1();
+        return IDENT;
     }
-    lvalp->ident = makeIdentifier(stringStart, currentCharacter() - stringStart);
-    shift1();
-    m_atLineStart = false;
-    m_delimited = false;
-    token = STRING;
-    goto returnToken;
 
-inString:
-    while (m_current != stringQuoteCharacter) {
-        if (m_current == '\\')
-            goto inStringEscapeSequence;
-        if (UNLIKELY(isLineTerminator(m_current)))
-            goto returnError;
-        if (UNLIKELY(m_current == -1))
-            goto returnError;
-        record16(m_current);
-        shift1();
-    }
-    goto doneString;
-
-inStringEscapeSequence:
-    shift1();
-    if (m_current == 'x') {
-        shift1();
-        if (isASCIIHexDigit(m_current) && isASCIIHexDigit(m_next1)) {
-            record16(convertHex(m_current, m_next1));
-            shift2();
-            goto inString;
-        }
-        record16('x');
-        if (m_current == stringQuoteCharacter)
-            goto doneString;
-        goto inString;
-    }
-    if (m_current == 'u') {
-        shift1();
-        if (isASCIIHexDigit(m_current) && isASCIIHexDigit(m_next1) && isASCIIHexDigit(m_next2) && isASCIIHexDigit(m_next3)) {
-            record16(convertUnicode(m_current, m_next1, m_next2, m_next3));
-            shift4();
-            goto inString;
-        }
-        if (m_current == stringQuoteCharacter) {
-            record16('u');
-            goto doneString;
+    return IDENT;
+}
+
+template <typename T>
+template <bool shouldCreateIdentifier> JSTokenType Lexer<T>::parseIdentifierSlowCase(JSTokenData* tokenData, unsigned lexerFlags, bool strictMode)
+{
+    const ptrdiff_t remaining = m_codeEnd - m_code;
+    const T* identifierStart = currentSourcePtr();
+    bool bufferRequired = false;
+
+    while (true) {
+        if (LIKELY(isIdentPart(m_current))) {
+            shift();
+            continue;
         }
-        goto returnError;
+        if (LIKELY(m_current != '\\'))
+            break;
+
+        // \uXXXX unicode characters.
+        bufferRequired = true;
+        if (identifierStart != currentSourcePtr())
+            m_buffer16.append(identifierStart, currentSourcePtr() - identifierStart);
+        shift();
+        if (UNLIKELY(m_current != 'u'))
+            return atEnd() ? UNTERMINATED_IDENTIFIER_ESCAPE_ERRORTOK : INVALID_IDENTIFIER_ESCAPE_ERRORTOK;
+        shift();
+        UnicodeHexValue character = parseFourDigitUnicodeHex();
+        if (UNLIKELY(!character.isValid()))
+            return character.valueType() == UnicodeHexValue::IncompleteHex ? UNTERMINATED_IDENTIFIER_UNICODE_ESCAPE_ERRORTOK : INVALID_IDENTIFIER_UNICODE_ESCAPE_ERRORTOK;
+        UChar ucharacter = static_cast<UChar>(character.value());
+        if (UNLIKELY(m_buffer16.size() ? !isIdentPart(ucharacter) : !isIdentStart(ucharacter)))
+            return INVALID_IDENTIFIER_UNICODE_ESCAPE_ERRORTOK;
+        if (shouldCreateIdentifier)
+            record16(ucharacter);
+        identifierStart = currentSourcePtr();
     }
-    if (isASCIIOctalDigit(m_current)) {
-        if (m_current >= '0' && m_current <= '3' && isASCIIOctalDigit(m_next1) && isASCIIOctalDigit(m_next2)) {
-            record16((m_current - '0') * 64 + (m_next1 - '0') * 8 + m_next2 - '0');
-            shift3();
-            goto inString;
+
+    int identifierLength;
+    const Identifier* ident = 0;
+    if (shouldCreateIdentifier) {
+        if (!bufferRequired) {
+            identifierLength = currentSourcePtr() - identifierStart;
+            ident = makeIdentifier(identifierStart, identifierLength);
+        } else {
+            if (identifierStart != currentSourcePtr())
+                m_buffer16.append(identifierStart, currentSourcePtr() - identifierStart);
+            ident = makeIdentifier(m_buffer16.data(), m_buffer16.size());
         }
-        if (isASCIIOctalDigit(m_next1)) {
-            record16((m_current - '0') * 8 + m_next1 - '0');
-            shift2();
-            goto inString;
+
+        tokenData->ident = ident;
+    } else
+        tokenData->ident = 0;
+
+    if (LIKELY(!bufferRequired && !(lexerFlags & LexerFlagsIgnoreReservedWords))) {
+        ASSERT(shouldCreateIdentifier);
+        // Keywords must not be recognized if there was an \uXXXX in the identifier.
+        if (remaining < maxTokenLength) {
+            const HashEntry* entry = m_vm->keywords->getKeyword(*ident);
+            ASSERT((remaining < maxTokenLength) || !entry);
+            if (!entry)
+                return IDENT;
+            JSTokenType token = static_cast<JSTokenType>(entry->lexerValue());
+            return (token != RESERVED_IF_STRICT) || strictMode ? token : IDENT;
         }
-        record16(m_current - '0');
-        shift1();
-        goto inString;
+        return IDENT;
     }
-    if (isLineTerminator(m_current)) {
-        shiftLineTerminator();
-        goto inString;
-    }
-    if (m_current == -1)
-        goto returnError;
-    record16(singleEscape(m_current));
-    shift1();
-    goto inString;
+
+    m_buffer16.resize(0);
+    return IDENT;
 }
 
-startIdentifierWithBackslash:
-    shift1();
-    if (UNLIKELY(m_current != 'u'))
-        goto returnError;
-    shift1();
-    if (UNLIKELY(!isASCIIHexDigit(m_current) || !isASCIIHexDigit(m_next1) || !isASCIIHexDigit(m_next2) || !isASCIIHexDigit(m_next3)))
-        goto returnError;
-    token = convertUnicode(m_current, m_next1, m_next2, m_next3);
-    if (UNLIKELY(!isIdentStart(token)))
-        goto returnError;
-    goto inIdentifierAfterCharacterCheck;
+static ALWAYS_INLINE bool characterRequiresParseStringSlowCase(LChar character)
+{
+    return character < 0xE;
+}
 
-startIdentifierOrKeyword: {
-    const UChar* identifierStart = currentCharacter();
-    shift1();
-    while (isIdentPart(m_current))
-        shift1();
-    if (LIKELY(m_current != '\\')) {
-        lvalp->ident = makeIdentifier(identifierStart, currentCharacter() - identifierStart);
-        goto doneIdentifierOrKeyword;
-    }
-    m_buffer16.append(identifierStart, currentCharacter() - identifierStart);
+static ALWAYS_INLINE bool characterRequiresParseStringSlowCase(UChar character)
+{
+    return character < 0xE || character > 0xFF;
 }
 
-    do {
-        shift1();
-        if (UNLIKELY(m_current != 'u'))
-            goto returnError;
-        shift1();
-        if (UNLIKELY(!isASCIIHexDigit(m_current) || !isASCIIHexDigit(m_next1) || !isASCIIHexDigit(m_next2) || !isASCIIHexDigit(m_next3)))
-            goto returnError;
-        token = convertUnicode(m_current, m_next1, m_next2, m_next3);
-        if (UNLIKELY(!isIdentPart(token)))
-            goto returnError;
-inIdentifierAfterCharacterCheck:
-        record16(token);
-        shift4();
+template <typename T>
+template <bool shouldBuildStrings> ALWAYS_INLINE typename Lexer<T>::StringParseResult Lexer<T>::parseString(JSTokenData* tokenData, bool strictMode)
+{
+    int startingOffset = currentOffset();
+    int startingLineStartOffset = currentLineStartOffset();
+    int startingLineNumber = lineNumber();
+    T stringQuoteCharacter = m_current;
+    shift();
 
-        while (isIdentPart(m_current)) {
-            record16(m_current);
-            shift1();
+    const T* stringStart = currentSourcePtr();
+
+    while (m_current != stringQuoteCharacter) {
+        if (UNLIKELY(m_current == '\\')) {
+            if (stringStart != currentSourcePtr() && shouldBuildStrings)
+                append8(stringStart, currentSourcePtr() - stringStart);
+            shift();
+
+            LChar escape = singleEscape(m_current);
+
+            // Most common escape sequences first
+            if (escape) {
+                if (shouldBuildStrings)
+                    record8(escape);
+                shift();
+            } else if (UNLIKELY(isLineTerminator(m_current)))
+                shiftLineTerminator();
+            else if (m_current == 'x') {
+                shift();
+                if (!isASCIIHexDigit(m_current) || !isASCIIHexDigit(peek(1))) {
+                    m_lexErrorMessage = "\\x can only be followed by a hex character sequence";
+                    return (atEnd() || (isASCIIHexDigit(m_current) && (m_code + 1 == m_codeEnd))) ? StringUnterminated : StringCannotBeParsed;
+                }
+                T prev = m_current;
+                shift();
+                if (shouldBuildStrings)
+                    record8(convertHex(prev, m_current));
+                shift();
+            } else {
+                setOffset(startingOffset, startingLineStartOffset);
+                setLineNumber(startingLineNumber);
+                m_buffer8.resize(0);
+                return parseStringSlowCase<shouldBuildStrings>(tokenData, strictMode);
+            }
+            stringStart = currentSourcePtr();
+            continue;
         }
-    } while (UNLIKELY(m_current == '\\'));
-    goto doneIdentifier;
 
-inSingleLineComment:
-    while (!isLineTerminator(m_current)) {
-        if (UNLIKELY(m_current == -1))
-            return 0;
-        shift1();
-    }
-    shiftLineTerminator();
-    m_atLineStart = true;
-    m_terminator = true;
-    if (lastTokenWasRestrKeyword())
-        goto doneSemicolon;
-    goto start;
-
-inMultiLineComment:
-    shift2();
-    while (m_current != '*' || m_next1 != '/') {
-        if (isLineTerminator(m_current))
-            shiftLineTerminator();
-        else {
-            shift1();
-            if (UNLIKELY(m_current == -1))
-                goto returnError;
+        if (UNLIKELY(characterRequiresParseStringSlowCase(m_current))) {
+            setOffset(startingOffset, startingLineStartOffset);
+            setLineNumber(startingLineNumber);
+            m_buffer8.resize(0);
+            return parseStringSlowCase<shouldBuildStrings>(tokenData, strictMode);
         }
-    }
-    shift2();
-    m_atLineStart = false;
-    goto start;
 
-startNumberWithZeroDigit:
-    shift1();
-    if ((m_current | 0x20) == 'x' && isASCIIHexDigit(m_next1)) {
-        shift1();
-        goto inHex;
-    }
-    if (m_current == '.') {
-        record8('0');
-        record8('.');
-        shift1();
-        goto inNumberAfterDecimalPoint;
+        shift();
     }
-    if ((m_current | 0x20) == 'e') {
-        record8('0');
-        record8('e');
-        shift1();
-        goto inExponentIndicator;
+
+    if (currentSourcePtr() != stringStart && shouldBuildStrings)
+        append8(stringStart, currentSourcePtr() - stringStart);
+    if (shouldBuildStrings) {
+        tokenData->ident = makeIdentifier(m_buffer8.data(), m_buffer8.size());
+        m_buffer8.resize(0);
+    } else
+        tokenData->ident = 0;
+
+    return StringParsedSuccessfully;
+}
+
+template <typename T>
+template <bool shouldBuildStrings> typename Lexer<T>::StringParseResult Lexer<T>::parseStringSlowCase(JSTokenData* tokenData, bool strictMode)
+{
+    T stringQuoteCharacter = m_current;
+    shift();
+
+    const T* stringStart = currentSourcePtr();
+
+    while (m_current != stringQuoteCharacter) {
+        if (UNLIKELY(m_current == '\\')) {
+            if (stringStart != currentSourcePtr() && shouldBuildStrings)
+                append16(stringStart, currentSourcePtr() - stringStart);
+            shift();
+
+            LChar escape = singleEscape(m_current);
+
+            // Most common escape sequences first
+            if (escape) {
+                if (shouldBuildStrings)
+                    record16(escape);
+                shift();
+            } else if (UNLIKELY(isLineTerminator(m_current)))
+                shiftLineTerminator();
+            else if (m_current == 'x') {
+                shift();
+                if (!isASCIIHexDigit(m_current) || !isASCIIHexDigit(peek(1))) {
+                    m_lexErrorMessage = "\\x can only be followed by a hex character sequence";
+                    return StringCannotBeParsed;
+                }
+                T prev = m_current;
+                shift();
+                if (shouldBuildStrings)
+                    record16(convertHex(prev, m_current));
+                shift();
+            } else if (m_current == 'u') {
+                shift();
+                UnicodeHexValue character = parseFourDigitUnicodeHex();
+                if (character.isValid()) {
+                    if (shouldBuildStrings)
+                        record16(character.value());
+                } else if (m_current == stringQuoteCharacter) {
+                    if (shouldBuildStrings)
+                        record16('u');
+                } else {
+                    m_lexErrorMessage = "\\u can only be followed by a Unicode character sequence";
+                    return character.valueType() == UnicodeHexValue::IncompleteHex ? StringUnterminated : StringCannotBeParsed;
+                }
+            } else if (strictMode && isASCIIDigit(m_current)) {
+                // The only valid numeric escape in strict mode is '\0', and this must not be followed by a decimal digit.
+                int character1 = m_current;
+                shift();
+                if (character1 != '0' || isASCIIDigit(m_current)) {
+                    m_lexErrorMessage = "The only valid numeric escape in strict mode is '\\0'";
+                    return StringCannotBeParsed;
+                }
+                if (shouldBuildStrings)
+                    record16(0);
+            } else if (!strictMode && isASCIIOctalDigit(m_current)) {
+                // Octal character sequences
+                T character1 = m_current;
+                shift();
+                if (isASCIIOctalDigit(m_current)) {
+                    // Two octal characters
+                    T character2 = m_current;
+                    shift();
+                    if (character1 >= '0' && character1 <= '3' && isASCIIOctalDigit(m_current)) {
+                        if (shouldBuildStrings)
+                            record16((character1 - '0') * 64 + (character2 - '0') * 8 + m_current - '0');
+                        shift();
+                    } else {
+                        if (shouldBuildStrings)
+                            record16((character1 - '0') * 8 + character2 - '0');
+                    }
+                } else {
+                    if (shouldBuildStrings)
+                        record16(character1 - '0');
+                }
+            } else if (!atEnd()) {
+                if (shouldBuildStrings)
+                    record16(m_current);
+                shift();
+            } else {
+                m_lexErrorMessage = "Unterminated string constant";
+                return StringUnterminated;
+            }
+
+            stringStart = currentSourcePtr();
+            continue;
+        }
+        // Fast check for characters that require special handling.
+        // Catches 0, \n, \r, 0x2028, and 0x2029 as efficiently
+        // as possible, and lets through all common ASCII characters.
+        if (UNLIKELY(((static_cast<unsigned>(m_current) - 0xE) & 0x2000))) {
+            // New-line or end of input is not allowed
+            if (atEnd() || isLineTerminator(m_current)) {
+                m_lexErrorMessage = "Unexpected EOF";
+                return atEnd() ? StringUnterminated : StringCannotBeParsed;
+            }
+            // Anything else is just a normal character
+        }
+        shift();
     }
-    if (isASCIIOctalDigit(m_current))
-        goto inOctal;
-    if (isASCIIDigit(m_current))
-        goto startNumber;
-    lvalp->doubleValue = 0;
-    goto doneNumeric;
 
-inNumberAfterDecimalPoint:
-    while (isASCIIDigit(m_current)) {
-        record8(m_current);
-        shift1();
+    if (currentSourcePtr() != stringStart && shouldBuildStrings)
+        append16(stringStart, currentSourcePtr() - stringStart);
+    if (shouldBuildStrings)
+        tokenData->ident = makeIdentifier(m_buffer16.data(), m_buffer16.size());
+    else
+        tokenData->ident = 0;
+
+    m_buffer16.resize(0);
+    return StringParsedSuccessfully;
+}
+
+template <typename T>
+ALWAYS_INLINE void Lexer<T>::parseHex(double& returnValue)
+{
+    // Optimization: most hexadecimal values fit into 4 bytes.
+    uint32_t hexValue = 0;
+    int maximumDigits = 7;
+
+    // Shift out the 'x' prefix.
+    shift();
+
+    do {
+        hexValue = (hexValue << 4) + toASCIIHexValue(m_current);
+        shift();
+        --maximumDigits;
+    } while (isASCIIHexDigit(m_current) && maximumDigits >= 0);
+
+    if (maximumDigits >= 0) {
+        returnValue = hexValue;
+        return;
     }
-    if ((m_current | 0x20) == 'e') {
-        record8('e');
-        shift1();
-        goto inExponentIndicator;
+
+    // No more place in the hexValue buffer.
+    // The values are shifted out and placed into the m_buffer8 vector.
+    for (int i = 0; i < 8; ++i) {
+         int digit = hexValue >> 28;
+         if (digit < 10)
+             record8(digit + '0');
+         else
+             record8(digit - 10 + 'a');
+         hexValue <<= 4;
     }
-    goto doneNumber;
 
-inExponentIndicator:
-    if (m_current == '+' || m_current == '-') {
+    while (isASCIIHexDigit(m_current)) {
         record8(m_current);
-        shift1();
+        shift();
     }
-    if (!isASCIIDigit(m_current))
-        goto returnError;
-    do {
-        record8(m_current);
-        shift1();
-    } while (isASCIIDigit(m_current));
-    goto doneNumber;
 
-inOctal: {
+    returnValue = parseIntOverflow(m_buffer8.data(), m_buffer8.size(), 16);
+}
+
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::parseOctal(double& returnValue)
+{
+    // Optimization: most octal values fit into 4 bytes.
+    uint32_t octalValue = 0;
+    int maximumDigits = 9;
+    // Temporary buffer for the digits. Makes easier
+    // to reconstruct the input characters when needed.
+    LChar digits[10];
+
     do {
-        record8(m_current);
-        shift1();
-    } while (isASCIIOctalDigit(m_current));
-    if (isASCIIDigit(m_current))
-        goto startNumber;
+        octalValue = octalValue * 8 + (m_current - '0');
+        digits[maximumDigits] = m_current;
+        shift();
+        --maximumDigits;
+    } while (isASCIIOctalDigit(m_current) && maximumDigits >= 0);
+
+    if (!isASCIIDigit(m_current) && maximumDigits >= 0) {
+        returnValue = octalValue;
+        return true;
+    }
 
-    double dval = 0;
+    for (int i = 9; i > maximumDigits; --i)
+         record8(digits[i]);
 
-    const char* end = m_buffer8.end();
-    for (const char* p = m_buffer8.data(); p < end; ++p) {
-        dval *= 8;
-        dval += *p - '0';
+    while (isASCIIOctalDigit(m_current)) {
+        record8(m_current);
+        shift();
     }
-    if (dval >= mantissaOverflowLowerBound)
-        dval = parseIntOverflow(m_buffer8.data(), end - m_buffer8.data(), 8);
 
-    m_buffer8.resize(0);
+    if (isASCIIDigit(m_current))
+        return false;
 
-    lvalp->doubleValue = dval;
-    goto doneNumeric;
+    returnValue = parseIntOverflow(m_buffer8.data(), m_buffer8.size(), 8);
+    return true;
 }
 
-inHex: {
-    do {
-        record8(m_current);
-        shift1();
-    } while (isASCIIHexDigit(m_current));
-
-    double dval = 0;
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::parseDecimal(double& returnValue)
+{
+    // Optimization: most decimal values fit into 4 bytes.
+    uint32_t decimalValue = 0;
+
+    // Since parseOctal may be executed before parseDecimal,
+    // the m_buffer8 may hold ascii digits.
+    if (!m_buffer8.size()) {
+        int maximumDigits = 9;
+        // Temporary buffer for the digits. Makes easier
+        // to reconstruct the input characters when needed.
+        LChar digits[10];
+
+        do {
+            decimalValue = decimalValue * 10 + (m_current - '0');
+            digits[maximumDigits] = m_current;
+            shift();
+            --maximumDigits;
+        } while (isASCIIDigit(m_current) && maximumDigits >= 0);
+
+        if (maximumDigits >= 0 && m_current != '.' && (m_current | 0x20) != 'e') {
+            returnValue = decimalValue;
+            return true;
+        }
 
-    const char* end = m_buffer8.end();
-    for (const char* p = m_buffer8.data(); p < end; ++p) {
-        dval *= 16;
-        dval += toASCIIHexValue(*p);
+        for (int i = 9; i > maximumDigits; --i)
+            record8(digits[i]);
     }
-    if (dval >= mantissaOverflowLowerBound)
-        dval = parseIntOverflow(m_buffer8.data(), end - m_buffer8.data(), 16);
 
-    m_buffer8.resize(0);
+    while (isASCIIDigit(m_current)) {
+        record8(m_current);
+        shift();
+    }
 
-    lvalp->doubleValue = dval;
-    goto doneNumeric;
+    return false;
 }
 
-startNumber:
-    record8(m_current);
-    shift1();
+template <typename T>
+ALWAYS_INLINE void Lexer<T>::parseNumberAfterDecimalPoint()
+{
+    record8('.');
     while (isASCIIDigit(m_current)) {
         record8(m_current);
-        shift1();
-    }
-    if (m_current == '.') {
-        record8('.');
-        shift1();
-        goto inNumberAfterDecimalPoint;
+        shift();
     }
-    if ((m_current | 0x20) == 'e') {
-        record8('e');
-        shift1();
-        goto inExponentIndicator;
+}
+
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::parseNumberAfterExponentIndicator()
+{
+    record8('e');
+    shift();
+    if (m_current == '+' || m_current == '-') {
+        record8(m_current);
+        shift();
     }
 
-    // Fall through into doneNumber.
+    if (!isASCIIDigit(m_current))
+        return false;
 
-doneNumber:
-    // Null-terminate string for strtod.
-    m_buffer8.append('\0');
-    lvalp->doubleValue = WTF::strtod(m_buffer8.data(), 0);
-    m_buffer8.resize(0);
+    do {
+        record8(m_current);
+        shift();
+    } while (isASCIIDigit(m_current));
+    return true;
+}
 
-    // Fall through into doneNumeric.
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::parseMultilineComment()
+{
+    while (true) {
+        while (UNLIKELY(m_current == '*')) {
+            shift();
+            if (m_current == '/') {
+                shift();
+                return true;
+            }
+        }
 
-doneNumeric:
-    // No identifiers allowed directly after numeric literal, e.g. "3in" is bad.
-    if (UNLIKELY(isIdentStart(m_current)))
-        goto returnError;
+        if (atEnd())
+            return false;
 
-    m_atLineStart = false;
-    m_delimited = false;
-    token = NUMBER;
-    goto returnToken;
+        if (isLineTerminator(m_current)) {
+            shiftLineTerminator();
+            m_terminator = true;
+        } else
+            shift();
+    }
+}
 
-doneSemicolon:
-    token = ';';
-    m_delimited = true;
-    goto returnToken;
+template <typename T>
+bool Lexer<T>::nextTokenIsColon()
+{
+    const T* code = m_code;
+    while (code < m_codeEnd && (isWhiteSpace(*code) || isLineTerminator(*code)))
+        code++;
+    
+    return code < m_codeEnd && *code == ':';
+}
 
-doneIdentifier:
-    m_atLineStart = false;
-    m_delimited = false;
-    lvalp->ident = makeIdentifier(m_buffer16.data(), m_buffer16.size());
-    m_buffer16.resize(0);
-    token = IDENT;
-    goto returnToken;
+template <typename T>
+JSTokenType Lexer<T>::lex(JSTokenData* tokenData, JSTokenLocation* tokenLocation, unsigned lexerFlags, bool strictMode)
+{
+    ASSERT(!m_error);
+    ASSERT(m_buffer8.isEmpty());
+    ASSERT(m_buffer16.isEmpty());
+
+    JSTokenType token = ERRORTOK;
+    m_terminator = false;
+
+start:
+    while (isWhiteSpace(m_current))
+        shift();
+
+    if (atEnd())
+        return EOFTOK;
+    
+    tokenLocation->startOffset = currentOffset();
+    ASSERT(currentOffset() >= currentLineStartOffset());
+
+    CharacterType type;
+    if (LIKELY(isLatin1(m_current)))
+        type = static_cast<CharacterType>(typesOfLatin1Characters[m_current]);
+    else if (isNonLatin1IdentStart(m_current))
+        type = CharacterIdentifierStart;
+    else if (isLineTerminator(m_current))
+        type = CharacterLineTerminator;
+    else
+        type = CharacterInvalid;
+
+    switch (type) {
+    case CharacterGreater:
+        shift();
+        if (m_current == '>') {
+            shift();
+            if (m_current == '>') {
+                shift();
+                if (m_current == '=') {
+                    shift();
+                    token = URSHIFTEQUAL;
+                    break;
+                }
+                token = URSHIFT;
+                break;
+            }
+            if (m_current == '=') {
+                shift();
+                token = RSHIFTEQUAL;
+                break;
+            }
+            token = RSHIFT;
+            break;
+        }
+        if (m_current == '=') {
+            shift();
+            token = GE;
+            break;
+        }
+        token = GT;
+        break;
+    case CharacterEqual:
+        shift();
+        if (m_current == '=') {
+            shift();
+            if (m_current == '=') {
+                shift();
+                token = STREQ;
+                break;
+            }
+            token = EQEQ;
+            break;
+        }
+        token = EQUAL;
+        break;
+    case CharacterLess:
+        shift();
+        if (m_current == '!' && peek(1) == '-' && peek(2) == '-') {
+            // <!-- marks the beginning of a line comment (for www usage)
+            goto inSingleLineComment;
+        }
+        if (m_current == '<') {
+            shift();
+            if (m_current == '=') {
+                shift();
+                token = LSHIFTEQUAL;
+                break;
+            }
+            token = LSHIFT;
+            break;
+        }
+        if (m_current == '=') {
+            shift();
+            token = LE;
+            break;
+        }
+        token = LT;
+        break;
+    case CharacterExclamationMark:
+        shift();
+        if (m_current == '=') {
+            shift();
+            if (m_current == '=') {
+                shift();
+                token = STRNEQ;
+                break;
+            }
+            token = NE;
+            break;
+        }
+        token = EXCLAMATION;
+        break;
+    case CharacterAdd:
+        shift();
+        if (m_current == '+') {
+            shift();
+            token = (!m_terminator) ? PLUSPLUS : AUTOPLUSPLUS;
+            break;
+        }
+        if (m_current == '=') {
+            shift();
+            token = PLUSEQUAL;
+            break;
+        }
+        token = PLUS;
+        break;
+    case CharacterSub:
+        shift();
+        if (m_current == '-') {
+            shift();
+            if (m_atLineStart && m_current == '>') {
+                shift();
+                goto inSingleLineComment;
+            }
+            token = (!m_terminator) ? MINUSMINUS : AUTOMINUSMINUS;
+            break;
+        }
+        if (m_current == '=') {
+            shift();
+            token = MINUSEQUAL;
+            break;
+        }
+        token = MINUS;
+        break;
+    case CharacterMultiply:
+        shift();
+        if (m_current == '=') {
+            shift();
+            token = MULTEQUAL;
+            break;
+        }
+        token = TIMES;
+        break;
+    case CharacterSlash:
+        shift();
+        if (m_current == '/') {
+            shift();
+            goto inSingleLineComment;
+        }
+        if (m_current == '*') {
+            shift();
+            if (parseMultilineComment())
+                goto start;
+            m_lexErrorMessage = "Multiline comment was not closed properly";
+            token = UNTERMINATED_MULTILINE_COMMENT_ERRORTOK;
+            goto returnError;
+        }
+        if (m_current == '=') {
+            shift();
+            token = DIVEQUAL;
+            break;
+        }
+        token = DIVIDE;
+        break;
+    case CharacterAnd:
+        shift();
+        if (m_current == '&') {
+            shift();
+            token = AND;
+            break;
+        }
+        if (m_current == '=') {
+            shift();
+            token = ANDEQUAL;
+            break;
+        }
+        token = BITAND;
+        break;
+    case CharacterXor:
+        shift();
+        if (m_current == '=') {
+            shift();
+            token = XOREQUAL;
+            break;
+        }
+        token = BITXOR;
+        break;
+    case CharacterModulo:
+        shift();
+        if (m_current == '=') {
+            shift();
+            token = MODEQUAL;
+            break;
+        }
+        token = MOD;
+        break;
+    case CharacterOr:
+        shift();
+        if (m_current == '=') {
+            shift();
+            token = OREQUAL;
+            break;
+        }
+        if (m_current == '|') {
+            shift();
+            token = OR;
+            break;
+        }
+        token = BITOR;
+        break;
+    case CharacterOpenParen:
+        token = OPENPAREN;
+        shift();
+        break;
+    case CharacterCloseParen:
+        token = CLOSEPAREN;
+        shift();
+        break;
+    case CharacterOpenBracket:
+        token = OPENBRACKET;
+        shift();
+        break;
+    case CharacterCloseBracket:
+        token = CLOSEBRACKET;
+        shift();
+        break;
+    case CharacterComma:
+        token = COMMA;
+        shift();
+        break;
+    case CharacterColon:
+        token = COLON;
+        shift();
+        break;
+    case CharacterQuestion:
+        token = QUESTION;
+        shift();
+        break;
+    case CharacterTilde:
+        token = TILDE;
+        shift();
+        break;
+    case CharacterSemicolon:
+        shift();
+        token = SEMICOLON;
+        break;
+    case CharacterOpenBrace:
+        tokenData->line = lineNumber();
+        tokenData->offset = currentOffset();
+        tokenData->lineStartOffset = currentLineStartOffset();
+        ASSERT(tokenData->offset >= tokenData->lineStartOffset);
+        shift();
+        token = OPENBRACE;
+        break;
+    case CharacterCloseBrace:
+        tokenData->line = lineNumber();
+        tokenData->offset = currentOffset();
+        tokenData->lineStartOffset = currentLineStartOffset();
+        ASSERT(tokenData->offset >= tokenData->lineStartOffset);
+        shift();
+        token = CLOSEBRACE;
+        break;
+    case CharacterDot:
+        shift();
+        if (!isASCIIDigit(m_current)) {
+            token = DOT;
+            break;
+        }
+        goto inNumberAfterDecimalPoint;
+    case CharacterZero:
+        shift();
+        if ((m_current | 0x20) == 'x' && isASCIIHexDigit(peek(1))) {
+            parseHex(tokenData->doubleValue);
+            token = NUMBER;
+        } else {
+            record8('0');
+            if (isASCIIOctalDigit(m_current)) {
+                if (parseOctal(tokenData->doubleValue)) {
+                    if (strictMode) {
+                        m_lexErrorMessage = "Octal escapes are forbidden in strict mode";
+                        token = INVALID_OCTAL_NUMBER_ERRORTOK;
+                        goto returnError;
+                    }
+                    token = NUMBER;
+                }
+            }
+        }
+        // Fall through into CharacterNumber
+    case CharacterNumber:
+        if (LIKELY(token != NUMBER)) {
+            if (!parseDecimal(tokenData->doubleValue)) {
+                if (m_current == '.') {
+                    shift();
+inNumberAfterDecimalPoint:
+                    parseNumberAfterDecimalPoint();
+                }
+                if ((m_current | 0x20) == 'e') {
+                    if (!parseNumberAfterExponentIndicator()) {
+                        m_lexErrorMessage = "Non-number found after exponent indicator";
+                        token = atEnd() ? UNTERMINATED_NUMERIC_LITERAL_ERRORTOK : INVALID_NUMERIC_LITERAL_ERRORTOK;
+                        goto returnError;
+                    }
+                }
+                size_t parsedLength;
+                tokenData->doubleValue = parseDouble(m_buffer8.data(), m_buffer8.size(), parsedLength);
+            }
+            token = NUMBER;
+        }
+
+        // No identifiers allowed directly after numeric literal, e.g. "3in" is bad.
+        if (UNLIKELY(isIdentStart(m_current))) {
+            m_lexErrorMessage = "At least one digit must occur after a decimal point";
+            token = atEnd() ? UNTERMINATED_NUMERIC_LITERAL_ERRORTOK : INVALID_NUMERIC_LITERAL_ERRORTOK;
+            goto returnError;
+        }
+        m_buffer8.resize(0);
+        break;
+    case CharacterQuote:
+        if (lexerFlags & LexerFlagsDontBuildStrings) {
+            StringParseResult result = parseString<false>(tokenData, strictMode);
+            if (UNLIKELY(result != StringParsedSuccessfully)) {
+                token = result == StringUnterminated ? UNTERMINATED_STRING_LITERAL_ERRORTOK : INVALID_STRING_LITERAL_ERRORTOK;
+                goto returnError;
+            }
+        } else {
+            StringParseResult result = parseString<true>(tokenData, strictMode);
+            if (UNLIKELY(result != StringParsedSuccessfully)) {
+                token = result == StringUnterminated ? UNTERMINATED_STRING_LITERAL_ERRORTOK : INVALID_STRING_LITERAL_ERRORTOK;
+                goto returnError;
+            }
+        }
+        shift();
+        token = STRING;
+        break;
+    case CharacterIdentifierStart:
+        ASSERT(isIdentStart(m_current));
+        // Fall through into CharacterBackSlash.
+    case CharacterBackSlash:
+        if (lexerFlags & LexexFlagsDontBuildKeywords)
+            token = parseIdentifier<false>(tokenData, lexerFlags, strictMode);
+        else
+            token = parseIdentifier<true>(tokenData, lexerFlags, strictMode);
+        break;
+    case CharacterLineTerminator:
+        ASSERT(isLineTerminator(m_current));
+        shiftLineTerminator();
+        m_atLineStart = true;
+        m_terminator = true;
+        m_lineStart = m_code;
+        goto start;
+    case CharacterInvalid:
+        m_lexErrorMessage = invalidCharacterMessage();
+        token = ERRORTOK;
+        goto returnError;
+    default:
+        RELEASE_ASSERT_NOT_REACHED();
+        m_lexErrorMessage = "Internal Error";
+        token = ERRORTOK;
+        goto returnError;
+    }
 
-doneIdentifierOrKeyword: {
     m_atLineStart = false;
-    m_delimited = false;
-    m_buffer16.resize(0);
-    const HashEntry* entry = m_keywordTable.entry(m_globalData, *lvalp->ident);
-    token = entry ? entry->lexerValue() : IDENT;
     goto returnToken;
-}
 
-doneString:
-    // Atomize constant strings in case they're later used in property lookup.
-    shift1();
-    m_atLineStart = false;
-    m_delimited = false;
-    lvalp->ident = makeIdentifier(m_buffer16.data(), m_buffer16.size());
-    m_buffer16.resize(0);
-    token = STRING;
+inSingleLineComment:
+    while (!isLineTerminator(m_current)) {
+        if (atEnd())
+            return EOFTOK;
+        shift();
+    }
+    shiftLineTerminator();
+    m_atLineStart = true;
+    m_terminator = true;
+    m_lineStart = m_code;
+    if (!lastTokenWasRestrKeyword())
+        goto start;
 
+    token = SEMICOLON;
     // Fall through into returnToken.
 
-returnToken: {
-    int lineNumber = m_lineNumber;
-    llocp->first_line = lineNumber;
-    llocp->last_line = lineNumber;
-    llocp->first_column = startOffset;
-    llocp->last_column = currentOffset();
-
+returnToken:
+    tokenLocation->line = m_lineNumber;
+    tokenLocation->endOffset = currentOffset();
+    tokenLocation->lineStartOffset = currentLineStartOffset();
+    ASSERT(tokenLocation->endOffset >= tokenLocation->lineStartOffset);
     m_lastToken = token;
     return token;
-}
 
 returnError:
     m_error = true;
-    return -1;
+    tokenLocation->line = m_lineNumber;
+    tokenLocation->endOffset = currentOffset();
+    tokenLocation->lineStartOffset = currentLineStartOffset();
+    ASSERT(tokenLocation->endOffset >= tokenLocation->lineStartOffset);
+    RELEASE_ASSERT(token & ErrorTokenFlag);
+    return token;
 }
 
-bool Lexer::scanRegExp(const Identifier*& pattern, const Identifier*& flags, UChar patternPrefix)
+template <typename T>
+static inline void orCharacter(UChar&, UChar);
+
+template <>
+inline void orCharacter<LChar>(UChar&, UChar) { }
+
+template <>
+inline void orCharacter<UChar>(UChar& orAccumulator, UChar character)
+{
+    orAccumulator |= character;
+}
+
+template <typename T>
+bool Lexer<T>::scanRegExp(const Identifier*& pattern, const Identifier*& flags, UChar patternPrefix)
 {
     ASSERT(m_buffer16.isEmpty());
 
     bool lastWasEscape = false;
     bool inBrackets = false;
+    UChar charactersOredTogether = 0;
 
     if (patternPrefix) {
         ASSERT(!isLineTerminator(patternPrefix));
@@ -918,26 +1741,27 @@ bool Lexer::scanRegExp(const Identifier*& pattern, const Identifier*& flags, UCh
     }
 
     while (true) {
-        int current = m_current;
-
-        if (isLineTerminator(current) || current == -1) {
+        if (isLineTerminator(m_current) || atEnd()) {
             m_buffer16.resize(0);
             return false;
         }
 
-        shift1();
+        T prev = m_current;
+        
+        shift();
 
-        if (current == '/' && !lastWasEscape && !inBrackets)
+        if (prev == '/' && !lastWasEscape && !inBrackets)
             break;
 
-        record16(current);
+        record16(prev);
+        orCharacter<T>(charactersOredTogether, prev);
 
         if (lastWasEscape) {
             lastWasEscape = false;
             continue;
         }
 
-        switch (current) {
+        switch (prev) {
         case '[':
             inBrackets = true;
             break;
@@ -950,34 +1774,38 @@ bool Lexer::scanRegExp(const Identifier*& pattern, const Identifier*& flags, UCh
         }
     }
 
-    pattern = makeIdentifier(m_buffer16.data(), m_buffer16.size());
+    pattern = makeRightSizedIdentifier(m_buffer16.data(), m_buffer16.size(), charactersOredTogether);
+
     m_buffer16.resize(0);
+    charactersOredTogether = 0;
 
     while (isIdentPart(m_current)) {
         record16(m_current);
-        shift1();
+        orCharacter<T>(charactersOredTogether, m_current);
+        shift();
     }
 
-    flags = makeIdentifier(m_buffer16.data(), m_buffer16.size());
+    flags = makeRightSizedIdentifier(m_buffer16.data(), m_buffer16.size(), charactersOredTogether);
     m_buffer16.resize(0);
 
     return true;
 }
 
-bool Lexer::skipRegExp()
+template <typename T>
+bool Lexer<T>::skipRegExp()
 {
     bool lastWasEscape = false;
     bool inBrackets = false;
 
     while (true) {
-        int current = m_current;
-
-        if (isLineTerminator(current) || current == -1)
+        if (isLineTerminator(m_current) || atEnd())
             return false;
 
-        shift1();
+        T prev = m_current;
+        
+        shift();
 
-        if (current == '/' && !lastWasEscape && !inBrackets)
+        if (prev == '/' && !lastWasEscape && !inBrackets)
             break;
 
         if (lastWasEscape) {
@@ -985,7 +1813,7 @@ bool Lexer::skipRegExp()
             continue;
         }
 
-        switch (current) {
+        switch (prev) {
         case '[':
             inBrackets = true;
             break;
@@ -999,17 +1827,17 @@ bool Lexer::skipRegExp()
     }
 
     while (isIdentPart(m_current))
-        shift1();
+        shift();
 
     return true;
 }
 
-void Lexer::clear()
+template <typename T>
+void Lexer<T>::clear()
 {
     m_arena = 0;
-    m_codeWithoutBOMs.clear();
 
-    Vector<char> newBuffer8;
+    Vector<LChar> newBuffer8;
     m_buffer8.swap(newBuffer8);
 
     Vector<UChar> newBuffer16;
@@ -1018,29 +1846,16 @@ void Lexer::clear()
     m_isReparsing = false;
 }
 
-SourceCode Lexer::sourceCode(int openBrace, int closeBrace, int firstLine)
+template <typename T>
+SourceCode Lexer<T>::sourceCode(int openBrace, int closeBrace, int firstLine, unsigned startColumn)
 {
-    if (m_codeWithoutBOMs.isEmpty())
-        return SourceCode(m_source->provider(), openBrace, closeBrace + 1, firstLine);
-
-    const UChar* data = m_source->provider()->data();
-    
-    ASSERT(openBrace < closeBrace);
-    int i;
-    for (i = m_source->startOffset(); i < openBrace; ++i) {
-        if (data[i] == byteOrderMark) {
-            openBrace++;
-            closeBrace++;
-        }
-    }
-    for (; i < closeBrace; ++i) {
-        if (data[i] == byteOrderMark)
-            closeBrace++;
-    }
-
-    ASSERT(openBrace < closeBrace);
-
-    return SourceCode(m_source->provider(), openBrace, closeBrace + 1, firstLine);
+    ASSERT(m_source->provider()->source()[openBrace] == '{');
+    ASSERT(m_source->provider()->source()[closeBrace] == '}');
+    return SourceCode(m_source->provider(), openBrace, closeBrace + 1, firstLine, startColumn);
 }
 
+// Instantiate the two flavors of Lexer we need instead of putting most of this file in Lexer.h
+template class Lexer<LChar>;
+template class Lexer<UChar>;
+
 } // namespace JSC