]> git.saurik.com Git - apple/javascriptcore.git/blame_incremental - parser/Lexer.cpp
projects / apple / javascriptcore.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
JavaScriptCore-1218.34.tar.gz
[apple/javascriptcore.git] / parser / Lexer.cpp
... / ...
CommitLineData
1/*
2 * Copyright (C) 1999-2000 Harri Porten (porten@kde.org)
3 * Copyright (C) 2006, 2007, 2008, 2009, 2011, 2012, 2013 Apple Inc. All Rights Reserved.
4 * Copyright (C) 2007 Cameron Zwarich (cwzwarich@uwaterloo.ca)
5 * Copyright (C) 2010 Zoltan Herczeg (zherczeg@inf.u-szeged.hu)
6 * Copyright (C) 2012 Mathias Bynens (mathias@qiwi.be)
7 *
8 * This library is free software; you can redistribute it and/or
9 * modify it under the terms of the GNU Library General Public
10 * License as published by the Free Software Foundation; either
11 * version 2 of the License, or (at your option) any later version.
12 *
13 * This library is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * Library General Public License for more details.
17 *
18 * You should have received a copy of the GNU Library General Public License
19 * along with this library; see the file COPYING.LIB. If not, write to
20 * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
21 * Boston, MA 02110-1301, USA.
22 *
23 */
24
25#include "config.h"
26#include "Lexer.h"
27
28#include "JSFunction.h"
29
30#include "JSGlobalObjectFunctions.h"
31#include "Identifier.h"
32#include "NodeInfo.h"
33#include "Nodes.h"
34#include <wtf/dtoa.h>
35#include <ctype.h>
36#include <limits.h>
37#include <string.h>
38#include <wtf/Assertions.h>
39
40using namespace WTF;
41using namespace Unicode;
42
43#include "KeywordLookup.h"
44#include "Lexer.lut.h"
45#include "Parser.h"
46
47namespace JSC {
48
49Keywords::Keywords(VM* vm)
50 : m_vm(vm)
51 , m_keywordTable(JSC::mainTable)
52{
53}
54
55enum CharacterType {
56 // Types for the main switch
57
58 // The first three types are fixed, and also used for identifying
59 // ASCII alpha and alphanumeric characters (see isIdentStart and isIdentPart).
60 CharacterIdentifierStart,
61 CharacterZero,
62 CharacterNumber,
63
64 CharacterInvalid,
65 CharacterLineTerminator,
66 CharacterExclamationMark,
67 CharacterOpenParen,
68 CharacterCloseParen,
69 CharacterOpenBracket,
70 CharacterCloseBracket,
71 CharacterComma,
72 CharacterColon,
73 CharacterQuestion,
74 CharacterTilde,
75 CharacterQuote,
76 CharacterDot,
77 CharacterSlash,
78 CharacterBackSlash,
79 CharacterSemicolon,
80 CharacterOpenBrace,
81 CharacterCloseBrace,
82
83 CharacterAdd,
84 CharacterSub,
85 CharacterMultiply,
86 CharacterModulo,
87 CharacterAnd,
88 CharacterXor,
89 CharacterOr,
90 CharacterLess,
91 CharacterGreater,
92 CharacterEqual,
93
94 // Other types (only one so far)
95 CharacterWhiteSpace,
96};
97
98// 256 Latin-1 codes
99static const unsigned short typesOfLatin1Characters[256] = {
100/* 0 - Null */ CharacterInvalid,
101/* 1 - Start of Heading */ CharacterInvalid,
102/* 2 - Start of Text */ CharacterInvalid,
103/* 3 - End of Text */ CharacterInvalid,
104/* 4 - End of Transm. */ CharacterInvalid,
105/* 5 - Enquiry */ CharacterInvalid,
106/* 6 - Acknowledgment */ CharacterInvalid,
107/* 7 - Bell */ CharacterInvalid,
108/* 8 - Back Space */ CharacterInvalid,
109/* 9 - Horizontal Tab */ CharacterWhiteSpace,
110/* 10 - Line Feed */ CharacterLineTerminator,
111/* 11 - Vertical Tab */ CharacterWhiteSpace,
112/* 12 - Form Feed */ CharacterWhiteSpace,
113/* 13 - Carriage Return */ CharacterLineTerminator,
114/* 14 - Shift Out */ CharacterInvalid,
115/* 15 - Shift In */ CharacterInvalid,
116/* 16 - Data Line Escape */ CharacterInvalid,
117/* 17 - Device Control 1 */ CharacterInvalid,
118/* 18 - Device Control 2 */ CharacterInvalid,
119/* 19 - Device Control 3 */ CharacterInvalid,
120/* 20 - Device Control 4 */ CharacterInvalid,
121/* 21 - Negative Ack. */ CharacterInvalid,
122/* 22 - Synchronous Idle */ CharacterInvalid,
123/* 23 - End of Transmit */ CharacterInvalid,
124/* 24 - Cancel */ CharacterInvalid,
125/* 25 - End of Medium */ CharacterInvalid,
126/* 26 - Substitute */ CharacterInvalid,
127/* 27 - Escape */ CharacterInvalid,
128/* 28 - File Separator */ CharacterInvalid,
129/* 29 - Group Separator */ CharacterInvalid,
130/* 30 - Record Separator */ CharacterInvalid,
131/* 31 - Unit Separator */ CharacterInvalid,
132/* 32 - Space */ CharacterWhiteSpace,
133/* 33 - ! */ CharacterExclamationMark,
134/* 34 - " */ CharacterQuote,
135/* 35 - # */ CharacterInvalid,
136/* 36 - $ */ CharacterIdentifierStart,
137/* 37 - % */ CharacterModulo,
138/* 38 - & */ CharacterAnd,
139/* 39 - ' */ CharacterQuote,
140/* 40 - ( */ CharacterOpenParen,
141/* 41 - ) */ CharacterCloseParen,
142/* 42 - * */ CharacterMultiply,
143/* 43 - + */ CharacterAdd,
144/* 44 - , */ CharacterComma,
145/* 45 - - */ CharacterSub,
146/* 46 - . */ CharacterDot,
147/* 47 - / */ CharacterSlash,
148/* 48 - 0 */ CharacterZero,
149/* 49 - 1 */ CharacterNumber,
150/* 50 - 2 */ CharacterNumber,
151/* 51 - 3 */ CharacterNumber,
152/* 52 - 4 */ CharacterNumber,
153/* 53 - 5 */ CharacterNumber,
154/* 54 - 6 */ CharacterNumber,
155/* 55 - 7 */ CharacterNumber,
156/* 56 - 8 */ CharacterNumber,
157/* 57 - 9 */ CharacterNumber,
158/* 58 - : */ CharacterColon,
159/* 59 - ; */ CharacterSemicolon,
160/* 60 - < */ CharacterLess,
161/* 61 - = */ CharacterEqual,
162/* 62 - > */ CharacterGreater,
163/* 63 - ? */ CharacterQuestion,
164/* 64 - @ */ CharacterInvalid,
165/* 65 - A */ CharacterIdentifierStart,
166/* 66 - B */ CharacterIdentifierStart,
167/* 67 - C */ CharacterIdentifierStart,
168/* 68 - D */ CharacterIdentifierStart,
169/* 69 - E */ CharacterIdentifierStart,
170/* 70 - F */ CharacterIdentifierStart,
171/* 71 - G */ CharacterIdentifierStart,
172/* 72 - H */ CharacterIdentifierStart,
173/* 73 - I */ CharacterIdentifierStart,
174/* 74 - J */ CharacterIdentifierStart,
175/* 75 - K */ CharacterIdentifierStart,
176/* 76 - L */ CharacterIdentifierStart,
177/* 77 - M */ CharacterIdentifierStart,
178/* 78 - N */ CharacterIdentifierStart,
179/* 79 - O */ CharacterIdentifierStart,
180/* 80 - P */ CharacterIdentifierStart,
181/* 81 - Q */ CharacterIdentifierStart,
182/* 82 - R */ CharacterIdentifierStart,
183/* 83 - S */ CharacterIdentifierStart,
184/* 84 - T */ CharacterIdentifierStart,
185/* 85 - U */ CharacterIdentifierStart,
186/* 86 - V */ CharacterIdentifierStart,
187/* 87 - W */ CharacterIdentifierStart,
188/* 88 - X */ CharacterIdentifierStart,
189/* 89 - Y */ CharacterIdentifierStart,
190/* 90 - Z */ CharacterIdentifierStart,
191/* 91 - [ */ CharacterOpenBracket,
192/* 92 - \ */ CharacterBackSlash,
193/* 93 - ] */ CharacterCloseBracket,
194/* 94 - ^ */ CharacterXor,
195/* 95 - _ */ CharacterIdentifierStart,
196/* 96 - ` */ CharacterInvalid,
197/* 97 - a */ CharacterIdentifierStart,
198/* 98 - b */ CharacterIdentifierStart,
199/* 99 - c */ CharacterIdentifierStart,
200/* 100 - d */ CharacterIdentifierStart,
201/* 101 - e */ CharacterIdentifierStart,
202/* 102 - f */ CharacterIdentifierStart,
203/* 103 - g */ CharacterIdentifierStart,
204/* 104 - h */ CharacterIdentifierStart,
205/* 105 - i */ CharacterIdentifierStart,
206/* 106 - j */ CharacterIdentifierStart,
207/* 107 - k */ CharacterIdentifierStart,
208/* 108 - l */ CharacterIdentifierStart,
209/* 109 - m */ CharacterIdentifierStart,
210/* 110 - n */ CharacterIdentifierStart,
211/* 111 - o */ CharacterIdentifierStart,
212/* 112 - p */ CharacterIdentifierStart,
213/* 113 - q */ CharacterIdentifierStart,
214/* 114 - r */ CharacterIdentifierStart,
215/* 115 - s */ CharacterIdentifierStart,
216/* 116 - t */ CharacterIdentifierStart,
217/* 117 - u */ CharacterIdentifierStart,
218/* 118 - v */ CharacterIdentifierStart,
219/* 119 - w */ CharacterIdentifierStart,
220/* 120 - x */ CharacterIdentifierStart,
221/* 121 - y */ CharacterIdentifierStart,
222/* 122 - z */ CharacterIdentifierStart,
223/* 123 - { */ CharacterOpenBrace,
224/* 124 - | */ CharacterOr,
225/* 125 - } */ CharacterCloseBrace,
226/* 126 - ~ */ CharacterTilde,
227/* 127 - Delete */ CharacterInvalid,
228/* 128 - Cc category */ CharacterInvalid,
229/* 129 - Cc category */ CharacterInvalid,
230/* 130 - Cc category */ CharacterInvalid,
231/* 131 - Cc category */ CharacterInvalid,
232/* 132 - Cc category */ CharacterInvalid,
233/* 133 - Cc category */ CharacterInvalid,
234/* 134 - Cc category */ CharacterInvalid,
235/* 135 - Cc category */ CharacterInvalid,
236/* 136 - Cc category */ CharacterInvalid,
237/* 137 - Cc category */ CharacterInvalid,
238/* 138 - Cc category */ CharacterInvalid,
239/* 139 - Cc category */ CharacterInvalid,
240/* 140 - Cc category */ CharacterInvalid,
241/* 141 - Cc category */ CharacterInvalid,
242/* 142 - Cc category */ CharacterInvalid,
243/* 143 - Cc category */ CharacterInvalid,
244/* 144 - Cc category */ CharacterInvalid,
245/* 145 - Cc category */ CharacterInvalid,
246/* 146 - Cc category */ CharacterInvalid,
247/* 147 - Cc category */ CharacterInvalid,
248/* 148 - Cc category */ CharacterInvalid,
249/* 149 - Cc category */ CharacterInvalid,
250/* 150 - Cc category */ CharacterInvalid,
251/* 151 - Cc category */ CharacterInvalid,
252/* 152 - Cc category */ CharacterInvalid,
253/* 153 - Cc category */ CharacterInvalid,
254/* 154 - Cc category */ CharacterInvalid,
255/* 155 - Cc category */ CharacterInvalid,
256/* 156 - Cc category */ CharacterInvalid,
257/* 157 - Cc category */ CharacterInvalid,
258/* 158 - Cc category */ CharacterInvalid,
259/* 159 - Cc category */ CharacterInvalid,
260/* 160 - Zs category (nbsp) */ CharacterWhiteSpace,
261/* 161 - Po category */ CharacterInvalid,
262/* 162 - Sc category */ CharacterInvalid,
263/* 163 - Sc category */ CharacterInvalid,
264/* 164 - Sc category */ CharacterInvalid,
265/* 165 - Sc category */ CharacterInvalid,
266/* 166 - So category */ CharacterInvalid,
267/* 167 - So category */ CharacterInvalid,
268/* 168 - Sk category */ CharacterInvalid,
269/* 169 - So category */ CharacterInvalid,
270/* 170 - Ll category */ CharacterIdentifierStart,
271/* 171 - Pi category */ CharacterInvalid,
272/* 172 - Sm category */ CharacterInvalid,
273/* 173 - Cf category */ CharacterInvalid,
274/* 174 - So category */ CharacterInvalid,
275/* 175 - Sk category */ CharacterInvalid,
276/* 176 - So category */ CharacterInvalid,
277/* 177 - Sm category */ CharacterInvalid,
278/* 178 - No category */ CharacterInvalid,
279/* 179 - No category */ CharacterInvalid,
280/* 180 - Sk category */ CharacterInvalid,
281/* 181 - Ll category */ CharacterIdentifierStart,
282/* 182 - So category */ CharacterInvalid,
283/* 183 - Po category */ CharacterInvalid,
284/* 184 - Sk category */ CharacterInvalid,
285/* 185 - No category */ CharacterInvalid,
286/* 186 - Ll category */ CharacterIdentifierStart,
287/* 187 - Pf category */ CharacterInvalid,
288/* 188 - No category */ CharacterInvalid,
289/* 189 - No category */ CharacterInvalid,
290/* 190 - No category */ CharacterInvalid,
291/* 191 - Po category */ CharacterInvalid,
292/* 192 - Lu category */ CharacterIdentifierStart,
293/* 193 - Lu category */ CharacterIdentifierStart,
294/* 194 - Lu category */ CharacterIdentifierStart,
295/* 195 - Lu category */ CharacterIdentifierStart,
296/* 196 - Lu category */ CharacterIdentifierStart,
297/* 197 - Lu category */ CharacterIdentifierStart,
298/* 198 - Lu category */ CharacterIdentifierStart,
299/* 199 - Lu category */ CharacterIdentifierStart,
300/* 200 - Lu category */ CharacterIdentifierStart,
301/* 201 - Lu category */ CharacterIdentifierStart,
302/* 202 - Lu category */ CharacterIdentifierStart,
303/* 203 - Lu category */ CharacterIdentifierStart,
304/* 204 - Lu category */ CharacterIdentifierStart,
305/* 205 - Lu category */ CharacterIdentifierStart,
306/* 206 - Lu category */ CharacterIdentifierStart,
307/* 207 - Lu category */ CharacterIdentifierStart,
308/* 208 - Lu category */ CharacterIdentifierStart,
309/* 209 - Lu category */ CharacterIdentifierStart,
310/* 210 - Lu category */ CharacterIdentifierStart,
311/* 211 - Lu category */ CharacterIdentifierStart,
312/* 212 - Lu category */ CharacterIdentifierStart,
313/* 213 - Lu category */ CharacterIdentifierStart,
314/* 214 - Lu category */ CharacterIdentifierStart,
315/* 215 - Sm category */ CharacterInvalid,
316/* 216 - Lu category */ CharacterIdentifierStart,
317/* 217 - Lu category */ CharacterIdentifierStart,
318/* 218 - Lu category */ CharacterIdentifierStart,
319/* 219 - Lu category */ CharacterIdentifierStart,
320/* 220 - Lu category */ CharacterIdentifierStart,
321/* 221 - Lu category */ CharacterIdentifierStart,
322/* 222 - Lu category */ CharacterIdentifierStart,
323/* 223 - Ll category */ CharacterIdentifierStart,
324/* 224 - Ll category */ CharacterIdentifierStart,
325/* 225 - Ll category */ CharacterIdentifierStart,
326/* 226 - Ll category */ CharacterIdentifierStart,
327/* 227 - Ll category */ CharacterIdentifierStart,
328/* 228 - Ll category */ CharacterIdentifierStart,
329/* 229 - Ll category */ CharacterIdentifierStart,
330/* 230 - Ll category */ CharacterIdentifierStart,
331/* 231 - Ll category */ CharacterIdentifierStart,
332/* 232 - Ll category */ CharacterIdentifierStart,
333/* 233 - Ll category */ CharacterIdentifierStart,
334/* 234 - Ll category */ CharacterIdentifierStart,
335/* 235 - Ll category */ CharacterIdentifierStart,
336/* 236 - Ll category */ CharacterIdentifierStart,
337/* 237 - Ll category */ CharacterIdentifierStart,
338/* 238 - Ll category */ CharacterIdentifierStart,
339/* 239 - Ll category */ CharacterIdentifierStart,
340/* 240 - Ll category */ CharacterIdentifierStart,
341/* 241 - Ll category */ CharacterIdentifierStart,
342/* 242 - Ll category */ CharacterIdentifierStart,
343/* 243 - Ll category */ CharacterIdentifierStart,
344/* 244 - Ll category */ CharacterIdentifierStart,
345/* 245 - Ll category */ CharacterIdentifierStart,
346/* 246 - Ll category */ CharacterIdentifierStart,
347/* 247 - Sm category */ CharacterInvalid,
348/* 248 - Ll category */ CharacterIdentifierStart,
349/* 249 - Ll category */ CharacterIdentifierStart,
350/* 250 - Ll category */ CharacterIdentifierStart,
351/* 251 - Ll category */ CharacterIdentifierStart,
352/* 252 - Ll category */ CharacterIdentifierStart,
353/* 253 - Ll category */ CharacterIdentifierStart,
354/* 254 - Ll category */ CharacterIdentifierStart,
355/* 255 - Ll category */ CharacterIdentifierStart
356};
357
358// This table provides the character that results from \X where X is the index in the table beginning
359// with SPACE. A table value of 0 means that more processing needs to be done.
360static const LChar singleCharacterEscapeValuesForASCII[128] = {
361/* 0 - Null */ 0,
362/* 1 - Start of Heading */ 0,
363/* 2 - Start of Text */ 0,
364/* 3 - End of Text */ 0,
365/* 4 - End of Transm. */ 0,
366/* 5 - Enquiry */ 0,
367/* 6 - Acknowledgment */ 0,
368/* 7 - Bell */ 0,
369/* 8 - Back Space */ 0,
370/* 9 - Horizontal Tab */ 0,
371/* 10 - Line Feed */ 0,
372/* 11 - Vertical Tab */ 0,
373/* 12 - Form Feed */ 0,
374/* 13 - Carriage Return */ 0,
375/* 14 - Shift Out */ 0,
376/* 15 - Shift In */ 0,
377/* 16 - Data Line Escape */ 0,
378/* 17 - Device Control 1 */ 0,
379/* 18 - Device Control 2 */ 0,
380/* 19 - Device Control 3 */ 0,
381/* 20 - Device Control 4 */ 0,
382/* 21 - Negative Ack. */ 0,
383/* 22 - Synchronous Idle */ 0,
384/* 23 - End of Transmit */ 0,
385/* 24 - Cancel */ 0,
386/* 25 - End of Medium */ 0,
387/* 26 - Substitute */ 0,
388/* 27 - Escape */ 0,
389/* 28 - File Separator */ 0,
390/* 29 - Group Separator */ 0,
391/* 30 - Record Separator */ 0,
392/* 31 - Unit Separator */ 0,
393/* 32 - Space */ ' ',
394/* 33 - ! */ '!',
395/* 34 - " */ '"',
396/* 35 - # */ '#',
397/* 36 - $ */ '$',
398/* 37 - % */ '%',
399/* 38 - & */ '&',
400/* 39 - ' */ '\'',
401/* 40 - ( */ '(',
402/* 41 - ) */ ')',
403/* 42 - * */ '*',
404/* 43 - + */ '+',
405/* 44 - , */ ',',
406/* 45 - - */ '-',
407/* 46 - . */ '.',
408/* 47 - / */ '/',
409/* 48 - 0 */ 0,
410/* 49 - 1 */ 0,
411/* 50 - 2 */ 0,
412/* 51 - 3 */ 0,
413/* 52 - 4 */ 0,
414/* 53 - 5 */ 0,
415/* 54 - 6 */ 0,
416/* 55 - 7 */ 0,
417/* 56 - 8 */ 0,
418/* 57 - 9 */ 0,
419/* 58 - : */ ':',
420/* 59 - ; */ ';',
421/* 60 - < */ '<',
422/* 61 - = */ '=',
423/* 62 - > */ '>',
424/* 63 - ? */ '?',
425/* 64 - @ */ '@',
426/* 65 - A */ 'A',
427/* 66 - B */ 'B',
428/* 67 - C */ 'C',
429/* 68 - D */ 'D',
430/* 69 - E */ 'E',
431/* 70 - F */ 'F',
432/* 71 - G */ 'G',
433/* 72 - H */ 'H',
434/* 73 - I */ 'I',
435/* 74 - J */ 'J',
436/* 75 - K */ 'K',
437/* 76 - L */ 'L',
438/* 77 - M */ 'M',
439/* 78 - N */ 'N',
440/* 79 - O */ 'O',
441/* 80 - P */ 'P',
442/* 81 - Q */ 'Q',
443/* 82 - R */ 'R',
444/* 83 - S */ 'S',
445/* 84 - T */ 'T',
446/* 85 - U */ 'U',
447/* 86 - V */ 'V',
448/* 87 - W */ 'W',
449/* 88 - X */ 'X',
450/* 89 - Y */ 'Y',
451/* 90 - Z */ 'Z',
452/* 91 - [ */ '[',
453/* 92 - \ */ '\\',
454/* 93 - ] */ ']',
455/* 94 - ^ */ '^',
456/* 95 - _ */ '_',
457/* 96 - ` */ '`',
458/* 97 - a */ 'a',
459/* 98 - b */ 0x08,
460/* 99 - c */ 'c',
461/* 100 - d */ 'd',
462/* 101 - e */ 'e',
463/* 102 - f */ 0x0C,
464/* 103 - g */ 'g',
465/* 104 - h */ 'h',
466/* 105 - i */ 'i',
467/* 106 - j */ 'j',
468/* 107 - k */ 'k',
469/* 108 - l */ 'l',
470/* 109 - m */ 'm',
471/* 110 - n */ 0x0A,
472/* 111 - o */ 'o',
473/* 112 - p */ 'p',
474/* 113 - q */ 'q',
475/* 114 - r */ 0x0D,
476/* 115 - s */ 's',
477/* 116 - t */ 0x09,
478/* 117 - u */ 0,
479/* 118 - v */ 0x0B,
480/* 119 - w */ 'w',
481/* 120 - x */ 0,
482/* 121 - y */ 'y',
483/* 122 - z */ 'z',
484/* 123 - { */ '{',
485/* 124 - | */ '|',
486/* 125 - } */ '}',
487/* 126 - ~ */ '~',
488/* 127 - Delete */ 0
489};
490
491template <typename T>
492Lexer<T>::Lexer(VM* vm)
493 : m_isReparsing(false)
494 , m_vm(vm)
495{
496}
497
498template <typename T>
499Lexer<T>::~Lexer()
500{
501}
502
503template <typename T>
504String Lexer<T>::invalidCharacterMessage() const
505{
506 switch (m_current) {
507 case 0:
508 return "Invalid character: '\\0'";
509 case 10:
510 return "Invalid character: '\\n'";
511 case 11:
512 return "Invalid character: '\\v'";
513 case 13:
514 return "Invalid character: '\\r'";
515 case 35:
516 return "Invalid character: '#'";
517 case 64:
518 return "Invalid character: '@'";
519 case 96:
520 return "Invalid character: '`'";
521 default:
522 return String::format("Invalid character '\\u%04u'", static_cast<unsigned>(m_current)).impl();
523 }
524}
525
526template <typename T>
527ALWAYS_INLINE const T* Lexer<T>::currentSourcePtr() const
528{
529 ASSERT(m_code <= m_codeEnd);
530 return m_code;
531}
532
533template <typename T>
534void Lexer<T>::setCode(const SourceCode& source, ParserArena* arena)
535{
536 m_arena = &arena->identifierArena();
537
538 m_lineNumber = source.firstLine();
539 m_lastToken = -1;
540
541 const String& sourceString = source.provider()->source();
542
543 if (!sourceString.isNull())
544 setCodeStart(sourceString.impl());
545 else
546 m_codeStart = 0;
547
548 m_source = &source;
549 m_sourceOffset = source.startOffset();
550 m_codeStartPlusOffset = m_codeStart + source.startOffset();
551 m_code = m_codeStartPlusOffset;
552 m_codeEnd = m_codeStart + source.endOffset();
553 m_error = false;
554 m_atLineStart = true;
555 m_lineStart = m_code;
556 m_lexErrorMessage = String();
557
558 m_buffer8.reserveInitialCapacity(initialReadBufferCapacity);
559 m_buffer16.reserveInitialCapacity((m_codeEnd - m_code) / 2);
560
561 if (LIKELY(m_code < m_codeEnd))
562 m_current = *m_code;
563 else
564 m_current = 0;
565 ASSERT(currentOffset() == source.startOffset());
566}
567
568template <typename T>
569template <int shiftAmount> ALWAYS_INLINE void Lexer<T>::internalShift()
570{
571 m_code += shiftAmount;
572 ASSERT(currentOffset() >= currentLineStartOffset());
573 m_current = *m_code;
574}
575
576template <typename T>
577ALWAYS_INLINE void Lexer<T>::shift()
578{
579 // At one point timing showed that setting m_current to 0 unconditionally was faster than an if-else sequence.
580 m_current = 0;
581 ++m_code;
582 if (LIKELY(m_code < m_codeEnd))
583 m_current = *m_code;
584}
585
586template <typename T>
587ALWAYS_INLINE bool Lexer<T>::atEnd() const
588{
589 ASSERT(!m_current || m_code < m_codeEnd);
590 return UNLIKELY(UNLIKELY(!m_current) && m_code == m_codeEnd);
591}
592
593template <typename T>
594ALWAYS_INLINE T Lexer<T>::peek(int offset) const
595{
596 ASSERT(offset > 0 && offset < 5);
597 const T* code = m_code + offset;
598 return (code < m_codeEnd) ? *code : 0;
599}
600
601template <typename T>
602typename Lexer<T>::UnicodeHexValue Lexer<T>::parseFourDigitUnicodeHex()
603{
604 T char1 = peek(1);
605 T char2 = peek(2);
606 T char3 = peek(3);
607
608 if (UNLIKELY(!isASCIIHexDigit(m_current) || !isASCIIHexDigit(char1) || !isASCIIHexDigit(char2) || !isASCIIHexDigit(char3)))
609 return UnicodeHexValue((m_code + 4) >= m_codeEnd ? UnicodeHexValue::IncompleteHex : UnicodeHexValue::InvalidHex);
610
611 int result = convertUnicode(m_current, char1, char2, char3);
612 shift();
613 shift();
614 shift();
615 shift();
616 return UnicodeHexValue(result);
617}
618
619template <typename T>
620void Lexer<T>::shiftLineTerminator()
621{
622 ASSERT(isLineTerminator(m_current));
623
624 T prev = m_current;
625 shift();
626
627 // Allow both CRLF and LFCR.
628 if (prev + m_current == '\n' + '\r')
629 shift();
630
631 ++m_lineNumber;
632}
633
634template <typename T>
635ALWAYS_INLINE bool Lexer<T>::lastTokenWasRestrKeyword() const
636{
637 return m_lastToken == CONTINUE || m_lastToken == BREAK || m_lastToken == RETURN || m_lastToken == THROW;
638}
639
640static NEVER_INLINE bool isNonLatin1IdentStart(int c)
641{
642 return category(c) & (Letter_Uppercase | Letter_Lowercase | Letter_Titlecase | Letter_Modifier | Letter_Other);
643}
644
645static ALWAYS_INLINE bool isLatin1(LChar)
646{
647 return true;
648}
649
650static ALWAYS_INLINE bool isLatin1(UChar c)
651{
652 return c < 256;
653}
654
655static inline bool isIdentStart(LChar c)
656{
657 return typesOfLatin1Characters[c] == CharacterIdentifierStart;
658}
659
660static inline bool isIdentStart(UChar c)
661{
662 return isLatin1(c) ? isIdentStart(static_cast<LChar>(c)) : isNonLatin1IdentStart(c);
663}
664
665static NEVER_INLINE bool isNonLatin1IdentPart(int c)
666{
667 return (category(c) & (Letter_Uppercase | Letter_Lowercase | Letter_Titlecase | Letter_Modifier | Letter_Other
668 | Mark_NonSpacing | Mark_SpacingCombining | Number_DecimalDigit | Punctuation_Connector)) || c == 0x200C || c == 0x200D;
669}
670
671static ALWAYS_INLINE bool isIdentPart(LChar c)
672{
673 // Character types are divided into two groups depending on whether they can be part of an
674 // identifier or not. Those whose type value is less or equal than CharacterNumber can be
675 // part of an identifier. (See the CharacterType definition for more details.)
676 return typesOfLatin1Characters[c] <= CharacterNumber;
677}
678
679static ALWAYS_INLINE bool isIdentPart(UChar c)
680{
681 return isLatin1(c) ? isIdentPart(static_cast<LChar>(c)) : isNonLatin1IdentPart(c);
682}
683
684static inline LChar singleEscape(int c)
685{
686 if (c < 128) {
687 ASSERT(static_cast<size_t>(c) < ARRAY_SIZE(singleCharacterEscapeValuesForASCII));
688 return singleCharacterEscapeValuesForASCII[c];
689 }
690 return 0;
691}
692
693template <typename T>
694inline void Lexer<T>::record8(int c)
695{
696 ASSERT(c >= 0);
697 ASSERT(c <= 0xFF);
698 m_buffer8.append(static_cast<LChar>(c));
699}
700
701template <typename T>
702inline void assertCharIsIn8BitRange(T c)
703{
704 UNUSED_PARAM(c);
705 ASSERT(c >= 0);
706 ASSERT(c <= 0xFF);
707}
708
709template <>
710inline void assertCharIsIn8BitRange(UChar c)
711{
712 UNUSED_PARAM(c);
713 ASSERT(c <= 0xFF);
714}
715
716template <>
717inline void assertCharIsIn8BitRange(LChar)
718{
719}
720
721template <typename T>
722inline void Lexer<T>::append8(const T* p, size_t length)
723{
724 size_t currentSize = m_buffer8.size();
725 m_buffer8.grow(currentSize + length);
726 LChar* rawBuffer = m_buffer8.data() + currentSize;
727
728 for (size_t i = 0; i < length; i++) {
729 T c = p[i];
730 assertCharIsIn8BitRange(c);
731 rawBuffer[i] = c;
732 }
733}
734
735template <typename T>
736inline void Lexer<T>::append16(const LChar* p, size_t length)
737{
738 size_t currentSize = m_buffer16.size();
739 m_buffer16.grow(currentSize + length);
740 UChar* rawBuffer = m_buffer16.data() + currentSize;
741
742 for (size_t i = 0; i < length; i++)
743 rawBuffer[i] = p[i];
744}
745
746template <typename T>
747inline void Lexer<T>::record16(T c)
748{
749 m_buffer16.append(c);
750}
751
752template <typename T>
753inline void Lexer<T>::record16(int c)
754{
755 ASSERT(c >= 0);
756 ASSERT(c <= static_cast<int>(USHRT_MAX));
757 m_buffer16.append(static_cast<UChar>(c));
758}
759
760template <>
761template <bool shouldCreateIdentifier> ALWAYS_INLINE JSTokenType Lexer<LChar>::parseIdentifier(JSTokenData* tokenData, unsigned lexerFlags, bool strictMode)
762{
763 const ptrdiff_t remaining = m_codeEnd - m_code;
764 if ((remaining >= maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords)) {
765 JSTokenType keyword = parseKeyword<shouldCreateIdentifier>(tokenData);
766 if (keyword != IDENT) {
767 ASSERT((!shouldCreateIdentifier) || tokenData->ident);
768 return keyword == RESERVED_IF_STRICT && !strictMode ? IDENT : keyword;
769 }
770 }
771
772 const LChar* identifierStart = currentSourcePtr();
773 unsigned identifierLineStart = currentLineStartOffset();
774
775 while (isIdentPart(m_current))
776 shift();
777
778 if (UNLIKELY(m_current == '\\')) {
779 setOffsetFromSourcePtr(identifierStart, identifierLineStart);
780 return parseIdentifierSlowCase<shouldCreateIdentifier>(tokenData, lexerFlags, strictMode);
781 }
782
783 const Identifier* ident = 0;
784
785 if (shouldCreateIdentifier) {
786 int identifierLength = currentSourcePtr() - identifierStart;
787 ident = makeIdentifier(identifierStart, identifierLength);
788
789 tokenData->ident = ident;
790 } else
791 tokenData->ident = 0;
792
793 if (UNLIKELY((remaining < maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords))) {
794 ASSERT(shouldCreateIdentifier);
795 if (remaining < maxTokenLength) {
796 const HashEntry* entry = m_vm->keywords->getKeyword(*ident);
797 ASSERT((remaining < maxTokenLength) || !entry);
798 if (!entry)
799 return IDENT;
800 JSTokenType token = static_cast<JSTokenType>(entry->lexerValue());
801 return (token != RESERVED_IF_STRICT) || strictMode ? token : IDENT;
802 }
803 return IDENT;
804 }
805
806 return IDENT;
807}
808
809template <>
810template <bool shouldCreateIdentifier> ALWAYS_INLINE JSTokenType Lexer<UChar>::parseIdentifier(JSTokenData* tokenData, unsigned lexerFlags, bool strictMode)
811{
812 const ptrdiff_t remaining = m_codeEnd - m_code;
813 if ((remaining >= maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords)) {
814 JSTokenType keyword = parseKeyword<shouldCreateIdentifier>(tokenData);
815 if (keyword != IDENT) {
816 ASSERT((!shouldCreateIdentifier) || tokenData->ident);
817 return keyword == RESERVED_IF_STRICT && !strictMode ? IDENT : keyword;
818 }
819 }
820
821 const UChar* identifierStart = currentSourcePtr();
822 int identifierLineStart = currentLineStartOffset();
823
824 UChar orAllChars = 0;
825
826 while (isIdentPart(m_current)) {
827 orAllChars |= m_current;
828 shift();
829 }
830
831 if (UNLIKELY(m_current == '\\')) {
832 setOffsetFromSourcePtr(identifierStart, identifierLineStart);
833 return parseIdentifierSlowCase<shouldCreateIdentifier>(tokenData, lexerFlags, strictMode);
834 }
835
836 bool isAll8Bit = false;
837
838 if (!(orAllChars & ~0xff))
839 isAll8Bit = true;
840
841 const Identifier* ident = 0;
842
843 if (shouldCreateIdentifier) {
844 int identifierLength = currentSourcePtr() - identifierStart;
845 if (isAll8Bit)
846 ident = makeIdentifierLCharFromUChar(identifierStart, identifierLength);
847 else
848 ident = makeIdentifier(identifierStart, identifierLength);
849
850 tokenData->ident = ident;
851 } else
852 tokenData->ident = 0;
853
854 if (UNLIKELY((remaining < maxTokenLength) && !(lexerFlags & LexerFlagsIgnoreReservedWords))) {
855 ASSERT(shouldCreateIdentifier);
856 if (remaining < maxTokenLength) {
857 const HashEntry* entry = m_vm->keywords->getKeyword(*ident);
858 ASSERT((remaining < maxTokenLength) || !entry);
859 if (!entry)
860 return IDENT;
861 JSTokenType token = static_cast<JSTokenType>(entry->lexerValue());
862 return (token != RESERVED_IF_STRICT) || strictMode ? token : IDENT;
863 }
864 return IDENT;
865 }
866
867 return IDENT;
868}
869
870template <typename T>
871template <bool shouldCreateIdentifier> JSTokenType Lexer<T>::parseIdentifierSlowCase(JSTokenData* tokenData, unsigned lexerFlags, bool strictMode)
872{
873 const ptrdiff_t remaining = m_codeEnd - m_code;
874 const T* identifierStart = currentSourcePtr();
875 bool bufferRequired = false;
876
877 while (true) {
878 if (LIKELY(isIdentPart(m_current))) {
879 shift();
880 continue;
881 }
882 if (LIKELY(m_current != '\\'))
883 break;
884
885 // \uXXXX unicode characters.
886 bufferRequired = true;
887 if (identifierStart != currentSourcePtr())
888 m_buffer16.append(identifierStart, currentSourcePtr() - identifierStart);
889 shift();
890 if (UNLIKELY(m_current != 'u'))
891 return atEnd() ? UNTERMINATED_IDENTIFIER_ESCAPE_ERRORTOK : INVALID_IDENTIFIER_ESCAPE_ERRORTOK;
892 shift();
893 UnicodeHexValue character = parseFourDigitUnicodeHex();
894 if (UNLIKELY(!character.isValid()))
895 return character.valueType() == UnicodeHexValue::IncompleteHex ? UNTERMINATED_IDENTIFIER_UNICODE_ESCAPE_ERRORTOK : INVALID_IDENTIFIER_UNICODE_ESCAPE_ERRORTOK;
896 UChar ucharacter = static_cast<UChar>(character.value());
897 if (UNLIKELY(m_buffer16.size() ? !isIdentPart(ucharacter) : !isIdentStart(ucharacter)))
898 return INVALID_IDENTIFIER_UNICODE_ESCAPE_ERRORTOK;
899 if (shouldCreateIdentifier)
900 record16(ucharacter);
901 identifierStart = currentSourcePtr();
902 }
903
904 int identifierLength;
905 const Identifier* ident = 0;
906 if (shouldCreateIdentifier) {
907 if (!bufferRequired) {
908 identifierLength = currentSourcePtr() - identifierStart;
909 ident = makeIdentifier(identifierStart, identifierLength);
910 } else {
911 if (identifierStart != currentSourcePtr())
912 m_buffer16.append(identifierStart, currentSourcePtr() - identifierStart);
913 ident = makeIdentifier(m_buffer16.data(), m_buffer16.size());
914 }
915
916 tokenData->ident = ident;
917 } else
918 tokenData->ident = 0;
919
920 if (LIKELY(!bufferRequired && !(lexerFlags & LexerFlagsIgnoreReservedWords))) {
921 ASSERT(shouldCreateIdentifier);
922 // Keywords must not be recognized if there was an \uXXXX in the identifier.
923 if (remaining < maxTokenLength) {
924 const HashEntry* entry = m_vm->keywords->getKeyword(*ident);
925 ASSERT((remaining < maxTokenLength) || !entry);
926 if (!entry)
927 return IDENT;
928 JSTokenType token = static_cast<JSTokenType>(entry->lexerValue());
929 return (token != RESERVED_IF_STRICT) || strictMode ? token : IDENT;
930 }
931 return IDENT;
932 }
933
934 m_buffer16.resize(0);
935 return IDENT;
936}
937
938static ALWAYS_INLINE bool characterRequiresParseStringSlowCase(LChar character)
939{
940 return character < 0xE;
941}
942
943static ALWAYS_INLINE bool characterRequiresParseStringSlowCase(UChar character)
944{
945 return character < 0xE || character > 0xFF;
946}
947
948template <typename T>
949template <bool shouldBuildStrings> ALWAYS_INLINE typename Lexer<T>::StringParseResult Lexer<T>::parseString(JSTokenData* tokenData, bool strictMode)
950{
951 int startingOffset = currentOffset();
952 int startingLineStartOffset = currentLineStartOffset();
953 int startingLineNumber = lineNumber();
954 T stringQuoteCharacter = m_current;
955 shift();
956
957 const T* stringStart = currentSourcePtr();
958
959 while (m_current != stringQuoteCharacter) {
960 if (UNLIKELY(m_current == '\\')) {
961 if (stringStart != currentSourcePtr() && shouldBuildStrings)
962 append8(stringStart, currentSourcePtr() - stringStart);
963 shift();
964
965 LChar escape = singleEscape(m_current);
966
967 // Most common escape sequences first
968 if (escape) {
969 if (shouldBuildStrings)
970 record8(escape);
971 shift();
972 } else if (UNLIKELY(isLineTerminator(m_current)))
973 shiftLineTerminator();
974 else if (m_current == 'x') {
975 shift();
976 if (!isASCIIHexDigit(m_current) || !isASCIIHexDigit(peek(1))) {
977 m_lexErrorMessage = "\\x can only be followed by a hex character sequence";
978 return (atEnd() || (isASCIIHexDigit(m_current) && (m_code + 1 == m_codeEnd))) ? StringUnterminated : StringCannotBeParsed;
979 }
980 T prev = m_current;
981 shift();
982 if (shouldBuildStrings)
983 record8(convertHex(prev, m_current));
984 shift();
985 } else {
986 setOffset(startingOffset, startingLineStartOffset);
987 setLineNumber(startingLineNumber);
988 m_buffer8.resize(0);
989 return parseStringSlowCase<shouldBuildStrings>(tokenData, strictMode);
990 }
991 stringStart = currentSourcePtr();
992 continue;
993 }
994
995 if (UNLIKELY(characterRequiresParseStringSlowCase(m_current))) {
996 setOffset(startingOffset, startingLineStartOffset);
997 setLineNumber(startingLineNumber);
998 m_buffer8.resize(0);
999 return parseStringSlowCase<shouldBuildStrings>(tokenData, strictMode);
1000 }
1001
1002 shift();
1003 }
1004
1005 if (currentSourcePtr() != stringStart && shouldBuildStrings)
1006 append8(stringStart, currentSourcePtr() - stringStart);
1007 if (shouldBuildStrings) {
1008 tokenData->ident = makeIdentifier(m_buffer8.data(), m_buffer8.size());
1009 m_buffer8.resize(0);
1010 } else
1011 tokenData->ident = 0;
1012
1013 return StringParsedSuccessfully;
1014}
1015
1016template <typename T>
1017template <bool shouldBuildStrings> typename Lexer<T>::StringParseResult Lexer<T>::parseStringSlowCase(JSTokenData* tokenData, bool strictMode)
1018{
1019 T stringQuoteCharacter = m_current;
1020 shift();
1021
1022 const T* stringStart = currentSourcePtr();
1023
1024 while (m_current != stringQuoteCharacter) {
1025 if (UNLIKELY(m_current == '\\')) {
1026 if (stringStart != currentSourcePtr() && shouldBuildStrings)
1027 append16(stringStart, currentSourcePtr() - stringStart);
1028 shift();
1029
1030 LChar escape = singleEscape(m_current);
1031
1032 // Most common escape sequences first
1033 if (escape) {
1034 if (shouldBuildStrings)
1035 record16(escape);
1036 shift();
1037 } else if (UNLIKELY(isLineTerminator(m_current)))
1038 shiftLineTerminator();
1039 else if (m_current == 'x') {
1040 shift();
1041 if (!isASCIIHexDigit(m_current) || !isASCIIHexDigit(peek(1))) {
1042 m_lexErrorMessage = "\\x can only be followed by a hex character sequence";
1043 return StringCannotBeParsed;
1044 }
1045 T prev = m_current;
1046 shift();
1047 if (shouldBuildStrings)
1048 record16(convertHex(prev, m_current));
1049 shift();
1050 } else if (m_current == 'u') {
1051 shift();
1052 UnicodeHexValue character = parseFourDigitUnicodeHex();
1053 if (character.isValid()) {
1054 if (shouldBuildStrings)
1055 record16(character.value());
1056 } else if (m_current == stringQuoteCharacter) {
1057 if (shouldBuildStrings)
1058 record16('u');
1059 } else {
1060 m_lexErrorMessage = "\\u can only be followed by a Unicode character sequence";
1061 return character.valueType() == UnicodeHexValue::IncompleteHex ? StringUnterminated : StringCannotBeParsed;
1062 }
1063 } else if (strictMode && isASCIIDigit(m_current)) {
1064 // The only valid numeric escape in strict mode is '\0', and this must not be followed by a decimal digit.
1065 int character1 = m_current;
1066 shift();
1067 if (character1 != '0' || isASCIIDigit(m_current)) {
1068 m_lexErrorMessage = "The only valid numeric escape in strict mode is '\\0'";
1069 return StringCannotBeParsed;
1070 }
1071 if (shouldBuildStrings)
1072 record16(0);
1073 } else if (!strictMode && isASCIIOctalDigit(m_current)) {
1074 // Octal character sequences
1075 T character1 = m_current;
1076 shift();
1077 if (isASCIIOctalDigit(m_current)) {
1078 // Two octal characters
1079 T character2 = m_current;
1080 shift();
1081 if (character1 >= '0' && character1 <= '3' && isASCIIOctalDigit(m_current)) {
1082 if (shouldBuildStrings)
1083 record16((character1 - '0') * 64 + (character2 - '0') * 8 + m_current - '0');
1084 shift();
1085 } else {
1086 if (shouldBuildStrings)
1087 record16((character1 - '0') * 8 + character2 - '0');
1088 }
1089 } else {
1090 if (shouldBuildStrings)
1091 record16(character1 - '0');
1092 }
1093 } else if (!atEnd()) {
1094 if (shouldBuildStrings)
1095 record16(m_current);
1096 shift();
1097 } else {
1098 m_lexErrorMessage = "Unterminated string constant";
1099 return StringUnterminated;
1100 }
1101
1102 stringStart = currentSourcePtr();
1103 continue;
1104 }
1105 // Fast check for characters that require special handling.
1106 // Catches 0, \n, \r, 0x2028, and 0x2029 as efficiently
1107 // as possible, and lets through all common ASCII characters.
1108 if (UNLIKELY(((static_cast<unsigned>(m_current) - 0xE) & 0x2000))) {
1109 // New-line or end of input is not allowed
1110 if (atEnd() || isLineTerminator(m_current)) {
1111 m_lexErrorMessage = "Unexpected EOF";
1112 return atEnd() ? StringUnterminated : StringCannotBeParsed;
1113 }
1114 // Anything else is just a normal character
1115 }
1116 shift();
1117 }
1118
1119 if (currentSourcePtr() != stringStart && shouldBuildStrings)
1120 append16(stringStart, currentSourcePtr() - stringStart);
1121 if (shouldBuildStrings)
1122 tokenData->ident = makeIdentifier(m_buffer16.data(), m_buffer16.size());
1123 else
1124 tokenData->ident = 0;
1125
1126 m_buffer16.resize(0);
1127 return StringParsedSuccessfully;
1128}
1129
1130template <typename T>
1131ALWAYS_INLINE void Lexer<T>::parseHex(double& returnValue)
1132{
1133 // Optimization: most hexadecimal values fit into 4 bytes.
1134 uint32_t hexValue = 0;
1135 int maximumDigits = 7;
1136
1137 // Shift out the 'x' prefix.
1138 shift();
1139
1140 do {
1141 hexValue = (hexValue << 4) + toASCIIHexValue(m_current);
1142 shift();
1143 --maximumDigits;
1144 } while (isASCIIHexDigit(m_current) && maximumDigits >= 0);
1145
1146 if (maximumDigits >= 0) {
1147 returnValue = hexValue;
1148 return;
1149 }
1150
1151 // No more place in the hexValue buffer.
1152 // The values are shifted out and placed into the m_buffer8 vector.
1153 for (int i = 0; i < 8; ++i) {
1154 int digit = hexValue >> 28;
1155 if (digit < 10)
1156 record8(digit + '0');
1157 else
1158 record8(digit - 10 + 'a');
1159 hexValue <<= 4;
1160 }
1161
1162 while (isASCIIHexDigit(m_current)) {
1163 record8(m_current);
1164 shift();
1165 }
1166
1167 returnValue = parseIntOverflow(m_buffer8.data(), m_buffer8.size(), 16);
1168}
1169
1170template <typename T>
1171ALWAYS_INLINE bool Lexer<T>::parseOctal(double& returnValue)
1172{
1173 // Optimization: most octal values fit into 4 bytes.
1174 uint32_t octalValue = 0;
1175 int maximumDigits = 9;
1176 // Temporary buffer for the digits. Makes easier
1177 // to reconstruct the input characters when needed.
1178 LChar digits[10];
1179
1180 do {
1181 octalValue = octalValue * 8 + (m_current - '0');
1182 digits[maximumDigits] = m_current;
1183 shift();
1184 --maximumDigits;
1185 } while (isASCIIOctalDigit(m_current) && maximumDigits >= 0);
1186
1187 if (!isASCIIDigit(m_current) && maximumDigits >= 0) {
1188 returnValue = octalValue;
1189 return true;
1190 }
1191
1192 for (int i = 9; i > maximumDigits; --i)
1193 record8(digits[i]);
1194
1195 while (isASCIIOctalDigit(m_current)) {
1196 record8(m_current);
1197 shift();
1198 }
1199
1200 if (isASCIIDigit(m_current))
1201 return false;
1202
1203 returnValue = parseIntOverflow(m_buffer8.data(), m_buffer8.size(), 8);
1204 return true;
1205}
1206
1207template <typename T>
1208ALWAYS_INLINE bool Lexer<T>::parseDecimal(double& returnValue)
1209{
1210 // Optimization: most decimal values fit into 4 bytes.
1211 uint32_t decimalValue = 0;
1212
1213 // Since parseOctal may be executed before parseDecimal,
1214 // the m_buffer8 may hold ascii digits.
1215 if (!m_buffer8.size()) {
1216 int maximumDigits = 9;
1217 // Temporary buffer for the digits. Makes easier
1218 // to reconstruct the input characters when needed.
1219 LChar digits[10];
1220
1221 do {
1222 decimalValue = decimalValue * 10 + (m_current - '0');
1223 digits[maximumDigits] = m_current;
1224 shift();
1225 --maximumDigits;
1226 } while (isASCIIDigit(m_current) && maximumDigits >= 0);
1227
1228 if (maximumDigits >= 0 && m_current != '.' && (m_current | 0x20) != 'e') {
1229 returnValue = decimalValue;
1230 return true;
1231 }
1232
1233 for (int i = 9; i > maximumDigits; --i)
1234 record8(digits[i]);
1235 }
1236
1237 while (isASCIIDigit(m_current)) {
1238 record8(m_current);
1239 shift();
1240 }
1241
1242 return false;
1243}
1244
1245template <typename T>
1246ALWAYS_INLINE void Lexer<T>::parseNumberAfterDecimalPoint()
1247{
1248 record8('.');
1249 while (isASCIIDigit(m_current)) {
1250 record8(m_current);
1251 shift();
1252 }
1253}
1254
1255template <typename T>
1256ALWAYS_INLINE bool Lexer<T>::parseNumberAfterExponentIndicator()
1257{
1258 record8('e');
1259 shift();
1260 if (m_current == '+' || m_current == '-') {
1261 record8(m_current);
1262 shift();
1263 }
1264
1265 if (!isASCIIDigit(m_current))
1266 return false;
1267
1268 do {
1269 record8(m_current);
1270 shift();
1271 } while (isASCIIDigit(m_current));
1272 return true;
1273}
1274
1275template <typename T>
1276ALWAYS_INLINE bool Lexer<T>::parseMultilineComment()
1277{
1278 while (true) {
1279 while (UNLIKELY(m_current == '*')) {
1280 shift();
1281 if (m_current == '/') {
1282 shift();
1283 return true;
1284 }
1285 }
1286
1287 if (atEnd())
1288 return false;
1289
1290 if (isLineTerminator(m_current)) {
1291 shiftLineTerminator();
1292 m_terminator = true;
1293 } else
1294 shift();
1295 }
1296}
1297
1298template <typename T>
1299bool Lexer<T>::nextTokenIsColon()
1300{
1301 const T* code = m_code;
1302 while (code < m_codeEnd && (isWhiteSpace(*code) || isLineTerminator(*code)))
1303 code++;
1304
1305 return code < m_codeEnd && *code == ':';
1306}
1307
1308template <typename T>
1309JSTokenType Lexer<T>::lex(JSTokenData* tokenData, JSTokenLocation* tokenLocation, unsigned lexerFlags, bool strictMode)
1310{
1311 ASSERT(!m_error);
1312 ASSERT(m_buffer8.isEmpty());
1313 ASSERT(m_buffer16.isEmpty());
1314
1315 JSTokenType token = ERRORTOK;
1316 m_terminator = false;
1317
1318start:
1319 while (isWhiteSpace(m_current))
1320 shift();
1321
1322 if (atEnd())
1323 return EOFTOK;
1324
1325 tokenLocation->startOffset = currentOffset();
1326 ASSERT(currentOffset() >= currentLineStartOffset());
1327
1328 CharacterType type;
1329 if (LIKELY(isLatin1(m_current)))
1330 type = static_cast<CharacterType>(typesOfLatin1Characters[m_current]);
1331 else if (isNonLatin1IdentStart(m_current))
1332 type = CharacterIdentifierStart;
1333 else if (isLineTerminator(m_current))
1334 type = CharacterLineTerminator;
1335 else
1336 type = CharacterInvalid;
1337
1338 switch (type) {
1339 case CharacterGreater:
1340 shift();
1341 if (m_current == '>') {
1342 shift();
1343 if (m_current == '>') {
1344 shift();
1345 if (m_current == '=') {
1346 shift();
1347 token = URSHIFTEQUAL;
1348 break;
1349 }
1350 token = URSHIFT;
1351 break;
1352 }
1353 if (m_current == '=') {
1354 shift();
1355 token = RSHIFTEQUAL;
1356 break;
1357 }
1358 token = RSHIFT;
1359 break;
1360 }
1361 if (m_current == '=') {
1362 shift();
1363 token = GE;
1364 break;
1365 }
1366 token = GT;
1367 break;
1368 case CharacterEqual:
1369 shift();
1370 if (m_current == '=') {
1371 shift();
1372 if (m_current == '=') {
1373 shift();
1374 token = STREQ;
1375 break;
1376 }
1377 token = EQEQ;
1378 break;
1379 }
1380 token = EQUAL;
1381 break;
1382 case CharacterLess:
1383 shift();
1384 if (m_current == '!' && peek(1) == '-' && peek(2) == '-') {
1385 // <!-- marks the beginning of a line comment (for www usage)
1386 goto inSingleLineComment;
1387 }
1388 if (m_current == '<') {
1389 shift();
1390 if (m_current == '=') {
1391 shift();
1392 token = LSHIFTEQUAL;
1393 break;
1394 }
1395 token = LSHIFT;
1396 break;
1397 }
1398 if (m_current == '=') {
1399 shift();
1400 token = LE;
1401 break;
1402 }
1403 token = LT;
1404 break;
1405 case CharacterExclamationMark:
1406 shift();
1407 if (m_current == '=') {
1408 shift();
1409 if (m_current == '=') {
1410 shift();
1411 token = STRNEQ;
1412 break;
1413 }
1414 token = NE;
1415 break;
1416 }
1417 token = EXCLAMATION;
1418 break;
1419 case CharacterAdd:
1420 shift();
1421 if (m_current == '+') {
1422 shift();
1423 token = (!m_terminator) ? PLUSPLUS : AUTOPLUSPLUS;
1424 break;
1425 }
1426 if (m_current == '=') {
1427 shift();
1428 token = PLUSEQUAL;
1429 break;
1430 }
1431 token = PLUS;
1432 break;
1433 case CharacterSub:
1434 shift();
1435 if (m_current == '-') {
1436 shift();
1437 if (m_atLineStart && m_current == '>') {
1438 shift();
1439 goto inSingleLineComment;
1440 }
1441 token = (!m_terminator) ? MINUSMINUS : AUTOMINUSMINUS;
1442 break;
1443 }
1444 if (m_current == '=') {
1445 shift();
1446 token = MINUSEQUAL;
1447 break;
1448 }
1449 token = MINUS;
1450 break;
1451 case CharacterMultiply:
1452 shift();
1453 if (m_current == '=') {
1454 shift();
1455 token = MULTEQUAL;
1456 break;
1457 }
1458 token = TIMES;
1459 break;
1460 case CharacterSlash:
1461 shift();
1462 if (m_current == '/') {
1463 shift();
1464 goto inSingleLineComment;
1465 }
1466 if (m_current == '*') {
1467 shift();
1468 if (parseMultilineComment())
1469 goto start;
1470 m_lexErrorMessage = "Multiline comment was not closed properly";
1471 token = UNTERMINATED_MULTILINE_COMMENT_ERRORTOK;
1472 goto returnError;
1473 }
1474 if (m_current == '=') {
1475 shift();
1476 token = DIVEQUAL;
1477 break;
1478 }
1479 token = DIVIDE;
1480 break;
1481 case CharacterAnd:
1482 shift();
1483 if (m_current == '&') {
1484 shift();
1485 token = AND;
1486 break;
1487 }
1488 if (m_current == '=') {
1489 shift();
1490 token = ANDEQUAL;
1491 break;
1492 }
1493 token = BITAND;
1494 break;
1495 case CharacterXor:
1496 shift();
1497 if (m_current == '=') {
1498 shift();
1499 token = XOREQUAL;
1500 break;
1501 }
1502 token = BITXOR;
1503 break;
1504 case CharacterModulo:
1505 shift();
1506 if (m_current == '=') {
1507 shift();
1508 token = MODEQUAL;
1509 break;
1510 }
1511 token = MOD;
1512 break;
1513 case CharacterOr:
1514 shift();
1515 if (m_current == '=') {
1516 shift();
1517 token = OREQUAL;
1518 break;
1519 }
1520 if (m_current == '|') {
1521 shift();
1522 token = OR;
1523 break;
1524 }
1525 token = BITOR;
1526 break;
1527 case CharacterOpenParen:
1528 token = OPENPAREN;
1529 shift();
1530 break;
1531 case CharacterCloseParen:
1532 token = CLOSEPAREN;
1533 shift();
1534 break;
1535 case CharacterOpenBracket:
1536 token = OPENBRACKET;
1537 shift();
1538 break;
1539 case CharacterCloseBracket:
1540 token = CLOSEBRACKET;
1541 shift();
1542 break;
1543 case CharacterComma:
1544 token = COMMA;
1545 shift();
1546 break;
1547 case CharacterColon:
1548 token = COLON;
1549 shift();
1550 break;
1551 case CharacterQuestion:
1552 token = QUESTION;
1553 shift();
1554 break;
1555 case CharacterTilde:
1556 token = TILDE;
1557 shift();
1558 break;
1559 case CharacterSemicolon:
1560 shift();
1561 token = SEMICOLON;
1562 break;
1563 case CharacterOpenBrace:
1564 tokenData->line = lineNumber();
1565 tokenData->offset = currentOffset();
1566 tokenData->lineStartOffset = currentLineStartOffset();
1567 ASSERT(tokenData->offset >= tokenData->lineStartOffset);
1568 shift();
1569 token = OPENBRACE;
1570 break;
1571 case CharacterCloseBrace:
1572 tokenData->line = lineNumber();
1573 tokenData->offset = currentOffset();
1574 tokenData->lineStartOffset = currentLineStartOffset();
1575 ASSERT(tokenData->offset >= tokenData->lineStartOffset);
1576 shift();
1577 token = CLOSEBRACE;
1578 break;
1579 case CharacterDot:
1580 shift();
1581 if (!isASCIIDigit(m_current)) {
1582 token = DOT;
1583 break;
1584 }
1585 goto inNumberAfterDecimalPoint;
1586 case CharacterZero:
1587 shift();
1588 if ((m_current | 0x20) == 'x' && isASCIIHexDigit(peek(1))) {
1589 parseHex(tokenData->doubleValue);
1590 token = NUMBER;
1591 } else {
1592 record8('0');
1593 if (isASCIIOctalDigit(m_current)) {
1594 if (parseOctal(tokenData->doubleValue)) {
1595 if (strictMode) {
1596 m_lexErrorMessage = "Octal escapes are forbidden in strict mode";
1597 token = INVALID_OCTAL_NUMBER_ERRORTOK;
1598 goto returnError;
1599 }
1600 token = NUMBER;
1601 }
1602 }
1603 }
1604 // Fall through into CharacterNumber
1605 case CharacterNumber:
1606 if (LIKELY(token != NUMBER)) {
1607 if (!parseDecimal(tokenData->doubleValue)) {
1608 if (m_current == '.') {
1609 shift();
1610inNumberAfterDecimalPoint:
1611 parseNumberAfterDecimalPoint();
1612 }
1613 if ((m_current | 0x20) == 'e') {
1614 if (!parseNumberAfterExponentIndicator()) {
1615 m_lexErrorMessage = "Non-number found after exponent indicator";
1616 token = atEnd() ? UNTERMINATED_NUMERIC_LITERAL_ERRORTOK : INVALID_NUMERIC_LITERAL_ERRORTOK;
1617 goto returnError;
1618 }
1619 }
1620 size_t parsedLength;
1621 tokenData->doubleValue = parseDouble(m_buffer8.data(), m_buffer8.size(), parsedLength);
1622 }
1623 token = NUMBER;
1624 }
1625
1626 // No identifiers allowed directly after numeric literal, e.g. "3in" is bad.
1627 if (UNLIKELY(isIdentStart(m_current))) {
1628 m_lexErrorMessage = "At least one digit must occur after a decimal point";
1629 token = atEnd() ? UNTERMINATED_NUMERIC_LITERAL_ERRORTOK : INVALID_NUMERIC_LITERAL_ERRORTOK;
1630 goto returnError;
1631 }
1632 m_buffer8.resize(0);
1633 break;
1634 case CharacterQuote:
1635 if (lexerFlags & LexerFlagsDontBuildStrings) {
1636 StringParseResult result = parseString<false>(tokenData, strictMode);
1637 if (UNLIKELY(result != StringParsedSuccessfully)) {
1638 token = result == StringUnterminated ? UNTERMINATED_STRING_LITERAL_ERRORTOK : INVALID_STRING_LITERAL_ERRORTOK;
1639 goto returnError;
1640 }
1641 } else {
1642 StringParseResult result = parseString<true>(tokenData, strictMode);
1643 if (UNLIKELY(result != StringParsedSuccessfully)) {
1644 token = result == StringUnterminated ? UNTERMINATED_STRING_LITERAL_ERRORTOK : INVALID_STRING_LITERAL_ERRORTOK;
1645 goto returnError;
1646 }
1647 }
1648 shift();
1649 token = STRING;
1650 break;
1651 case CharacterIdentifierStart:
1652 ASSERT(isIdentStart(m_current));
1653 // Fall through into CharacterBackSlash.
1654 case CharacterBackSlash:
1655 if (lexerFlags & LexexFlagsDontBuildKeywords)
1656 token = parseIdentifier<false>(tokenData, lexerFlags, strictMode);
1657 else
1658 token = parseIdentifier<true>(tokenData, lexerFlags, strictMode);
1659 break;
1660 case CharacterLineTerminator:
1661 ASSERT(isLineTerminator(m_current));
1662 shiftLineTerminator();
1663 m_atLineStart = true;
1664 m_terminator = true;
1665 m_lineStart = m_code;
1666 goto start;
1667 case CharacterInvalid:
1668 m_lexErrorMessage = invalidCharacterMessage();
1669 token = ERRORTOK;
1670 goto returnError;
1671 default:
1672 RELEASE_ASSERT_NOT_REACHED();
1673 m_lexErrorMessage = "Internal Error";
1674 token = ERRORTOK;
1675 goto returnError;
1676 }
1677
1678 m_atLineStart = false;
1679 goto returnToken;
1680
1681inSingleLineComment:
1682 while (!isLineTerminator(m_current)) {
1683 if (atEnd())
1684 return EOFTOK;
1685 shift();
1686 }
1687 shiftLineTerminator();
1688 m_atLineStart = true;
1689 m_terminator = true;
1690 m_lineStart = m_code;
1691 if (!lastTokenWasRestrKeyword())
1692 goto start;
1693
1694 token = SEMICOLON;
1695 // Fall through into returnToken.
1696
1697returnToken:
1698 tokenLocation->line = m_lineNumber;
1699 tokenLocation->endOffset = currentOffset();
1700 tokenLocation->lineStartOffset = currentLineStartOffset();
1701 ASSERT(tokenLocation->endOffset >= tokenLocation->lineStartOffset);
1702 m_lastToken = token;
1703 return token;
1704
1705returnError:
1706 m_error = true;
1707 tokenLocation->line = m_lineNumber;
1708 tokenLocation->endOffset = currentOffset();
1709 tokenLocation->lineStartOffset = currentLineStartOffset();
1710 ASSERT(tokenLocation->endOffset >= tokenLocation->lineStartOffset);
1711 RELEASE_ASSERT(token & ErrorTokenFlag);
1712 return token;
1713}
1714
1715template <typename T>
1716static inline void orCharacter(UChar&, UChar);
1717
1718template <>
1719inline void orCharacter<LChar>(UChar&, UChar) { }
1720
1721template <>
1722inline void orCharacter<UChar>(UChar& orAccumulator, UChar character)
1723{
1724 orAccumulator |= character;
1725}
1726
1727template <typename T>
1728bool Lexer<T>::scanRegExp(const Identifier*& pattern, const Identifier*& flags, UChar patternPrefix)
1729{
1730 ASSERT(m_buffer16.isEmpty());
1731
1732 bool lastWasEscape = false;
1733 bool inBrackets = false;
1734 UChar charactersOredTogether = 0;
1735
1736 if (patternPrefix) {
1737 ASSERT(!isLineTerminator(patternPrefix));
1738 ASSERT(patternPrefix != '/');
1739 ASSERT(patternPrefix != '[');
1740 record16(patternPrefix);
1741 }
1742
1743 while (true) {
1744 if (isLineTerminator(m_current) || atEnd()) {
1745 m_buffer16.resize(0);
1746 return false;
1747 }
1748
1749 T prev = m_current;
1750
1751 shift();
1752
1753 if (prev == '/' && !lastWasEscape && !inBrackets)
1754 break;
1755
1756 record16(prev);
1757 orCharacter<T>(charactersOredTogether, prev);
1758
1759 if (lastWasEscape) {
1760 lastWasEscape = false;
1761 continue;
1762 }
1763
1764 switch (prev) {
1765 case '[':
1766 inBrackets = true;
1767 break;
1768 case ']':
1769 inBrackets = false;
1770 break;
1771 case '\\':
1772 lastWasEscape = true;
1773 break;
1774 }
1775 }
1776
1777 pattern = makeRightSizedIdentifier(m_buffer16.data(), m_buffer16.size(), charactersOredTogether);
1778
1779 m_buffer16.resize(0);
1780 charactersOredTogether = 0;
1781
1782 while (isIdentPart(m_current)) {
1783 record16(m_current);
1784 orCharacter<T>(charactersOredTogether, m_current);
1785 shift();
1786 }
1787
1788 flags = makeRightSizedIdentifier(m_buffer16.data(), m_buffer16.size(), charactersOredTogether);
1789 m_buffer16.resize(0);
1790
1791 return true;
1792}
1793
1794template <typename T>
1795bool Lexer<T>::skipRegExp()
1796{
1797 bool lastWasEscape = false;
1798 bool inBrackets = false;
1799
1800 while (true) {
1801 if (isLineTerminator(m_current) || atEnd())
1802 return false;
1803
1804 T prev = m_current;
1805
1806 shift();
1807
1808 if (prev == '/' && !lastWasEscape && !inBrackets)
1809 break;
1810
1811 if (lastWasEscape) {
1812 lastWasEscape = false;
1813 continue;
1814 }
1815
1816 switch (prev) {
1817 case '[':
1818 inBrackets = true;
1819 break;
1820 case ']':
1821 inBrackets = false;
1822 break;
1823 case '\\':
1824 lastWasEscape = true;
1825 break;
1826 }
1827 }
1828
1829 while (isIdentPart(m_current))
1830 shift();
1831
1832 return true;
1833}
1834
1835template <typename T>
1836void Lexer<T>::clear()
1837{
1838 m_arena = 0;
1839
1840 Vector<LChar> newBuffer8;
1841 m_buffer8.swap(newBuffer8);
1842
1843 Vector<UChar> newBuffer16;
1844 m_buffer16.swap(newBuffer16);
1845
1846 m_isReparsing = false;
1847}
1848
1849template <typename T>
1850SourceCode Lexer<T>::sourceCode(int openBrace, int closeBrace, int firstLine, unsigned startColumn)
1851{
1852 ASSERT(m_source->provider()->source()[openBrace] == '{');
1853 ASSERT(m_source->provider()->source()[closeBrace] == '}');
1854 return SourceCode(m_source->provider(), openBrace, closeBrace + 1, firstLine, startColumn);
1855}
1856
1857// Instantiate the two flavors of Lexer we need instead of putting most of this file in Lexer.h
1858template class Lexer<LChar>;
1859template class Lexer<UChar>;
1860
1861} // namespace JSC
mirror of JavaScriptCore