+#if ENABLE(ES6_TEMPLATE_LITERAL_SYNTAX)
+// While the lexer accepts <LF><CR> (not <CR><LF>) sequence
+// as one line terminator and increments one line number,
+// TemplateLiteral considers it as two line terminators <LF> and <CR>.
+//
+// TemplateLiteral normalizes line terminators as follows.
+//
+// <LF> => <LF>
+// <CR> => <LF>
+// <CR><LF> => <LF>
+// <\u2028> => <\u2028>
+// <\u2029> => <\u2029>
+//
+// So, <LF><CR> should be normalized to <LF><LF>.
+// However, the lexer should increment the line number only once for <LF><CR>.
+//
+// To achieve this, LineNumberAdder holds the current status of line terminator sequence.
+// When TemplateLiteral lexer encounters a line terminator, it notifies to LineNumberAdder.
+// LineNumberAdder maintains the status and increments the line number when it's necessary.
+// For example, LineNumberAdder increments the line number only once for <LF><CR> and <CR><LF>.
+template<typename CharacterType>
+class LineNumberAdder {
+public:
+ LineNumberAdder(int& lineNumber)
+ : m_lineNumber(lineNumber)
+ {
+ }
+
+ void clear()
+ {
+ m_previous = 0;
+ }
+
+ void add(CharacterType character)
+ {
+ ASSERT(Lexer<CharacterType>::isLineTerminator(character));
+ if ((character + m_previous) == ('\n' + '\r'))
+ m_previous = 0;
+ else {
+ ++m_lineNumber;
+ m_previous = character;
+ }
+ }
+
+private:
+ int& m_lineNumber;
+ CharacterType m_previous { 0 };
+};
+
+template <typename T>
+template <bool shouldBuildStrings> typename Lexer<T>::StringParseResult Lexer<T>::parseTemplateLiteral(JSTokenData* tokenData, RawStringsBuildMode rawStringsBuildMode)
+{
+ const T* stringStart = currentSourcePtr();
+ const T* rawStringStart = currentSourcePtr();
+
+ LineNumberAdder<T> lineNumberAdder(m_lineNumber);
+
+ while (m_current != '`') {
+ if (UNLIKELY(m_current == '\\')) {
+ lineNumberAdder.clear();
+ if (stringStart != currentSourcePtr() && shouldBuildStrings)
+ append16(stringStart, currentSourcePtr() - stringStart);
+ shift();
+
+ LChar escape = singleEscape(m_current);
+
+ // Most common escape sequences first.
+ if (escape) {
+ if (shouldBuildStrings)
+ record16(escape);
+ shift();
+ } else if (UNLIKELY(isLineTerminator(m_current))) {
+ if (m_current == '\r') {
+ lineNumberAdder.add(m_current);
+ shift();
+ if (m_current == '\n') {
+ lineNumberAdder.add(m_current);
+ shift();
+ }
+ } else {
+ lineNumberAdder.add(m_current);
+ shift();
+ }
+ } else {
+ bool strictMode = true;
+ StringParseResult result = parseComplexEscape<shouldBuildStrings>(EscapeParseMode::Template, strictMode, '`');
+ if (result != StringParsedSuccessfully)
+ return result;
+ }
+
+ stringStart = currentSourcePtr();
+ continue;
+ }
+
+ if (m_current == '$' && peek(1) == '{')
+ break;
+
+ // Fast check for characters that require special handling.
+ // Catches 0, \n, \r, 0x2028, and 0x2029 as efficiently
+ // as possible, and lets through all common ASCII characters.
+ if (UNLIKELY(((static_cast<unsigned>(m_current) - 0xE) & 0x2000))) {
+ // End of input is not allowed.
+ // Unlike String, line terminator is allowed.
+ if (atEnd()) {
+ m_lexErrorMessage = ASCIILiteral("Unexpected EOF");
+ return atEnd() ? StringUnterminated : StringCannotBeParsed;
+ }
+
+ if (isLineTerminator(m_current)) {
+ if (m_current == '\r') {
+ // Normalize <CR>, <CR><LF> to <LF>.
+ if (shouldBuildStrings) {
+ if (stringStart != currentSourcePtr())
+ append16(stringStart, currentSourcePtr() - stringStart);
+ if (rawStringStart != currentSourcePtr() && rawStringsBuildMode == RawStringsBuildMode::BuildRawStrings)
+ m_bufferForRawTemplateString16.append(rawStringStart, currentSourcePtr() - rawStringStart);
+
+ record16('\n');
+ if (rawStringsBuildMode == RawStringsBuildMode::BuildRawStrings)
+ m_bufferForRawTemplateString16.append('\n');
+ }
+ lineNumberAdder.add(m_current);
+ shift();
+ if (m_current == '\n') {
+ lineNumberAdder.add(m_current);
+ shift();
+ }
+ stringStart = currentSourcePtr();
+ rawStringStart = currentSourcePtr();
+ } else {
+ lineNumberAdder.add(m_current);
+ shift();
+ }
+ continue;
+ }
+ // Anything else is just a normal character
+ }
+
+ lineNumberAdder.clear();
+ shift();
+ }
+
+ bool isTail = m_current == '`';
+
+ if (shouldBuildStrings) {
+ if (currentSourcePtr() != stringStart)
+ append16(stringStart, currentSourcePtr() - stringStart);
+ if (rawStringStart != currentSourcePtr() && rawStringsBuildMode == RawStringsBuildMode::BuildRawStrings)
+ m_bufferForRawTemplateString16.append(rawStringStart, currentSourcePtr() - rawStringStart);
+ }
+
+ if (shouldBuildStrings) {
+ tokenData->cooked = makeIdentifier(m_buffer16.data(), m_buffer16.size());
+ // Line terminator normalization (e.g. <CR> => <LF>) should be applied to both the raw and cooked representations.
+ if (rawStringsBuildMode == RawStringsBuildMode::BuildRawStrings)
+ tokenData->raw = makeIdentifier(m_bufferForRawTemplateString16.data(), m_bufferForRawTemplateString16.size());
+ else
+ tokenData->raw = makeEmptyIdentifier();
+ } else {
+ tokenData->cooked = makeEmptyIdentifier();
+ tokenData->raw = makeEmptyIdentifier();
+ }
+ tokenData->isTail = isTail;
+
+ m_buffer16.shrink(0);
+ m_bufferForRawTemplateString16.shrink(0);
+
+ if (isTail) {
+ // Skip `
+ shift();
+ } else {
+ // Skip $ and {
+ shift();
+ shift();
+ }
+
+ return StringParsedSuccessfully;
+}
+#endif
+
+template <typename T>
+ALWAYS_INLINE void Lexer<T>::parseHex(double& returnValue)
+{
+ // Optimization: most hexadecimal values fit into 4 bytes.
+ uint32_t hexValue = 0;
+ int maximumDigits = 7;
+
+ do {
+ hexValue = (hexValue << 4) + toASCIIHexValue(m_current);
+ shift();
+ --maximumDigits;
+ } while (isASCIIHexDigit(m_current) && maximumDigits >= 0);
+
+ if (maximumDigits >= 0) {
+ returnValue = hexValue;
+ return;
+ }
+
+ // No more place in the hexValue buffer.
+ // The values are shifted out and placed into the m_buffer8 vector.
+ for (int i = 0; i < 8; ++i) {
+ int digit = hexValue >> 28;
+ if (digit < 10)
+ record8(digit + '0');
+ else
+ record8(digit - 10 + 'a');
+ hexValue <<= 4;
+ }
+
+ while (isASCIIHexDigit(m_current)) {
+ record8(m_current);
+ shift();
+ }
+
+ returnValue = parseIntOverflow(m_buffer8.data(), m_buffer8.size(), 16);
+}
+
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::parseBinary(double& returnValue)
+{
+ // Optimization: most binary values fit into 4 bytes.
+ uint32_t binaryValue = 0;
+ const unsigned maximumDigits = 32;
+ int digit = maximumDigits - 1;
+ // Temporary buffer for the digits. Makes easier
+ // to reconstruct the input characters when needed.
+ LChar digits[maximumDigits];
+
+ do {
+ binaryValue = (binaryValue << 1) + (m_current - '0');
+ digits[digit] = m_current;
+ shift();
+ --digit;
+ } while (isASCIIBinaryDigit(m_current) && digit >= 0);
+
+ if (!isASCIIDigit(m_current) && digit >= 0) {
+ returnValue = binaryValue;
+ return true;
+ }
+
+ for (int i = maximumDigits - 1; i > digit; --i)
+ record8(digits[i]);
+
+ while (isASCIIBinaryDigit(m_current)) {
+ record8(m_current);
+ shift();
+ }
+
+ if (isASCIIDigit(m_current))
+ return false;
+
+ returnValue = parseIntOverflow(m_buffer8.data(), m_buffer8.size(), 2);
+ return true;
+}
+
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::parseOctal(double& returnValue)
+{
+ // Optimization: most octal values fit into 4 bytes.
+ uint32_t octalValue = 0;
+ const unsigned maximumDigits = 10;
+ int digit = maximumDigits - 1;
+ // Temporary buffer for the digits. Makes easier
+ // to reconstruct the input characters when needed.
+ LChar digits[maximumDigits];
+
+ do {
+ octalValue = octalValue * 8 + (m_current - '0');
+ digits[digit] = m_current;
+ shift();
+ --digit;
+ } while (isASCIIOctalDigit(m_current) && digit >= 0);
+
+ if (!isASCIIDigit(m_current) && digit >= 0) {
+ returnValue = octalValue;
+ return true;
+ }
+
+ for (int i = maximumDigits - 1; i > digit; --i)
+ record8(digits[i]);
+
+ while (isASCIIOctalDigit(m_current)) {
+ record8(m_current);
+ shift();
+ }
+
+ if (isASCIIDigit(m_current))
+ return false;
+
+ returnValue = parseIntOverflow(m_buffer8.data(), m_buffer8.size(), 8);
+ return true;
+}
+
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::parseDecimal(double& returnValue)
+{
+ // Optimization: most decimal values fit into 4 bytes.
+ uint32_t decimalValue = 0;
+
+ // Since parseOctal may be executed before parseDecimal,
+ // the m_buffer8 may hold ascii digits.
+ if (!m_buffer8.size()) {
+ const unsigned maximumDigits = 10;
+ int digit = maximumDigits - 1;
+ // Temporary buffer for the digits. Makes easier
+ // to reconstruct the input characters when needed.
+ LChar digits[maximumDigits];
+
+ do {
+ decimalValue = decimalValue * 10 + (m_current - '0');
+ digits[digit] = m_current;
+ shift();
+ --digit;
+ } while (isASCIIDigit(m_current) && digit >= 0);
+
+ if (digit >= 0 && m_current != '.' && (m_current | 0x20) != 'e') {
+ returnValue = decimalValue;
+ return true;
+ }
+
+ for (int i = maximumDigits - 1; i > digit; --i)
+ record8(digits[i]);
+ }
+
+ while (isASCIIDigit(m_current)) {
+ record8(m_current);
+ shift();
+ }
+
+ return false;
+}
+
+template <typename T>
+ALWAYS_INLINE void Lexer<T>::parseNumberAfterDecimalPoint()
+{
+ record8('.');
+ while (isASCIIDigit(m_current)) {
+ record8(m_current);
+ shift();
+ }
+}
+
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::parseNumberAfterExponentIndicator()
+{
+ record8('e');
+ shift();
+ if (m_current == '+' || m_current == '-') {
+ record8(m_current);
+ shift();
+ }
+
+ if (!isASCIIDigit(m_current))
+ return false;
+
+ do {
+ record8(m_current);
+ shift();
+ } while (isASCIIDigit(m_current));
+ return true;
+}
+
+template <typename T>
+ALWAYS_INLINE bool Lexer<T>::parseMultilineComment()
+{
+ while (true) {
+ while (UNLIKELY(m_current == '*')) {
+ shift();
+ if (m_current == '/') {
+ shift();
+ return true;
+ }
+ }
+
+ if (atEnd())
+ return false;
+
+ if (isLineTerminator(m_current)) {
+ shiftLineTerminator();
+ m_terminator = true;
+ } else
+ shift();
+ }
+}
+
+template <typename T>
+bool Lexer<T>::nextTokenIsColon()
+{
+ const T* code = m_code;
+ while (code < m_codeEnd && (isWhiteSpace(*code) || isLineTerminator(*code)))
+ code++;
+
+ return code < m_codeEnd && *code == ':';
+}
+
+#if ENABLE(ES6_ARROWFUNCTION_SYNTAX)
+template <typename T>
+void Lexer<T>::setTokenPosition(JSToken* tokenRecord)
+{
+ JSTokenData* tokenData = &tokenRecord->m_data;
+ tokenData->line = lineNumber();
+ tokenData->offset = currentOffset();
+ tokenData->lineStartOffset = currentLineStartOffset();
+ ASSERT(tokenData->offset >= tokenData->lineStartOffset);
+}
+#endif
+
+template <typename T>
+JSTokenType Lexer<T>::lex(JSToken* tokenRecord, unsigned lexerFlags, bool strictMode)
+{
+ JSTokenData* tokenData = &tokenRecord->m_data;
+ JSTokenLocation* tokenLocation = &tokenRecord->m_location;
+ m_lastTockenLocation = JSTokenLocation(tokenRecord->m_location);
+
+ ASSERT(!m_error);
+ ASSERT(m_buffer8.isEmpty());
+ ASSERT(m_buffer16.isEmpty());
+
+ JSTokenType token = ERRORTOK;
+ m_terminator = false;
+
+start:
+ while (isWhiteSpace(m_current))
+ shift();
+
+ if (atEnd())
+ return EOFTOK;
+
+ tokenLocation->startOffset = currentOffset();
+ ASSERT(currentOffset() >= currentLineStartOffset());
+ tokenRecord->m_startPosition = currentPosition();
+
+ CharacterType type;
+ if (LIKELY(isLatin1(m_current)))
+ type = static_cast<CharacterType>(typesOfLatin1Characters[m_current]);
+ else if (isNonLatin1IdentStart(m_current))
+ type = CharacterIdentifierStart;
+ else if (isLineTerminator(m_current))
+ type = CharacterLineTerminator;
+ else
+ type = CharacterInvalid;
+
+ switch (type) {
+ case CharacterGreater:
+ shift();
+ if (m_current == '>') {
+ shift();
+ if (m_current == '>') {
+ shift();
+ if (m_current == '=') {
+ shift();
+ token = URSHIFTEQUAL;
+ break;
+ }
+ token = URSHIFT;
+ break;
+ }
+ if (m_current == '=') {
+ shift();
+ token = RSHIFTEQUAL;
+ break;
+ }
+ token = RSHIFT;
+ break;
+ }
+ if (m_current == '=') {
+ shift();
+ token = GE;
+ break;
+ }
+ token = GT;
+ break;
+ case CharacterEqual: {
+#if ENABLE(ES6_ARROWFUNCTION_SYNTAX)
+ if (peek(1) == '>') {
+ token = ARROWFUNCTION;
+ tokenData->line = lineNumber();
+ tokenData->offset = currentOffset();
+ tokenData->lineStartOffset = currentLineStartOffset();
+ ASSERT(tokenData->offset >= tokenData->lineStartOffset);
+ shift();
+ shift();
+ break;
+ }
+#endif
+ shift();
+ if (m_current == '=') {
+ shift();
+ if (m_current == '=') {
+ shift();
+ token = STREQ;
+ break;
+ }
+ token = EQEQ;
+ break;
+ }
+ token = EQUAL;
+ break;
+ }
+ case CharacterLess:
+ shift();
+ if (m_current == '!' && peek(1) == '-' && peek(2) == '-') {
+ // <!-- marks the beginning of a line comment (for www usage)
+ goto inSingleLineComment;
+ }
+ if (m_current == '<') {
+ shift();
+ if (m_current == '=') {
+ shift();
+ token = LSHIFTEQUAL;
+ break;
+ }
+ token = LSHIFT;
+ break;
+ }
+ if (m_current == '=') {
+ shift();
+ token = LE;
+ break;
+ }
+ token = LT;
+ break;
+ case CharacterExclamationMark:
+ shift();
+ if (m_current == '=') {
+ shift();
+ if (m_current == '=') {
+ shift();
+ token = STRNEQ;
+ break;
+ }
+ token = NE;
+ break;
+ }
+ token = EXCLAMATION;
+ break;
+ case CharacterAdd:
+ shift();
+ if (m_current == '+') {
+ shift();
+ token = (!m_terminator) ? PLUSPLUS : AUTOPLUSPLUS;
+ break;
+ }
+ if (m_current == '=') {
+ shift();
+ token = PLUSEQUAL;
+ break;
+ }
+ token = PLUS;
+ break;
+ case CharacterSub:
+ shift();
+ if (m_current == '-') {
+ shift();
+ if (m_atLineStart && m_current == '>') {
+ shift();
+ goto inSingleLineComment;
+ }
+ token = (!m_terminator) ? MINUSMINUS : AUTOMINUSMINUS;
+ break;
+ }
+ if (m_current == '=') {
+ shift();
+ token = MINUSEQUAL;
+ break;
+ }
+ token = MINUS;
+ break;
+ case CharacterMultiply:
+ shift();
+ if (m_current == '=') {
+ shift();
+ token = MULTEQUAL;
+ break;
+ }
+ token = TIMES;
+ break;
+ case CharacterSlash:
+ shift();
+ if (m_current == '/') {
+ shift();
+ goto inSingleLineComment;
+ }
+ if (m_current == '*') {
+ shift();
+ if (parseMultilineComment())
+ goto start;
+ m_lexErrorMessage = ASCIILiteral("Multiline comment was not closed properly");
+ token = UNTERMINATED_MULTILINE_COMMENT_ERRORTOK;
+ goto returnError;
+ }
+ if (m_current == '=') {
+ shift();
+ token = DIVEQUAL;
+ break;
+ }
+ token = DIVIDE;
+ break;
+ case CharacterAnd:
+ shift();
+ if (m_current == '&') {
+ shift();
+ token = AND;
+ break;
+ }
+ if (m_current == '=') {
+ shift();
+ token = ANDEQUAL;
+ break;
+ }
+ token = BITAND;
+ break;
+ case CharacterXor:
+ shift();
+ if (m_current == '=') {
+ shift();
+ token = XOREQUAL;
+ break;
+ }
+ token = BITXOR;
+ break;
+ case CharacterModulo:
+ shift();
+ if (m_current == '=') {
+ shift();
+ token = MODEQUAL;
+ break;
+ }
+ token = MOD;
+ break;
+ case CharacterOr:
+ shift();
+ if (m_current == '=') {
+ shift();
+ token = OREQUAL;
+ break;
+ }
+ if (m_current == '|') {
+ shift();
+ token = OR;
+ break;
+ }
+ token = BITOR;
+ break;
+ case CharacterOpenParen:
+ token = OPENPAREN;
+ shift();
+ break;
+ case CharacterCloseParen:
+ token = CLOSEPAREN;
+ shift();
+ break;
+ case CharacterOpenBracket:
+ token = OPENBRACKET;
+ shift();
+ break;
+ case CharacterCloseBracket:
+ token = CLOSEBRACKET;
+ shift();
+ break;
+ case CharacterComma:
+ token = COMMA;
+ shift();
+ break;
+ case CharacterColon:
+ token = COLON;
+ shift();
+ break;
+ case CharacterQuestion:
+ token = QUESTION;
+ shift();
+ break;
+ case CharacterTilde:
+ token = TILDE;
+ shift();
+ break;
+ case CharacterSemicolon:
+ shift();
+ token = SEMICOLON;
+ break;
+ case CharacterOpenBrace:
+ tokenData->line = lineNumber();
+ tokenData->offset = currentOffset();
+ tokenData->lineStartOffset = currentLineStartOffset();
+ ASSERT(tokenData->offset >= tokenData->lineStartOffset);
+ shift();
+ token = OPENBRACE;
+ break;
+ case CharacterCloseBrace:
+ tokenData->line = lineNumber();
+ tokenData->offset = currentOffset();
+ tokenData->lineStartOffset = currentLineStartOffset();
+ ASSERT(tokenData->offset >= tokenData->lineStartOffset);
+ shift();
+ token = CLOSEBRACE;
+ break;
+ case CharacterDot:
+ shift();
+ if (!isASCIIDigit(m_current)) {
+ if (UNLIKELY((m_current == '.') && (peek(1) == '.'))) {
+ shift();
+ shift();
+ token = DOTDOTDOT;
+ break;
+ }
+ token = DOT;
+ break;
+ }
+ goto inNumberAfterDecimalPoint;
+ case CharacterZero:
+ shift();
+ if ((m_current | 0x20) == 'x') {
+ if (!isASCIIHexDigit(peek(1))) {
+ m_lexErrorMessage = ASCIILiteral("No hexadecimal digits after '0x'");
+ token = INVALID_HEX_NUMBER_ERRORTOK;
+ goto returnError;
+ }
+
+ // Shift out the 'x' prefix.
+ shift();
+
+ parseHex(tokenData->doubleValue);
+ if (isIdentStart(m_current)) {
+ m_lexErrorMessage = ASCIILiteral("No space between hexadecimal literal and identifier");
+ token = INVALID_HEX_NUMBER_ERRORTOK;
+ goto returnError;
+ }
+ token = tokenTypeForIntegerLikeToken(tokenData->doubleValue);
+ m_buffer8.shrink(0);
+ break;
+ }
+ if ((m_current | 0x20) == 'b') {
+ if (!isASCIIBinaryDigit(peek(1))) {
+ m_lexErrorMessage = ASCIILiteral("No binary digits after '0b'");
+ token = INVALID_BINARY_NUMBER_ERRORTOK;
+ goto returnError;
+ }
+
+ // Shift out the 'b' prefix.
+ shift();
+
+ parseBinary(tokenData->doubleValue);
+ if (isIdentStart(m_current)) {
+ m_lexErrorMessage = ASCIILiteral("No space between binary literal and identifier");
+ token = INVALID_BINARY_NUMBER_ERRORTOK;
+ goto returnError;
+ }
+ token = tokenTypeForIntegerLikeToken(tokenData->doubleValue);
+ m_buffer8.shrink(0);
+ break;
+ }
+
+ if ((m_current | 0x20) == 'o') {
+ if (!isASCIIOctalDigit(peek(1))) {
+ m_lexErrorMessage = ASCIILiteral("No octal digits after '0o'");
+ token = INVALID_OCTAL_NUMBER_ERRORTOK;
+ goto returnError;
+ }
+
+ // Shift out the 'o' prefix.
+ shift();
+
+ parseOctal(tokenData->doubleValue);
+ if (isIdentStart(m_current)) {
+ m_lexErrorMessage = ASCIILiteral("No space between octal literal and identifier");
+ token = INVALID_OCTAL_NUMBER_ERRORTOK;
+ goto returnError;
+ }
+ token = tokenTypeForIntegerLikeToken(tokenData->doubleValue);
+ m_buffer8.shrink(0);
+ break;
+ }
+
+ record8('0');
+ if (strictMode && isASCIIDigit(m_current)) {
+ m_lexErrorMessage = ASCIILiteral("Decimal integer literals with a leading zero are forbidden in strict mode");
+ token = INVALID_OCTAL_NUMBER_ERRORTOK;
+ goto returnError;
+ }
+ if (isASCIIOctalDigit(m_current)) {
+ if (parseOctal(tokenData->doubleValue)) {
+ token = tokenTypeForIntegerLikeToken(tokenData->doubleValue);
+ }
+ }
+ FALLTHROUGH;
+ case CharacterNumber:
+ if (LIKELY(token != INTEGER && token != DOUBLE)) {
+ if (!parseDecimal(tokenData->doubleValue)) {
+ token = INTEGER;
+ if (m_current == '.') {
+ shift();
+inNumberAfterDecimalPoint:
+ parseNumberAfterDecimalPoint();
+ token = DOUBLE;
+ }
+ if ((m_current | 0x20) == 'e') {
+ if (!parseNumberAfterExponentIndicator()) {
+ m_lexErrorMessage = ASCIILiteral("Non-number found after exponent indicator");
+ token = atEnd() ? UNTERMINATED_NUMERIC_LITERAL_ERRORTOK : INVALID_NUMERIC_LITERAL_ERRORTOK;
+ goto returnError;
+ }
+ }
+ size_t parsedLength;
+ tokenData->doubleValue = parseDouble(m_buffer8.data(), m_buffer8.size(), parsedLength);
+ if (token == INTEGER)
+ token = tokenTypeForIntegerLikeToken(tokenData->doubleValue);
+ } else
+ token = tokenTypeForIntegerLikeToken(tokenData->doubleValue);
+ }
+
+ // No identifiers allowed directly after numeric literal, e.g. "3in" is bad.
+ if (UNLIKELY(isIdentStart(m_current))) {
+ m_lexErrorMessage = ASCIILiteral("At least one digit must occur after a decimal point");
+ token = atEnd() ? UNTERMINATED_NUMERIC_LITERAL_ERRORTOK : INVALID_NUMERIC_LITERAL_ERRORTOK;
+ goto returnError;
+ }
+ m_buffer8.shrink(0);
+ break;
+ case CharacterQuote: {
+ StringParseResult result = StringCannotBeParsed;
+ if (lexerFlags & LexerFlagsDontBuildStrings)
+ result = parseString<false>(tokenData, strictMode);
+ else
+ result = parseString<true>(tokenData, strictMode);
+
+ if (UNLIKELY(result != StringParsedSuccessfully)) {
+ token = result == StringUnterminated ? UNTERMINATED_STRING_LITERAL_ERRORTOK : INVALID_STRING_LITERAL_ERRORTOK;
+ goto returnError;
+ }
+ shift();
+ token = STRING;
+ break;
+ }
+#if ENABLE(ES6_TEMPLATE_LITERAL_SYNTAX)
+ case CharacterBackQuote: {
+ // Skip backquote.
+ shift();
+ StringParseResult result = StringCannotBeParsed;
+ if (lexerFlags & LexerFlagsDontBuildStrings)
+ result = parseTemplateLiteral<false>(tokenData, RawStringsBuildMode::BuildRawStrings);
+ else
+ result = parseTemplateLiteral<true>(tokenData, RawStringsBuildMode::BuildRawStrings);
+
+ if (UNLIKELY(result != StringParsedSuccessfully)) {
+ token = result == StringUnterminated ? UNTERMINATED_TEMPLATE_LITERAL_ERRORTOK : INVALID_TEMPLATE_LITERAL_ERRORTOK;
+ goto returnError;
+ }
+ token = TEMPLATE;
+ break;
+ }
+#endif
+ case CharacterIdentifierStart:
+ ASSERT(isIdentStart(m_current));
+ FALLTHROUGH;
+ case CharacterBackSlash:
+ parseIdent:
+ if (lexerFlags & LexexFlagsDontBuildKeywords)
+ token = parseIdentifier<false>(tokenData, lexerFlags, strictMode);
+ else
+ token = parseIdentifier<true>(tokenData, lexerFlags, strictMode);
+ break;
+ case CharacterLineTerminator:
+ ASSERT(isLineTerminator(m_current));
+ shiftLineTerminator();
+ m_atLineStart = true;
+ m_terminator = true;
+ m_lineStart = m_code;
+ goto start;
+ case CharacterPrivateIdentifierStart:
+ if (m_parsingBuiltinFunction)
+ goto parseIdent;
+
+ FALLTHROUGH;
+ case CharacterInvalid:
+ m_lexErrorMessage = invalidCharacterMessage();
+ token = ERRORTOK;
+ goto returnError;
+ default:
+ RELEASE_ASSERT_NOT_REACHED();
+ m_lexErrorMessage = ASCIILiteral("Internal Error");
+ token = ERRORTOK;
+ goto returnError;
+ }
+
+ m_atLineStart = false;
+ goto returnToken;
+
+inSingleLineComment:
+ while (!isLineTerminator(m_current)) {
+ if (atEnd())
+ return EOFTOK;
+ shift();
+ }
+ shiftLineTerminator();
+ m_atLineStart = true;
+ m_terminator = true;
+ m_lineStart = m_code;
+ if (!lastTokenWasRestrKeyword())
+ goto start;
+
+ token = SEMICOLON;
+ // Fall through into returnToken.
+
+returnToken:
+ tokenLocation->line = m_lineNumber;
+ tokenLocation->endOffset = currentOffset();
+ tokenLocation->lineStartOffset = currentLineStartOffset();
+ ASSERT(tokenLocation->endOffset >= tokenLocation->lineStartOffset);
+ tokenRecord->m_endPosition = currentPosition();
+ m_lastToken = token;
+ return token;
+
+returnError:
+ m_error = true;
+ tokenLocation->line = m_lineNumber;
+ tokenLocation->endOffset = currentOffset();
+ tokenLocation->lineStartOffset = currentLineStartOffset();
+ ASSERT(tokenLocation->endOffset >= tokenLocation->lineStartOffset);
+ tokenRecord->m_endPosition = currentPosition();
+ RELEASE_ASSERT(token & ErrorTokenFlag);
+ return token;
+}
+
+template <typename T>
+static inline void orCharacter(UChar&, UChar);
+
+template <>
+inline void orCharacter<LChar>(UChar&, UChar) { }
+
+template <>
+inline void orCharacter<UChar>(UChar& orAccumulator, UChar character)
+{
+ orAccumulator |= character;
+}
+
+template <typename T>
+bool Lexer<T>::scanRegExp(const Identifier*& pattern, const Identifier*& flags, UChar patternPrefix)
+{
+ ASSERT(m_buffer16.isEmpty());
+
+ bool lastWasEscape = false;
+ bool inBrackets = false;
+ UChar charactersOredTogether = 0;
+
+ if (patternPrefix) {
+ ASSERT(!isLineTerminator(patternPrefix));
+ ASSERT(patternPrefix != '/');
+ ASSERT(patternPrefix != '[');
+ record16(patternPrefix);
+ }
+
+ while (true) {
+ if (isLineTerminator(m_current) || atEnd()) {
+ m_buffer16.shrink(0);
+ return false;
+ }
+
+ T prev = m_current;
+
+ shift();
+
+ if (prev == '/' && !lastWasEscape && !inBrackets)
+ break;
+
+ record16(prev);
+ orCharacter<T>(charactersOredTogether, prev);
+
+ if (lastWasEscape) {
+ lastWasEscape = false;
+ continue;
+ }
+
+ switch (prev) {
+ case '[':
+ inBrackets = true;
+ break;
+ case ']':
+ inBrackets = false;
+ break;
+ case '\\':
+ lastWasEscape = true;
+ break;
+ }
+ }
+
+ pattern = makeRightSizedIdentifier(m_buffer16.data(), m_buffer16.size(), charactersOredTogether);
+
+ m_buffer16.shrink(0);
+ charactersOredTogether = 0;
+
+ while (isIdentPart(m_current)) {
+ record16(m_current);
+ orCharacter<T>(charactersOredTogether, m_current);
+ shift();
+ }
+
+ flags = makeRightSizedIdentifier(m_buffer16.data(), m_buffer16.size(), charactersOredTogether);
+ m_buffer16.shrink(0);
+
+ return true;
+}
+
+template <typename T>
+bool Lexer<T>::skipRegExp()
+{
+ bool lastWasEscape = false;
+ bool inBrackets = false;
+
+ while (true) {
+ if (isLineTerminator(m_current) || atEnd())
+ return false;
+
+ T prev = m_current;
+
+ shift();
+
+ if (prev == '/' && !lastWasEscape && !inBrackets)
+ break;
+
+ if (lastWasEscape) {
+ lastWasEscape = false;
+ continue;
+ }
+
+ switch (prev) {
+ case '[':
+ inBrackets = true;
+ break;
+ case ']':
+ inBrackets = false;
+ break;
+ case '\\':
+ lastWasEscape = true;
+ break;
+ }
+ }
+
+ while (isIdentPart(m_current))
+ shift();
+
+ return true;
+}
+
+#if ENABLE(ES6_TEMPLATE_LITERAL_SYNTAX)
+template <typename T>
+JSTokenType Lexer<T>::scanTrailingTemplateString(JSToken* tokenRecord, RawStringsBuildMode rawStringsBuildMode)
+{
+ JSTokenData* tokenData = &tokenRecord->m_data;
+ JSTokenLocation* tokenLocation = &tokenRecord->m_location;
+ ASSERT(!m_error);
+ ASSERT(m_buffer16.isEmpty());
+
+ // Leading closing brace } is already shifted in the previous token scan.
+ // So in this re-scan phase, shift() is not needed here.
+ StringParseResult result = parseTemplateLiteral<true>(tokenData, rawStringsBuildMode);
+ JSTokenType token = ERRORTOK;
+ if (UNLIKELY(result != StringParsedSuccessfully)) {
+ token = result == StringUnterminated ? UNTERMINATED_TEMPLATE_LITERAL_ERRORTOK : INVALID_TEMPLATE_LITERAL_ERRORTOK;
+ m_error = true;
+ } else {
+ token = TEMPLATE;
+ m_lastToken = token;
+ }
+
+ // Since TemplateString always ends with ` or }, m_atLineStart always becomes false.
+ m_atLineStart = false;
+
+ // Adjust current tokenLocation data for TemplateString.
+ tokenLocation->line = m_lineNumber;
+ tokenLocation->endOffset = currentOffset();
+ tokenLocation->lineStartOffset = currentLineStartOffset();
+ ASSERT(tokenLocation->endOffset >= tokenLocation->lineStartOffset);
+ tokenRecord->m_endPosition = currentPosition();
+ return token;
+}
+#endif
+
+template <typename T>
+void Lexer<T>::clear()
+{
+ m_arena = 0;
+
+ Vector<LChar> newBuffer8;
+ m_buffer8.swap(newBuffer8);
+
+ Vector<UChar> newBuffer16;
+ m_buffer16.swap(newBuffer16);
+
+ Vector<UChar> newBufferForRawTemplateString16;
+ m_bufferForRawTemplateString16.swap(newBufferForRawTemplateString16);
+
+ m_isReparsing = false;
+}
+
+// Instantiate the two flavors of Lexer we need instead of putting most of this file in Lexer.h
+template class Lexer<LChar>;
+template class Lexer<UChar>;
+
projects / apple / javascriptcore.git / blobdiff