[apple/javascriptcore.git] / parser / ParserTokens.h

/*
 * Copyright (C) 2010, 2013 Apple Inc. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
 * THE POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef ParserTokens_h
#define ParserTokens_h

#include "ParserModes.h"
#include <limits.h>
#include <stdint.h>

namespace JSC {

class Identifier;

enum {
    UnaryOpTokenFlag = 64,
    KeywordTokenFlag = 128,
    BinaryOpTokenPrecedenceShift = 8,
    BinaryOpTokenAllowsInPrecedenceAdditionalShift = 4,
    BinaryOpTokenPrecedenceMask = 15 << BinaryOpTokenPrecedenceShift,
    ErrorTokenFlag = 1 << (BinaryOpTokenAllowsInPrecedenceAdditionalShift + BinaryOpTokenPrecedenceShift + 7),
    UnterminatedErrorTokenFlag = ErrorTokenFlag << 1
};

#define BINARY_OP_PRECEDENCE(prec) (((prec) << BinaryOpTokenPrecedenceShift) | ((prec) << (BinaryOpTokenPrecedenceShift + BinaryOpTokenAllowsInPrecedenceAdditionalShift)))
#define IN_OP_PRECEDENCE(prec) ((prec) << (BinaryOpTokenPrecedenceShift + BinaryOpTokenAllowsInPrecedenceAdditionalShift))

enum JSTokenType {
    NULLTOKEN = KeywordTokenFlag,
    TRUETOKEN,
    FALSETOKEN,
    BREAK,
    CASE,
    DEFAULT,
    FOR,
    NEW,
    VAR,
    CONSTTOKEN,
    CONTINUE,
    FUNCTION,
    RETURN,
    IF,
    THISTOKEN,
    DO,
    WHILE,
    SWITCH,
    WITH,
    RESERVED,
    RESERVED_IF_STRICT,
    THROW,
    TRY,
    CATCH,
    FINALLY,
    DEBUGGER,
    ELSE,
#if ENABLE(ES6_ARROWFUNCTION_SYNTAX)
    ARROWFUNCTION,
#endif
#if ENABLE(ES6_CLASS_SYNTAX)
    CLASSTOKEN,
    EXTENDS,
    SUPER,
#else
    CLASSTOKEN = RESERVED,
    EXTENDS = RESERVED,
    SUPER = RESERVED,
#endif
    OPENBRACE = 0,
    CLOSEBRACE,
    OPENPAREN,
    CLOSEPAREN,
    OPENBRACKET,
    CLOSEBRACKET,
    COMMA,
    QUESTION,
    INTEGER,
    DOUBLE,
    IDENT,
    STRING,
    TEMPLATE,
    SEMICOLON,
    COLON,
    DOT,
    EOFTOK,
    EQUAL,
    PLUSEQUAL,
    MINUSEQUAL,
    MULTEQUAL,
    DIVEQUAL,
    LSHIFTEQUAL,
    RSHIFTEQUAL,
    URSHIFTEQUAL,
    ANDEQUAL,
    MODEQUAL,
    XOREQUAL,
    OREQUAL,
    DOTDOTDOT,
    LastUntaggedToken,

    // Begin tagged tokens
    PLUSPLUS = 0 | UnaryOpTokenFlag,
    MINUSMINUS = 1 | UnaryOpTokenFlag,
    EXCLAMATION = 2 | UnaryOpTokenFlag,
    TILDE = 3 | UnaryOpTokenFlag,
    AUTOPLUSPLUS = 4 | UnaryOpTokenFlag,
    AUTOMINUSMINUS = 5 | UnaryOpTokenFlag,
    TYPEOF = 6 | UnaryOpTokenFlag | KeywordTokenFlag,
    VOIDTOKEN = 7 | UnaryOpTokenFlag | KeywordTokenFlag,
    DELETETOKEN = 8 | UnaryOpTokenFlag | KeywordTokenFlag,
    OR = 0 | BINARY_OP_PRECEDENCE(1),
    AND = 1 | BINARY_OP_PRECEDENCE(2),
    BITOR = 2 | BINARY_OP_PRECEDENCE(3),
    BITXOR = 3 | BINARY_OP_PRECEDENCE(4),
    BITAND = 4 | BINARY_OP_PRECEDENCE(5),
    EQEQ = 5 | BINARY_OP_PRECEDENCE(6),
    NE = 6 | BINARY_OP_PRECEDENCE(6),
    STREQ = 7 | BINARY_OP_PRECEDENCE(6),
    STRNEQ = 8 | BINARY_OP_PRECEDENCE(6),
    LT = 9 | BINARY_OP_PRECEDENCE(7),
    GT = 10 | BINARY_OP_PRECEDENCE(7),
    LE = 11 | BINARY_OP_PRECEDENCE(7),
    GE = 12 | BINARY_OP_PRECEDENCE(7),
    INSTANCEOF = 13 | BINARY_OP_PRECEDENCE(7) | KeywordTokenFlag,
    INTOKEN = 14 | IN_OP_PRECEDENCE(7) | KeywordTokenFlag,
    LSHIFT = 15 | BINARY_OP_PRECEDENCE(8),
    RSHIFT = 16 | BINARY_OP_PRECEDENCE(8),
    URSHIFT = 17 | BINARY_OP_PRECEDENCE(8),
    PLUS = 18 | BINARY_OP_PRECEDENCE(9) | UnaryOpTokenFlag,
    MINUS = 19 | BINARY_OP_PRECEDENCE(9) | UnaryOpTokenFlag,
    TIMES = 20 | BINARY_OP_PRECEDENCE(10),
    DIVIDE = 21 | BINARY_OP_PRECEDENCE(10),
    MOD = 22 | BINARY_OP_PRECEDENCE(10),
    ERRORTOK = 0 | ErrorTokenFlag,
    UNTERMINATED_IDENTIFIER_ESCAPE_ERRORTOK = 0 | ErrorTokenFlag | UnterminatedErrorTokenFlag,
    INVALID_IDENTIFIER_ESCAPE_ERRORTOK = 1 | ErrorTokenFlag,
    UNTERMINATED_IDENTIFIER_UNICODE_ESCAPE_ERRORTOK = 2 | ErrorTokenFlag | UnterminatedErrorTokenFlag,
    INVALID_IDENTIFIER_UNICODE_ESCAPE_ERRORTOK = 3 | ErrorTokenFlag,
    UNTERMINATED_MULTILINE_COMMENT_ERRORTOK = 4 | ErrorTokenFlag | UnterminatedErrorTokenFlag,
    UNTERMINATED_NUMERIC_LITERAL_ERRORTOK = 5 | ErrorTokenFlag | UnterminatedErrorTokenFlag,
    INVALID_OCTAL_NUMBER_ERRORTOK = 6 | ErrorTokenFlag | UnterminatedErrorTokenFlag,
    INVALID_NUMERIC_LITERAL_ERRORTOK = 7 | ErrorTokenFlag,
    UNTERMINATED_STRING_LITERAL_ERRORTOK = 8 | ErrorTokenFlag | UnterminatedErrorTokenFlag,
    INVALID_STRING_LITERAL_ERRORTOK = 9 | ErrorTokenFlag,
    INVALID_PRIVATE_NAME_ERRORTOK = 10 | ErrorTokenFlag,
    INVALID_HEX_NUMBER_ERRORTOK = 11 | ErrorTokenFlag,
    INVALID_BINARY_NUMBER_ERRORTOK = 12 | ErrorTokenFlag,
    UNTERMINATED_TEMPLATE_LITERAL_ERRORTOK = 13 | ErrorTokenFlag | UnterminatedErrorTokenFlag,
    INVALID_TEMPLATE_LITERAL_ERRORTOK = 14 | ErrorTokenFlag,
};

struct JSTextPosition {
    JSTextPosition() : line(0), offset(0), lineStartOffset(0) { }
    JSTextPosition(int _line, int _offset, int _lineStartOffset) : line(_line), offset(_offset), lineStartOffset(_lineStartOffset) { }
    JSTextPosition(const JSTextPosition& other) : line(other.line), offset(other.offset), lineStartOffset(other.lineStartOffset) { }

    JSTextPosition operator+(int adjustment) const { return JSTextPosition(line, offset + adjustment, lineStartOffset); }
    JSTextPosition operator+(unsigned adjustment) const { return *this + static_cast<int>(adjustment); }
    JSTextPosition operator-(int adjustment) const { return *this + (- adjustment); }
    JSTextPosition operator-(unsigned adjustment) const { return *this + (- static_cast<int>(adjustment)); }

    operator int() const { return offset; }

    int line;
    int offset;
    int lineStartOffset;
};

union JSTokenData {
    struct {
        uint32_t line;
        uint32_t offset;
        uint32_t lineStartOffset;
    };
    double doubleValue;
    const Identifier* ident;
    struct {
        const Identifier* cooked;
        const Identifier* raw;
        bool isTail;
    };
};

struct JSTokenLocation {
    JSTokenLocation() : line(0), lineStartOffset(0), startOffset(0) { }
    JSTokenLocation(const JSTokenLocation& location)
    {
        line = location.line;
        lineStartOffset = location.lineStartOffset;
        startOffset = location.startOffset;
        endOffset = location.endOffset;
    }

    int line;
    unsigned lineStartOffset;
    unsigned startOffset;
    unsigned endOffset;
};

struct JSToken {
    JSTokenType m_type;
    JSTokenData m_data;
    JSTokenLocation m_location;
    JSTextPosition m_startPosition;
    JSTextPosition m_endPosition;
};

} // namespace JSC

#endif // ParserTokens_h
Commit	Line	Data
14957cd0	1	/*
93a37866	2	* Copyright (C) 2010, 2013 Apple Inc. All rights reserved.
14957cd0 A	3	*
	4	* Redistribution and use in source and binary forms, with or without
	5	* modification, are permitted provided that the following conditions
	6	* are met:
	7	* 1. Redistributions of source code must retain the above copyright
	8	* notice, this list of conditions and the following disclaimer.
	9	* 2. Redistributions in binary form must reproduce the above copyright
	10	* notice, this list of conditions and the following disclaimer in the
	11	* documentation and/or other materials provided with the distribution.
	12	*
	13	* THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
	14	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
	15	* THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
	16	* PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
	17	* BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	18	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	19	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
	20	* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
	21	* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	22	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
	23	* THE POSSIBILITY OF SUCH DAMAGE.
	24	*/
	25
6fe7ccc8 A	26	#ifndef ParserTokens_h
6fe7ccc8 A	27	#define ParserTokens_h
14957cd0	28
93a37866 A	29	#include "ParserModes.h"
	30	#include <limits.h>
	31	#include <stdint.h>
	32
14957cd0 A	33	namespace JSC {
14957cd0 A	34
14957cd0	35	class Identifier;
14957cd0 A	36
	37	enum {
	38	UnaryOpTokenFlag = 64,
	39	KeywordTokenFlag = 128,
	40	BinaryOpTokenPrecedenceShift = 8,
	41	BinaryOpTokenAllowsInPrecedenceAdditionalShift = 4,
	42	BinaryOpTokenPrecedenceMask = 15 << BinaryOpTokenPrecedenceShift,
93a37866 A	43	ErrorTokenFlag = 1 << (BinaryOpTokenAllowsInPrecedenceAdditionalShift + BinaryOpTokenPrecedenceShift + 7),
93a37866 A	44	UnterminatedErrorTokenFlag = ErrorTokenFlag << 1
14957cd0 A	45	};
	46
	47	#define BINARY_OP_PRECEDENCE(prec) (((prec) << BinaryOpTokenPrecedenceShift) \| ((prec) << (BinaryOpTokenPrecedenceShift + BinaryOpTokenAllowsInPrecedenceAdditionalShift)))
	48	#define IN_OP_PRECEDENCE(prec) ((prec) << (BinaryOpTokenPrecedenceShift + BinaryOpTokenAllowsInPrecedenceAdditionalShift))
	49
	50	enum JSTokenType {
	51	NULLTOKEN = KeywordTokenFlag,
	52	TRUETOKEN,
	53	FALSETOKEN,
	54	BREAK,
	55	CASE,
	56	DEFAULT,
	57	FOR,
	58	NEW,
	59	VAR,
	60	CONSTTOKEN,
	61	CONTINUE,
	62	FUNCTION,
	63	RETURN,
	64	IF,
	65	THISTOKEN,
	66	DO,
	67	WHILE,
	68	SWITCH,
	69	WITH,
	70	RESERVED,
6fe7ccc8	71	RESERVED_IF_STRICT,
14957cd0 A	72	THROW,
	73	TRY,
	74	CATCH,
	75	FINALLY,
	76	DEBUGGER,
	77	ELSE,
ed1e77d3 A	78	#if ENABLE(ES6_ARROWFUNCTION_SYNTAX)
	79	ARROWFUNCTION,
	80	#endif
	81	#if ENABLE(ES6_CLASS_SYNTAX)
	82	CLASSTOKEN,
	83	EXTENDS,
	84	SUPER,
	85	#else
	86	CLASSTOKEN = RESERVED,
	87	EXTENDS = RESERVED,
	88	SUPER = RESERVED,
	89	#endif
14957cd0 A	90	OPENBRACE = 0,
	91	CLOSEBRACE,
	92	OPENPAREN,
	93	CLOSEPAREN,
	94	OPENBRACKET,
	95	CLOSEBRACKET,
	96	COMMA,
	97	QUESTION,
ed1e77d3 A	98	INTEGER,
ed1e77d3 A	99	DOUBLE,
14957cd0 A	100	IDENT,
14957cd0 A	101	STRING,
ed1e77d3	102	TEMPLATE,
14957cd0 A	103	SEMICOLON,
	104	COLON,
	105	DOT,
14957cd0 A	106	EOFTOK,
	107	EQUAL,
	108	PLUSEQUAL,
	109	MINUSEQUAL,
	110	MULTEQUAL,
	111	DIVEQUAL,
	112	LSHIFTEQUAL,
	113	RSHIFTEQUAL,
	114	URSHIFTEQUAL,
	115	ANDEQUAL,
	116	MODEQUAL,
	117	XOREQUAL,
	118	OREQUAL,
81345200	119	DOTDOTDOT,
14957cd0 A	120	LastUntaggedToken,
	121
	122	// Begin tagged tokens
	123	PLUSPLUS = 0 \| UnaryOpTokenFlag,
	124	MINUSMINUS = 1 \| UnaryOpTokenFlag,
	125	EXCLAMATION = 2 \| UnaryOpTokenFlag,
	126	TILDE = 3 \| UnaryOpTokenFlag,
	127	AUTOPLUSPLUS = 4 \| UnaryOpTokenFlag,
	128	AUTOMINUSMINUS = 5 \| UnaryOpTokenFlag,
	129	TYPEOF = 6 \| UnaryOpTokenFlag \| KeywordTokenFlag,
	130	VOIDTOKEN = 7 \| UnaryOpTokenFlag \| KeywordTokenFlag,
	131	DELETETOKEN = 8 \| UnaryOpTokenFlag \| KeywordTokenFlag,
	132	OR = 0 \| BINARY_OP_PRECEDENCE(1),
	133	AND = 1 \| BINARY_OP_PRECEDENCE(2),
	134	BITOR = 2 \| BINARY_OP_PRECEDENCE(3),
	135	BITXOR = 3 \| BINARY_OP_PRECEDENCE(4),
	136	BITAND = 4 \| BINARY_OP_PRECEDENCE(5),
	137	EQEQ = 5 \| BINARY_OP_PRECEDENCE(6),
	138	NE = 6 \| BINARY_OP_PRECEDENCE(6),
	139	STREQ = 7 \| BINARY_OP_PRECEDENCE(6),
	140	STRNEQ = 8 \| BINARY_OP_PRECEDENCE(6),
	141	LT = 9 \| BINARY_OP_PRECEDENCE(7),
	142	GT = 10 \| BINARY_OP_PRECEDENCE(7),
	143	LE = 11 \| BINARY_OP_PRECEDENCE(7),
	144	GE = 12 \| BINARY_OP_PRECEDENCE(7),
	145	INSTANCEOF = 13 \| BINARY_OP_PRECEDENCE(7) \| KeywordTokenFlag,
	146	INTOKEN = 14 \| IN_OP_PRECEDENCE(7) \| KeywordTokenFlag,
	147	LSHIFT = 15 \| BINARY_OP_PRECEDENCE(8),
	148	RSHIFT = 16 \| BINARY_OP_PRECEDENCE(8),
	149	URSHIFT = 17 \| BINARY_OP_PRECEDENCE(8),
	150	PLUS = 18 \| BINARY_OP_PRECEDENCE(9) \| UnaryOpTokenFlag,
	151	MINUS = 19 \| BINARY_OP_PRECEDENCE(9) \| UnaryOpTokenFlag,
	152	TIMES = 20 \| BINARY_OP_PRECEDENCE(10),
	153	DIVIDE = 21 \| BINARY_OP_PRECEDENCE(10),
93a37866 A	154	MOD = 22 \| BINARY_OP_PRECEDENCE(10),
	155	ERRORTOK = 0 \| ErrorTokenFlag,
	156	UNTERMINATED_IDENTIFIER_ESCAPE_ERRORTOK = 0 \| ErrorTokenFlag \| UnterminatedErrorTokenFlag,
	157	INVALID_IDENTIFIER_ESCAPE_ERRORTOK = 1 \| ErrorTokenFlag,
	158	UNTERMINATED_IDENTIFIER_UNICODE_ESCAPE_ERRORTOK = 2 \| ErrorTokenFlag \| UnterminatedErrorTokenFlag,
	159	INVALID_IDENTIFIER_UNICODE_ESCAPE_ERRORTOK = 3 \| ErrorTokenFlag,
	160	UNTERMINATED_MULTILINE_COMMENT_ERRORTOK = 4 \| ErrorTokenFlag \| UnterminatedErrorTokenFlag,
	161	UNTERMINATED_NUMERIC_LITERAL_ERRORTOK = 5 \| ErrorTokenFlag \| UnterminatedErrorTokenFlag,
	162	INVALID_OCTAL_NUMBER_ERRORTOK = 6 \| ErrorTokenFlag \| UnterminatedErrorTokenFlag,
	163	INVALID_NUMERIC_LITERAL_ERRORTOK = 7 \| ErrorTokenFlag,
	164	UNTERMINATED_STRING_LITERAL_ERRORTOK = 8 \| ErrorTokenFlag \| UnterminatedErrorTokenFlag,
	165	INVALID_STRING_LITERAL_ERRORTOK = 9 \| ErrorTokenFlag,
81345200	166	INVALID_PRIVATE_NAME_ERRORTOK = 10 \| ErrorTokenFlag,
ed1e77d3 A	167	INVALID_HEX_NUMBER_ERRORTOK = 11 \| ErrorTokenFlag,
	168	INVALID_BINARY_NUMBER_ERRORTOK = 12 \| ErrorTokenFlag,
	169	UNTERMINATED_TEMPLATE_LITERAL_ERRORTOK = 13 \| ErrorTokenFlag \| UnterminatedErrorTokenFlag,
	170	INVALID_TEMPLATE_LITERAL_ERRORTOK = 14 \| ErrorTokenFlag,
81345200 A	171	};
	172
	173	struct JSTextPosition {
	174	JSTextPosition() : line(0), offset(0), lineStartOffset(0) { }
	175	JSTextPosition(int _line, int _offset, int _lineStartOffset) : line(_line), offset(_offset), lineStartOffset(_lineStartOffset) { }
	176	JSTextPosition(const JSTextPosition& other) : line(other.line), offset(other.offset), lineStartOffset(other.lineStartOffset) { }
	177
	178	JSTextPosition operator+(int adjustment) const { return JSTextPosition(line, offset + adjustment, lineStartOffset); }
	179	JSTextPosition operator+(unsigned adjustment) const { return *this + static_cast<int>(adjustment); }
	180	JSTextPosition operator-(int adjustment) const { return *this + (- adjustment); }
	181	JSTextPosition operator-(unsigned adjustment) const { return *this + (- static_cast<int>(adjustment)); }
	182
	183	operator int() const { return offset; }
	184
	185	int line;
	186	int offset;
	187	int lineStartOffset;
14957cd0 A	188	};
	189
	190	union JSTokenData {
93a37866 A	191	struct {
	192	uint32_t line;
	193	uint32_t offset;
	194	uint32_t lineStartOffset;
	195	};
14957cd0 A	196	double doubleValue;
14957cd0 A	197	const Identifier* ident;
ed1e77d3 A	198	struct {
	199	const Identifier* cooked;
	200	const Identifier* raw;
	201	bool isTail;
	202	};
14957cd0 A	203	};
14957cd0 A	204
93a37866 A	205	struct JSTokenLocation {
	206	JSTokenLocation() : line(0), lineStartOffset(0), startOffset(0) { }
	207	JSTokenLocation(const JSTokenLocation& location)
	208	{
	209	line = location.line;
	210	lineStartOffset = location.lineStartOffset;
	211	startOffset = location.startOffset;
	212	endOffset = location.endOffset;
	213	}
	214
14957cd0	215	int line;
93a37866 A	216	unsigned lineStartOffset;
	217	unsigned startOffset;
	218	unsigned endOffset;
14957cd0 A	219	};
	220
	221	struct JSToken {
	222	JSTokenType m_type;
	223	JSTokenData m_data;
93a37866	224	JSTokenLocation m_location;
81345200 A	225	JSTextPosition m_startPosition;
81345200 A	226	JSTextPosition m_endPosition;
14957cd0 A	227	};
14957cd0 A	228
93a37866	229	} // namespace JSC
6fe7ccc8 A	230
6fe7ccc8 A	231	#endif // ParserTokens_h