From b23692f3038296c94d25c84c55ed4c1bb49619cf Mon Sep 17 00:00:00 2001 From: "Jay Freeman (saurik)" Date: Wed, 16 Dec 2015 04:31:34 -0800 Subject: [PATCH] Move non-RegEx lexer hacks from Scanner to Parser. --- Driver.cpp | 2 +- Driver.hpp | 7 ++++++- Parser.ypp.in | 55 +++++++++++++++++++++++++++++++++++++++++++++++++- Scanner.lpp.in | 19 ++++++++--------- 4 files changed, 69 insertions(+), 14 deletions(-) diff --git a/Driver.cpp b/Driver.cpp index 83b9462..fef1eb1 100644 --- a/Driver.cpp +++ b/Driver.cpp @@ -26,7 +26,7 @@ bool CYParser(CYPool &pool, bool debug); CYDriver::CYDriver(CYPool &pool, std::istream &data, const std::string &filename) : pool_(pool), - newline_(false), + newline_(NewLineNone), last_(false), next_(false), data_(data), diff --git a/Driver.hpp b/Driver.hpp index efd2bc6..f0d2732 100644 --- a/Driver.hpp +++ b/Driver.hpp @@ -60,7 +60,12 @@ class _visible CYDriver { std::stack class_; - bool newline_; + enum { + NewLineNone, + NewLineLast, + NewLineHere, + } newline_; + bool last_; bool next_; diff --git a/Parser.ypp.in b/Parser.ypp.in index f7f0f0f..d2d2775 100644 --- a/Parser.ypp.in +++ b/Parser.ypp.in @@ -113,6 +113,8 @@ int cylex(YYSTYPE *, CYLocation *, void *); %code { +typedef cy::parser::token tk; + #undef yylex _finline int yylex(cy::parser::semantic_type *semantic, CYLocation *location, CYDriver &driver) { if (driver.mark_ == CYMarkIgnore); @@ -124,9 +126,60 @@ _finline int yylex(cy::parser::semantic_type *semantic, CYLocation *location, CY return cy::parser::token::MarkModule; } + lex: + if (driver.newline_ == CYDriver::NewLineHere) + driver.newline_ = CYDriver::NewLineLast; + else if (driver.newline_ == CYDriver::NewLineLast) + driver.newline_ = CYDriver::NewLineNone; + YYSTYPE data; int token(cylex(&data, location, driver.scanner_)); *semantic = data.semantic_; + + switch (token) { + case tk::_in_: + if (driver.in_.top()) + token = tk::_in__; + break; + + case tk::_return_: + if (driver.return_.top()) + token = tk::_return__; + break; + + case tk::_yield_: + if (driver.yield_.top()) + token = tk::_yield__; + break; + + case tk::EqualRight: + if (driver.newline_ == CYDriver::NewLineLast) + token = tk::EqualRight_; + break; + + case tk::HyphenHyphen: + if (driver.newline_ == CYDriver::NewLineLast) + token = tk::HyphenHyphen_; + break; + + case tk::NewLine: + driver.newline_ = CYDriver::NewLineHere; + if (!driver.next_) + goto lex; + break; + + case tk::OpenBrace: + if (driver.newline_ == CYDriver::NewLineLast) + token = tk::OpenBrace_; + break; + + case tk::PlusPlus: + if (driver.newline_ == CYDriver::NewLineLast) + token = tk::PlusPlus_; + break; + } + + driver.next_ = false; return token; } @@ -793,7 +846,7 @@ TerminatorSoft Terminator : ";" - | error { if (yyla.type_get() != yyeof_ && yyla.type != yytranslate_(token::CloseBrace) && !driver.newline_) { CYERR(@1, "required semi-colon"); } else { yyerrok; driver.errors_.pop_back(); } } StrictSemi + | error { if (yyla.type_get() != yyeof_ && yyla.type != yytranslate_(token::CloseBrace) && driver.newline_ == CYDriver::NewLineNone) { CYERR(@1, "required semi-colon"); } else { yyerrok; driver.errors_.pop_back(); } } StrictSemi ; TerminatorOpt diff --git a/Scanner.lpp.in b/Scanner.lpp.in index 1fb16aa..399feb8 100644 --- a/Scanner.lpp.in +++ b/Scanner.lpp.in @@ -47,9 +47,6 @@ typedef cy::parser::token tk; #define YY_EXTRA_TYPE CYDriver * #define F(value, highlight) do { \ - yyextra->newline_ = yyextra->last_; \ - yyextra->last_ = false; \ - yyextra->next_ = false; \ BEGIN(yyextra->template_.top() ? DivOrTemplateTail : Div); \ yylval->highlight_ = highlight; \ return value; \ @@ -72,7 +69,7 @@ typedef cy::parser::token tk; } while (false) #define N \ - if (yyextra->last_ && yyextra->next_) { \ + if (yyextra->last_) { \ yyextra->last_ = false; \ F(tk::NewLine, hi::Nothing); \ } @@ -366,13 +363,13 @@ XMLName {XMLNameStart}{XMLNamePart}* "=" L F(tk::Equal, hi::Operator); "==" L F(tk::EqualEqual, hi::Operator); "===" L F(tk::EqualEqualEqual, hi::Operator); -"=>" L F(yyextra->newline_ ? tk::EqualRight_ : tk::EqualRight, hi::Operator); +"=>" L F(tk::EqualRight, hi::Operator); "!" L F(tk::Exclamation, hi::Operator); "!=" L F(tk::ExclamationEqual, hi::Operator); "!==" L F(tk::ExclamationEqualEqual, hi::Operator); "-" L F(tk::Hyphen, hi::Operator); "-=" L F(tk::HyphenEqual, hi::Operator); -"--" L F(yyextra->newline_ ? tk::HyphenHyphen_ : tk::HyphenHyphen, hi::Operator); +"--" L F(tk::HyphenHyphen, hi::Operator); "->" L F(tk::HyphenRight, hi::Operator); "<" L F(tk::Left, hi::Operator); "<=" L F(tk::LeftEqual, hi::Operator); @@ -386,7 +383,7 @@ XMLName {XMLNameStart}{XMLNamePart}* "||" L F(tk::PipePipe, hi::Operator); "+" L F(tk::Plus, hi::Operator); "+=" L F(tk::PlusEqual, hi::Operator); -"++" L F(yyextra->newline_ ? tk::PlusPlus_ : tk::PlusPlus, hi::Operator); +"++" L F(tk::PlusPlus, hi::Operator); ">" L F(tk::Right, hi::Operator); ">=" L F(tk::RightEqual, hi::Operator); ">>" L F(tk::RightRight, hi::Operator); @@ -408,7 +405,7 @@ XMLName {XMLNameStart}{XMLNamePart}* "(" L F(tk::OpenParen, hi::Structure); ")" L F(tk::CloseParen, hi::Structure); -"{" L yyextra->template_.push(false); F(yyextra->newline_ ? tk::OpenBrace_ : tk::OpenBrace, hi::Structure); +"{" L yyextra->template_.push(false); F(tk::OpenBrace, hi::Structure);
"}" L S(template_); F(tk::CloseBrace, hi::Structure); "[" L F(tk::OpenBracket, hi::Structure); @@ -486,7 +483,7 @@ XMLName {XMLNameStart}{XMLNamePart}* "if" L /*KKK*/ F(tk::_if_, hi::Control); "implements" L /*FSS*/ F(tk::_implements_, hi::Meta); "import" L /*FFK*/ F(tk::_import_, hi::Meta); -"in" L /*KKK*/ F(yyextra->in_.top() ? tk::_in__ : tk::_in_, hi::Operator); +"in" L /*KKK*/ F(tk::_in_, hi::Operator); "Infinity" L /*III*/ F(tk::_Infinity_, hi::Constant); "instanceof" L /*KKK*/ F(tk::_instanceof_, hi::Operator); "int" L /*FII*/ F(tk::_int_, hi::Type); @@ -501,7 +498,7 @@ XMLName {XMLNameStart}{XMLNamePart}* "protected" L /*FSS*/ F(tk::_protected_, hi::Meta); "prototype" L /*III*/ F(tk::_prototype_, hi::Special); "public" L /*FSS*/ F(tk::_public_, hi::Meta); -"return" L /*KKK*/ F(yyextra->return_.top() ? tk::_return__ : tk::_return_, hi::Control); +"return" L /*KKK*/ F(tk::_return_, hi::Control); "set" L /*III*/ F(tk::_set_, hi::Meta); "short" L /*FII*/ F(tk::_short_, hi::Type); "static" L /*FS?*/ F(tk::_static_, hi::Meta); @@ -521,7 +518,7 @@ XMLName {XMLNameStart}{XMLNamePart}* "volatile" L /*FII*/ F(tk::_volatile_, hi::Meta); "while" L /*KKK*/ F(tk::_while_, hi::Control); "with" L /*KKK*/ F(tk::_with_, hi::Control); -"yield" L /*IS?*/ F(yyextra->yield_.top() ? tk::_yield__ : tk::_yield_, hi::Control); +"yield" L /*IS?*/ F(tk::_yield_, hi::Control); "auto" L F(tk::_auto_, hi::Meta); "each" L F(tk::_each_, hi::Control); -- 2.45.2