Move non-RegEx lexer hacks from Scanner to Parser.

diff --git a/Driver.cpp b/Driver.cpp
index 83b9462f9199e65d1689a3bc18b99c0fb9c45b75..fef1eb124d118dfd38a9604903910b868fb8cea8 100644 (file)
--- a/Driver.cpp
+++ b/Driver.cpp
@@ -26,7 +26,7 @@ bool CYParser(CYPool &pool, bool debug);
 
 CYDriver::CYDriver(CYPool &pool, std::istream &data, const std::string &filename) :
     pool_(pool),
 
 CYDriver::CYDriver(CYPool &pool, std::istream &data, const std::string &filename) :
     pool_(pool),

-    newline_(false),
+    newline_(NewLineNone),

     last_(false),
     next_(false),
     data_(data),
     last_(false),
     next_(false),
     data_(data),

diff --git a/Driver.hpp b/Driver.hpp
index efd2bc63ec6a28db9ca7230cf1afaeb01f38a68e..f0d27322877133ec4e548be4eda98e1359c9c4b6 100644 (file)
--- a/Driver.hpp
+++ b/Driver.hpp
@@ -60,7 +60,12 @@ class _visible CYDriver {
 
     std::stack<CYClassTail *> class_;
 
 
     std::stack<CYClassTail *> class_;
 

-    bool newline_;
+    enum {
+        NewLineNone,
+        NewLineLast,
+        NewLineHere,
+    } newline_;
+

     bool last_;
     bool next_;
 
     bool last_;
     bool next_;
 

diff --git a/Parser.ypp.in b/Parser.ypp.in
index f7f0f0f82ca2edaddd7db5a572bc0059050d5493..d2d2775ae07eae8800542fb4722494ced677b7ba 100644 (file)
--- a/Parser.ypp.in
+++ b/Parser.ypp.in
@@ -113,6 +113,8 @@ int cylex(YYSTYPE *, CYLocation *, void *);
 
 %code {
 
 
 %code {
 

+typedef cy::parser::token tk;
+

 #undef yylex
 _finline int yylex(cy::parser::semantic_type *semantic, CYLocation *location, CYDriver &driver) {
     if (driver.mark_ == CYMarkIgnore);
 #undef yylex
 _finline int yylex(cy::parser::semantic_type *semantic, CYLocation *location, CYDriver &driver) {
     if (driver.mark_ == CYMarkIgnore);


@@ -124,9 +126,60 @@ _finline int yylex(cy::parser::semantic_type *semantic, CYLocation *location, CY
         return cy::parser::token::MarkModule;
     }
 
         return cy::parser::token::MarkModule;
     }
 

+  lex:
+    if (driver.newline_ == CYDriver::NewLineHere)
+        driver.newline_ = CYDriver::NewLineLast;
+    else if (driver.newline_ == CYDriver::NewLineLast)
+        driver.newline_ = CYDriver::NewLineNone;
+

     YYSTYPE data;
     int token(cylex(&data, location, driver.scanner_));
     *semantic = data.semantic_;
     YYSTYPE data;
     int token(cylex(&data, location, driver.scanner_));
     *semantic = data.semantic_;

+
+    switch (token) {
+        case tk::_in_:
+            if (driver.in_.top())
+                token = tk::_in__;
+        break;
+
+        case tk::_return_:
+            if (driver.return_.top())
+                token = tk::_return__;
+        break;
+
+        case tk::_yield_:
+            if (driver.yield_.top())
+                token = tk::_yield__;
+        break;
+
+        case tk::EqualRight:
+            if (driver.newline_ == CYDriver::NewLineLast)
+                token = tk::EqualRight_;
+        break;
+
+        case tk::HyphenHyphen:
+            if (driver.newline_ == CYDriver::NewLineLast)
+                token = tk::HyphenHyphen_;
+        break;
+
+        case tk::NewLine:
+            driver.newline_ = CYDriver::NewLineHere;
+            if (!driver.next_)
+                goto lex;
+        break;
+
+        case tk::OpenBrace:
+            if (driver.newline_ == CYDriver::NewLineLast)
+                token = tk::OpenBrace_;
+        break;
+
+        case tk::PlusPlus:
+            if (driver.newline_ == CYDriver::NewLineLast)
+                token = tk::PlusPlus_;
+        break;
+    }
+
+    driver.next_ = false;

     return token;
 }
 
     return token;
 }
 


@@ -793,7 +846,7 @@ TerminatorSoft
 
 Terminator
     : ";"
 
 Terminator
     : ";"

-    | error { if (yyla.type_get() != yyeof_ && yyla.type != yytranslate_(token::CloseBrace) && !driver.newline_) { CYERR(@1, "required semi-colon"); } else { yyerrok; driver.errors_.pop_back(); } } StrictSemi
+    | error { if (yyla.type_get() != yyeof_ && yyla.type != yytranslate_(token::CloseBrace) && driver.newline_ == CYDriver::NewLineNone) { CYERR(@1, "required semi-colon"); } else { yyerrok; driver.errors_.pop_back(); } } StrictSemi

     ;
 
 TerminatorOpt
     ;
 
 TerminatorOpt

diff --git a/Scanner.lpp.in b/Scanner.lpp.in
index 1fb16aa5362c3a521b8662ec9efc3b1c08b584a0..399feb828e273a44cdb3d45f360a29e4ed4cfa62 100644 (file)
--- a/Scanner.lpp.in
+++ b/Scanner.lpp.in
@@ -47,9 +47,6 @@ typedef cy::parser::token tk;
 #define YY_EXTRA_TYPE CYDriver *
 
 #define F(value, highlight) do { \
 #define YY_EXTRA_TYPE CYDriver *
 
 #define F(value, highlight) do { \

-    yyextra->newline_ = yyextra->last_; \
-    yyextra->last_ = false; \
-    yyextra->next_ = false; \

     BEGIN(yyextra->template_.top() ? DivOrTemplateTail : Div); \
     yylval->highlight_ = highlight; \
     return value; \
     BEGIN(yyextra->template_.top() ? DivOrTemplateTail : Div); \
     yylval->highlight_ = highlight; \
     return value; \


@@ -72,7 +69,7 @@ typedef cy::parser::token tk;
 } while (false)
 
 #define N \
 } while (false)
 
 #define N \

-    if (yyextra->last_ && yyextra->next_) { \
+    if (yyextra->last_) { \

         yyextra->last_ = false; \
         F(tk::NewLine, hi::Nothing); \
     }
         yyextra->last_ = false; \
         F(tk::NewLine, hi::Nothing); \
     }


@@ -366,13 +363,13 @@ XMLName {XMLNameStart}{XMLNamePart}*
 "="    L F(tk::Equal, hi::Operator);
 "=="   L F(tk::EqualEqual, hi::Operator);
 "==="  L F(tk::EqualEqualEqual, hi::Operator);
 "="    L F(tk::Equal, hi::Operator);
 "=="   L F(tk::EqualEqual, hi::Operator);
 "==="  L F(tk::EqualEqualEqual, hi::Operator);

-"=>"   L F(yyextra->newline_ ? tk::EqualRight_ : tk::EqualRight, hi::Operator);
+"=>"   L F(tk::EqualRight, hi::Operator);

 "!"    L F(tk::Exclamation, hi::Operator);
 "!="   L F(tk::ExclamationEqual, hi::Operator);
 "!=="  L F(tk::ExclamationEqualEqual, hi::Operator);
 "-"    L F(tk::Hyphen, hi::Operator);
 "-="   L F(tk::HyphenEqual, hi::Operator);
 "!"    L F(tk::Exclamation, hi::Operator);
 "!="   L F(tk::ExclamationEqual, hi::Operator);
 "!=="  L F(tk::ExclamationEqualEqual, hi::Operator);
 "-"    L F(tk::Hyphen, hi::Operator);
 "-="   L F(tk::HyphenEqual, hi::Operator);

-"--"   L F(yyextra->newline_ ? tk::HyphenHyphen_ : tk::HyphenHyphen, hi::Operator);
+"--"   L F(tk::HyphenHyphen, hi::Operator);

 "->"   L F(tk::HyphenRight, hi::Operator);
 "<"    L F(tk::Left, hi::Operator);
 "<="   L F(tk::LeftEqual, hi::Operator);
 "->"   L F(tk::HyphenRight, hi::Operator);
 "<"    L F(tk::Left, hi::Operator);
 "<="   L F(tk::LeftEqual, hi::Operator);


@@ -386,7 +383,7 @@ XMLName {XMLNameStart}{XMLNamePart}*
 "||"   L F(tk::PipePipe, hi::Operator);
 "+"    L F(tk::Plus, hi::Operator);
 "+="   L F(tk::PlusEqual, hi::Operator);
 "||"   L F(tk::PipePipe, hi::Operator);
 "+"    L F(tk::Plus, hi::Operator);
 "+="   L F(tk::PlusEqual, hi::Operator);

-"++"   L F(yyextra->newline_ ? tk::PlusPlus_ : tk::PlusPlus, hi::Operator);
+"++"   L F(tk::PlusPlus, hi::Operator);

 ">"    L F(tk::Right, hi::Operator);
 ">="   L F(tk::RightEqual, hi::Operator);
 ">>"   L F(tk::RightRight, hi::Operator);
 ">"    L F(tk::Right, hi::Operator);
 ">="   L F(tk::RightEqual, hi::Operator);
 ">>"   L F(tk::RightRight, hi::Operator);


@@ -408,7 +405,7 @@ XMLName {XMLNameStart}{XMLNamePart}*
 "("    L F(tk::OpenParen, hi::Structure);
 ")"    L F(tk::CloseParen, hi::Structure);
 
 "("    L F(tk::OpenParen, hi::Structure);
 ")"    L F(tk::CloseParen, hi::Structure);
 

-"{"    L yyextra->template_.push(false); F(yyextra->newline_ ? tk::OpenBrace_ : tk::OpenBrace, hi::Structure);
+"{"    L yyextra->template_.push(false); F(tk::OpenBrace, hi::Structure);

 <Div>"}" L S(template_); F(tk::CloseBrace, hi::Structure);
 
 "["    L F(tk::OpenBracket, hi::Structure);
 <Div>"}" L S(template_); F(tk::CloseBrace, hi::Structure);
 
 "["    L F(tk::OpenBracket, hi::Structure);


@@ -486,7 +483,7 @@ XMLName {XMLNameStart}{XMLNamePart}*
 "if"              L /*KKK*/ F(tk::_if_, hi::Control);
 "implements"      L /*FSS*/ F(tk::_implements_, hi::Meta);
 "import"          L /*FFK*/ F(tk::_import_, hi::Meta);
 "if"              L /*KKK*/ F(tk::_if_, hi::Control);
 "implements"      L /*FSS*/ F(tk::_implements_, hi::Meta);
 "import"          L /*FFK*/ F(tk::_import_, hi::Meta);

-"in"              L /*KKK*/ F(yyextra->in_.top() ? tk::_in__ : tk::_in_, hi::Operator);
+"in"              L /*KKK*/ F(tk::_in_, hi::Operator);

 "Infinity"        L /*III*/ F(tk::_Infinity_, hi::Constant);
 "instanceof"      L /*KKK*/ F(tk::_instanceof_, hi::Operator);
 "int"             L /*FII*/ F(tk::_int_, hi::Type);
 "Infinity"        L /*III*/ F(tk::_Infinity_, hi::Constant);
 "instanceof"      L /*KKK*/ F(tk::_instanceof_, hi::Operator);
 "int"             L /*FII*/ F(tk::_int_, hi::Type);


@@ -501,7 +498,7 @@ XMLName {XMLNameStart}{XMLNamePart}*
 "protected"       L /*FSS*/ F(tk::_protected_, hi::Meta);
 "prototype"       L /*III*/ F(tk::_prototype_, hi::Special);
 "public"          L /*FSS*/ F(tk::_public_, hi::Meta);
 "protected"       L /*FSS*/ F(tk::_protected_, hi::Meta);
 "prototype"       L /*III*/ F(tk::_prototype_, hi::Special);
 "public"          L /*FSS*/ F(tk::_public_, hi::Meta);

-"return"          L /*KKK*/ F(yyextra->return_.top() ? tk::_return__ : tk::_return_, hi::Control);
+"return"          L /*KKK*/ F(tk::_return_, hi::Control);

 "set"             L /*III*/ F(tk::_set_, hi::Meta);
 "short"           L /*FII*/ F(tk::_short_, hi::Type);
 "static"          L /*FS?*/ F(tk::_static_, hi::Meta);
 "set"             L /*III*/ F(tk::_set_, hi::Meta);
 "short"           L /*FII*/ F(tk::_short_, hi::Type);
 "static"          L /*FS?*/ F(tk::_static_, hi::Meta);


@@ -521,7 +518,7 @@ XMLName {XMLNameStart}{XMLNamePart}*
 "volatile"        L /*FII*/ F(tk::_volatile_, hi::Meta);
 "while"           L /*KKK*/ F(tk::_while_, hi::Control);
 "with"            L /*KKK*/ F(tk::_with_, hi::Control);
 "volatile"        L /*FII*/ F(tk::_volatile_, hi::Meta);
 "while"           L /*KKK*/ F(tk::_while_, hi::Control);
 "with"            L /*KKK*/ F(tk::_with_, hi::Control);

-"yield"           L /*IS?*/ F(yyextra->yield_.top() ? tk::_yield__ : tk::_yield_, hi::Control);
+"yield"           L /*IS?*/ F(tk::_yield_, hi::Control);

 
 "auto"            L F(tk::_auto_, hi::Meta);
 "each"            L F(tk::_each_, hi::Control);
 
 "auto"            L F(tk::_auto_, hi::Meta);
 "each"            L F(tk::_each_, hi::Control);