Add src/system.h; this isn't really needed, but it keeps "make update"

[bison.git] / src / scan-gram.l

diff --git a/src/scan-gram.l b/src/scan-gram.l
index 02d47e5aeb5ec92f7aacf5a247e0a43083aefdbe..8f18c355f40a4301a8958dca5d985a6fbf7215ee 100644 (file)
--- a/src/scan-gram.l
+++ b/src/scan-gram.l
@@ -19,11 +19,12 @@
    02111-1307  USA
 */
 
    02111-1307  USA
 */
 

-%option debug nodefault noyywrap nounput never-interactive stack
+%option debug nodefault noyywrap never-interactive

 %option prefix="gram_" outfile="lex.yy.c"
 
 %{
 #include "system.h"
 %option prefix="gram_" outfile="lex.yy.c"
 
 %{
 #include "system.h"

+#include "mbswidth.h"

 #include "complain.h"
 #include "quote.h"
 #include "getargs.h"
 #include "complain.h"
 #include "quote.h"
 #include "getargs.h"


@@ -34,14 +35,100 @@
 #define YY_USER_INIT                           \
 do {                                           \
   LOCATION_RESET (*yylloc);                    \
 #define YY_USER_INIT                           \
 do {                                           \
   LOCATION_RESET (*yylloc);                    \

+  yylloc->file = infile;                       \

    /* This is only to avoid GCC warnings. */   \
   if (yycontrol) {;};                          \
 } while (0)
 
    /* This is only to avoid GCC warnings. */   \
   if (yycontrol) {;};                          \
 } while (0)
 

-#define YY_USER_ACTION  LOCATION_COLUMNS (*yylloc, yyleng)
-#define YY_LINES        LOCATION_LINES (*yylloc, yyleng); lineno += yyleng;
+#define YY_USER_ACTION  extend_location (yylloc, yytext, yyleng);

 #define YY_STEP         LOCATION_STEP (*yylloc)
 
 #define YY_STEP         LOCATION_STEP (*yylloc)
 

+#define YY_INPUT(buf, result, size) ((result) = no_cr_read (yyin, buf, size))
+
+
+/* Read bytes from FP into buffer BUF of size SIZE.  Return the
+   number of bytes read.  Remove '\r' from input, treating \r\n
+   and isolated \r as \n.  */
+
+static size_t
+no_cr_read (FILE *fp, char *buf, size_t size)
+{
+  size_t s = fread (buf, 1, size, fp);
+  if (s)
+    {
+      char *w = memchr (buf, '\r', s);
+      if (w)
+       {
+         char const *r = ++w;
+         char const *lim = buf + s;
+
+         for (;;)
+           {
+             /* Found an '\r'.  Treat it like '\n', but ignore any
+                '\n' that immediately follows.  */
+             w[-1] = '\n';
+             if (r == lim)
+               {
+                 int ch = getc (fp);
+                 if (ch != '\n' && ungetc (ch, fp) != ch)
+                   break;
+               }
+             else if (*r == '\n')
+               r++;
+
+             /* Copy until the next '\r'.  */
+             do
+               {
+                 if (r == lim)
+                   return w - buf;
+               }
+             while ((*w++ = *r++) != '\r');
+           }
+
+         return w - buf;
+       }
+    }
+
+  return s;
+}
+
+
+/* Extend *LOC to account for token TOKEN of size SIZE.  */
+
+static void
+extend_location (location_t *loc, char const *token, int size)
+{
+  int line = loc->last_line;
+  int column = loc->last_column;
+  char const *p0 = token;
+  char const *p = token;
+  char const *lim = token + size;
+
+  for (p = token; p < lim; p++)
+    switch (*p)
+      {
+      case '\r':
+       /* \r shouldn't survive no_cr_read.  */
+       abort ();
+
+      case '\n':
+       line++;
+       column = 1;
+       p0 = p + 1;
+       break;
+
+      case '\t':
+       column += mbsnwidth (p0, p - p0, 0);
+       column += 8 - ((column - 1) & 7);
+       p0 = p + 1;
+       break;
+      }
+
+  loc->last_line = line;
+  loc->last_column = column + mbsnwidth (p0, p - p0, 0);
+}
+
+

 
 /* STRING_OBSTACK -- Used to store all the characters that we need to
    keep (to construct ID, STRINGS etc.).  Use the following macros to
 
 /* STRING_OBSTACK -- Used to store all the characters that we need to
    keep (to construct ID, STRINGS etc.).  Use the following macros to


@@ -53,7 +140,6 @@ do {                                         \
    used, and which is used by YY_OBS_FREE to free the last string.  */
 
 static struct obstack string_obstack;
    used, and which is used by YY_OBS_FREE to free the last string.  */
 
 static struct obstack string_obstack;

-char *last_string;

 
 #define YY_OBS_GROW   \
   obstack_grow (&string_obstack, yytext, yyleng)
 
 #define YY_OBS_GROW   \
   obstack_grow (&string_obstack, yytext, yyleng)


@@ -64,53 +150,60 @@ char *last_string;
     last_string = obstack_finish (&string_obstack);    \
   } while (0)
 
     last_string = obstack_finish (&string_obstack);    \
   } while (0)
 

-#define YY_OBS_FREE                                            \
-  do {                                                         \
-    obstack_free (&string_obstack, last_string);               \
-  } while (0)
+#define YY_OBS_FREE \
+  obstack_free (&string_obstack, last_string)

 
 

-void
-scanner_last_string_free (void)
-{
-  YY_OBS_FREE;
-}

 
 

+/* Within well-formed rules, RULE_LENGTH is the number of values in
+   the current rule so far, which says where to find `$0' with respect
+   to the top of the stack.  It is not the same as the rule->length in
+   the case of mid rule actions.

 
 

-static int braces_level = 0;
-static int percent_percent_count = 0;
+   Outside of well-formed rules, RULE_LENGTH has an undefined value.  */
+static int rule_length;

 
 

-static void handle_dollar PARAMS ((braced_code_t code_kind,
-                                  char *cp, location_t location));
-static void handle_at PARAMS ((braced_code_t code_kind,
-                              char *cp, location_t location));
+static void handle_dollar (braced_code_t code_kind,
+                          char *cp, location_t location);
+static void handle_at (braced_code_t code_kind,
+                      char *cp, location_t location);
+static void handle_syncline (char *args, location_t *location);
+static int convert_ucn_to_byte (char const *hex_text);
+static void unexpected_end_of_file (location_t *, char const *);

 
 %}
 
 %}

-%x SC_COMMENT
+%x SC_COMMENT SC_LINE_COMMENT SC_YACC_COMMENT

 %x SC_STRING SC_CHARACTER
 %x SC_ESCAPED_STRING SC_ESCAPED_CHARACTER
 %x SC_BRACED_CODE SC_PROLOGUE SC_EPILOGUE
 
 %x SC_STRING SC_CHARACTER
 %x SC_ESCAPED_STRING SC_ESCAPED_CHARACTER
 %x SC_BRACED_CODE SC_PROLOGUE SC_EPILOGUE
 

-id      [.a-zA-Z][.a-zA-Z_0-9]*
-int     [0-9]+
-eols     (\n|\r|\n\r|\r\n)+
-blanks   [ \t\f]+
+letter   [.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_]
+id       {letter}({letter}|[0-9])*
+directive %{letter}({letter}|[0-9]|-)*
+int      [0-9]+
+
+/* POSIX says that a tag must be both an id and a C union member, but
+   historically almost any character is allowed in a tag.  We disallow
+   NUL and newline, as this simplifies our implementation.  */
+tag     [^\0\n>]+
+
+/* Zero or more instances of backslash-newline.  Following GCC, allow
+   white space between the backslash and the newline.  */
+splice  (\\[ \f\t\v]*\n)*

 
 %%
 %{
 
 %%
 %{

+  /* Nesting level of the current code in braces.  */
+  int braces_level IF_LINT (= 0);
+
+  /* Scanner context when scanning C code.  */
+  int c_context IF_LINT (= 0);
+
+  /* A string representing the most recently saved token.  */
+  char *last_string;
+

   /* At each yylex invocation, mark the current position as the
      start of the next token.  */
   /* At each yylex invocation, mark the current position as the
      start of the next token.  */

-#define TR_POS 0
-#if TR_POS
-  fprintf (stderr, "FOO1: %p: ", yylloc);
-  LOCATION_PRINT (stderr, *yylloc);
-  fprintf (stderr, "\n");
-#endif

   YY_STEP;
   YY_STEP;

-#if TR_POS
-  fprintf (stderr, "BAR1: ");
-  LOCATION_PRINT (stderr, *yylloc);
-  fprintf (stderr, "\n");
-#endif

 %}
 
 
 %}
 
 


@@ -124,21 +217,26 @@ blanks   [ \t\f]+
   "%define"               return PERCENT_DEFINE;
   "%defines"              return PERCENT_DEFINES;
   "%destructor"           return PERCENT_DESTRUCTOR;
   "%define"               return PERCENT_DEFINE;
   "%defines"              return PERCENT_DEFINES;
   "%destructor"           return PERCENT_DESTRUCTOR;

+  "%dprec"               return PERCENT_DPREC;

   "%error"[-_]"verbose"   return PERCENT_ERROR_VERBOSE;
   "%expect"               return PERCENT_EXPECT;
   "%file-prefix"          return PERCENT_FILE_PREFIX;
   "%fixed"[-_]"output"[-_]"files"   return PERCENT_YACC;
   "%error"[-_]"verbose"   return PERCENT_ERROR_VERBOSE;
   "%expect"               return PERCENT_EXPECT;
   "%file-prefix"          return PERCENT_FILE_PREFIX;
   "%fixed"[-_]"output"[-_]"files"   return PERCENT_YACC;

+  "%glr-parser"           return PERCENT_GLR_PARSER;

   "%left"                 return PERCENT_LEFT;
   "%locations"            return PERCENT_LOCATIONS;
   "%left"                 return PERCENT_LEFT;
   "%locations"            return PERCENT_LOCATIONS;

+  "%merge"               return PERCENT_MERGE;

   "%name"[-_]"prefix"     return PERCENT_NAME_PREFIX;
   "%no"[-_]"lines"        return PERCENT_NO_LINES;
   "%nonassoc"             return PERCENT_NONASSOC;
   "%nterm"                return PERCENT_NTERM;
   "%output"               return PERCENT_OUTPUT;
   "%name"[-_]"prefix"     return PERCENT_NAME_PREFIX;
   "%no"[-_]"lines"        return PERCENT_NO_LINES;
   "%nonassoc"             return PERCENT_NONASSOC;
   "%nterm"                return PERCENT_NTERM;
   "%output"               return PERCENT_OUTPUT;

-  "%prec"                 return PERCENT_PREC;
+  "%parse-param"          return PERCENT_PARSE_PARAM;
+  "%prec"                 rule_length--; return PERCENT_PREC;

   "%printer"              return PERCENT_PRINTER;
   "%pure"[-_]"parser"     return PERCENT_PURE_PARSER;
   "%right"                return PERCENT_RIGHT;
   "%printer"              return PERCENT_PRINTER;
   "%pure"[-_]"parser"     return PERCENT_PURE_PARSER;
   "%right"                return PERCENT_RIGHT;

+  "%lex-param"            return PERCENT_LEX_PARAM;

   "%skeleton"             return PERCENT_SKELETON;
   "%start"                return PERCENT_START;
   "%term"                 return PERCENT_TOKEN;
   "%skeleton"             return PERCENT_SKELETON;
   "%start"                return PERCENT_START;
   "%term"                 return PERCENT_TOKEN;


@@ -149,100 +247,114 @@ blanks   [ \t\f]+
   "%verbose"              return PERCENT_VERBOSE;
   "%yacc"                 return PERCENT_YACC;
 
   "%verbose"              return PERCENT_VERBOSE;
   "%yacc"                 return PERCENT_YACC;
 

+  {directive}             {
+    complain_at (*yylloc, _("invalid directive: %s"), quote (yytext));
+    YY_STEP;
+  }
+
+  ^"#line "{int}" \""[^\"]*"\"\n"   handle_syncline (yytext + strlen ("#line "), yylloc); YY_STEP;
+

   "="                     return EQUAL;
   "="                     return EQUAL;

-  ":"                     return COLON;
-  "|"                     return PIPE;
+  ":"                     rule_length = 0; return COLON;
+  "|"                     rule_length = 0; return PIPE;
+  ","                     return COMMA;

   ";"                     return SEMICOLON;
 
   ";"                     return SEMICOLON;
 

-  {eols}      YY_LINES; YY_STEP;
-  {blanks}    YY_STEP;
+  [ \f\n\t\v]  YY_STEP;
+

   {id}        {
   {id}        {

-    yylval->symbol = getsym (yytext, *yylloc);
+    yylval->symbol = symbol_get (yytext, *yylloc);
+    rule_length++;

     return ID;
   }
 
     return ID;
   }
 

-  {int}       yylval->integer = strtol (yytext, 0, 10); return INT;
+  {int} {
+    unsigned long num;
+    errno = 0;
+    num = strtoul (yytext, 0, 10);
+    if (INT_MAX < num || errno)
+      {
+       complain_at (*yylloc, _("integer out of range: %s"), quote (yytext));
+       num = INT_MAX;
+      }
+    yylval->integer = num;
+    return INT;
+  }

 
   /* Characters.  We don't check there is only one.  */
 
   /* Characters.  We don't check there is only one.  */

-  \'          YY_OBS_GROW; yy_push_state (SC_ESCAPED_CHARACTER);
+  "'"         YY_OBS_GROW; BEGIN SC_ESCAPED_CHARACTER;

 
   /* Strings. */
 
   /* Strings. */

-  \"          YY_OBS_GROW; yy_push_state (SC_ESCAPED_STRING);
+  "\""        YY_OBS_GROW; BEGIN SC_ESCAPED_STRING;

 
   /* Comments. */
 
   /* Comments. */

-  "/*"        yy_push_state (SC_COMMENT);
+  "/*"        BEGIN SC_YACC_COMMENT;

   "//".*      YY_STEP;
 
   /* Prologue. */
   "//".*      YY_STEP;
 
   /* Prologue. */

-  "%{"        yy_push_state (SC_PROLOGUE);
+  "%{"        BEGIN SC_PROLOGUE;

 
   /* Code in between braces.  */
 
   /* Code in between braces.  */

-  "{"         YY_OBS_GROW; ++braces_level; yy_push_state (SC_BRACED_CODE);
+  "{"         YY_OBS_GROW; braces_level = 0; BEGIN SC_BRACED_CODE;

 
   /* A type. */
 
   /* A type. */

-  "<"[^>]+">" {
+  "<"{tag}">" {

     obstack_grow (&string_obstack, yytext + 1, yyleng - 2);
     YY_OBS_FINISH;
     yylval->string = last_string;
     return TYPE;
   }
 
     obstack_grow (&string_obstack, yytext + 1, yyleng - 2);
     YY_OBS_FINISH;
     yylval->string = last_string;
     return TYPE;
   }
 

-
-  "%%"   {
+  "%%" {
+    static int percent_percent_count;

     if (++percent_percent_count == 2)
     if (++percent_percent_count == 2)

-      yy_push_state (SC_EPILOGUE);
+      BEGIN SC_EPILOGUE;

     return PERCENT_PERCENT;
   }
 
     return PERCENT_PERCENT;
   }
 

-  .           {
-    LOCATION_PRINT (stderr, *yylloc);
-    fprintf (stderr, ": invalid character: `%c'\n", *yytext);
+  . {
+    complain_at (*yylloc, _("invalid character: %s"), quote (yytext));

     YY_STEP;
   }
 }
 
 
     YY_STEP;
   }
 }
 
 

-  /*------------------------------------------------------------.
-  | Whatever the start condition (but those which correspond to |
-  | entity `swallowed' by Bison: SC_ESCAPED_STRING and          |
-  | SC_ESCAPED_CHARACTER), no M4 character must escape as is.   |
-  `------------------------------------------------------------*/
+  /*---------------------------------------------------------------.
+  | Scanning a Yacc comment.  The initial `/ *' is already eaten.  |
+  `---------------------------------------------------------------*/

 
 

-<SC_COMMENT,SC_STRING,SC_CHARACTER,SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>
+<SC_YACC_COMMENT>

 {
 {

-  \[          if (YY_START != SC_COMMENT) obstack_sgrow (&string_obstack, "@<:@");
-  \]          if (YY_START != SC_COMMENT) obstack_sgrow (&string_obstack, "@:>@");
-}
+  "*/" {
+    YY_STEP;
+    BEGIN INITIAL;
+  }

 
 

+  .|\n    ;
+  <<EOF>>  unexpected_end_of_file (yylloc, "*/");
+}

 
 
 
 

-  /*-----------------------------------------------------------.
-  | Scanning a C comment. The initial `/ *' is already eaten.  |
-  `-----------------------------------------------------------*/
+  /*------------------------------------------------------------.
+  | Scanning a C comment.  The initial `/ *' is already eaten.  |
+  `------------------------------------------------------------*/

 
 <SC_COMMENT>
 {
 
 <SC_COMMENT>
 {

-  "*/" { /* End of the comment. */
-    if (yy_top_state () == INITIAL)
-      {
-       YY_STEP;
-      }
-    else
-      {
-       YY_OBS_GROW;
-      }
-    yy_pop_state ();
-  }
+  "*"{splice}"/"  YY_OBS_GROW; BEGIN c_context;
+  <<EOF>>        unexpected_end_of_file (yylloc, "*/");
+}

 
 

-  [^\[\]*\n\r]+        if (yy_top_state () != INITIAL) YY_OBS_GROW;
-  {eols}       if (yy_top_state () != INITIAL) YY_OBS_GROW; YY_LINES;
-  .             /* Stray `*'. */if (yy_top_state () != INITIAL) YY_OBS_GROW;

 
 

-  <<EOF>> {
-    LOCATION_PRINT (stderr, *yylloc);
-    fprintf (stderr, ": unexpected end of file in a comment\n");
-    yy_pop_state ();
-  }
+  /*--------------------------------------------------------------.
+  | Scanning a line comment.  The initial `//' is already eaten.  |
+  `--------------------------------------------------------------*/
+
+<SC_LINE_COMMENT>
+{
+  "\n"          YY_OBS_GROW; BEGIN c_context;
+  {splice}      YY_OBS_GROW;
+  <<EOF>>       BEGIN c_context;

 }
 
 
 }
 
 


@@ -253,28 +365,17 @@ blanks   [ \t\f]+
 
 <SC_ESCAPED_STRING>
 {
 
 <SC_ESCAPED_STRING>
 {

-  \" {
-    assert (yy_top_state () == INITIAL);
+  "\"" {

     YY_OBS_GROW;
     YY_OBS_FINISH;
     yylval->string = last_string;
     YY_OBS_GROW;
     YY_OBS_FINISH;
     yylval->string = last_string;

-    yy_pop_state ();
+    rule_length++;
+    BEGIN INITIAL;

     return STRING;
   }
 
     return STRING;
   }
 

-  [^\"\n\r\\]+      YY_OBS_GROW;
-
-  {eols}    obstack_1grow (&string_obstack, '\n'); YY_LINES;
-
-  <<EOF>> {
-    LOCATION_PRINT (stderr, *yylloc);
-    fprintf (stderr, ": unexpected end of file in a string\n");
-    assert (yy_top_state () == INITIAL);
-    YY_OBS_FINISH;
-    yylval->string = last_string;
-    yy_pop_state ();
-    return STRING;
-  }
+  .|\n     YY_OBS_GROW;
+  <<EOF>>   unexpected_end_of_file (yylloc, "\"");

 }
 
   /*---------------------------------------------------------------.
 }
 
   /*---------------------------------------------------------------.


@@ -284,33 +385,21 @@ blanks   [ \t\f]+
 
 <SC_ESCAPED_CHARACTER>
 {
 
 <SC_ESCAPED_CHARACTER>
 {

-  \' {
+  "'" {

     YY_OBS_GROW;
     YY_OBS_GROW;

-    assert (yy_top_state () == INITIAL);
-    {
-      YY_OBS_FINISH;
-      yylval->symbol = getsym (last_string, *yylloc);
-      symbol_class_set (yylval->symbol, token_sym, *yylloc);
-      symbol_user_token_number_set (yylval->symbol, last_string[1], *yylloc);
-      YY_OBS_FREE;
-      yy_pop_state ();
-      return ID;
-    }
-  }
-
-  [^\'\n\r\\]      YY_OBS_GROW;
-
-  {eols}    obstack_1grow (&string_obstack, '\n'); YY_LINES;
-
-  <<EOF>> {
-    LOCATION_PRINT (stderr, *yylloc);
-    fprintf (stderr, ": unexpected end of file in a character\n");
-    assert (yy_top_state () == INITIAL);

     YY_OBS_FINISH;
     YY_OBS_FINISH;

-    yylval->string = last_string;
-    yy_pop_state ();
-    return CHARACTER;
+    yylval->symbol = symbol_get (last_string, *yylloc);
+    symbol_class_set (yylval->symbol, token_sym, *yylloc);
+    symbol_user_token_number_set (yylval->symbol,
+                                 (unsigned char) last_string[1], *yylloc);
+    YY_OBS_FREE;
+    rule_length++;
+    BEGIN INITIAL;
+    return ID;

   }
   }

+
+  .|\n     YY_OBS_GROW;
+  <<EOF>>   unexpected_end_of_file (yylloc, "'");

 }
 
 
 }
 
 


@@ -320,20 +409,30 @@ blanks   [ \t\f]+
 
 <SC_ESCAPED_STRING,SC_ESCAPED_CHARACTER>
 {
 
 <SC_ESCAPED_STRING,SC_ESCAPED_CHARACTER>
 {

-  \\[0-7]{3}           {
-    long c = strtol (yytext + 1, 0, 8);
-    if (c > 255)
+  \\[0-7]{1,3} {
+    unsigned long c = strtoul (yytext + 1, 0, 8);
+    if (UCHAR_MAX < c)

       {
       {

-       LOCATION_PRINT (stderr, *yylloc);
-       fprintf (stderr, ": invalid escape: %s\n", quote (yytext));
+       complain_at (*yylloc, _("invalid escape sequence: %s"),
+                    quote (yytext));

        YY_STEP;
       }
     else
       obstack_1grow (&string_obstack, c);
   }
 
        YY_STEP;
       }
     else
       obstack_1grow (&string_obstack, c);
   }
 

-  \\x[0-9a-fA-F]{2}    {
-    obstack_1grow (&string_obstack, strtol (yytext + 2, 0, 16));
+  \\x[0-9a-fA-F]+ {
+    unsigned long c;
+    errno = 0;
+    c = strtoul (yytext + 2, 0, 16);
+    if (UCHAR_MAX < c || errno)
+      {
+       complain_at (*yylloc, _("invalid escape sequence: %s"),
+                    quote (yytext));
+       YY_STEP;
+      }
+    else
+      obstack_1grow (&string_obstack, c);

   }
 
   \\a  obstack_1grow (&string_obstack, '\a');
   }
 
   \\a  obstack_1grow (&string_obstack, '\a');


@@ -343,14 +442,23 @@ blanks   [ \t\f]+
   \\r  obstack_1grow (&string_obstack, '\r');
   \\t  obstack_1grow (&string_obstack, '\t');
   \\v  obstack_1grow (&string_obstack, '\v');
   \\r  obstack_1grow (&string_obstack, '\r');
   \\t  obstack_1grow (&string_obstack, '\t');
   \\v  obstack_1grow (&string_obstack, '\v');

-  \\[\\""]   obstack_1grow (&string_obstack, yytext[1]);
+  \\[\"\'?\\]  obstack_1grow (&string_obstack, yytext[1]);
+  \\(u|U[0-9a-fA-F]{4})[0-9a-fA-F]{4} {
+    int c = convert_ucn_to_byte (yytext);
+    if (c < 0)
+      {
+       complain_at (*yylloc, _("invalid escape sequence: %s"),
+                    quote (yytext));
+       YY_STEP;
+      }
+    else
+      obstack_1grow (&string_obstack, c);
+  }

   \\(.|\n)     {
   \\(.|\n)     {

-    LOCATION_PRINT (stderr, *yylloc);
-    fprintf (stderr, ": unrecognized escape: %s\n", quote (yytext));
+    complain_at (*yylloc, _("unrecognized escape sequence: %s"),
+                quote (yytext));

     YY_OBS_GROW;
   }
     YY_OBS_GROW;
   }

-  /* FLex wants this rule, in case of a `\<<EOF>>'. */
-  \\                   YY_OBS_GROW;

 }
 
 
 }
 
 


@@ -361,25 +469,9 @@ blanks   [ \t\f]+
 
 <SC_CHARACTER>
 {
 
 <SC_CHARACTER>
 {

-  \' {
-    YY_OBS_GROW;
-    assert (yy_top_state () != INITIAL);
-    yy_pop_state ();
-  }
-
-  [^\[\]\'\n\r\\]+     YY_OBS_GROW;
-  \\(.|\n)             YY_OBS_GROW;
-  /* FLex wants this rule, in case of a `\<<EOF>>'. */
-  \\                   YY_OBS_GROW;
-
-  {eols}               YY_OBS_GROW; YY_LINES;
-
-  <<EOF>> {
-    LOCATION_PRINT (stderr, *yylloc);
-    fprintf (stderr, ": unexpected end of file in a character\n");
-    assert (yy_top_state () != INITIAL);
-    yy_pop_state ();
-  }
+  "'"              YY_OBS_GROW; BEGIN c_context;
+  \\{splice}[\'\\]  YY_OBS_GROW;
+  <<EOF>>          unexpected_end_of_file (yylloc, "'");

 }
 
 
 }
 
 


@@ -390,25 +482,9 @@ blanks   [ \t\f]+
 
 <SC_STRING>
 {
 
 <SC_STRING>
 {

-  \" {
-    assert (yy_top_state () != INITIAL);
-    YY_OBS_GROW;
-    yy_pop_state ();
-  }
-
-  [^\[\]\"\n\r\\]+      YY_OBS_GROW;
-  \\(.|\n)              YY_OBS_GROW;
-  /* FLex wants this rule, in case of a `\<<EOF>>'. */
-  \\                   YY_OBS_GROW;
-
-  {eols}                YY_OBS_GROW; YY_LINES;
-
-  <<EOF>> {
-    LOCATION_PRINT (stderr, *yylloc);
-    fprintf (stderr, ": unexpected end of file in a string\n");
-    assert (yy_top_state () != INITIAL);
-    yy_pop_state ();
-  }
+  "\""             YY_OBS_GROW; BEGIN c_context;
+  \\{splice}[\"\\]  YY_OBS_GROW;
+  <<EOF>>          unexpected_end_of_file (yylloc, "\"");

 }
 
 
 }
 
 


@@ -418,18 +494,10 @@ blanks   [ \t\f]+
 
 <SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>
 {
 
 <SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>
 {

-  /* Characters.  We don't check there is only one.  */
-  \'          YY_OBS_GROW; yy_push_state (SC_CHARACTER);
-
-  /* Strings. */
-  \"          YY_OBS_GROW; yy_push_state (SC_STRING);
-
-  /* Comments. */
-  "/*"        YY_OBS_GROW; yy_push_state (SC_COMMENT);
-  "//".*      YY_OBS_GROW;
-
-  /* Not comments. */
-  "/"         YY_OBS_GROW;
+  "'"            YY_OBS_GROW; c_context = YY_START; BEGIN SC_CHARACTER;
+  "\""           YY_OBS_GROW; c_context = YY_START; BEGIN SC_STRING;
+  "/"{splice}"*"  YY_OBS_GROW; c_context = YY_START; BEGIN SC_COMMENT;
+  "/"{splice}"/"  YY_OBS_GROW; c_context = YY_START; BEGIN SC_LINE_COMMENT;

 }
 
 
 }
 
 


@@ -440,39 +508,31 @@ blanks   [ \t\f]+
 
 <SC_BRACED_CODE>
 {
 
 <SC_BRACED_CODE>
 {

+  "{"|"<"{splice}"%"  YY_OBS_GROW; braces_level++;
+  "%"{splice}">"      YY_OBS_GROW; braces_level--;

   "}" {
     YY_OBS_GROW;
   "}" {
     YY_OBS_GROW;

-    if (--braces_level == 0)
+    braces_level--;
+    if (braces_level < 0)

       {
       {

-       yy_pop_state ();

        YY_OBS_FINISH;
        yylval->string = last_string;
        YY_OBS_FINISH;
        yylval->string = last_string;

+       rule_length++;
+       BEGIN INITIAL;

        return BRACED_CODE;
       }
   }
 
        return BRACED_CODE;
       }
   }
 

-  "{"                  YY_OBS_GROW; braces_level++;
+  /* Tokenize `<<%' correctly (as `<<' `%') rather than incorrrectly
+     (as `<' `<%').  */
+  "<"{splice}"<"  YY_OBS_GROW;

 
 

-  "$"("<"[^>]+">")?(-?[0-9]+|"$") { handle_dollar (current_braced_code,
+  "$"("<"{tag}">")?(-?[0-9]+|"$") { handle_dollar (current_braced_code,

                                                   yytext, *yylloc); }
   "@"(-?[0-9]+|"$")               { handle_at (current_braced_code,
                                               yytext, *yylloc); }
 
                                                   yytext, *yylloc); }
   "@"(-?[0-9]+|"$")               { handle_at (current_braced_code,
                                               yytext, *yylloc); }
 

-  [^$@\[\]/\'\"\{\}\n\r]+ YY_OBS_GROW;
-  {eols}       YY_OBS_GROW; YY_LINES;
-
-  /* A lose $, or /, or etc. */
-  .             YY_OBS_GROW;
-
-  <<EOF>> {
-    LOCATION_PRINT (stderr, *yylloc);
-    fprintf (stderr, ": unexpected end of file in a braced code\n");
-    yy_pop_state ();
-    YY_OBS_FINISH;
-    yylval->string = last_string;
-    return BRACED_CODE;
-  }
-
+  <<EOF>>  unexpected_end_of_file (yylloc, "}");

 }
 
 
 }
 
 


@@ -483,45 +543,47 @@ blanks   [ \t\f]+
 <SC_PROLOGUE>
 {
   "%}" {
 <SC_PROLOGUE>
 {
   "%}" {

-    yy_pop_state ();

     YY_OBS_FINISH;
     yylval->string = last_string;
     YY_OBS_FINISH;
     yylval->string = last_string;

+    BEGIN INITIAL;

     return PROLOGUE;
   }
 
     return PROLOGUE;
   }
 

-  [^%\[\]/\'\"\n\r]+ YY_OBS_GROW;
-  "%"                YY_OBS_GROW;
-  {eols}            YY_OBS_GROW; YY_LINES;
-
-  <<EOF>> {
-    LOCATION_PRINT (stderr, *yylloc);
-    fprintf (stderr, ": unexpected end of file in a prologue\n");
-    yy_pop_state ();
-    YY_OBS_FINISH;
-    yylval->string = last_string;
-    return PROLOGUE;
-  }
+  <<EOF>>  unexpected_end_of_file (yylloc, "%}");

 }
 
 
   /*---------------------------------------------------------------.
   | Scanning the epilogue (everything after the second "%%", which |
 }
 
 
   /*---------------------------------------------------------------.
   | Scanning the epilogue (everything after the second "%%", which |

-  | has already been eaten.                                        |
+  | has already been eaten).                                       |

   `---------------------------------------------------------------*/
 
 <SC_EPILOGUE>
 {
   `---------------------------------------------------------------*/
 
 <SC_EPILOGUE>
 {

-  ([^\[\]]|{eols})+  YY_OBS_GROW;
-

   <<EOF>> {
   <<EOF>> {

-    yy_pop_state ();

     YY_OBS_FINISH;
     yylval->string = last_string;
     YY_OBS_FINISH;
     yylval->string = last_string;

+    BEGIN INITIAL;

     return EPILOGUE;
   }
 }
 
 
     return EPILOGUE;
   }
 }
 
 

+  /*----------------------------------------------------------------.
+  | By default, grow the string obstack with the input, escaping M4 |
+  | quoting characters.                                                    |
+  `----------------------------------------------------------------*/
+
+<SC_COMMENT,SC_LINE_COMMENT,SC_STRING,SC_CHARACTER,SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>
+{
+  \$   obstack_sgrow (&string_obstack, "$][");
+  \@   obstack_sgrow (&string_obstack, "@@");
+  \[   obstack_sgrow (&string_obstack, "@{");
+  \]   obstack_sgrow (&string_obstack, "@}");
+  .|\n  YY_OBS_GROW;
+}
+
+

 %%
 
 /*------------------------------------------------------------------.
 %%
 
 /*------------------------------------------------------------------.


@@ -554,47 +616,41 @@ handle_action_dollar (char *text, location_t location)
        type_name = symbol_list_n_type_name_get (current_rule, location, 0);
       if (!type_name && typed)
        complain_at (location, _("$$ of `%s' has no declared type"),
        type_name = symbol_list_n_type_name_get (current_rule, location, 0);
       if (!type_name && typed)
        complain_at (location, _("$$ of `%s' has no declared type"),

-                    symbol_tag_get (current_rule->sym));
+                    current_rule->sym->tag);

       if (!type_name)
        type_name = "";
       obstack_fgrow1 (&string_obstack,
                      "]b4_lhs_value([%s])[", type_name);
     }
       if (!type_name)
        type_name = "";
       obstack_fgrow1 (&string_obstack,
                      "]b4_lhs_value([%s])[", type_name);
     }

-  else if (isdigit (*cp) || *cp == '-')
+  else

     {
     {

-      /* RULE_LENGTH is the number of values in the current rule so
-        far, which says where to find `$0' with respect to the top of
-        the stack.  It is not the same as the rule->length in the
-        case of mid rule actions.  */
-      int rule_length = symbol_list_length (current_rule->next);
-      int n = strtol (cp, &cp, 10);
-
-      if (n > rule_length)
-       complain_at (location, _("invalid value: %s%d"), "$", n);
-      else
+      long num;
+      errno = 0;
+      num = strtol (cp, 0, 10);
+
+      if (INT_MIN <= num && num <= rule_length && ! errno)

        {
        {

+         int n = num;

          if (!type_name && n > 0)
            type_name = symbol_list_n_type_name_get (current_rule, location,
                                                     n);
          if (!type_name && typed)
            complain_at (location, _("$%d of `%s' has no declared type"),
          if (!type_name && n > 0)
            type_name = symbol_list_n_type_name_get (current_rule, location,
                                                     n);
          if (!type_name && typed)
            complain_at (location, _("$%d of `%s' has no declared type"),

-                     n, symbol_tag_get (current_rule->sym));
+                     n, current_rule->sym->tag);

          if (!type_name)
            type_name = "";
          obstack_fgrow3 (&string_obstack,
                          "]b4_rhs_value([%d], [%d], [%s])[",
                          rule_length, n, type_name);
        }
          if (!type_name)
            type_name = "";
          obstack_fgrow3 (&string_obstack,
                          "]b4_rhs_value([%d], [%d], [%s])[",
                          rule_length, n, type_name);
        }

-    }
-  else
-    {
-      complain_at (location, _("%s is invalid"), quote (text));
+      else
+       complain_at (location, _("integer out of range: %s"), quote (text));

     }
 }
 
 
 /*---------------------------------------------------------------.
     }
 }
 
 
 /*---------------------------------------------------------------.

-| TEXT is expexted tp be $$ in some code associated to a symbol: |
+| TEXT is expected to be $$ in some code associated to a symbol: |

 | destructor or printer.                                         |
 `---------------------------------------------------------------*/
 
 | destructor or printer.                                         |
 `---------------------------------------------------------------*/
 


@@ -605,7 +661,7 @@ handle_symbol_code_dollar (char *text, location_t location)
   if (*cp == '$')
     obstack_sgrow (&string_obstack, "]b4_dollar_dollar[");
   else
   if (*cp == '$')
     obstack_sgrow (&string_obstack, "]b4_dollar_dollar[");
   else

-    complain_at (location, _("%s is invalid"), quote (text));
+    complain_at (location, _("invalid value: %s"), quote (text));

 }
 
 
 }
 
 


@@ -647,30 +703,26 @@ handle_action_at (char *text, location_t location)
     {
       obstack_sgrow (&string_obstack, "]b4_lhs_location[");
     }
     {
       obstack_sgrow (&string_obstack, "]b4_lhs_location[");
     }

-  else if (isdigit (*cp) || *cp == '-')
-    {
-      /* RULE_LENGTH is the number of values in the current rule so
-        far, which says where to find `$0' with respect to the top of
-        the stack.  It is not the same as the rule->length in the
-        case of mid rule actions.  */
-      int rule_length = symbol_list_length (current_rule->next);
-      int n = strtol (cp, &cp, 10);
-
-      if (n > rule_length)
-       complain_at (location, _("invalid value: %s%d"), "@", n);
-      else
-       obstack_fgrow2 (&string_obstack, "]b4_rhs_location([%d], [%d])[",
-                       rule_length, n);
-    }

   else
     {
   else
     {

-      complain_at (location, _("%s is invalid"), quote (text));
+      long num;
+      errno = 0;
+      num = strtol (cp, 0, 10);
+
+      if (INT_MIN <= num && num <= rule_length && ! errno)
+       {
+         int n = num;
+         obstack_fgrow2 (&string_obstack, "]b4_rhs_location([%d], [%d])[",
+                         rule_length, n);
+       }
+      else
+       complain_at (location, _("integer out of range: %s"), quote (text));

     }
 }
 
 
 /*---------------------------------------------------------------.
     }
 }
 
 
 /*---------------------------------------------------------------.

-| TEXT is expexted tp be @$ in some code associated to a symbol: |
+| TEXT is expected to be @$ in some code associated to a symbol: |

 | destructor or printer.                                         |
 `---------------------------------------------------------------*/
 
 | destructor or printer.                                         |
 `---------------------------------------------------------------*/
 


@@ -681,7 +733,7 @@ handle_symbol_code_at (char *text, location_t location)
   if (*cp == '$')
     obstack_sgrow (&string_obstack, "]b4_at_dollar[");
   else
   if (*cp == '$')
     obstack_sgrow (&string_obstack, "]b4_at_dollar[");
   else

-    complain_at (location, _("%s is invalid"), quote (text));
+    complain_at (location, _("invalid value: %s"), quote (text));

 }
 
 
 }
 
 


@@ -708,6 +760,103 @@ handle_at (braced_code_t braced_code_kind,
 }
 
 
 }
 
 

+/*------------------------------------------------------------------.
+| Convert universal character name UCN to a single-byte character,  |
+| and return that character.  Return -1 if UCN does not correspond  |
+| to a single-byte character.                                      |
+`------------------------------------------------------------------*/
+
+static int
+convert_ucn_to_byte (char const *ucn)
+{
+  unsigned long code = strtoul (ucn + 2, 0, 16);
+
+  /* FIXME: Currently we assume Unicode-compatible unibyte characters
+     on ASCII hosts (i.e., Latin-1 on hosts with 8-bit bytes).  On
+     non-ASCII hosts we support only the portable C character set.
+     These limitations should be removed once we add support for
+     multibyte characters.  */
+
+  if (UCHAR_MAX < code)
+    return -1;
+
+#if ! ('$' == 0x24 && '@' == 0x40 && '`' == 0x60 && '~' == 0x7e)
+  {
+    /* A non-ASCII host.  Use CODE to index into a table of the C
+       basic execution character set, which is guaranteed to exist on
+       all Standard C platforms.  This table also includes '$', '@',
+       and '`', which are not in the basic execution character set but
+       which are unibyte characters on all the platforms that we know
+       about.  */
+    static signed char const table[] =
+      {
+       '\0',   -1,   -1,   -1,   -1,   -1,   -1, '\a',
+       '\b', '\t', '\n', '\v', '\f', '\r',   -1,   -1,
+         -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+         -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
+        ' ',  '!',  '"',  '#',  '$',  '%',  '&', '\'',
+        '(',  ')',  '*',  '+',  ',',  '-',  '.',  '/',
+        '0',  '1',  '2',  '3',  '4',  '5',  '6',  '7',
+        '8',  '9',  ':',  ';',  '<',  '=',  '>',  '?',
+        '@',  'A',  'B',  'C',  'D',  'E',  'F',  'G',
+        'H',  'I',  'J',  'K',  'L',  'M',  'N',  'O',
+        'P',  'Q',  'R',  'S',  'T',  'U',  'V',  'W',
+        'X',  'Y',  'Z',  '[', '\\',  ']',  '^',  '_',
+        '`',  'a',  'b',  'c',  'd',  'e',  'f',  'g',
+        'h',  'i',  'j',  'k',  'l',  'm',  'n',  'o',
+        'p',  'q',  'r',  's',  't',  'u',  'v',  'w',
+        'x',  'y',  'z',  '{',  '|',  '}',  '~'
+      };
+
+    code = code < sizeof table ? table[code] : -1;
+  }
+#endif
+
+  return code;
+}
+
+
+/*----------------------------------------------------------------.
+| Handle `#line INT "FILE"'.  ARGS has already skipped `#line '.  |
+`----------------------------------------------------------------*/
+
+static void
+handle_syncline (char *args, location_t *location)
+{
+  int lineno = strtol (args, &args, 10);
+  const char *file = NULL;
+  file = strchr (args, '"') + 1;
+  *strchr (file, '"') = 0;
+  /* FIXME: Leaking...  Can't free, as some locations are still
+     pointing to the old file name.  */
+  infile = xstrdup (file);
+  location->file = infile;
+  location->last_line = lineno;
+}
+
+
+/*-------------------------------------------------------------.
+| Report an unexpected end of file at LOC.  An end of file was |
+| encountered and the expected TOKEN_END was missing.  After   |
+| reporting the problem, pretend that TOKEN_END was found.     |
+`-------------------------------------------------------------*/
+
+static void
+unexpected_end_of_file (location_t *loc, char const *token_end)
+{
+  size_t i = strlen (token_end);
+
+  complain_at (*loc, _("missing `%s' at end of file"), token_end);
+
+  /* Adjust location's last column so that any later message does not
+     mention the characters just inserted.  */
+  loc->last_column -= i;
+
+  while (i != 0)
+    unput (token_end[--i]);
+}
+
+

 /*-------------------------.
 | Initialize the scanner.  |
 `-------------------------*/
 /*-------------------------.
 | Initialize the scanner.  |
 `-------------------------*/


@@ -727,4 +876,6 @@ void
 scanner_free (void)
 {
   obstack_free (&string_obstack, 0);
 scanner_free (void)
 {
   obstack_free (&string_obstack, 0);

+  /* Reclaim Flex's buffers.  */
+  yy_delete_buffer (YY_CURRENT_BUFFER);

 }
 }