identifiers: dashes are letters.

[bison.git] / src / scan-gram.l

diff --git a/src/scan-gram.l b/src/scan-gram.l
index d5d92c4c51e439f87f6e3d2f09e4c8d4ad969c87..5d4b4b306f20746ba1885b6487c3b6ca15b19d12 100644 (file)
--- a/src/scan-gram.l
+++ b/src/scan-gram.l
@@ -1,12 +1,13 @@
 /* Bison Grammar Scanner                             -*- C -*-
 
 /* Bison Grammar Scanner                             -*- C -*-
 

-   Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
+   Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009
+   Free Software Foundation, Inc.

 
    This file is part of Bison, the GNU Compiler Compiler.
 
 
    This file is part of Bison, the GNU Compiler Compiler.
 

-   This program is free software; you can redistribute it and/or modify
+   This program is free software: you can redistribute it and/or modify

    it under the terms of the GNU General Public License as published by
    it under the terms of the GNU General Public License as published by

-   the Free Software Foundation; either version 2 of the License, or
+   the Free Software Foundation, either version 3 of the License, or

    (at your option) any later version.
 
    This program is distributed in the hope that it will be useful,
    (at your option) any later version.
 
    This program is distributed in the hope that it will be useful,


@@ -15,38 +16,31 @@
    GNU General Public License for more details.
 
    You should have received a copy of the GNU General Public License
    GNU General Public License for more details.
 
    You should have received a copy of the GNU General Public License

-   along with this program; if not, write to the Free Software
-   Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
-   02110-1301  USA
-*/
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.  */

 
 %option debug nodefault nounput noyywrap never-interactive
 %option prefix="gram_" outfile="lex.yy.c"
 
 %{
 
 %option debug nodefault nounput noyywrap never-interactive
 %option prefix="gram_" outfile="lex.yy.c"
 
 %{

-#include <config.h>
-#include "system.h"
-

 /* Work around a bug in flex 2.5.31.  See Debian bug 333231
    <http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=333231>.  */
 #undef gram_wrap
 #define gram_wrap() 1
 
 #define FLEX_PREFIX(Id) gram_ ## Id
 /* Work around a bug in flex 2.5.31.  See Debian bug 333231
    <http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=333231>.  */
 #undef gram_wrap
 #define gram_wrap() 1
 
 #define FLEX_PREFIX(Id) gram_ ## Id

-#include "flex-scanner.h"
+#include <src/flex-scanner.h>

 
 

-#include "complain.h"
-#include "files.h"
-#include "getargs.h"    /* yacc_flag */
-#include "gram.h"
-#include "quotearg.h"
-#include "reader.h"
-#include "uniqstr.h"
+#include <src/complain.h>
+#include <src/files.h>
+#include <src/gram.h>
+#include <quotearg.h>
+#include <src/reader.h>
+#include <src/uniqstr.h>

 
 #include <mbswidth.h>
 #include <quote.h>
 
 
 #include <mbswidth.h>
 #include <quote.h>
 

-#include "scan-gram.h"
+#include <src/scan-gram.h>

 
 #define YY_DECL GRAM_LEX_DECL
 
 
 #define YY_DECL GRAM_LEX_DECL
 


@@ -61,8 +55,15 @@ static boundary scanner_cursor;
 static size_t no_cr_read (FILE *, char *, size_t);
 #define YY_INPUT(buf, result, size) ((result) = no_cr_read (yyin, buf, size))
 
 static size_t no_cr_read (FILE *, char *, size_t);
 #define YY_INPUT(buf, result, size) ((result) = no_cr_read (yyin, buf, size))
 

+#define RETURN_PERCENT_FLAG(Value)              \
+  do {                                          \
+    val->uniqstr = uniqstr_new (Value);         \
+    return PERCENT_FLAG;                        \
+  } while (0)
+
+

 /* A string representing the most recently saved token.  */
 /* A string representing the most recently saved token.  */

-char *last_string;
+static char *last_string;

 
 void
 gram_scanner_last_string_free (void)
 
 void
 gram_scanner_last_string_free (void)


@@ -70,10 +71,6 @@ gram_scanner_last_string_free (void)
   STRING_FREE;
 }
 
   STRING_FREE;
 }
 

-/* The location of the most recently saved token, if it was a
-   BRACED_CODE token; otherwise, this has an unspecified value.  */
-location gram_last_braced_code_loc;
-

 static void handle_syncline (char *, location);
 static unsigned long int scan_integer (char const *p, int base, location loc);
 static int convert_ucn_to_byte (char const *hex_text);
 static void handle_syncline (char *, location);
 static unsigned long int scan_integer (char const *p, int base, location loc);
 static int convert_ucn_to_byte (char const *hex_text);


@@ -88,6 +85,8 @@ static void unexpected_newline (boundary, char const *);
  /* A identifier was just read in directives/rules.  Special state
     to capture the sequence `identifier :'. */
 %x SC_AFTER_IDENTIFIER
  /* A identifier was just read in directives/rules.  Special state
     to capture the sequence `identifier :'. */
 %x SC_AFTER_IDENTIFIER

+ /* A complex tag, with nested angles brackets. */
+%x SC_TAG

 
  /* Three types of user code:
     - prologue (code between `%{' `%}' in the first section, before %%);
 
  /* Three types of user code:
     - prologue (code between `%{' `%}' in the first section, before %%);


@@ -99,15 +98,17 @@ static void unexpected_newline (boundary, char const *);
  /* Strings and characters in code. */
 %x SC_STRING SC_CHARACTER
 
  /* Strings and characters in code. */
 %x SC_STRING SC_CHARACTER
 

-letter   [.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_]
+letter   [-.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_]

 id       {letter}({letter}|[0-9])*
 id       {letter}({letter}|[0-9])*

-directive %{letter}({letter}|[0-9]|-)*
+directive %{id}

 int      [0-9]+
 
 /* POSIX says that a tag must be both an id and a C union member, but
    historically almost any character is allowed in a tag.  We disallow
 int      [0-9]+
 
 /* POSIX says that a tag must be both an id and a C union member, but
    historically almost any character is allowed in a tag.  We disallow

-   NUL and newline, as this simplifies our implementation.  */
-tag     [^\0\n>]+
+   NUL, as this simplifies our implementation.  We disallow angle
+   bracket to match them in nested pairs: several languages use them
+   for generics/template types.  */
+tag     [^\0<>]+

 
 /* Zero or more instances of backslash-newline.  Following GCC, allow
    white space between the backslash and the newline.  */
 
 /* Zero or more instances of backslash-newline.  Following GCC, allow
    white space between the backslash and the newline.  */


@@ -115,8 +116,9 @@ splice       (\\[ \f\t\v]*\n)*
 
 %%
 %{
 
 %%
 %{

-  /* Nesting level of the current code in braces.  */
-  int braces_level IF_LINT (= 0);
+  /* Nesting level.  Either for nested braces, or nested angle brackets
+     (but not mixed).  */
+  int nesting IF_LINT (= 0);

 
   /* Parent context state, when applicable.  */
   int context_state IF_LINT (= 0);
 
   /* Parent context state, when applicable.  */
   int context_state IF_LINT (= 0);


@@ -162,52 +164,58 @@ splice     (\\[ \f\t\v]*\n)*
   /*----------------------------.
   | Scanning Bison directives.  |
   `----------------------------*/
   /*----------------------------.
   | Scanning Bison directives.  |
   `----------------------------*/

+
+  /* For directives that are also command line options, the regex must be
+       "%..."
+     after "[-_]"s are removed, and the directive must match the --long
+     option name, with a single string argument.  Otherwise, add exceptions
+     to ../build-aux/cross-options.pl.  */
+

 <INITIAL>
 {
 <INITIAL>
 {

-  "%after-header"                   return PERCENT_AFTER_HEADER;
-  "%before-header"                  return PERCENT_BEFORE_HEADER;
-  "%binary"                        return PERCENT_NONASSOC;
-  "%debug"                         return PERCENT_DEBUG;
-  "%default"[-_]"prec"             return PERCENT_DEFAULT_PREC;
-  "%define"                        return PERCENT_DEFINE;
-  "%defines"                       return PERCENT_DEFINES;
-  "%destructor"                            return PERCENT_DESTRUCTOR;
-  "%dprec"                         return PERCENT_DPREC;
-  "%end-header"                     return PERCENT_END_HEADER;
-  "%error"[-_]"verbose"            return PERCENT_ERROR_VERBOSE;
-  "%expect"                        return PERCENT_EXPECT;
-  "%expect"[-_]"rr"                return PERCENT_EXPECT_RR;
-  "%file-prefix"                   return PERCENT_FILE_PREFIX;
+  "%binary"                         return PERCENT_NONASSOC;
+  "%code"                           return PERCENT_CODE;
+  "%debug"                          RETURN_PERCENT_FLAG("parse.trace");
+  "%default"[-_]"prec"              return PERCENT_DEFAULT_PREC;
+  "%define"                         return PERCENT_DEFINE;
+  "%defines"                        return PERCENT_DEFINES;
+  "%destructor"                     return PERCENT_DESTRUCTOR;
+  "%dprec"                          return PERCENT_DPREC;
+  "%error"[-_]"verbose"             RETURN_PERCENT_FLAG("error-verbose");
+  "%expect"                         return PERCENT_EXPECT;
+  "%expect"[-_]"rr"                 return PERCENT_EXPECT_RR;
+  "%file-prefix"                    return PERCENT_FILE_PREFIX;

   "%fixed"[-_]"output"[-_]"files"   return PERCENT_YACC;
   "%fixed"[-_]"output"[-_]"files"   return PERCENT_YACC;

-  "%initial-action"                return PERCENT_INITIAL_ACTION;
-  "%glr-parser"                    return PERCENT_GLR_PARSER;
-  "%left"                          return PERCENT_LEFT;
-  "%lex-param"                     return PERCENT_LEX_PARAM;
-  "%locations"                     return PERCENT_LOCATIONS;
-  "%merge"                         return PERCENT_MERGE;
-  "%name"[-_]"prefix"              return PERCENT_NAME_PREFIX;
-  "%no"[-_]"default"[-_]"prec"     return PERCENT_NO_DEFAULT_PREC;
-  "%no"[-_]"lines"                 return PERCENT_NO_LINES;
-  "%nonassoc"                      return PERCENT_NONASSOC;
-  "%nondeterministic-parser"               return PERCENT_NONDETERMINISTIC_PARSER;
-  "%nterm"                         return PERCENT_NTERM;
-  "%output"                        return PERCENT_OUTPUT;
-  "%parse-param"                   return PERCENT_PARSE_PARAM;
-  "%prec"                          return PERCENT_PREC;
-  "%printer"                       return PERCENT_PRINTER;
-  "%pure"[-_]"parser"              return PERCENT_PURE_PARSER;
-  "%require"                       return PERCENT_REQUIRE;
-  "%right"                         return PERCENT_RIGHT;
-  "%skeleton"                      return PERCENT_SKELETON;
-  "%start"                         return PERCENT_START;
-  "%start-header"                   return PERCENT_START_HEADER;
-  "%term"                          return PERCENT_TOKEN;
-  "%token"                         return PERCENT_TOKEN;
-  "%token"[-_]"table"              return PERCENT_TOKEN_TABLE;
-  "%type"                          return PERCENT_TYPE;
-  "%union"                         return PERCENT_UNION;
-  "%verbose"                       return PERCENT_VERBOSE;
-  "%yacc"                          return PERCENT_YACC;
+  "%initial-action"                 return PERCENT_INITIAL_ACTION;
+  "%glr-parser"                     return PERCENT_GLR_PARSER;
+  "%language"                       return PERCENT_LANGUAGE;
+  "%left"                           return PERCENT_LEFT;
+  "%lex-param"                      return PERCENT_LEX_PARAM;
+  "%locations"                      RETURN_PERCENT_FLAG("locations");
+  "%merge"                          return PERCENT_MERGE;
+  "%name"[-_]"prefix"               return PERCENT_NAME_PREFIX;
+  "%no"[-_]"default"[-_]"prec"      return PERCENT_NO_DEFAULT_PREC;
+  "%no"[-_]"lines"                  return PERCENT_NO_LINES;
+  "%nonassoc"                       return PERCENT_NONASSOC;
+  "%nondeterministic-parser"        return PERCENT_NONDETERMINISTIC_PARSER;
+  "%nterm"                          return PERCENT_NTERM;
+  "%output"                         return PERCENT_OUTPUT;
+  "%parse-param"                    return PERCENT_PARSE_PARAM;
+  "%prec"                           return PERCENT_PREC;
+  "%precedence"                     return PERCENT_PRECEDENCE;
+  "%printer"                        return PERCENT_PRINTER;
+  "%pure"[-_]"parser"               RETURN_PERCENT_FLAG("api.pure");
+  "%require"                        return PERCENT_REQUIRE;
+  "%right"                          return PERCENT_RIGHT;
+  "%skeleton"                       return PERCENT_SKELETON;
+  "%start"                          return PERCENT_START;
+  "%term"                           return PERCENT_TOKEN;
+  "%token"                          return PERCENT_TOKEN;
+  "%token"[-_]"table"               return PERCENT_TOKEN_TABLE;
+  "%type"                           return PERCENT_TYPE;
+  "%union"                          return PERCENT_UNION;
+  "%verbose"                        return PERCENT_VERBOSE;
+  "%yacc"                           return PERCENT_YACC;

 
   {directive} {
     complain_at (*loc, _("invalid directive: %s"), quote (yytext));
 
   {directive} {
     complain_at (*loc, _("invalid directive: %s"), quote (yytext));


@@ -244,18 +252,25 @@ splice     (\\[ \f\t\v]*\n)*
   /* Code in between braces.  */
   "{" {
     STRING_GROW;
   /* Code in between braces.  */
   "{" {
     STRING_GROW;

-    braces_level = 0;
+    nesting = 0;

     code_start = loc->start;
     BEGIN SC_BRACED_CODE;
   }
 
   /* A type. */
     code_start = loc->start;
     BEGIN SC_BRACED_CODE;
   }
 
   /* A type. */

+  "<*>"       return TAG_ANY;
+  "<>"        return TAG_NONE;

   "<"{tag}">" {
     obstack_grow (&obstack_for_string, yytext + 1, yyleng - 2);
     STRING_FINISH;
     val->uniqstr = uniqstr_new (last_string);
     STRING_FREE;
   "<"{tag}">" {
     obstack_grow (&obstack_for_string, yytext + 1, yyleng - 2);
     STRING_FINISH;
     val->uniqstr = uniqstr_new (last_string);
     STRING_FREE;

-    return TYPE;
+    return TAG;
+  }
+  "<"         {
+    nesting = 0;
+    token_start = loc->start;
+    BEGIN SC_TAG;

   }
 
   "%%" {
   }
 
   "%%" {


@@ -276,6 +291,17 @@ splice      (\\[ \f\t\v]*\n)*
 }
 
 
 }
 
 

+  /*--------------------------------------------------------------.
+  | Supporting \0 complexifies our implementation for no expected |
+  | added value.                                                  |
+  `--------------------------------------------------------------*/
+
+<SC_ESCAPED_CHARACTER,SC_ESCAPED_STRING,SC_TAG>
+{
+  \0       complain_at (*loc, _("invalid null character"));
+}
+
+

   /*-----------------------------------------------------------------.
   | Scanning after an identifier, checking whether a colon is next.  |
   `-----------------------------------------------------------------*/
   /*-----------------------------------------------------------------.
   | Scanning after an identifier, checking whether a colon is next.  |
   `-----------------------------------------------------------------*/


@@ -344,15 +370,23 @@ splice     (\\[ \f\t\v]*\n)*
 
 <SC_ESCAPED_STRING>
 {
 
 <SC_ESCAPED_STRING>
 {

-  "\"" {
+  "\""|"\n" {
+    if (yytext[0] == '\n')
+      unexpected_newline (token_start, "\"");
+    STRING_FINISH;
+    loc->start = token_start;
+    val->chars = last_string;
+    BEGIN INITIAL;
+    return STRING;
+  }
+  <<EOF>> {
+    unexpected_eof (token_start, "\"");

     STRING_FINISH;
     loc->start = token_start;
     val->chars = last_string;
     BEGIN INITIAL;
     return STRING;
   }
     STRING_FINISH;
     loc->start = token_start;
     val->chars = last_string;
     BEGIN INITIAL;
     return STRING;
   }

-  \n           unexpected_newline (token_start, "\""); BEGIN INITIAL;
-  <<EOF>>      unexpected_eof (token_start, "\"");     BEGIN INITIAL;

 }
 
   /*----------------------------------------------------------.
 }
 
   /*----------------------------------------------------------.


@@ -362,7 +396,9 @@ splice       (\\[ \f\t\v]*\n)*
 
 <SC_ESCAPED_CHARACTER>
 {
 
 <SC_ESCAPED_CHARACTER>
 {

-  "'" {
+  "'"|"\n" {
+    if (yytext[0] == '\n')
+      unexpected_newline (token_start, "'");

     STRING_GROW;
     STRING_FINISH;
     loc->start = token_start;
     STRING_GROW;
     STRING_FINISH;
     loc->start = token_start;


@@ -371,15 +407,54 @@ splice     (\\[ \f\t\v]*\n)*
     BEGIN INITIAL;
     return CHAR;
   }
     BEGIN INITIAL;
     return CHAR;
   }

-  \n           unexpected_newline (token_start, "'");  BEGIN INITIAL;
-  <<EOF>>      unexpected_eof (token_start, "'");      BEGIN INITIAL;
+  <<EOF>> {
+    unexpected_eof (token_start, "'");
+    STRING_FINISH;
+    loc->start = token_start;
+    if (strlen (last_string) > 1)
+      val->character = last_string[1];
+    else
+      val->character = last_string[0];
+    STRING_FREE;
+    BEGIN INITIAL;
+    return CHAR;
+  }

 }
 
 }
 

-<SC_ESCAPED_CHARACTER,SC_ESCAPED_STRING>
+  /*-----------------------------------------------------------.
+  | Scanning a Bison nested tag.  The initial angle bracket is |
+  | already eaten.                                             |
+  `-----------------------------------------------------------*/
+
+<SC_TAG>

 {
 {

-  \0       complain_at (*loc, _("invalid null character"));
-}
+  ">" {
+    --nesting;
+    if (nesting < 0)
+      {
+        STRING_FINISH;
+        loc->start = token_start;
+        val->uniqstr = uniqstr_new (last_string);
+        STRING_FREE;
+        BEGIN INITIAL;
+        return TAG;
+      }
+    STRING_GROW;
+  }
+
+  [^<>]+ STRING_GROW;
+  "<"+   STRING_GROW; nesting += yyleng;

 
 

+  <<EOF>> {
+    unexpected_eof (token_start, ">");
+    STRING_FINISH;
+    loc->start = token_start;
+    val->uniqstr = uniqstr_new (last_string);
+    STRING_FREE;
+    BEGIN INITIAL;
+    return TAG;
+  }
+}

 
   /*----------------------------.
   | Decode escaped characters.  |
 
   /*----------------------------.
   | Decode escaped characters.  |


@@ -498,18 +573,17 @@ splice     (\\[ \f\t\v]*\n)*
 
 <SC_BRACED_CODE>
 {
 
 <SC_BRACED_CODE>
 {

-  "{"|"<"{splice}"%"  STRING_GROW; braces_level++;
-  "%"{splice}">"      STRING_GROW; braces_level--;
+  "{"|"<"{splice}"%"  STRING_GROW; nesting++;
+  "%"{splice}">"      STRING_GROW; nesting--;

   "}" {
     obstack_1grow (&obstack_for_string, '}');
 
   "}" {
     obstack_1grow (&obstack_for_string, '}');
 

-    --braces_level;
-    if (braces_level < 0)
+    --nesting;
+    if (nesting < 0)

       {
        STRING_FINISH;
        loc->start = code_start;
       {
        STRING_FINISH;
        loc->start = code_start;

-       val->chars = last_string;
-       gram_last_braced_code_loc = *loc;
+       val->code = last_string;

        BEGIN INITIAL;
        return BRACED_CODE;
       }
        BEGIN INITIAL;
        return BRACED_CODE;
       }


@@ -519,7 +593,14 @@ splice      (\\[ \f\t\v]*\n)*
      (as `<' `<%').  */
   "<"{splice}"<"  STRING_GROW;
 
      (as `<' `<%').  */
   "<"{splice}"<"  STRING_GROW;
 

-  <<EOF>>  unexpected_eof (code_start, "}"); BEGIN INITIAL;
+  <<EOF>> {
+    unexpected_eof (code_start, "}");
+    STRING_FINISH;
+    loc->start = code_start;
+    val->code = last_string;
+    BEGIN INITIAL;
+    return BRACED_CODE;
+  }

 }
 
 
 }
 
 


@@ -537,7 +618,14 @@ splice      (\\[ \f\t\v]*\n)*
     return PROLOGUE;
   }
 
     return PROLOGUE;
   }
 

-  <<EOF>>  unexpected_eof (code_start, "%}"); BEGIN INITIAL;
+  <<EOF>> {
+    unexpected_eof (code_start, "%}");
+    STRING_FINISH;
+    loc->start = code_start;
+    val->chars = last_string;
+    BEGIN INITIAL;
+    return PROLOGUE;
+  }

 }
 
 
 }
 
 


@@ -772,5 +860,5 @@ gram_scanner_free (void)
 {
   obstack_free (&obstack_for_string, 0);
   /* Reclaim Flex's buffers.  */
 {
   obstack_free (&obstack_for_string, 0);
   /* Reclaim Flex's buffers.  */

-  yy_delete_buffer (YY_CURRENT_BUFFER);
+  yylex_destroy ();

 }
 }