X-Git-Url: https://git.saurik.com/bison.git/blobdiff_plain/f9bfc42aa704603fc3aeb18be00c78871569a0b4..05c93b7d844e59ecaa5dec3bd6d9091f5aa5d1b0:/src/scan-gram.l?ds=sidebyside diff --git a/src/scan-gram.l b/src/scan-gram.l index 8193ddf2..ad061cc6 100644 --- a/src/scan-gram.l +++ b/src/scan-gram.l @@ -1,12 +1,12 @@ /* Bison Grammar Scanner -*- C -*- - Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc. + Copyright (C) 2002-2012 Free Software Foundation, Inc. This file is part of Bison, the GNU Compiler Compiler. - This program is free software; you can redistribute it and/or modify + This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or + the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, @@ -15,12 +15,9 @@ GNU General Public License for more details. You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA - 02110-1301 USA -*/ + along with this program. If not, see . */ -%option debug nodefault nounput noyywrap never-interactive +%option debug nodefault noinput nounput noyywrap never-interactive %option prefix="gram_" outfile="lex.yy.c" %{ @@ -39,6 +36,7 @@ #include "reader.h" #include "uniqstr.h" +#include #include #include @@ -57,8 +55,20 @@ static boundary scanner_cursor; static size_t no_cr_read (FILE *, char *, size_t); #define YY_INPUT(buf, result, size) ((result) = no_cr_read (yyin, buf, size)) +#define ROLLBACK_CURRENT_TOKEN \ + do { \ + scanner_cursor.column -= mbsnwidth (yytext, yyleng, 0); \ + yyless (0); \ + } while (0) + /* A string representing the most recently saved token. */ -char *last_string; +static char *last_string; + +/* Bracketed identifier. */ +static uniqstr bracketed_id_str = 0; +static location bracketed_id_loc; +static boundary bracketed_id_start; +static int bracketed_id_context_state = 0; void gram_scanner_last_string_free (void) @@ -66,10 +76,6 @@ gram_scanner_last_string_free (void) STRING_FREE; } -/* The location of the most recently saved token, if it was a - BRACED_CODE token; otherwise, this has an unspecified value. */ -location gram_last_braced_code_loc; - static void handle_syncline (char *, location); static unsigned long int scan_integer (char const *p, int base, location loc); static int convert_ucn_to_byte (char const *hex_text); @@ -82,11 +88,11 @@ static void unexpected_newline (boundary, char const *); /* Strings and characters in directives/rules. */ %x SC_ESCAPED_STRING SC_ESCAPED_CHARACTER /* A identifier was just read in directives/rules. Special state - to capture the sequence `identifier :'. */ + to capture the sequence 'identifier :'. */ %x SC_AFTER_IDENTIFIER /* Three types of user code: - - prologue (code between `%{' `%}' in the first section, before %%); + - prologue (code between '%{' '%}' in the first section, before %%); - actions, printers, union, etc, (between braced in the middle section); - epilogue (everything after the second %%). */ %x SC_PROLOGUE SC_BRACED_CODE SC_EPILOGUE @@ -94,10 +100,12 @@ static void unexpected_newline (boundary, char const *); %x SC_COMMENT SC_LINE_COMMENT /* Strings and characters in code. */ %x SC_STRING SC_CHARACTER + /* Bracketed identifiers support. */ +%x SC_BRACKETED_ID SC_RETURN_BRACKETED_ID letter [.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_] -id {letter}({letter}|[0-9])* -directive %{letter}({letter}|[0-9]|-)* +id {letter}({letter}|[-0-9])* +directive %{id} int [0-9]+ /* POSIX says that a tag must be both an id and a C union member, but @@ -112,13 +120,13 @@ splice (\\[ \f\t\v]*\n)* %% %{ /* Nesting level of the current code in braces. */ - int braces_level IF_LINT (= 0); + int braces_level PACIFY_CC (= 0); /* Parent context state, when applicable. */ - int context_state IF_LINT (= 0); + int context_state PACIFY_CC (= 0); /* Location of most recent identifier, when applicable. */ - location id_loc IF_LINT (= empty_location); + location id_loc PACIFY_CC (= empty_location); /* Where containing code started, when applicable. Its initial value is relevant only when yylex is invoked in the SC_EPILOGUE @@ -127,7 +135,7 @@ splice (\\[ \f\t\v]*\n)* /* Where containing comment or string or character literal started, when applicable. */ - boundary token_start IF_LINT (= scanner_cursor); + boundary token_start PACIFY_CC (= scanner_cursor); %} @@ -135,10 +143,10 @@ splice (\\[ \f\t\v]*\n)* | Scanning white space. | `-----------------------*/ - + { /* Comments and white space. */ - "," warn_at (*loc, _("stray `,' treated as white space")); + "," warn_at (*loc, _("stray ',' treated as white space")); [ \f\n\t\v] | "//".* ; "/*" { @@ -158,52 +166,57 @@ splice (\\[ \f\t\v]*\n)* /*----------------------------. | Scanning Bison directives. | `----------------------------*/ + + /* For directives that are also command line options, the regex must be + "%..." + after "[-_]"s are removed, and the directive must match the --long + option name, with a single string argument. Otherwise, add exceptions + to ../build-aux/cross-options.pl. */ + { - "%after-header" return PERCENT_AFTER_HEADER; - "%before-header" return PERCENT_BEFORE_HEADER; - "%binary" return PERCENT_NONASSOC; - "%debug" return PERCENT_DEBUG; - "%default"[-_]"prec" return PERCENT_DEFAULT_PREC; - "%define" return PERCENT_DEFINE; - "%defines" return PERCENT_DEFINES; - "%destructor" return PERCENT_DESTRUCTOR; - "%dprec" return PERCENT_DPREC; - "%end-header" return PERCENT_END_HEADER; - "%error"[-_]"verbose" return PERCENT_ERROR_VERBOSE; - "%expect" return PERCENT_EXPECT; - "%expect"[-_]"rr" return PERCENT_EXPECT_RR; - "%file-prefix" return PERCENT_FILE_PREFIX; + "%binary" return PERCENT_NONASSOC; + "%code" return PERCENT_CODE; + "%debug" return PERCENT_DEBUG; + "%default"[-_]"prec" return PERCENT_DEFAULT_PREC; + "%define" return PERCENT_DEFINE; + "%defines" return PERCENT_DEFINES; + "%destructor" return PERCENT_DESTRUCTOR; + "%dprec" return PERCENT_DPREC; + "%error"[-_]"verbose" return PERCENT_ERROR_VERBOSE; + "%expect" return PERCENT_EXPECT; + "%expect"[-_]"rr" return PERCENT_EXPECT_RR; + "%file-prefix" return PERCENT_FILE_PREFIX; "%fixed"[-_]"output"[-_]"files" return PERCENT_YACC; - "%initial-action" return PERCENT_INITIAL_ACTION; - "%glr-parser" return PERCENT_GLR_PARSER; - "%left" return PERCENT_LEFT; - "%lex-param" return PERCENT_LEX_PARAM; - "%locations" return PERCENT_LOCATIONS; - "%merge" return PERCENT_MERGE; - "%name"[-_]"prefix" return PERCENT_NAME_PREFIX; - "%no"[-_]"default"[-_]"prec" return PERCENT_NO_DEFAULT_PREC; - "%no"[-_]"lines" return PERCENT_NO_LINES; - "%nonassoc" return PERCENT_NONASSOC; - "%nondeterministic-parser" return PERCENT_NONDETERMINISTIC_PARSER; - "%nterm" return PERCENT_NTERM; - "%output" return PERCENT_OUTPUT; - "%parse-param" return PERCENT_PARSE_PARAM; - "%prec" return PERCENT_PREC; - "%printer" return PERCENT_PRINTER; - "%pure"[-_]"parser" return PERCENT_PURE_PARSER; - "%require" return PERCENT_REQUIRE; - "%right" return PERCENT_RIGHT; - "%skeleton" return PERCENT_SKELETON; - "%start" return PERCENT_START; - "%start-header" return PERCENT_START_HEADER; - "%term" return PERCENT_TOKEN; - "%token" return PERCENT_TOKEN; - "%token"[-_]"table" return PERCENT_TOKEN_TABLE; - "%type" return PERCENT_TYPE; - "%union" return PERCENT_UNION; - "%verbose" return PERCENT_VERBOSE; - "%yacc" return PERCENT_YACC; + "%initial-action" return PERCENT_INITIAL_ACTION; + "%glr-parser" return PERCENT_GLR_PARSER; + "%language" return PERCENT_LANGUAGE; + "%left" return PERCENT_LEFT; + "%lex-param" return PERCENT_LEX_PARAM; + "%locations" return PERCENT_LOCATIONS; + "%merge" return PERCENT_MERGE; + "%name"[-_]"prefix" return PERCENT_NAME_PREFIX; + "%no"[-_]"default"[-_]"prec" return PERCENT_NO_DEFAULT_PREC; + "%no"[-_]"lines" return PERCENT_NO_LINES; + "%nonassoc" return PERCENT_NONASSOC; + "%nondeterministic-parser" return PERCENT_NONDETERMINISTIC_PARSER; + "%nterm" return PERCENT_NTERM; + "%output" return PERCENT_OUTPUT; + "%parse-param" return PERCENT_PARSE_PARAM; + "%prec" return PERCENT_PREC; + "%printer" return PERCENT_PRINTER; + "%pure"[-_]"parser" return PERCENT_PURE_PARSER; + "%require" return PERCENT_REQUIRE; + "%right" return PERCENT_RIGHT; + "%skeleton" return PERCENT_SKELETON; + "%start" return PERCENT_START; + "%term" return PERCENT_TOKEN; + "%token" return PERCENT_TOKEN; + "%token"[-_]"table" return PERCENT_TOKEN_TABLE; + "%type" return PERCENT_TYPE; + "%union" return PERCENT_UNION; + "%verbose" return PERCENT_VERBOSE; + "%yacc" return PERCENT_YACC; {directive} { complain_at (*loc, _("invalid directive: %s"), quote (yytext)); @@ -212,10 +225,13 @@ splice (\\[ \f\t\v]*\n)* "=" return EQUAL; "|" return PIPE; ";" return SEMICOLON; + "<*>" return TYPE_TAG_ANY; + "<>" return TYPE_TAG_NONE; {id} { val->uniqstr = uniqstr_new (yytext); id_loc = *loc; + bracketed_id_str = NULL; BEGIN SC_AFTER_IDENTIFIER; } @@ -228,8 +244,14 @@ splice (\\[ \f\t\v]*\n)* return INT; } - /* Characters. We don't check there is only one. */ - "'" STRING_GROW; token_start = loc->start; BEGIN SC_ESCAPED_CHARACTER; + /* Identifiers may not start with a digit. Yet, don't silently + accept "1FOO" as "1 FOO". */ + {int}{id} { + complain_at (*loc, _("invalid identifier: %s"), quote (yytext)); + } + + /* Characters. */ + "'" token_start = loc->start; BEGIN SC_ESCAPED_CHARACTER; /* Strings. */ "\"" token_start = loc->start; BEGIN SC_ESCAPED_STRING; @@ -261,8 +283,15 @@ splice (\\[ \f\t\v]*\n)* return PERCENT_PERCENT; } + "[" { + bracketed_id_str = NULL; + bracketed_id_start = loc->start; + bracketed_id_context_state = YY_START; + BEGIN SC_BRACKETED_ID; + } + . { - complain_at (*loc, _("invalid character: %s"), quote (yytext)); + complain_at (*loc, _("invalid character: %s"), quote_mem (yytext, yyleng)); } <> { @@ -278,28 +307,97 @@ splice (\\[ \f\t\v]*\n)* { + "[" { + if (bracketed_id_str) + { + ROLLBACK_CURRENT_TOKEN; + BEGIN SC_RETURN_BRACKETED_ID; + *loc = id_loc; + return ID; + } + else + { + bracketed_id_start = loc->start; + bracketed_id_context_state = YY_START; + BEGIN SC_BRACKETED_ID; + } + } ":" { + BEGIN (bracketed_id_str ? SC_RETURN_BRACKETED_ID : INITIAL); *loc = id_loc; - BEGIN INITIAL; return ID_COLON; } . { - scanner_cursor.column -= mbsnwidth (yytext, yyleng, 0); - yyless (0); + ROLLBACK_CURRENT_TOKEN; + BEGIN (bracketed_id_str ? SC_RETURN_BRACKETED_ID : INITIAL); *loc = id_loc; - BEGIN INITIAL; return ID; } <> { + BEGIN (bracketed_id_str ? SC_RETURN_BRACKETED_ID : INITIAL); *loc = id_loc; - BEGIN INITIAL; return ID; } } + /*--------------------------------. + | Scanning bracketed identifiers. | + `--------------------------------*/ + + +{ + {id} { + if (bracketed_id_str) + { + complain_at (*loc, _("unexpected identifier in bracketed name: %s"), + quote (yytext)); + } + else + { + bracketed_id_str = uniqstr_new (yytext); + bracketed_id_loc = *loc; + } + } + "]" { + BEGIN bracketed_id_context_state; + if (bracketed_id_str) + { + if (INITIAL == bracketed_id_context_state) + { + val->uniqstr = bracketed_id_str; + bracketed_id_str = 0; + *loc = bracketed_id_loc; + return BRACKETED_ID; + } + } + else + complain_at (*loc, _("an identifier expected")); + } + . { + complain_at (*loc, _("invalid character in bracketed name: %s"), + quote_mem (yytext, yyleng)); + } + <> { + BEGIN bracketed_id_context_state; + unexpected_eof (bracketed_id_start, "]"); + } +} + + +{ + . { + ROLLBACK_CURRENT_TOKEN; + val->uniqstr = bracketed_id_str; + bracketed_id_str = 0; + *loc = bracketed_id_loc; + BEGIN INITIAL; + return BRACKETED_ID; + } +} + /*---------------------------------------------------------------. - | Scanning a Yacc comment. The initial `/ *' is already eaten. | + | Scanning a Yacc comment. The initial '/ *' is already eaten. | `---------------------------------------------------------------*/ @@ -311,7 +409,7 @@ splice (\\[ \f\t\v]*\n)* /*------------------------------------------------------------. - | Scanning a C comment. The initial `/ *' is already eaten. | + | Scanning a C comment. The initial '/ *' is already eaten. | `------------------------------------------------------------*/ @@ -322,7 +420,7 @@ splice (\\[ \f\t\v]*\n)* /*--------------------------------------------------------------. - | Scanning a line comment. The initial `//' is already eaten. | + | Scanning a line comment. The initial '//' is already eaten. | `--------------------------------------------------------------*/ @@ -340,15 +438,23 @@ splice (\\[ \f\t\v]*\n)* { - "\"" { + "\""|"\n" { + if (yytext[0] == '\n') + unexpected_newline (token_start, "\""); + STRING_FINISH; + loc->start = token_start; + val->chars = last_string; + BEGIN INITIAL; + return STRING; + } + <> { + unexpected_eof (token_start, "\""); STRING_FINISH; loc->start = token_start; val->chars = last_string; BEGIN INITIAL; return STRING; } - \n unexpected_newline (token_start, "\""); BEGIN INITIAL; - <> unexpected_eof (token_start, "\""); BEGIN INITIAL; } /*----------------------------------------------------------. @@ -358,17 +464,47 @@ splice (\\[ \f\t\v]*\n)* { - "'" { - STRING_GROW; + "'"|"\n" { STRING_FINISH; loc->start = token_start; - val->character = last_string[1]; + val->character = last_string[0]; + { + /* FIXME: Eventually, make these errors. */ + if (last_string[0] == '\0') + { + warn_at (*loc, _("empty character literal")); + /* '\0' seems dangerous even if we are about to complain. */ + val->character = '\''; + } + else if (last_string[1] != '\0') + warn_at (*loc, _("extra characters in character literal")); + } + if (yytext[0] == '\n') + unexpected_newline (token_start, "'"); + STRING_FREE; + BEGIN INITIAL; + return CHAR; + } + <> { + STRING_FINISH; + loc->start = token_start; + val->character = last_string[0]; + { + /* FIXME: Eventually, make these errors. */ + if (last_string[0] == '\0') + { + warn_at (*loc, _("empty character literal")); + /* '\0' seems dangerous even if we are about to complain. */ + val->character = '\''; + } + else if (last_string[1] != '\0') + warn_at (*loc, _("extra characters in character literal")); + } + unexpected_eof (token_start, "'"); STRING_FREE; BEGIN INITIAL; return CHAR; } - \n unexpected_newline (token_start, "'"); BEGIN INITIAL; - <> unexpected_eof (token_start, "'"); BEGIN INITIAL; } @@ -385,10 +521,9 @@ splice (\\[ \f\t\v]*\n)* { \\[0-7]{1,3} { unsigned long int c = strtoul (yytext + 1, NULL, 8); - if (UCHAR_MAX < c) - complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext)); - else if (! c) - complain_at (*loc, _("invalid null character: %s"), quote (yytext)); + if (!c || UCHAR_MAX < c) + complain_at (*loc, _("invalid number after \\-escape: %s"), + yytext+1); else obstack_1grow (&obstack_for_string, c); } @@ -396,10 +531,9 @@ splice (\\[ \f\t\v]*\n)* \\x[0-9abcdefABCDEF]+ { verify (UCHAR_MAX < ULONG_MAX); unsigned long int c = strtoul (yytext + 2, NULL, 16); - if (UCHAR_MAX < c) - complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext)); - else if (! c) - complain_at (*loc, _("invalid null character: %s"), quote (yytext)); + if (!c || UCHAR_MAX < c) + complain_at (*loc, _("invalid number after \\-escape: %s"), + yytext+1); else obstack_1grow (&obstack_for_string, c); } @@ -417,16 +551,20 @@ splice (\\[ \f\t\v]*\n)* \\(u|U[0-9abcdefABCDEF]{4})[0-9abcdefABCDEF]{4} { int c = convert_ucn_to_byte (yytext); - if (c < 0) - complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext)); - else if (! c) - complain_at (*loc, _("invalid null character: %s"), quote (yytext)); + if (c <= 0) + complain_at (*loc, _("invalid number after \\-escape: %s"), + yytext+1); else obstack_1grow (&obstack_for_string, c); } \\(.|\n) { - complain_at (*loc, _("unrecognized escape sequence: %s"), quote (yytext)); - STRING_GROW; + char const *p = yytext + 1; + /* Quote only if escaping won't make the character visible. */ + if (c_isspace ((unsigned char) *p) && c_isprint ((unsigned char) *p)) + p = quote (p); + else + p = quotearg_style_mem (escape_quoting_style, p, 1); + complain_at (*loc, _("invalid character after \\-escape: %s"), p); } } @@ -504,18 +642,24 @@ splice (\\[ \f\t\v]*\n)* { STRING_FINISH; loc->start = code_start; - val->chars = last_string; - gram_last_braced_code_loc = *loc; + val->code = last_string; BEGIN INITIAL; return BRACED_CODE; } } - /* Tokenize `<<%' correctly (as `<<' `%') rather than incorrrectly - (as `<' `<%'). */ + /* Tokenize '<<%' correctly (as '<<' '%') rather than incorrrectly + (as '<' '<%'). */ "<"{splice}"<" STRING_GROW; - <> unexpected_eof (code_start, "}"); BEGIN INITIAL; + <> { + unexpected_eof (code_start, "}"); + STRING_FINISH; + loc->start = code_start; + val->code = last_string; + BEGIN INITIAL; + return BRACED_CODE; + } } @@ -533,7 +677,14 @@ splice (\\[ \f\t\v]*\n)* return PROLOGUE; } - <> unexpected_eof (code_start, "%}"); BEGIN INITIAL; + <> { + unexpected_eof (code_start, "%}"); + STRING_FINISH; + loc->start = code_start; + val->chars = last_string; + BEGIN INITIAL; + return PROLOGUE; + } } @@ -689,7 +840,7 @@ convert_ucn_to_byte (char const *ucn) /*----------------------------------------------------------------. -| Handle `#line INT "FILE"'. ARGS has already skipped `#line '. | +| Handle '#line INT "FILE"'. ARGS has already skipped '#line '. | `----------------------------------------------------------------*/ static void @@ -697,8 +848,8 @@ handle_syncline (char *args, location loc) { char *after_num; unsigned long int lineno = strtoul (args, &after_num, 10); - char *file = strchr (after_num, '"') + 1; - *strchr (file, '"') = '\0'; + char *file = mbschr (after_num, '"') + 1; + *mbschr (file, '"') = '\0'; if (INT_MAX <= lineno) { warn_at (loc, _("line number overflow")); @@ -721,6 +872,10 @@ unexpected_end (boundary start, char const *msgid, char const *token_end) location loc; loc.start = start; loc.end = scanner_cursor; + token_end = quote (token_end); + // Instead of '\'', display "'". + if (!strcmp (token_end, "'\\''")) + token_end = "\"'\""; complain_at (loc, _(msgid), token_end); } @@ -733,7 +888,7 @@ unexpected_end (boundary start, char const *msgid, char const *token_end) static void unexpected_eof (boundary start, char const *token_end) { - unexpected_end (start, N_("missing `%s' at end of file"), token_end); + unexpected_end (start, N_("missing %s at end of file"), token_end); } @@ -744,7 +899,7 @@ unexpected_eof (boundary start, char const *token_end) static void unexpected_newline (boundary start, char const *token_end) { - unexpected_end (start, N_("missing `%s' at end of line"), token_end); + unexpected_end (start, N_("missing %s at end of line"), token_end); } @@ -768,5 +923,5 @@ gram_scanner_free (void) { obstack_free (&obstack_for_string, 0); /* Reclaim Flex's buffers. */ - yy_delete_buffer (YY_CURRENT_BUFFER); + yylex_destroy (); }