X-Git-Url: https://git.saurik.com/bison.git/blobdiff_plain/4f646c3794c45940aaf96d5409eff02a2c74978e..ba60c39547a445dee3e07920931b4d7a81843868:/src/scan-gram.l diff --git a/src/scan-gram.l b/src/scan-gram.l index bc7592ec..15b6b31d 100644 --- a/src/scan-gram.l +++ b/src/scan-gram.l @@ -1,7 +1,6 @@ /* Bison Grammar Scanner -*- C -*- - Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 - Free Software Foundation, Inc. + Copyright (C) 2002-2011 Free Software Foundation, Inc. This file is part of Bison, the GNU Compiler Compiler. @@ -18,7 +17,7 @@ You should have received a copy of the GNU General Public License along with this program. If not, see . */ -%option debug nodefault nounput noyywrap never-interactive +%option debug nodefault noinput nounput noyywrap never-interactive %option prefix="gram_" outfile="lex.yy.c" %{ @@ -37,6 +36,7 @@ #include #include +#include #include #include @@ -55,16 +55,33 @@ static boundary scanner_cursor; static size_t no_cr_read (FILE *, char *, size_t); #define YY_INPUT(buf, result, size) ((result) = no_cr_read (yyin, buf, size)) -#define RETURN_PERCENT_FLAG(Value) \ +#define RETURN_PERCENT_PARAM(Value) \ + RETURN_VALUE(PERCENT_PARAM, param, param_ ## Value) + +#define RETURN_PERCENT_FLAG(Value) \ + RETURN_VALUE(PERCENT_FLAG, uniqstr, uniqstr_new (Value)) + +#define RETURN_VALUE(Token, Field, Value) \ do { \ - val->uniqstr = uniqstr_new (Value); \ - return PERCENT_FLAG; \ + val->Field = Value; \ + return Token; \ } while (0) +#define ROLLBACK_CURRENT_TOKEN \ + do { \ + scanner_cursor.column -= mbsnwidth (yytext, yyleng, 0); \ + yyless (0); \ + } while (0) /* A string representing the most recently saved token. */ static char *last_string; +/* Bracketed identifier. */ +static uniqstr bracketed_id_str = 0; +static location bracketed_id_loc; +static boundary bracketed_id_start; +static int bracketed_id_context_state = 0; + void gram_scanner_last_string_free (void) { @@ -88,18 +105,21 @@ static void unexpected_newline (boundary, char const *); /* A complex tag, with nested angles brackets. */ %x SC_TAG - /* Three types of user code: + /* Four types of user code: - prologue (code between `%{' `%}' in the first section, before %%); - actions, printers, union, etc, (between braced in the middle section); - - epilogue (everything after the second %%). */ -%x SC_PROLOGUE SC_BRACED_CODE SC_EPILOGUE + - epilogue (everything after the second %%). + - predicate (code between `%?{' and `{' in middle section); */ +%x SC_PROLOGUE SC_BRACED_CODE SC_EPILOGUE SC_PREDICATE /* C and C++ comments in code. */ %x SC_COMMENT SC_LINE_COMMENT /* Strings and characters in code. */ %x SC_STRING SC_CHARACTER + /* Bracketed identifiers support. */ +%x SC_BRACKETED_ID SC_RETURN_BRACKETED_ID letter [.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_] -id {letter}({letter}|[0-9]|-)* +id {letter}({letter}|[-0-9])* directive %{id} int [0-9]+ @@ -141,7 +161,7 @@ splice (\\[ \f\t\v]*\n)* | Scanning white space. | `-----------------------*/ - + { /* Comments and white space. */ "," warn_at (*loc, _("stray `,' treated as white space")); @@ -181,7 +201,7 @@ splice (\\[ \f\t\v]*\n)* "%defines" return PERCENT_DEFINES; "%destructor" return PERCENT_DESTRUCTOR; "%dprec" return PERCENT_DPREC; - "%error"[-_]"verbose" RETURN_PERCENT_FLAG("error_verbose"); + "%error"[-_]"verbose" return PERCENT_ERROR_VERBOSE; "%expect" return PERCENT_EXPECT; "%expect"[-_]"rr" return PERCENT_EXPECT_RR; "%file-prefix" return PERCENT_FILE_PREFIX; @@ -190,7 +210,7 @@ splice (\\[ \f\t\v]*\n)* "%glr-parser" return PERCENT_GLR_PARSER; "%language" return PERCENT_LANGUAGE; "%left" return PERCENT_LEFT; - "%lex-param" return PERCENT_LEX_PARAM; + "%lex-param" RETURN_PERCENT_PARAM(lex); "%locations" RETURN_PERCENT_FLAG("locations"); "%merge" return PERCENT_MERGE; "%name"[-_]"prefix" return PERCENT_NAME_PREFIX; @@ -200,7 +220,8 @@ splice (\\[ \f\t\v]*\n)* "%nondeterministic-parser" return PERCENT_NONDETERMINISTIC_PARSER; "%nterm" return PERCENT_NTERM; "%output" return PERCENT_OUTPUT; - "%parse-param" return PERCENT_PARSE_PARAM; + "%param" RETURN_PERCENT_PARAM(both); + "%parse-param" RETURN_PERCENT_PARAM(parse); "%prec" return PERCENT_PREC; "%precedence" return PERCENT_PRECEDENCE; "%printer" return PERCENT_PRINTER; @@ -228,6 +249,7 @@ splice (\\[ \f\t\v]*\n)* {id} { val->uniqstr = uniqstr_new (yytext); id_loc = *loc; + bracketed_id_str = NULL; BEGIN SC_AFTER_IDENTIFIER; } @@ -240,8 +262,14 @@ splice (\\[ \f\t\v]*\n)* return INT; } - /* Characters. We don't check there is only one. */ - "'" STRING_GROW; token_start = loc->start; BEGIN SC_ESCAPED_CHARACTER; + /* Identifiers may not start with a digit. Yet, don't silently + accept "1FOO" as "1 FOO". */ + {int}{id} { + complain_at (*loc, _("invalid identifier: %s"), quote (yytext)); + } + + /* Characters. */ + "'" token_start = loc->start; BEGIN SC_ESCAPED_CHARACTER; /* Strings. */ "\"" token_start = loc->start; BEGIN SC_ESCAPED_STRING; @@ -257,6 +285,13 @@ splice (\\[ \f\t\v]*\n)* BEGIN SC_BRACED_CODE; } + /* Semantic predicate. */ + "%?"[ \f\n\t\v]*"{" { + nesting = 0; + code_start = loc->start; + BEGIN SC_PREDICATE; + } + /* A type. */ "<*>" return TAG_ANY; "<>" return TAG_NONE; @@ -280,6 +315,13 @@ splice (\\[ \f\t\v]*\n)* return PERCENT_PERCENT; } + "[" { + bracketed_id_str = NULL; + bracketed_id_start = loc->start; + bracketed_id_context_state = YY_START; + BEGIN SC_BRACKETED_ID; + } + . { complain_at (*loc, _("invalid character: %s"), quote (yytext)); } @@ -308,25 +350,94 @@ splice (\\[ \f\t\v]*\n)* { + "[" { + if (bracketed_id_str) + { + ROLLBACK_CURRENT_TOKEN; + BEGIN SC_RETURN_BRACKETED_ID; + *loc = id_loc; + return ID; + } + else + { + bracketed_id_start = loc->start; + bracketed_id_context_state = YY_START; + BEGIN SC_BRACKETED_ID; + } + } ":" { + BEGIN (bracketed_id_str ? SC_RETURN_BRACKETED_ID : INITIAL); *loc = id_loc; - BEGIN INITIAL; return ID_COLON; } . { - scanner_cursor.column -= mbsnwidth (yytext, yyleng, 0); - yyless (0); + ROLLBACK_CURRENT_TOKEN; + BEGIN (bracketed_id_str ? SC_RETURN_BRACKETED_ID : INITIAL); *loc = id_loc; - BEGIN INITIAL; return ID; } <> { + BEGIN (bracketed_id_str ? SC_RETURN_BRACKETED_ID : INITIAL); *loc = id_loc; - BEGIN INITIAL; return ID; } } + /*--------------------------------. + | Scanning bracketed identifiers. | + `--------------------------------*/ + + +{ + {id} { + if (bracketed_id_str) + { + complain_at (*loc, _("unexpected identifier in bracketed name: %s"), + quote (yytext)); + } + else + { + bracketed_id_str = uniqstr_new (yytext); + bracketed_id_loc = *loc; + } + } + "]" { + BEGIN bracketed_id_context_state; + if (bracketed_id_str) + { + if (INITIAL == bracketed_id_context_state) + { + val->uniqstr = bracketed_id_str; + bracketed_id_str = 0; + *loc = bracketed_id_loc; + return BRACKETED_ID; + } + } + else + complain_at (*loc, _("an identifier expected")); + } + . { + complain_at (*loc, _("invalid character in bracketed name: %s"), + quote (yytext)); + } + <> { + BEGIN bracketed_id_context_state; + unexpected_eof (bracketed_id_start, "]"); + } +} + + +{ + . { + ROLLBACK_CURRENT_TOKEN; + val->uniqstr = bracketed_id_str; + bracketed_id_str = 0; + *loc = bracketed_id_loc; + BEGIN INITIAL; + return BRACKETED_ID; + } +} + /*---------------------------------------------------------------. | Scanning a Yacc comment. The initial `/ *' is already eaten. | @@ -397,24 +508,42 @@ splice (\\[ \f\t\v]*\n)* { "'"|"\n" { - if (yytext[0] == '\n') - unexpected_newline (token_start, "'"); - STRING_GROW; STRING_FINISH; loc->start = token_start; - val->character = last_string[1]; + val->character = last_string[0]; + { + /* FIXME: Eventually, make these errors. */ + if (last_string[0] == '\0') + { + warn_at (*loc, _("empty character literal")); + /* '\0' seems dangerous even if we are about to complain. */ + val->character = '\''; + } + else if (last_string[1] != '\0') + warn_at (*loc, _("extra characters in character literal")); + } + if (yytext[0] == '\n') + unexpected_newline (token_start, "'"); STRING_FREE; BEGIN INITIAL; return CHAR; } <> { - unexpected_eof (token_start, "'"); STRING_FINISH; loc->start = token_start; - if (strlen (last_string) > 1) - val->character = last_string[1]; - else - val->character = last_string[0]; + val->character = last_string[0]; + { + /* FIXME: Eventually, make these errors. */ + if (last_string[0] == '\0') + { + warn_at (*loc, _("empty character literal")); + /* '\0' seems dangerous even if we are about to complain. */ + val->character = '\''; + } + else if (last_string[1] != '\0') + warn_at (*loc, _("extra characters in character literal")); + } + unexpected_eof (token_start, "'"); STRING_FREE; BEGIN INITIAL; return CHAR; @@ -464,10 +593,9 @@ splice (\\[ \f\t\v]*\n)* { \\[0-7]{1,3} { unsigned long int c = strtoul (yytext + 1, NULL, 8); - if (UCHAR_MAX < c) - complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext)); - else if (! c) - complain_at (*loc, _("invalid null character: %s"), quote (yytext)); + if (!c || UCHAR_MAX < c) + complain_at (*loc, _("invalid number after \\-escape: %s"), + yytext+1); else obstack_1grow (&obstack_for_string, c); } @@ -475,10 +603,9 @@ splice (\\[ \f\t\v]*\n)* \\x[0-9abcdefABCDEF]+ { verify (UCHAR_MAX < ULONG_MAX); unsigned long int c = strtoul (yytext + 2, NULL, 16); - if (UCHAR_MAX < c) - complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext)); - else if (! c) - complain_at (*loc, _("invalid null character: %s"), quote (yytext)); + if (!c || UCHAR_MAX < c) + complain_at (*loc, _("invalid number after \\-escape: %s"), + yytext+1); else obstack_1grow (&obstack_for_string, c); } @@ -496,16 +623,20 @@ splice (\\[ \f\t\v]*\n)* \\(u|U[0-9abcdefABCDEF]{4})[0-9abcdefABCDEF]{4} { int c = convert_ucn_to_byte (yytext); - if (c < 0) - complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext)); - else if (! c) - complain_at (*loc, _("invalid null character: %s"), quote (yytext)); + if (c <= 0) + complain_at (*loc, _("invalid number after \\-escape: %s"), + yytext+1); else obstack_1grow (&obstack_for_string, c); } \\(.|\n) { - complain_at (*loc, _("unrecognized escape sequence: %s"), quote (yytext)); - STRING_GROW; + char const *p = yytext + 1; + /* Quote only if escaping won't make the character visible. */ + if (isspace ((unsigned char) *p) && isprint ((unsigned char) *p)) + p = quote (p); + else + p = quotearg_style_mem (escape_quoting_style, p, 1); + complain_at (*loc, _("invalid character after \\-escape: %s"), p); } } @@ -537,7 +668,7 @@ splice (\\[ \f\t\v]*\n)* | Strings, comments etc. can be found in user code. | `---------------------------------------------------*/ - + { "'" { STRING_GROW; @@ -567,14 +698,32 @@ splice (\\[ \f\t\v]*\n)* /*-----------------------------------------------------------. - | Scanning some code in braces (actions). The initial "{" is | - | already eaten. | + | Scanning some code in braces (actions, predicates). The | + | initial "{" is already eaten. | `-----------------------------------------------------------*/ - + { "{"|"<"{splice}"%" STRING_GROW; nesting++; "%"{splice}">" STRING_GROW; nesting--; + + /* Tokenize `<<%' correctly (as `<<' `%') rather than incorrrectly + (as `<' `<%'). */ + "<"{splice}"<" STRING_GROW; + + <> { + int token = (YY_START == SC_BRACED_CODE) ? BRACED_CODE : BRACED_PREDICATE; + unexpected_eof (code_start, "}"); + STRING_FINISH; + loc->start = code_start; + val->code = last_string; + BEGIN INITIAL; + return token; + } +} + + +{ "}" { obstack_1grow (&obstack_for_string, '}'); @@ -588,22 +737,25 @@ splice (\\[ \f\t\v]*\n)* return BRACED_CODE; } } +} - /* Tokenize `<<%' correctly (as `<<' `%') rather than incorrrectly - (as `<' `<%'). */ - "<"{splice}"<" STRING_GROW; - - <> { - unexpected_eof (code_start, "}"); - STRING_FINISH; - loc->start = code_start; - val->code = last_string; - BEGIN INITIAL; - return BRACED_CODE; + +{ + "}" { + --nesting; + if (nesting < 0) + { + STRING_FINISH; + loc->start = code_start; + val->code = last_string; + BEGIN INITIAL; + return BRACED_PREDICATE; + } + else + obstack_1grow (&obstack_for_string, '}'); } } - /*--------------------------------------------------------------. | Scanning some prologue: from "%{" (already scanned) to "%}". | `--------------------------------------------------------------*/ @@ -650,8 +802,8 @@ splice (\\[ \f\t\v]*\n)* | By default, grow the string obstack with the input. | `-----------------------------------------------------*/ -. | -\n STRING_GROW; +. | + \n STRING_GROW; %% @@ -789,8 +941,8 @@ handle_syncline (char *args, location loc) { char *after_num; unsigned long int lineno = strtoul (args, &after_num, 10); - char *file = strchr (after_num, '"') + 1; - *strchr (file, '"') = '\0'; + char *file = mbschr (after_num, '"') + 1; + *mbschr (file, '"') = '\0'; if (INT_MAX <= lineno) { warn_at (loc, _("line number overflow"));