X-Git-Url: https://git.saurik.com/bison.git/blobdiff_plain/a706a1cc03800f9bedd3658c6e53f17fc1f5d350..a0b76449c7ebb01064f6e091715819f7519c90b2:/src/scan-gram.l diff --git a/src/scan-gram.l b/src/scan-gram.l index ad200267..a96b18b9 100644 --- a/src/scan-gram.l +++ b/src/scan-gram.l @@ -26,133 +26,64 @@ #include "system.h" #include "mbswidth.h" #include "complain.h" +#include "files.h" #include "quote.h" +#include "struniq.h" #include "getargs.h" #include "gram.h" #include "reader.h" -/* Each time we match a string, move the end cursor to its end. */ -#define YY_USER_INIT \ -do { \ - LOCATION_RESET (*yylloc); \ - yylloc->file = infile; \ - /* This is only to avoid GCC warnings. */ \ - if (yycontrol) {;}; \ -} while (0) +#define YY_USER_INIT \ + do \ + { \ + scanner_cursor.file = current_file; \ + scanner_cursor.line = 1; \ + scanner_cursor.column = 1; \ + } \ + while (0) -#define YY_USER_ACTION extend_location (yylloc, yytext, yyleng); -#define YY_STEP LOCATION_STEP (*yylloc) +/* Location of scanner cursor. */ +boundary scanner_cursor; -#define YY_INPUT(buf, result, size) ((result) = no_cr_read (yyin, buf, size)) - - -/* Read bytes from FP into buffer BUF of size SIZE. Return the - number of bytes read. Remove '\r' from input, treating \r\n - and isolated \r as \n. */ - -static size_t -no_cr_read (FILE *fp, char *buf, size_t size) -{ - size_t s = fread (buf, 1, size, fp); - if (s) - { - char *w = memchr (buf, '\r', s); - if (w) - { - char const *r = ++w; - char const *lim = buf + s; - - for (;;) - { - /* Found an '\r'. Treat it like '\n', but ignore any - '\n' that immediately follows. */ - w[-1] = '\n'; - if (r == lim) - { - int ch = getc (fp); - if (ch != '\n' && ungetc (ch, fp) != ch) - break; - } - else if (*r == '\n') - r++; - - /* Copy until the next '\r'. */ - do - { - if (r == lim) - return w - buf; - } - while ((*w++ = *r++) != '\r'); - } - - return w - buf; - } - } - - return s; -} - - -/* Extend *LOC to account for token TOKEN of size SIZE. */ - -static void -extend_location (location_t *loc, char const *token, int size) -{ - int line = loc->last_line; - int column = loc->last_column; - char const *p0 = token; - char const *p = token; - char const *lim = token + size; - - for (p = token; p < lim; p++) - switch (*p) - { - case '\r': - /* \r shouldn't survive no_cr_read. */ - abort (); - - case '\n': - line++; - column = 1; - p0 = p + 1; - break; - - case '\t': - column += mbsnwidth (p0, p - p0, 0); - column += 8 - ((column - 1) & 7); - p0 = p + 1; - break; - } - - loc->last_line = line; - loc->last_column = column + mbsnwidth (p0, p - p0, 0); -} +static void adjust_location (location_t *, char const *, size_t); +#define YY_USER_ACTION adjust_location (loc, yytext, yyleng); +static size_t no_cr_read (FILE *, char *, size_t); +#define YY_INPUT(buf, result, size) ((result) = no_cr_read (yyin, buf, size)) /* STRING_OBSTACK -- Used to store all the characters that we need to keep (to construct ID, STRINGS etc.). Use the following macros to use it. - Use YY_OBS_GROW to append what has just been matched, and - YY_OBS_FINISH to end the string (it puts the ending 0). - YY_OBS_FINISH also stores this string in LAST_STRING, which can be - used, and which is used by YY_OBS_FREE to free the last string. */ + Use STRING_GROW to append what has just been matched, and + STRING_FINISH to end the string (it puts the ending 0). + STRING_FINISH also stores this string in LAST_STRING, which can be + used, and which is used by STRING_FREE to free the last string. */ static struct obstack string_obstack; -#define YY_OBS_GROW \ +/* A string representing the most recently saved token. */ +static char *last_string; + + +#define STRING_GROW \ obstack_grow (&string_obstack, yytext, yyleng) -#define YY_OBS_FINISH \ +#define STRING_FINISH \ do { \ obstack_1grow (&string_obstack, '\0'); \ last_string = obstack_finish (&string_obstack); \ } while (0) -#define YY_OBS_FREE \ +#define STRING_FREE \ obstack_free (&string_obstack, last_string) +void +scanner_last_string_free (void) +{ + STRING_FREE; +} /* Within well-formed rules, RULE_LENGTH is the number of values in the current rule so far, which says where to find `$0' with respect @@ -166,13 +97,14 @@ static void handle_dollar (braced_code_t code_kind, char *cp, location_t location); static void handle_at (braced_code_t code_kind, char *cp, location_t location); -static void handle_syncline (char *args, location_t *location); +static void handle_syncline (char *args); static int convert_ucn_to_byte (char const *hex_text); -static void unexpected_end_of_file (location_t, char const *); +static void unexpected_end_of_file (boundary, char const *); %} %x SC_COMMENT SC_LINE_COMMENT SC_YACC_COMMENT %x SC_STRING SC_CHARACTER +%x SC_AFTER_IDENTIFIER %x SC_ESCAPED_STRING SC_ESCAPED_CHARACTER %x SC_BRACED_CODE SC_PROLOGUE SC_EPILOGUE @@ -195,18 +127,41 @@ splice (\\[ \f\t\v]*\n)* /* Nesting level of the current code in braces. */ int braces_level IF_LINT (= 0); - /* Scanner context when scanning C code. */ - int c_context IF_LINT (= 0); + /* Parent context state, when applicable. */ + int context_state IF_LINT (= 0); + + /* Location of most recent identifier, when applicable. */ + location_t id_loc IF_LINT (= *loc); - /* A string representing the most recently saved token. */ - char *last_string; + /* Location where containing code started, when applicable. */ + boundary code_start IF_LINT (= loc->start); - /* At each yylex invocation, mark the current position as the - start of the next token. */ - YY_STEP; + /* Location where containing comment or string or character literal + started, when applicable. */ + boundary token_start IF_LINT (= loc->start); %} + /*-----------------------. + | Scanning white space. | + `-----------------------*/ + + +{ + [ \f\n\t\v] ; + + /* Comments. */ + "/*" token_start = loc->start; context_state = YY_START; BEGIN SC_YACC_COMMENT; + "//".* ; + + /* #line directives are not documented, and may be withdrawn or + modified in future versions of Bison. */ + ^"#line "{int}" \"".*"\"\n" { + handle_syncline (yytext + sizeof "#line " - 1); + } +} + + /*----------------------------. | Scanning Bison directives. | `----------------------------*/ @@ -247,25 +202,23 @@ splice (\\[ \f\t\v]*\n)* "%verbose" return PERCENT_VERBOSE; "%yacc" return PERCENT_YACC; - {directive} { - complain_at (*yylloc, _("invalid directive: %s"), quote (yytext)); - YY_STEP; + {directive} { + complain_at (*loc, _("invalid directive: %s"), quote (yytext)); } - ^"#line "{int}" \""[^\"]*"\"\n" handle_syncline (yytext + strlen ("#line "), yylloc); YY_STEP; - "=" return EQUAL; - ":" rule_length = 0; return COLON; "|" rule_length = 0; return PIPE; - "," return COMMA; ";" return SEMICOLON; - [ \f\n\t\v] YY_STEP; + "," { + warn_at (*loc, _("stray `,' treated as white space")); + } - {id} { - yylval->symbol = symbol_get (yytext, *yylloc); + {id} { + val->symbol = symbol_get (yytext, *loc); + id_loc = *loc; rule_length++; - return ID; + BEGIN SC_AFTER_IDENTIFIER; } {int} { @@ -274,47 +227,78 @@ splice (\\[ \f\t\v]*\n)* num = strtoul (yytext, 0, 10); if (INT_MAX < num || errno) { - complain_at (*yylloc, _("integer out of range: %s"), quote (yytext)); + complain_at (*loc, _("integer out of range: %s"), quote (yytext)); num = INT_MAX; } - yylval->integer = num; + val->integer = num; return INT; } /* Characters. We don't check there is only one. */ - "'" YY_OBS_GROW; BEGIN SC_ESCAPED_CHARACTER; + "'" STRING_GROW; token_start = loc->start; BEGIN SC_ESCAPED_CHARACTER; /* Strings. */ - "\"" YY_OBS_GROW; BEGIN SC_ESCAPED_STRING; - - /* Comments. */ - "/*" BEGIN SC_YACC_COMMENT; - "//".* YY_STEP; + "\"" STRING_GROW; token_start = loc->start; BEGIN SC_ESCAPED_STRING; /* Prologue. */ - "%{" BEGIN SC_PROLOGUE; + "%{" code_start = loc->start; BEGIN SC_PROLOGUE; /* Code in between braces. */ - "{" YY_OBS_GROW; braces_level = 0; BEGIN SC_BRACED_CODE; + "{" { + STRING_GROW; + braces_level = 0; + code_start = loc->start; + BEGIN SC_BRACED_CODE; + } /* A type. */ "<"{tag}">" { obstack_grow (&string_obstack, yytext + 1, yyleng - 2); - YY_OBS_FINISH; - yylval->string = last_string; + STRING_FINISH; + val->struniq = struniq_new (last_string); + STRING_FREE; return TYPE; } "%%" { static int percent_percent_count; if (++percent_percent_count == 2) - BEGIN SC_EPILOGUE; + { + code_start = loc->start; + BEGIN SC_EPILOGUE; + } return PERCENT_PERCENT; } . { - complain_at (*yylloc, _("invalid character: %s"), quote (yytext)); - YY_STEP; + complain_at (*loc, _("invalid character: %s"), quote (yytext)); + } +} + + + /*-----------------------------------------------------------------. + | Scanning after an identifier, checking whether a colon is next. | + `-----------------------------------------------------------------*/ + + +{ + ":" { + rule_length = 0; + *loc = id_loc; + BEGIN INITIAL; + return ID_COLON; + } + . { + scanner_cursor.column -= mbsnwidth (yytext, yyleng, 0); + yyless (0); + *loc = id_loc; + BEGIN INITIAL; + return ID; + } + <> { + *loc = id_loc; + BEGIN INITIAL; + return ID; } } @@ -325,13 +309,9 @@ splice (\\[ \f\t\v]*\n)* { - "*/" { - YY_STEP; - BEGIN INITIAL; - } - + "*/" BEGIN context_state; .|\n ; - <> unexpected_end_of_file (*yylloc, "*/"); + <> unexpected_end_of_file (token_start, "*/"); } @@ -341,8 +321,8 @@ splice (\\[ \f\t\v]*\n)* { - "*"{splice}"/" YY_OBS_GROW; BEGIN c_context; - <> unexpected_end_of_file (*yylloc, "*/"); + "*"{splice}"/" STRING_GROW; BEGIN context_state; + <> unexpected_end_of_file (token_start, "*/"); } @@ -352,9 +332,9 @@ splice (\\[ \f\t\v]*\n)* { - "\n" YY_OBS_GROW; BEGIN c_context; - {splice} YY_OBS_GROW; - <> BEGIN c_context; + "\n" STRING_GROW; BEGIN context_state; + {splice} STRING_GROW; + <> BEGIN context_state; } @@ -366,16 +346,17 @@ splice (\\[ \f\t\v]*\n)* { "\"" { - YY_OBS_GROW; - YY_OBS_FINISH; - yylval->string = last_string; + STRING_GROW; + STRING_FINISH; + loc->start = token_start; + val->string = last_string; rule_length++; BEGIN INITIAL; return STRING; } - .|\n YY_OBS_GROW; - <> unexpected_end_of_file (*yylloc, "\""); + .|\n STRING_GROW; + <> unexpected_end_of_file (token_start, "\""); } /*---------------------------------------------------------------. @@ -386,20 +367,21 @@ splice (\\[ \f\t\v]*\n)* { "'" { - YY_OBS_GROW; - YY_OBS_FINISH; - yylval->symbol = symbol_get (last_string, *yylloc); - symbol_class_set (yylval->symbol, token_sym, *yylloc); - symbol_user_token_number_set (yylval->symbol, - (unsigned char) last_string[1], *yylloc); - YY_OBS_FREE; + STRING_GROW; + STRING_FINISH; + loc->start = token_start; + val->symbol = symbol_get (last_string, *loc); + symbol_class_set (val->symbol, token_sym, *loc); + symbol_user_token_number_set (val->symbol, + (unsigned char) last_string[1], *loc); + STRING_FREE; rule_length++; BEGIN INITIAL; return ID; } - .|\n YY_OBS_GROW; - <> unexpected_end_of_file (*yylloc, "'"); + .|\n STRING_GROW; + <> unexpected_end_of_file (token_start, "'"); } @@ -412,25 +394,17 @@ splice (\\[ \f\t\v]*\n)* \\[0-7]{1,3} { unsigned long c = strtoul (yytext + 1, 0, 8); if (UCHAR_MAX < c) - { - complain_at (*yylloc, _("invalid escape sequence: %s"), - quote (yytext)); - YY_STEP; - } + complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext)); else obstack_1grow (&string_obstack, c); } - \\x[0-9a-fA-F]+ { + \\x[0-9abcdefABCDEF]+ { unsigned long c; errno = 0; c = strtoul (yytext + 2, 0, 16); if (UCHAR_MAX < c || errno) - { - complain_at (*yylloc, _("invalid escape sequence: %s"), - quote (yytext)); - YY_STEP; - } + complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext)); else obstack_1grow (&string_obstack, c); } @@ -442,22 +416,20 @@ splice (\\[ \f\t\v]*\n)* \\r obstack_1grow (&string_obstack, '\r'); \\t obstack_1grow (&string_obstack, '\t'); \\v obstack_1grow (&string_obstack, '\v'); - \\[\"\'?\\] obstack_1grow (&string_obstack, yytext[1]); - \\(u|U[0-9a-fA-F]{4})[0-9a-fA-F]{4} { + + /* \\[\"\'?\\] would be shorter, but it confuses xgettext. */ + \\("\""|"'"|"?"|"\\") obstack_1grow (&string_obstack, yytext[1]); + + \\(u|U[0-9abcdefABCDEF]{4})[0-9abcdefABCDEF]{4} { int c = convert_ucn_to_byte (yytext); if (c < 0) - { - complain_at (*yylloc, _("invalid escape sequence: %s"), - quote (yytext)); - YY_STEP; - } + complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext)); else obstack_1grow (&string_obstack, c); } \\(.|\n) { - complain_at (*yylloc, _("unrecognized escape sequence: %s"), - quote (yytext)); - YY_OBS_GROW; + complain_at (*loc, _("unrecognized escape sequence: %s"), quote (yytext)); + STRING_GROW; } } @@ -469,10 +441,9 @@ splice (\\[ \f\t\v]*\n)* { - "'" YY_OBS_GROW; BEGIN c_context; - \\{splice}[^\[\]] YY_OBS_GROW; - {splice} YY_OBS_GROW; - <> unexpected_end_of_file (*yylloc, "'"); + "'" STRING_GROW; BEGIN context_state; + \\{splice}[^$@\[\]] STRING_GROW; + <> unexpected_end_of_file (token_start, "'"); } @@ -483,10 +454,9 @@ splice (\\[ \f\t\v]*\n)* { - "\"" YY_OBS_GROW; BEGIN c_context; - \\{splice}[^\[\]] YY_OBS_GROW; - {splice} YY_OBS_GROW; - <> unexpected_end_of_file (*yylloc, "\""); + "\"" STRING_GROW; BEGIN context_state; + \\{splice}[^$@\[\]] STRING_GROW; + <> unexpected_end_of_file (token_start, "\""); } @@ -496,10 +466,29 @@ splice (\\[ \f\t\v]*\n)* { - "'" YY_OBS_GROW; c_context = YY_START; BEGIN SC_CHARACTER; - "\"" YY_OBS_GROW; c_context = YY_START; BEGIN SC_STRING; - "/"{splice}"*" YY_OBS_GROW; c_context = YY_START; BEGIN SC_COMMENT; - "/"{splice}"/" YY_OBS_GROW; c_context = YY_START; BEGIN SC_LINE_COMMENT; + "'" { + STRING_GROW; + context_state = YY_START; + token_start = loc->start; + BEGIN SC_CHARACTER; + } + "\"" { + STRING_GROW; + context_state = YY_START; + token_start = loc->start; + BEGIN SC_STRING; + } + "/"{splice}"*" { + STRING_GROW; + context_state = YY_START; + token_start = loc->start; + BEGIN SC_COMMENT; + } + "/"{splice}"/" { + STRING_GROW; + context_state = YY_START; + BEGIN SC_LINE_COMMENT; + } } @@ -510,15 +499,16 @@ splice (\\[ \f\t\v]*\n)* { - "{"|"<"{splice}"%" YY_OBS_GROW; braces_level++; - "%"{splice}">" YY_OBS_GROW; braces_level--; + "{"|"<"{splice}"%" STRING_GROW; braces_level++; + "%"{splice}">" STRING_GROW; braces_level--; "}" { - YY_OBS_GROW; + STRING_GROW; braces_level--; if (braces_level < 0) { - YY_OBS_FINISH; - yylval->string = last_string; + STRING_FINISH; + loc->start = code_start; + val->string = last_string; rule_length++; BEGIN INITIAL; return BRACED_CODE; @@ -527,14 +517,14 @@ splice (\\[ \f\t\v]*\n)* /* Tokenize `<<%' correctly (as `<<' `%') rather than incorrrectly (as `<' `<%'). */ - "<"{splice}"<" YY_OBS_GROW; + "<"{splice}"<" STRING_GROW; "$"("<"{tag}">")?(-?[0-9]+|"$") { handle_dollar (current_braced_code, - yytext, *yylloc); } + yytext, *loc); } "@"(-?[0-9]+|"$") { handle_at (current_braced_code, - yytext, *yylloc); } + yytext, *loc); } - <> unexpected_end_of_file (*yylloc, "}"); + <> unexpected_end_of_file (code_start, "}"); } @@ -545,13 +535,14 @@ splice (\\[ \f\t\v]*\n)* { "%}" { - YY_OBS_FINISH; - yylval->string = last_string; + STRING_FINISH; + loc->start = code_start; + val->string = last_string; BEGIN INITIAL; return PROLOGUE; } - <> unexpected_end_of_file (*yylloc, "%}"); + <> unexpected_end_of_file (code_start, "%}"); } @@ -563,8 +554,9 @@ splice (\\[ \f\t\v]*\n)* { <> { - YY_OBS_FINISH; - yylval->string = last_string; + STRING_FINISH; + loc->start = code_start; + val->string = last_string; BEGIN INITIAL; return EPILOGUE; } @@ -578,14 +570,100 @@ splice (\\[ \f\t\v]*\n)* { - \[ obstack_sgrow (&string_obstack, "@<:@"); - \] obstack_sgrow (&string_obstack, "@:>@"); - .|\n YY_OBS_GROW; + \$ obstack_sgrow (&string_obstack, "$]["); + \@ obstack_sgrow (&string_obstack, "@@"); + \[ obstack_sgrow (&string_obstack, "@{"); + \] obstack_sgrow (&string_obstack, "@}"); + .|\n STRING_GROW; } %% +/* Set *LOC and adjust scanner cursor to account for token TOKEN of + size SIZE. */ + +static void +adjust_location (location_t *loc, char const *token, size_t size) +{ + int line = scanner_cursor.line; + int column = scanner_cursor.column; + char const *p0 = token; + char const *p = token; + char const *lim = token + size; + + loc->start = scanner_cursor; + + for (p = token; p < lim; p++) + switch (*p) + { + case '\n': + line++; + column = 1; + p0 = p + 1; + break; + + case '\t': + column += mbsnwidth (p0, p - p0, 0); + column += 8 - ((column - 1) & 7); + p0 = p + 1; + break; + } + + scanner_cursor.line = line; + scanner_cursor.column = column + mbsnwidth (p0, p - p0, 0); + + loc->end = scanner_cursor; +} + + +/* Read bytes from FP into buffer BUF of size SIZE. Return the + number of bytes read. Remove '\r' from input, treating \r\n + and isolated \r as \n. */ + +static size_t +no_cr_read (FILE *fp, char *buf, size_t size) +{ + size_t s = fread (buf, 1, size, fp); + if (s) + { + char *w = memchr (buf, '\r', s); + if (w) + { + char const *r = ++w; + char const *lim = buf + s; + + for (;;) + { + /* Found an '\r'. Treat it like '\n', but ignore any + '\n' that immediately follows. */ + w[-1] = '\n'; + if (r == lim) + { + int ch = getc (fp); + if (ch != '\n' && ungetc (ch, fp) != ch) + break; + } + else if (*r == '\n') + r++; + + /* Copy until the next '\r'. */ + do + { + if (r == lim) + return w - buf; + } + while ((*w++ = *r++) != '\r'); + } + + return w - buf; + } + } + + return s; +} + + /*------------------------------------------------------------------. | TEXT is pointing to a wannabee semantic value (i.e., a `$'). | | | @@ -821,34 +899,40 @@ convert_ucn_to_byte (char const *ucn) `----------------------------------------------------------------*/ static void -handle_syncline (char *args, location_t *location) +handle_syncline (char *args) { int lineno = strtol (args, &args, 10); const char *file = NULL; file = strchr (args, '"') + 1; *strchr (file, '"') = 0; - /* FIXME: Leaking... Can't free, as some locations are still - pointing to the old file name. */ - infile = xstrdup (file); - location->file = infile; - location->last_line = lineno; + scanner_cursor.file = current_file = xstrdup (file); + scanner_cursor.line = lineno; + scanner_cursor.column = 1; } -/*-------------------------------------------------------------. -| Report an unexpected end of file at LOC. An end of file was | -| encountered and the expected TOKEN_END was missing. After | -| reporting the problem, pretend that TOKEN_END was found. | -`-------------------------------------------------------------*/ +/*------------------------------------------------------------------------. +| Report an unexpected EOF in a token or comment starting at START. | +| An end of file was encountered and the expected TOKEN_END was missing. | +| After reporting the problem, pretend that TOKEN_END was found. | +`------------------------------------------------------------------------*/ static void -unexpected_end_of_file (location_t loc, char const *token_end) +unexpected_end_of_file (boundary start, char const *token_end) { - size_t i; + size_t i = strlen (token_end); + + location_t location; + location.start = start; + location.end = scanner_cursor; + complain_at (location, _("missing `%s' at end of file"), token_end); + + /* Adjust scanner cursor so that any later message does not count + the characters about to be inserted. */ + scanner_cursor.column -= i; - complain_at (loc, _("missing `%s' at end of file"), token_end); - for (i = strlen (token_end); i != 0; i--) - unput (token_end[i - 1]); + while (i != 0) + unput (token_end[--i]); }