src/scan-gram.l

   1 /* Bison Grammar Scanner                             -*- C -*-
   2
   3    Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation,
   4    Inc.
   5
   6    This file is part of Bison, the GNU Compiler Compiler.
   7
   8    This program is free software: you can redistribute it and/or modify
   9    it under the terms of the GNU General Public License as published by
  10    the Free Software Foundation, either version 3 of the License, or
  11    (at your option) any later version.
  12
  13    This program is distributed in the hope that it will be useful,
  14    but WITHOUT ANY WARRANTY; without even the implied warranty of
  15    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16    GNU General Public License for more details.
  17
  18    You should have received a copy of the GNU General Public License
  19    along with this program.  If not, see <http://www.gnu.org/licenses/>.  */
  20
  21 %option debug nodefault nounput noyywrap never-interactive
  22 %option prefix="gram_" outfile="lex.yy.c"
  23
  24 %{
  25 /* Work around a bug in flex 2.5.31.  See Debian bug 333231
  26    <http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=333231>.  */
  27 #undef gram_wrap
  28 #define gram_wrap() 1
  29
  30 #define FLEX_PREFIX(Id) gram_ ## Id
  31 #include <src/flex-scanner.h>
  32
  33 #include <src/complain.h>
  34 #include <src/files.h>
  35 #include <src/gram.h>
  36 #include <quotearg.h>
  37 #include <src/reader.h>
  38 #include <src/uniqstr.h>
  39
  40 #include <mbswidth.h>
  41 #include <quote.h>
  42
  43 #include <src/scan-gram.h>
  44
  45 #define YY_DECL GRAM_LEX_DECL
  46
  47 #define YY_USER_INIT                                    \
  48    code_start = scanner_cursor = loc->start;            \
  49
  50 /* Location of scanner cursor.  */
  51 static boundary scanner_cursor;
  52
  53 #define YY_USER_ACTION  location_compute (loc, &scanner_cursor, yytext, yyleng);
  54
  55 static size_t no_cr_read (FILE *, char *, size_t);
  56 #define YY_INPUT(buf, result, size) ((result) = no_cr_read (yyin, buf, size))
  57
  58 /* A string representing the most recently saved token.  */
  59 static char *last_string;
  60
  61 void
  62 gram_scanner_last_string_free (void)
  63 {
  64   STRING_FREE;
  65 }
  66
  67 static void handle_syncline (char *, location);
  68 static unsigned long int scan_integer (char const *p, int base, location loc);
  69 static int convert_ucn_to_byte (char const *hex_text);
  70 static void unexpected_eof (boundary, char const *);
  71 static void unexpected_newline (boundary, char const *);
  72
  73 %}
  74  /* A C-like comment in directives/rules. */
  75 %x SC_YACC_COMMENT
  76  /* Strings and characters in directives/rules. */
  77 %x SC_ESCAPED_STRING SC_ESCAPED_CHARACTER
  78  /* A identifier was just read in directives/rules.  Special state
  79     to capture the sequence `identifier :'. */
  80 %x SC_AFTER_IDENTIFIER
  81  /* A complex tag, with nested angles brackets. */
  82 %x SC_TAG
  83
  84  /* Three types of user code:
  85     - prologue (code between `%{' `%}' in the first section, before %%);
  86     - actions, printers, union, etc, (between braced in the middle section);
  87     - epilogue (everything after the second %%). */
  88 %x SC_PROLOGUE SC_BRACED_CODE SC_EPILOGUE
  89  /* C and C++ comments in code. */
  90 %x SC_COMMENT SC_LINE_COMMENT
  91  /* Strings and characters in code. */
  92 %x SC_STRING SC_CHARACTER
  93
  94 letter    [.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_]
  95 id        {letter}({letter}|[0-9])*
  96 directive %{letter}({letter}|[0-9]|-)*
  97 int       [0-9]+
  98
  99 /* POSIX says that a tag must be both an id and a C union member, but
 100    historically almost any character is allowed in a tag.  We disallow
 101    NUL, as this simplifies our implementation.  We disallow angle
 102    bracket to match them in nested pairs: several languages use them
 103    for generics/template types.  */
 104 tag      [^\0<>]+
 105
 106 /* Zero or more instances of backslash-newline.  Following GCC, allow
 107    white space between the backslash and the newline.  */
 108 splice   (\\[ \f\t\v]*\n)*
 109
 110 %%
 111 %{
 112   /* Nesting level.  Either for nested braces, or nested angle brackets
 113      (but not mixed).  */
 114   int nesting IF_LINT (= 0);
 115
 116   /* Parent context state, when applicable.  */
 117   int context_state IF_LINT (= 0);
 118
 119   /* Location of most recent identifier, when applicable.  */
 120   location id_loc IF_LINT (= empty_location);
 121
 122   /* Where containing code started, when applicable.  Its initial
 123      value is relevant only when yylex is invoked in the SC_EPILOGUE
 124      start condition.  */
 125   boundary code_start = scanner_cursor;
 126
 127   /* Where containing comment or string or character literal started,
 128      when applicable.  */
 129   boundary token_start IF_LINT (= scanner_cursor);
 130 %}
 131
 132
 133   /*-----------------------.
 134   | Scanning white space.  |
 135   `-----------------------*/
 136
 137 <INITIAL,SC_AFTER_IDENTIFIER>
 138 {
 139   /* Comments and white space.  */
 140   ","          warn_at (*loc, _("stray `,' treated as white space"));
 141   [ \f\n\t\v]  |
 142   "//".*       ;
 143   "/*" {
 144     token_start = loc->start;
 145     context_state = YY_START;
 146     BEGIN SC_YACC_COMMENT;
 147   }
 148
 149   /* #line directives are not documented, and may be withdrawn or
 150      modified in future versions of Bison.  */
 151   ^"#line "{int}" \"".*"\"\n" {
 152     handle_syncline (yytext + sizeof "#line " - 1, *loc);
 153   }
 154 }
 155
 156
 157   /*----------------------------.
 158   | Scanning Bison directives.  |
 159   `----------------------------*/
 160
 161   /* For directives that are also command line options, the regex must be
 162         "%..."
 163      after "[-_]"s are removed, and the directive must match the --long
 164      option name, with a single string argument.  Otherwise, add exceptions
 165      to ../build-aux/cross-options.pl.  */
 166
 167 <INITIAL>
 168 {
 169   "%binary"                         return PERCENT_NONASSOC;
 170   "%code"                           return PERCENT_CODE;
 171   "%debug"                          return PERCENT_DEBUG;
 172   "%default"[-_]"prec"              return PERCENT_DEFAULT_PREC;
 173   "%define"                         return PERCENT_DEFINE;
 174   "%defines"                        return PERCENT_DEFINES;
 175   "%destructor"                     return PERCENT_DESTRUCTOR;
 176   "%dprec"                          return PERCENT_DPREC;
 177   "%error"[-_]"verbose"             return PERCENT_ERROR_VERBOSE;
 178   "%expect"                         return PERCENT_EXPECT;
 179   "%expect"[-_]"rr"                 return PERCENT_EXPECT_RR;
 180   "%file-prefix"                    return PERCENT_FILE_PREFIX;
 181   "%fixed"[-_]"output"[-_]"files"   return PERCENT_YACC;
 182   "%initial-action"                 return PERCENT_INITIAL_ACTION;
 183   "%glr-parser"                     return PERCENT_GLR_PARSER;
 184   "%language"                       return PERCENT_LANGUAGE;
 185   "%left"                           return PERCENT_LEFT;
 186   "%lex-param"                      return PERCENT_LEX_PARAM;
 187   "%locations"                      return PERCENT_LOCATIONS;
 188   "%merge"                          return PERCENT_MERGE;
 189   "%name"[-_]"prefix"               return PERCENT_NAME_PREFIX;
 190   "%no"[-_]"default"[-_]"prec"      return PERCENT_NO_DEFAULT_PREC;
 191   "%no"[-_]"lines"                  return PERCENT_NO_LINES;
 192   "%nonassoc"                       return PERCENT_NONASSOC;
 193   "%nondeterministic-parser"        return PERCENT_NONDETERMINISTIC_PARSER;
 194   "%nterm"                          return PERCENT_NTERM;
 195   "%output"                         return PERCENT_OUTPUT;
 196   "%parse-param"                    return PERCENT_PARSE_PARAM;
 197   "%prec"                           return PERCENT_PREC;
 198   "%precedence"                     return PERCENT_PRECEDENCE;
 199   "%printer"                        return PERCENT_PRINTER;
 200   "%pure"[-_]"parser"               return PERCENT_PURE_PARSER;
 201   "%require"                        return PERCENT_REQUIRE;
 202   "%right"                          return PERCENT_RIGHT;
 203   "%skeleton"                       return PERCENT_SKELETON;
 204   "%start"                          return PERCENT_START;
 205   "%term"                           return PERCENT_TOKEN;
 206   "%token"                          return PERCENT_TOKEN;
 207   "%token"[-_]"table"               return PERCENT_TOKEN_TABLE;
 208   "%type"                           return PERCENT_TYPE;
 209   "%union"                          return PERCENT_UNION;
 210   "%verbose"                        return PERCENT_VERBOSE;
 211   "%yacc"                           return PERCENT_YACC;
 212
 213   {directive} {
 214     complain_at (*loc, _("invalid directive: %s"), quote (yytext));
 215   }
 216
 217   "="                     return EQUAL;
 218   "|"                     return PIPE;
 219   ";"                     return SEMICOLON;
 220
 221   {id} {
 222     val->uniqstr = uniqstr_new (yytext);
 223     id_loc = *loc;
 224     BEGIN SC_AFTER_IDENTIFIER;
 225   }
 226
 227   {int} {
 228     val->integer = scan_integer (yytext, 10, *loc);
 229     return INT;
 230   }
 231   0[xX][0-9abcdefABCDEF]+ {
 232     val->integer = scan_integer (yytext, 16, *loc);
 233     return INT;
 234   }
 235
 236   /* Characters.  We don't check there is only one.  */
 237   "'"         STRING_GROW; token_start = loc->start; BEGIN SC_ESCAPED_CHARACTER;
 238
 239   /* Strings. */
 240   "\""        token_start = loc->start; BEGIN SC_ESCAPED_STRING;
 241
 242   /* Prologue. */
 243   "%{"        code_start = loc->start; BEGIN SC_PROLOGUE;
 244
 245   /* Code in between braces.  */
 246   "{" {
 247     STRING_GROW;
 248     nesting = 0;
 249     code_start = loc->start;
 250     BEGIN SC_BRACED_CODE;
 251   }
 252
 253   /* A type. */
 254   "<*>"       return TAG_ANY;
 255   "<>"        return TAG_NONE;
 256   "<"{tag}">" {
 257     obstack_grow (&obstack_for_string, yytext + 1, yyleng - 2);
 258     STRING_FINISH;
 259     val->uniqstr = uniqstr_new (last_string);
 260     STRING_FREE;
 261     return TAG;
 262   }
 263   "<"         {
 264     nesting = 0;
 265     token_start = loc->start;
 266     BEGIN SC_TAG;
 267   }
 268
 269   "%%" {
 270     static int percent_percent_count;
 271     if (++percent_percent_count == 2)
 272       BEGIN SC_EPILOGUE;
 273     return PERCENT_PERCENT;
 274   }
 275
 276   . {
 277     complain_at (*loc, _("invalid character: %s"), quote (yytext));
 278   }
 279
 280   <<EOF>> {
 281     loc->start = loc->end = scanner_cursor;
 282     yyterminate ();
 283   }
 284 }
 285
 286
 287   /*--------------------------------------------------------------.
 288   | Supporting \0 complexifies our implementation for no expected |
 289   | added value.                                                  |
 290   `--------------------------------------------------------------*/
 291
 292 <SC_ESCAPED_CHARACTER,SC_ESCAPED_STRING,SC_TAG>
 293 {
 294   \0        complain_at (*loc, _("invalid null character"));
 295 }
 296
 297
 298   /*-----------------------------------------------------------------.
 299   | Scanning after an identifier, checking whether a colon is next.  |
 300   `-----------------------------------------------------------------*/
 301
 302 <SC_AFTER_IDENTIFIER>
 303 {
 304   ":" {
 305     *loc = id_loc;
 306     BEGIN INITIAL;
 307     return ID_COLON;
 308   }
 309   . {
 310     scanner_cursor.column -= mbsnwidth (yytext, yyleng, 0);
 311     yyless (0);
 312     *loc = id_loc;
 313     BEGIN INITIAL;
 314     return ID;
 315   }
 316   <<EOF>> {
 317     *loc = id_loc;
 318     BEGIN INITIAL;
 319     return ID;
 320   }
 321 }
 322
 323
 324   /*---------------------------------------------------------------.
 325   | Scanning a Yacc comment.  The initial `/ *' is already eaten.  |
 326   `---------------------------------------------------------------*/
 327
 328 <SC_YACC_COMMENT>
 329 {
 330   "*/"     BEGIN context_state;
 331   .|\n     ;
 332   <<EOF>>  unexpected_eof (token_start, "*/"); BEGIN context_state;
 333 }
 334
 335
 336   /*------------------------------------------------------------.
 337   | Scanning a C comment.  The initial `/ *' is already eaten.  |
 338   `------------------------------------------------------------*/
 339
 340 <SC_COMMENT>
 341 {
 342   "*"{splice}"/"  STRING_GROW; BEGIN context_state;
 343   <<EOF>>         unexpected_eof (token_start, "*/"); BEGIN context_state;
 344 }
 345
 346
 347   /*--------------------------------------------------------------.
 348   | Scanning a line comment.  The initial `//' is already eaten.  |
 349   `--------------------------------------------------------------*/
 350
 351 <SC_LINE_COMMENT>
 352 {
 353   "\n"           STRING_GROW; BEGIN context_state;
 354   {splice}       STRING_GROW;
 355   <<EOF>>        BEGIN context_state;
 356 }
 357
 358
 359   /*------------------------------------------------.
 360   | Scanning a Bison string, including its escapes. |
 361   | The initial quote is already eaten.             |
 362   `------------------------------------------------*/
 363
 364 <SC_ESCAPED_STRING>
 365 {
 366   "\""|"\n" {
 367     if (yytext[0] == '\n')
 368       unexpected_newline (token_start, "\"");
 369     STRING_FINISH;
 370     loc->start = token_start;
 371     val->chars = last_string;
 372     BEGIN INITIAL;
 373     return STRING;
 374   }
 375   <<EOF>> {
 376     unexpected_eof (token_start, "\"");
 377     STRING_FINISH;
 378     loc->start = token_start;
 379     val->chars = last_string;
 380     BEGIN INITIAL;
 381     return STRING;
 382   }
 383 }
 384
 385   /*----------------------------------------------------------.
 386   | Scanning a Bison character literal, decoding its escapes. |
 387   | The initial quote is already eaten.                       |
 388   `----------------------------------------------------------*/
 389
 390 <SC_ESCAPED_CHARACTER>
 391 {
 392   "'"|"\n" {
 393     if (yytext[0] == '\n')
 394       unexpected_newline (token_start, "'");
 395     STRING_GROW;
 396     STRING_FINISH;
 397     loc->start = token_start;
 398     val->character = last_string[1];
 399     STRING_FREE;
 400     BEGIN INITIAL;
 401     return CHAR;
 402   }
 403   <<EOF>> {
 404     unexpected_eof (token_start, "'");
 405     STRING_FINISH;
 406     loc->start = token_start;
 407     if (strlen(last_string) > 1)
 408       val->character = last_string[1];
 409     else
 410       val->character = last_string[0];
 411     STRING_FREE;
 412     BEGIN INITIAL;
 413     return CHAR;
 414   }
 415 }
 416
 417   /*-----------------------------------------------------------.
 418   | Scanning a Bison nested tag.  The initial angle bracket is |
 419   | already eaten.                                             |
 420   `-----------------------------------------------------------*/
 421
 422 <SC_TAG>
 423 {
 424   ">" {
 425     --nesting;
 426     if (nesting < 0)
 427       {
 428         STRING_FINISH;
 429         loc->start = token_start;
 430         val->uniqstr = uniqstr_new (last_string);
 431         STRING_FREE;
 432         BEGIN INITIAL;
 433         return TAG;
 434       }
 435     STRING_GROW;
 436   }
 437
 438   [^<>]+ STRING_GROW;
 439   "<"+   STRING_GROW; nesting += yyleng;
 440
 441   <<EOF>> {
 442     unexpected_eof (token_start, ">");
 443     STRING_FINISH;
 444     loc->start = token_start;
 445     val->uniqstr = uniqstr_new (last_string);
 446     STRING_FREE;
 447     BEGIN INITIAL;
 448     return TAG;
 449   }
 450 }
 451
 452   /*----------------------------.
 453   | Decode escaped characters.  |
 454   `----------------------------*/
 455
 456 <SC_ESCAPED_STRING,SC_ESCAPED_CHARACTER>
 457 {
 458   \\[0-7]{1,3} {
 459     unsigned long int c = strtoul (yytext + 1, NULL, 8);
 460     if (UCHAR_MAX < c)
 461       complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext));
 462     else if (! c)
 463       complain_at (*loc, _("invalid null character: %s"), quote (yytext));
 464     else
 465       obstack_1grow (&obstack_for_string, c);
 466   }
 467
 468   \\x[0-9abcdefABCDEF]+ {
 469     verify (UCHAR_MAX < ULONG_MAX);
 470     unsigned long int c = strtoul (yytext + 2, NULL, 16);
 471     if (UCHAR_MAX < c)
 472       complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext));
 473     else if (! c)
 474       complain_at (*loc, _("invalid null character: %s"), quote (yytext));
 475     else
 476       obstack_1grow (&obstack_for_string, c);
 477   }
 478
 479   \\a   obstack_1grow (&obstack_for_string, '\a');
 480   \\b   obstack_1grow (&obstack_for_string, '\b');
 481   \\f   obstack_1grow (&obstack_for_string, '\f');
 482   \\n   obstack_1grow (&obstack_for_string, '\n');
 483   \\r   obstack_1grow (&obstack_for_string, '\r');
 484   \\t   obstack_1grow (&obstack_for_string, '\t');
 485   \\v   obstack_1grow (&obstack_for_string, '\v');
 486
 487   /* \\[\"\'?\\] would be shorter, but it confuses xgettext.  */
 488   \\("\""|"'"|"?"|"\\")  obstack_1grow (&obstack_for_string, yytext[1]);
 489
 490   \\(u|U[0-9abcdefABCDEF]{4})[0-9abcdefABCDEF]{4} {
 491     int c = convert_ucn_to_byte (yytext);
 492     if (c < 0)
 493       complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext));
 494     else if (! c)
 495       complain_at (*loc, _("invalid null character: %s"), quote (yytext));
 496     else
 497       obstack_1grow (&obstack_for_string, c);
 498   }
 499   \\(.|\n)      {
 500     complain_at (*loc, _("unrecognized escape sequence: %s"), quote (yytext));
 501     STRING_GROW;
 502   }
 503 }
 504
 505   /*--------------------------------------------.
 506   | Scanning user-code characters and strings.  |
 507   `--------------------------------------------*/
 508
 509 <SC_CHARACTER,SC_STRING>
 510 {
 511   {splice}|\\{splice}[^\n\[\]]  STRING_GROW;
 512 }
 513
 514 <SC_CHARACTER>
 515 {
 516   "'"           STRING_GROW; BEGIN context_state;
 517   \n            unexpected_newline (token_start, "'"); BEGIN context_state;
 518   <<EOF>>       unexpected_eof (token_start, "'"); BEGIN context_state;
 519 }
 520
 521 <SC_STRING>
 522 {
 523   "\""          STRING_GROW; BEGIN context_state;
 524   \n            unexpected_newline (token_start, "\""); BEGIN context_state;
 525   <<EOF>>       unexpected_eof (token_start, "\""); BEGIN context_state;
 526 }
 527
 528
 529   /*---------------------------------------------------.
 530   | Strings, comments etc. can be found in user code.  |
 531   `---------------------------------------------------*/
 532
 533 <SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>
 534 {
 535   "'" {
 536     STRING_GROW;
 537     context_state = YY_START;
 538     token_start = loc->start;
 539     BEGIN SC_CHARACTER;
 540   }
 541   "\"" {
 542     STRING_GROW;
 543     context_state = YY_START;
 544     token_start = loc->start;
 545     BEGIN SC_STRING;
 546   }
 547   "/"{splice}"*" {
 548     STRING_GROW;
 549     context_state = YY_START;
 550     token_start = loc->start;
 551     BEGIN SC_COMMENT;
 552   }
 553   "/"{splice}"/" {
 554     STRING_GROW;
 555     context_state = YY_START;
 556     BEGIN SC_LINE_COMMENT;
 557   }
 558 }
 559
 560
 561
 562   /*-----------------------------------------------------------.
 563   | Scanning some code in braces (actions). The initial "{" is |
 564   | already eaten.                                             |
 565   `-----------------------------------------------------------*/
 566
 567 <SC_BRACED_CODE>
 568 {
 569   "{"|"<"{splice}"%"  STRING_GROW; nesting++;
 570   "%"{splice}">"      STRING_GROW; nesting--;
 571   "}" {
 572     obstack_1grow (&obstack_for_string, '}');
 573
 574     --nesting;
 575     if (nesting < 0)
 576       {
 577         STRING_FINISH;
 578         loc->start = code_start;
 579         val->code = last_string;
 580         BEGIN INITIAL;
 581         return BRACED_CODE;
 582       }
 583   }
 584
 585   /* Tokenize `<<%' correctly (as `<<' `%') rather than incorrrectly
 586      (as `<' `<%').  */
 587   "<"{splice}"<"  STRING_GROW;
 588
 589   <<EOF>> {
 590     unexpected_eof (code_start, "}");
 591     STRING_FINISH;
 592     loc->start = code_start;
 593     val->code = last_string;
 594     BEGIN INITIAL;
 595     return BRACED_CODE;
 596   }
 597 }
 598
 599
 600   /*--------------------------------------------------------------.
 601   | Scanning some prologue: from "%{" (already scanned) to "%}".  |
 602   `--------------------------------------------------------------*/
 603
 604 <SC_PROLOGUE>
 605 {
 606   "%}" {
 607     STRING_FINISH;
 608     loc->start = code_start;
 609     val->chars = last_string;
 610     BEGIN INITIAL;
 611     return PROLOGUE;
 612   }
 613
 614   <<EOF>> {
 615     unexpected_eof (code_start, "%}");
 616     STRING_FINISH;
 617     loc->start = code_start;
 618     val->chars = last_string;
 619     BEGIN INITIAL;
 620     return PROLOGUE;
 621   }
 622 }
 623
 624
 625   /*---------------------------------------------------------------.
 626   | Scanning the epilogue (everything after the second "%%", which |
 627   | has already been eaten).                                       |
 628   `---------------------------------------------------------------*/
 629
 630 <SC_EPILOGUE>
 631 {
 632   <<EOF>> {
 633     STRING_FINISH;
 634     loc->start = code_start;
 635     val->chars = last_string;
 636     BEGIN INITIAL;
 637     return EPILOGUE;
 638   }
 639 }
 640
 641
 642   /*-----------------------------------------------------.
 643   | By default, grow the string obstack with the input.  |
 644   `-----------------------------------------------------*/
 645
 646 <SC_COMMENT,SC_LINE_COMMENT,SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE,SC_STRING,SC_CHARACTER,SC_ESCAPED_STRING,SC_ESCAPED_CHARACTER>.      |
 647 <SC_COMMENT,SC_LINE_COMMENT,SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>\n   STRING_GROW;
 648
 649 %%
 650
 651 /* Read bytes from FP into buffer BUF of size SIZE.  Return the
 652    number of bytes read.  Remove '\r' from input, treating \r\n
 653    and isolated \r as \n.  */
 654
 655 static size_t
 656 no_cr_read (FILE *fp, char *buf, size_t size)
 657 {
 658   size_t bytes_read = fread (buf, 1, size, fp);
 659   if (bytes_read)
 660     {
 661       char *w = memchr (buf, '\r', bytes_read);
 662       if (w)
 663         {
 664           char const *r = ++w;
 665           char const *lim = buf + bytes_read;
 666
 667           for (;;)
 668             {
 669               /* Found an '\r'.  Treat it like '\n', but ignore any
 670                  '\n' that immediately follows.  */
 671               w[-1] = '\n';
 672               if (r == lim)
 673                 {
 674                   int ch = getc (fp);
 675                   if (ch != '\n' && ungetc (ch, fp) != ch)
 676                     break;
 677                 }
 678               else if (*r == '\n')
 679                 r++;
 680
 681               /* Copy until the next '\r'.  */
 682               do
 683                 {
 684                   if (r == lim)
 685                     return w - buf;
 686                 }
 687               while ((*w++ = *r++) != '\r');
 688             }
 689
 690           return w - buf;
 691         }
 692     }
 693
 694   return bytes_read;
 695 }
 696
 697
 698
 699 /*------------------------------------------------------.
 700 | Scan NUMBER for a base-BASE integer at location LOC.  |
 701 `------------------------------------------------------*/
 702
 703 static unsigned long int
 704 scan_integer (char const *number, int base, location loc)
 705 {
 706   verify (INT_MAX < ULONG_MAX);
 707   unsigned long int num = strtoul (number, NULL, base);
 708
 709   if (INT_MAX < num)
 710     {
 711       complain_at (loc, _("integer out of range: %s"), quote (number));
 712       num = INT_MAX;
 713     }
 714
 715   return num;
 716 }
 717
 718
 719 /*------------------------------------------------------------------.
 720 | Convert universal character name UCN to a single-byte character,  |
 721 | and return that character.  Return -1 if UCN does not correspond  |
 722 | to a single-byte character.                                       |
 723 `------------------------------------------------------------------*/
 724
 725 static int
 726 convert_ucn_to_byte (char const *ucn)
 727 {
 728   verify (UCHAR_MAX <= INT_MAX);
 729   unsigned long int code = strtoul (ucn + 2, NULL, 16);
 730
 731   /* FIXME: Currently we assume Unicode-compatible unibyte characters
 732      on ASCII hosts (i.e., Latin-1 on hosts with 8-bit bytes).  On
 733      non-ASCII hosts we support only the portable C character set.
 734      These limitations should be removed once we add support for
 735      multibyte characters.  */
 736
 737   if (UCHAR_MAX < code)
 738     return -1;
 739
 740 #if ! ('$' == 0x24 && '@' == 0x40 && '`' == 0x60 && '~' == 0x7e)
 741   {
 742     /* A non-ASCII host.  Use CODE to index into a table of the C
 743        basic execution character set, which is guaranteed to exist on
 744        all Standard C platforms.  This table also includes '$', '@',
 745        and '`', which are not in the basic execution character set but
 746        which are unibyte characters on all the platforms that we know
 747        about.  */
 748     static signed char const table[] =
 749       {
 750         '\0',   -1,   -1,   -1,   -1,   -1,   -1, '\a',
 751         '\b', '\t', '\n', '\v', '\f', '\r',   -1,   -1,
 752           -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
 753           -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
 754          ' ',  '!',  '"',  '#',  '$',  '%',  '&', '\'',
 755          '(',  ')',  '*',  '+',  ',',  '-',  '.',  '/',
 756          '0',  '1',  '2',  '3',  '4',  '5',  '6',  '7',
 757          '8',  '9',  ':',  ';',  '<',  '=',  '>',  '?',
 758          '@',  'A',  'B',  'C',  'D',  'E',  'F',  'G',
 759          'H',  'I',  'J',  'K',  'L',  'M',  'N',  'O',
 760          'P',  'Q',  'R',  'S',  'T',  'U',  'V',  'W',
 761          'X',  'Y',  'Z',  '[', '\\',  ']',  '^',  '_',
 762          '`',  'a',  'b',  'c',  'd',  'e',  'f',  'g',
 763          'h',  'i',  'j',  'k',  'l',  'm',  'n',  'o',
 764          'p',  'q',  'r',  's',  't',  'u',  'v',  'w',
 765          'x',  'y',  'z',  '{',  '|',  '}',  '~'
 766       };
 767
 768     code = code < sizeof table ? table[code] : -1;
 769   }
 770 #endif
 771
 772   return code;
 773 }
 774
 775
 776 /*----------------------------------------------------------------.
 777 | Handle `#line INT "FILE"'.  ARGS has already skipped `#line '.  |
 778 `----------------------------------------------------------------*/
 779
 780 static void
 781 handle_syncline (char *args, location loc)
 782 {
 783   char *after_num;
 784   unsigned long int lineno = strtoul (args, &after_num, 10);
 785   char *file = strchr (after_num, '"') + 1;
 786   *strchr (file, '"') = '\0';
 787   if (INT_MAX <= lineno)
 788     {
 789       warn_at (loc, _("line number overflow"));
 790       lineno = INT_MAX;
 791     }
 792   current_file = uniqstr_new (file);
 793   boundary_set (&scanner_cursor, current_file, lineno, 1);
 794 }
 795
 796
 797 /*----------------------------------------------------------------.
 798 | For a token or comment starting at START, report message MSGID, |
 799 | which should say that an end marker was found before            |
 800 | the expected TOKEN_END.                                         |
 801 `----------------------------------------------------------------*/
 802
 803 static void
 804 unexpected_end (boundary start, char const *msgid, char const *token_end)
 805 {
 806   location loc;
 807   loc.start = start;
 808   loc.end = scanner_cursor;
 809   complain_at (loc, _(msgid), token_end);
 810 }
 811
 812
 813 /*------------------------------------------------------------------------.
 814 | Report an unexpected EOF in a token or comment starting at START.       |
 815 | An end of file was encountered and the expected TOKEN_END was missing.  |
 816 `------------------------------------------------------------------------*/
 817
 818 static void
 819 unexpected_eof (boundary start, char const *token_end)
 820 {
 821   unexpected_end (start, N_("missing `%s' at end of file"), token_end);
 822 }
 823
 824
 825 /*----------------------------------------.
 826 | Likewise, but for unexpected newlines.  |
 827 `----------------------------------------*/
 828
 829 static void
 830 unexpected_newline (boundary start, char const *token_end)
 831 {
 832   unexpected_end (start, N_("missing `%s' at end of line"), token_end);
 833 }
 834
 835
 836 /*-------------------------.
 837 | Initialize the scanner.  |
 838 `-------------------------*/
 839
 840 void
 841 gram_scanner_initialize (void)
 842 {
 843   obstack_init (&obstack_for_string);
 844 }
 845
 846
 847 /*-----------------------------------------------.
 848 | Free all the memory allocated to the scanner.  |
 849 `-----------------------------------------------*/
 850
 851 void
 852 gram_scanner_free (void)
 853 {
 854   obstack_free (&obstack_for_string, 0);
 855   /* Reclaim Flex's buffers.  */
 856   yylex_destroy ();
 857 }