src/scan-gram.l

   1 /* Bison Grammar Scanner                             -*- C -*-
   2
   3    Copyright (C) 2002, 2003, 2004, 2005, 2006 Free Software Foundation, Inc.
   4
   5    This file is part of Bison, the GNU Compiler Compiler.
   6
   7    This program is free software; you can redistribute it and/or modify
   8    it under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 2 of the License, or
  10    (at your option) any later version.
  11
  12    This program is distributed in the hope that it will be useful,
  13    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15    GNU General Public License for more details.
  16
  17    You should have received a copy of the GNU General Public License
  18    along with this program; if not, write to the Free Software
  19    Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  20    02110-1301  USA
  21 */
  22
  23 %option debug nodefault nounput noyywrap never-interactive
  24 %option prefix="gram_" outfile="lex.yy.c"
  25
  26 %{
  27 /* Work around a bug in flex 2.5.31.  See Debian bug 333231
  28    <http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=333231>.  */
  29 #undef gram_wrap
  30 #define gram_wrap() 1
  31
  32 #define FLEX_PREFIX(Id) gram_ ## Id
  33 #include "flex-scanner.h"
  34
  35 #include "complain.h"
  36 #include "files.h"
  37 #include "gram.h"
  38 #include "quotearg.h"
  39 #include "reader.h"
  40 #include "uniqstr.h"
  41
  42 #include <mbswidth.h>
  43 #include <quote.h>
  44
  45 #include "scan-gram.h"
  46
  47 #define YY_DECL GRAM_LEX_DECL
  48
  49 #define YY_USER_INIT                                    \
  50    code_start = scanner_cursor = loc->start;            \
  51
  52 /* Location of scanner cursor.  */
  53 static boundary scanner_cursor;
  54
  55 #define YY_USER_ACTION  location_compute (loc, &scanner_cursor, yytext, yyleng);
  56
  57 static size_t no_cr_read (FILE *, char *, size_t);
  58 #define YY_INPUT(buf, result, size) ((result) = no_cr_read (yyin, buf, size))
  59
  60 /* A string representing the most recently saved token.  */
  61 char *last_string;
  62
  63 void
  64 gram_scanner_last_string_free (void)
  65 {
  66   STRING_FREE;
  67 }
  68
  69 static void handle_syncline (char *, location);
  70 static unsigned long int scan_integer (char const *p, int base, location loc);
  71 static int convert_ucn_to_byte (char const *hex_text);
  72 static void unexpected_eof (boundary, char const *);
  73 static void unexpected_newline (boundary, char const *);
  74
  75 %}
  76  /* A C-like comment in directives/rules. */
  77 %x SC_YACC_COMMENT
  78  /* Strings and characters in directives/rules. */
  79 %x SC_ESCAPED_STRING SC_ESCAPED_CHARACTER
  80  /* A identifier was just read in directives/rules.  Special state
  81     to capture the sequence `identifier :'. */
  82 %x SC_AFTER_IDENTIFIER
  83
  84  /* Three types of user code:
  85     - prologue (code between `%{' `%}' in the first section, before %%);
  86     - actions, printers, union, etc, (between braced in the middle section);
  87     - epilogue (everything after the second %%). */
  88 %x SC_PROLOGUE SC_BRACED_CODE SC_EPILOGUE
  89  /* C and C++ comments in code. */
  90 %x SC_COMMENT SC_LINE_COMMENT
  91  /* Strings and characters in code. */
  92 %x SC_STRING SC_CHARACTER
  93
  94 letter    [.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_]
  95 id        {letter}({letter}|[0-9])*
  96 directive %{letter}({letter}|[0-9]|-)*
  97 int       [0-9]+
  98
  99 /* POSIX says that a tag must be both an id and a C union member, but
 100    historically almost any character is allowed in a tag.  We disallow
 101    NUL and newline, as this simplifies our implementation.  */
 102 tag      [^\0\n>]+
 103
 104 /* Zero or more instances of backslash-newline.  Following GCC, allow
 105    white space between the backslash and the newline.  */
 106 splice   (\\[ \f\t\v]*\n)*
 107
 108 %%
 109 %{
 110   /* Nesting level of the current code in braces.  */
 111   int braces_level IF_LINT (= 0);
 112
 113   /* Parent context state, when applicable.  */
 114   int context_state IF_LINT (= 0);
 115
 116   /* Location of most recent identifier, when applicable.  */
 117   location id_loc IF_LINT (= empty_location);
 118
 119   /* Where containing code started, when applicable.  Its initial
 120      value is relevant only when yylex is invoked in the SC_EPILOGUE
 121      start condition.  */
 122   boundary code_start = scanner_cursor;
 123
 124   /* Where containing comment or string or character literal started,
 125      when applicable.  */
 126   boundary token_start IF_LINT (= scanner_cursor);
 127 %}
 128
 129
 130   /*-----------------------.
 131   | Scanning white space.  |
 132   `-----------------------*/
 133
 134 <INITIAL,SC_AFTER_IDENTIFIER>
 135 {
 136   /* Comments and white space.  */
 137   ","          warn_at (*loc, _("stray `,' treated as white space"));
 138   [ \f\n\t\v]  |
 139   "//".*       ;
 140   "/*" {
 141     token_start = loc->start;
 142     context_state = YY_START;
 143     BEGIN SC_YACC_COMMENT;
 144   }
 145
 146   /* #line directives are not documented, and may be withdrawn or
 147      modified in future versions of Bison.  */
 148   ^"#line "{int}" \"".*"\"\n" {
 149     handle_syncline (yytext + sizeof "#line " - 1, *loc);
 150   }
 151 }
 152
 153
 154   /*----------------------------.
 155   | Scanning Bison directives.  |
 156   `----------------------------*/
 157 <INITIAL>
 158 {
 159   "%after-header"                   return PERCENT_AFTER_HEADER;
 160   "%before-header"                  return PERCENT_BEFORE_HEADER;
 161   "%binary"                         return PERCENT_NONASSOC;
 162   "%debug"                          return PERCENT_DEBUG;
 163   "%default"[-_]"prec"              return PERCENT_DEFAULT_PREC;
 164   "%define"                         return PERCENT_DEFINE;
 165   "%defines"                        return PERCENT_DEFINES;
 166   "%destructor"                     return PERCENT_DESTRUCTOR;
 167   "%dprec"                          return PERCENT_DPREC;
 168   "%end-header"                     return PERCENT_END_HEADER;
 169   "%error"[-_]"verbose"             return PERCENT_ERROR_VERBOSE;
 170   "%expect"                         return PERCENT_EXPECT;
 171   "%expect"[-_]"rr"                 return PERCENT_EXPECT_RR;
 172   "%file-prefix"                    return PERCENT_FILE_PREFIX;
 173   "%fixed"[-_]"output"[-_]"files"   return PERCENT_YACC;
 174   "%initial-action"                 return PERCENT_INITIAL_ACTION;
 175   "%glr-parser"                     return PERCENT_GLR_PARSER;
 176   "%left"                           return PERCENT_LEFT;
 177   "%lex-param"                      return PERCENT_LEX_PARAM;
 178   "%locations"                      return PERCENT_LOCATIONS;
 179   "%merge"                          return PERCENT_MERGE;
 180   "%name"[-_]"prefix"               return PERCENT_NAME_PREFIX;
 181   "%no"[-_]"default"[-_]"prec"      return PERCENT_NO_DEFAULT_PREC;
 182   "%no"[-_]"lines"                  return PERCENT_NO_LINES;
 183   "%nonassoc"                       return PERCENT_NONASSOC;
 184   "%nondeterministic-parser"        return PERCENT_NONDETERMINISTIC_PARSER;
 185   "%nterm"                          return PERCENT_NTERM;
 186   "%output"                         return PERCENT_OUTPUT;
 187   "%parse-param"                    return PERCENT_PARSE_PARAM;
 188   "%prec"                           return PERCENT_PREC;
 189   "%printer"                        return PERCENT_PRINTER;
 190   "%pure"[-_]"parser"               return PERCENT_PURE_PARSER;
 191   "%push"[-_]"parser"               return PERCENT_PUSH_PARSER;
 192   "%require"                        return PERCENT_REQUIRE;
 193   "%right"                          return PERCENT_RIGHT;
 194   "%skeleton"                       return PERCENT_SKELETON;
 195   "%start"                          return PERCENT_START;
 196   "%start-header"                   return PERCENT_START_HEADER;
 197   "%symbol-default"                 return PERCENT_SYMBOL_DEFAULT;
 198   "%term"                           return PERCENT_TOKEN;
 199   "%token"                          return PERCENT_TOKEN;
 200   "%token"[-_]"table"               return PERCENT_TOKEN_TABLE;
 201   "%type"                           return PERCENT_TYPE;
 202   "%union"                          return PERCENT_UNION;
 203   "%verbose"                        return PERCENT_VERBOSE;
 204   "%yacc"                           return PERCENT_YACC;
 205
 206   {directive} {
 207     complain_at (*loc, _("invalid directive: %s"), quote (yytext));
 208   }
 209
 210   "="                     return EQUAL;
 211   "|"                     return PIPE;
 212   ";"                     return SEMICOLON;
 213
 214   {id} {
 215     val->uniqstr = uniqstr_new (yytext);
 216     id_loc = *loc;
 217     BEGIN SC_AFTER_IDENTIFIER;
 218   }
 219
 220   {int} {
 221     val->integer = scan_integer (yytext, 10, *loc);
 222     return INT;
 223   }
 224   0[xX][0-9abcdefABCDEF]+ {
 225     val->integer = scan_integer (yytext, 16, *loc);
 226     return INT;
 227   }
 228
 229   /* Characters.  We don't check there is only one.  */
 230   "'"         STRING_GROW; token_start = loc->start; BEGIN SC_ESCAPED_CHARACTER;
 231
 232   /* Strings. */
 233   "\""        token_start = loc->start; BEGIN SC_ESCAPED_STRING;
 234
 235   /* Prologue. */
 236   "%{"        code_start = loc->start; BEGIN SC_PROLOGUE;
 237
 238   /* Code in between braces.  */
 239   "{" {
 240     STRING_GROW;
 241     braces_level = 0;
 242     code_start = loc->start;
 243     BEGIN SC_BRACED_CODE;
 244   }
 245
 246   /* A type. */
 247   "<"{tag}">" {
 248     obstack_grow (&obstack_for_string, yytext + 1, yyleng - 2);
 249     STRING_FINISH;
 250     val->uniqstr = uniqstr_new (last_string);
 251     STRING_FREE;
 252     return TYPE;
 253   }
 254
 255   "%%" {
 256     static int percent_percent_count;
 257     if (++percent_percent_count == 2)
 258       BEGIN SC_EPILOGUE;
 259     return PERCENT_PERCENT;
 260   }
 261
 262   . {
 263     complain_at (*loc, _("invalid character: %s"), quote (yytext));
 264   }
 265
 266   <<EOF>> {
 267     loc->start = loc->end = scanner_cursor;
 268     yyterminate ();
 269   }
 270 }
 271
 272
 273   /*-----------------------------------------------------------------.
 274   | Scanning after an identifier, checking whether a colon is next.  |
 275   `-----------------------------------------------------------------*/
 276
 277 <SC_AFTER_IDENTIFIER>
 278 {
 279   ":" {
 280     *loc = id_loc;
 281     BEGIN INITIAL;
 282     return ID_COLON;
 283   }
 284   . {
 285     scanner_cursor.column -= mbsnwidth (yytext, yyleng, 0);
 286     yyless (0);
 287     *loc = id_loc;
 288     BEGIN INITIAL;
 289     return ID;
 290   }
 291   <<EOF>> {
 292     *loc = id_loc;
 293     BEGIN INITIAL;
 294     return ID;
 295   }
 296 }
 297
 298
 299   /*---------------------------------------------------------------.
 300   | Scanning a Yacc comment.  The initial `/ *' is already eaten.  |
 301   `---------------------------------------------------------------*/
 302
 303 <SC_YACC_COMMENT>
 304 {
 305   "*/"     BEGIN context_state;
 306   .|\n     ;
 307   <<EOF>>  unexpected_eof (token_start, "*/"); BEGIN context_state;
 308 }
 309
 310
 311   /*------------------------------------------------------------.
 312   | Scanning a C comment.  The initial `/ *' is already eaten.  |
 313   `------------------------------------------------------------*/
 314
 315 <SC_COMMENT>
 316 {
 317   "*"{splice}"/"  STRING_GROW; BEGIN context_state;
 318   <<EOF>>         unexpected_eof (token_start, "*/"); BEGIN context_state;
 319 }
 320
 321
 322   /*--------------------------------------------------------------.
 323   | Scanning a line comment.  The initial `//' is already eaten.  |
 324   `--------------------------------------------------------------*/
 325
 326 <SC_LINE_COMMENT>
 327 {
 328   "\n"           STRING_GROW; BEGIN context_state;
 329   {splice}       STRING_GROW;
 330   <<EOF>>        BEGIN context_state;
 331 }
 332
 333
 334   /*------------------------------------------------.
 335   | Scanning a Bison string, including its escapes. |
 336   | The initial quote is already eaten.             |
 337   `------------------------------------------------*/
 338
 339 <SC_ESCAPED_STRING>
 340 {
 341   "\""|"\n" {
 342     if (yytext[0] == '\n')
 343       unexpected_newline (token_start, "\"");
 344     STRING_FINISH;
 345     loc->start = token_start;
 346     val->chars = last_string;
 347     BEGIN INITIAL;
 348     return STRING;
 349   }
 350   <<EOF>> {
 351     unexpected_eof (token_start, "\"");
 352     STRING_FINISH;
 353     loc->start = token_start;
 354     val->chars = last_string;
 355     BEGIN INITIAL;
 356     return STRING;
 357   }
 358 }
 359
 360   /*----------------------------------------------------------.
 361   | Scanning a Bison character literal, decoding its escapes. |
 362   | The initial quote is already eaten.                       |
 363   `----------------------------------------------------------*/
 364
 365 <SC_ESCAPED_CHARACTER>
 366 {
 367   "'"|"\n" {
 368     if (yytext[0] == '\n')
 369       unexpected_newline (token_start, "'");
 370     STRING_GROW;
 371     STRING_FINISH;
 372     loc->start = token_start;
 373     val->character = last_string[1];
 374     STRING_FREE;
 375     BEGIN INITIAL;
 376     return CHAR;
 377   }
 378   <<EOF>> {
 379     unexpected_eof (token_start, "'");
 380     STRING_FINISH;
 381     loc->start = token_start;
 382     if (strlen(last_string) > 1)
 383       val->character = last_string[1];
 384     else
 385       val->character = last_string[0];
 386     STRING_FREE;
 387     BEGIN INITIAL;
 388     return CHAR;
 389   }
 390 }
 391
 392 <SC_ESCAPED_CHARACTER,SC_ESCAPED_STRING>
 393 {
 394   \0        complain_at (*loc, _("invalid null character"));
 395 }
 396
 397
 398   /*----------------------------.
 399   | Decode escaped characters.  |
 400   `----------------------------*/
 401
 402 <SC_ESCAPED_STRING,SC_ESCAPED_CHARACTER>
 403 {
 404   \\[0-7]{1,3} {
 405     unsigned long int c = strtoul (yytext + 1, NULL, 8);
 406     if (UCHAR_MAX < c)
 407       complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext));
 408     else if (! c)
 409       complain_at (*loc, _("invalid null character: %s"), quote (yytext));
 410     else
 411       obstack_1grow (&obstack_for_string, c);
 412   }
 413
 414   \\x[0-9abcdefABCDEF]+ {
 415     verify (UCHAR_MAX < ULONG_MAX);
 416     unsigned long int c = strtoul (yytext + 2, NULL, 16);
 417     if (UCHAR_MAX < c)
 418       complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext));
 419     else if (! c)
 420       complain_at (*loc, _("invalid null character: %s"), quote (yytext));
 421     else
 422       obstack_1grow (&obstack_for_string, c);
 423   }
 424
 425   \\a   obstack_1grow (&obstack_for_string, '\a');
 426   \\b   obstack_1grow (&obstack_for_string, '\b');
 427   \\f   obstack_1grow (&obstack_for_string, '\f');
 428   \\n   obstack_1grow (&obstack_for_string, '\n');
 429   \\r   obstack_1grow (&obstack_for_string, '\r');
 430   \\t   obstack_1grow (&obstack_for_string, '\t');
 431   \\v   obstack_1grow (&obstack_for_string, '\v');
 432
 433   /* \\[\"\'?\\] would be shorter, but it confuses xgettext.  */
 434   \\("\""|"'"|"?"|"\\")  obstack_1grow (&obstack_for_string, yytext[1]);
 435
 436   \\(u|U[0-9abcdefABCDEF]{4})[0-9abcdefABCDEF]{4} {
 437     int c = convert_ucn_to_byte (yytext);
 438     if (c < 0)
 439       complain_at (*loc, _("invalid escape sequence: %s"), quote (yytext));
 440     else if (! c)
 441       complain_at (*loc, _("invalid null character: %s"), quote (yytext));
 442     else
 443       obstack_1grow (&obstack_for_string, c);
 444   }
 445   \\(.|\n)      {
 446     complain_at (*loc, _("unrecognized escape sequence: %s"), quote (yytext));
 447     STRING_GROW;
 448   }
 449 }
 450
 451   /*--------------------------------------------.
 452   | Scanning user-code characters and strings.  |
 453   `--------------------------------------------*/
 454
 455 <SC_CHARACTER,SC_STRING>
 456 {
 457   {splice}|\\{splice}[^\n\[\]]  STRING_GROW;
 458 }
 459
 460 <SC_CHARACTER>
 461 {
 462   "'"           STRING_GROW; BEGIN context_state;
 463   \n            unexpected_newline (token_start, "'"); BEGIN context_state;
 464   <<EOF>>       unexpected_eof (token_start, "'"); BEGIN context_state;
 465 }
 466
 467 <SC_STRING>
 468 {
 469   "\""          STRING_GROW; BEGIN context_state;
 470   \n            unexpected_newline (token_start, "\""); BEGIN context_state;
 471   <<EOF>>       unexpected_eof (token_start, "\""); BEGIN context_state;
 472 }
 473
 474
 475   /*---------------------------------------------------.
 476   | Strings, comments etc. can be found in user code.  |
 477   `---------------------------------------------------*/
 478
 479 <SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>
 480 {
 481   "'" {
 482     STRING_GROW;
 483     context_state = YY_START;
 484     token_start = loc->start;
 485     BEGIN SC_CHARACTER;
 486   }
 487   "\"" {
 488     STRING_GROW;
 489     context_state = YY_START;
 490     token_start = loc->start;
 491     BEGIN SC_STRING;
 492   }
 493   "/"{splice}"*" {
 494     STRING_GROW;
 495     context_state = YY_START;
 496     token_start = loc->start;
 497     BEGIN SC_COMMENT;
 498   }
 499   "/"{splice}"/" {
 500     STRING_GROW;
 501     context_state = YY_START;
 502     BEGIN SC_LINE_COMMENT;
 503   }
 504 }
 505
 506
 507
 508   /*-----------------------------------------------------------.
 509   | Scanning some code in braces (actions). The initial "{" is |
 510   | already eaten.                                             |
 511   `-----------------------------------------------------------*/
 512
 513 <SC_BRACED_CODE>
 514 {
 515   "{"|"<"{splice}"%"  STRING_GROW; braces_level++;
 516   "%"{splice}">"      STRING_GROW; braces_level--;
 517   "}" {
 518     obstack_1grow (&obstack_for_string, '}');
 519
 520     --braces_level;
 521     if (braces_level < 0)
 522       {
 523         STRING_FINISH;
 524         loc->start = code_start;
 525         val->chars = last_string;
 526         BEGIN INITIAL;
 527         return BRACED_CODE;
 528       }
 529   }
 530
 531   /* Tokenize `<<%' correctly (as `<<' `%') rather than incorrrectly
 532      (as `<' `<%').  */
 533   "<"{splice}"<"  STRING_GROW;
 534
 535   <<EOF>> {
 536     unexpected_eof (code_start, "}");
 537     STRING_FINISH;
 538     loc->start = code_start;
 539     val->chars = last_string;
 540     BEGIN INITIAL;
 541     return BRACED_CODE;
 542   }
 543 }
 544
 545
 546   /*--------------------------------------------------------------.
 547   | Scanning some prologue: from "%{" (already scanned) to "%}".  |
 548   `--------------------------------------------------------------*/
 549
 550 <SC_PROLOGUE>
 551 {
 552   "%}" {
 553     STRING_FINISH;
 554     loc->start = code_start;
 555     val->chars = last_string;
 556     BEGIN INITIAL;
 557     return PROLOGUE;
 558   }
 559
 560   <<EOF>> {
 561     unexpected_eof (code_start, "%}");
 562     STRING_FINISH;
 563     loc->start = code_start;
 564     val->chars = last_string;
 565     BEGIN INITIAL;
 566     return PROLOGUE;
 567   }
 568 }
 569
 570
 571   /*---------------------------------------------------------------.
 572   | Scanning the epilogue (everything after the second "%%", which |
 573   | has already been eaten).                                       |
 574   `---------------------------------------------------------------*/
 575
 576 <SC_EPILOGUE>
 577 {
 578   <<EOF>> {
 579     STRING_FINISH;
 580     loc->start = code_start;
 581     val->chars = last_string;
 582     BEGIN INITIAL;
 583     return EPILOGUE;
 584   }
 585 }
 586
 587
 588   /*-----------------------------------------------------.
 589   | By default, grow the string obstack with the input.  |
 590   `-----------------------------------------------------*/
 591
 592 <SC_COMMENT,SC_LINE_COMMENT,SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE,SC_STRING,SC_CHARACTER,SC_ESCAPED_STRING,SC_ESCAPED_CHARACTER>.      |
 593 <SC_COMMENT,SC_LINE_COMMENT,SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE>\n   STRING_GROW;
 594
 595 %%
 596
 597 /* Read bytes from FP into buffer BUF of size SIZE.  Return the
 598    number of bytes read.  Remove '\r' from input, treating \r\n
 599    and isolated \r as \n.  */
 600
 601 static size_t
 602 no_cr_read (FILE *fp, char *buf, size_t size)
 603 {
 604   size_t bytes_read = fread (buf, 1, size, fp);
 605   if (bytes_read)
 606     {
 607       char *w = memchr (buf, '\r', bytes_read);
 608       if (w)
 609         {
 610           char const *r = ++w;
 611           char const *lim = buf + bytes_read;
 612
 613           for (;;)
 614             {
 615               /* Found an '\r'.  Treat it like '\n', but ignore any
 616                  '\n' that immediately follows.  */
 617               w[-1] = '\n';
 618               if (r == lim)
 619                 {
 620                   int ch = getc (fp);
 621                   if (ch != '\n' && ungetc (ch, fp) != ch)
 622                     break;
 623                 }
 624               else if (*r == '\n')
 625                 r++;
 626
 627               /* Copy until the next '\r'.  */
 628               do
 629                 {
 630                   if (r == lim)
 631                     return w - buf;
 632                 }
 633               while ((*w++ = *r++) != '\r');
 634             }
 635
 636           return w - buf;
 637         }
 638     }
 639
 640   return bytes_read;
 641 }
 642
 643
 644
 645 /*------------------------------------------------------.
 646 | Scan NUMBER for a base-BASE integer at location LOC.  |
 647 `------------------------------------------------------*/
 648
 649 static unsigned long int
 650 scan_integer (char const *number, int base, location loc)
 651 {
 652   verify (INT_MAX < ULONG_MAX);
 653   unsigned long int num = strtoul (number, NULL, base);
 654
 655   if (INT_MAX < num)
 656     {
 657       complain_at (loc, _("integer out of range: %s"), quote (number));
 658       num = INT_MAX;
 659     }
 660
 661   return num;
 662 }
 663
 664
 665 /*------------------------------------------------------------------.
 666 | Convert universal character name UCN to a single-byte character,  |
 667 | and return that character.  Return -1 if UCN does not correspond  |
 668 | to a single-byte character.                                       |
 669 `------------------------------------------------------------------*/
 670
 671 static int
 672 convert_ucn_to_byte (char const *ucn)
 673 {
 674   verify (UCHAR_MAX <= INT_MAX);
 675   unsigned long int code = strtoul (ucn + 2, NULL, 16);
 676
 677   /* FIXME: Currently we assume Unicode-compatible unibyte characters
 678      on ASCII hosts (i.e., Latin-1 on hosts with 8-bit bytes).  On
 679      non-ASCII hosts we support only the portable C character set.
 680      These limitations should be removed once we add support for
 681      multibyte characters.  */
 682
 683   if (UCHAR_MAX < code)
 684     return -1;
 685
 686 #if ! ('$' == 0x24 && '@' == 0x40 && '`' == 0x60 && '~' == 0x7e)
 687   {
 688     /* A non-ASCII host.  Use CODE to index into a table of the C
 689        basic execution character set, which is guaranteed to exist on
 690        all Standard C platforms.  This table also includes '$', '@',
 691        and '`', which are not in the basic execution character set but
 692        which are unibyte characters on all the platforms that we know
 693        about.  */
 694     static signed char const table[] =
 695       {
 696         '\0',   -1,   -1,   -1,   -1,   -1,   -1, '\a',
 697         '\b', '\t', '\n', '\v', '\f', '\r',   -1,   -1,
 698           -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
 699           -1,   -1,   -1,   -1,   -1,   -1,   -1,   -1,
 700          ' ',  '!',  '"',  '#',  '$',  '%',  '&', '\'',
 701          '(',  ')',  '*',  '+',  ',',  '-',  '.',  '/',
 702          '0',  '1',  '2',  '3',  '4',  '5',  '6',  '7',
 703          '8',  '9',  ':',  ';',  '<',  '=',  '>',  '?',
 704          '@',  'A',  'B',  'C',  'D',  'E',  'F',  'G',
 705          'H',  'I',  'J',  'K',  'L',  'M',  'N',  'O',
 706          'P',  'Q',  'R',  'S',  'T',  'U',  'V',  'W',
 707          'X',  'Y',  'Z',  '[', '\\',  ']',  '^',  '_',
 708          '`',  'a',  'b',  'c',  'd',  'e',  'f',  'g',
 709          'h',  'i',  'j',  'k',  'l',  'm',  'n',  'o',
 710          'p',  'q',  'r',  's',  't',  'u',  'v',  'w',
 711          'x',  'y',  'z',  '{',  '|',  '}',  '~'
 712       };
 713
 714     code = code < sizeof table ? table[code] : -1;
 715   }
 716 #endif
 717
 718   return code;
 719 }
 720
 721
 722 /*----------------------------------------------------------------.
 723 | Handle `#line INT "FILE"'.  ARGS has already skipped `#line '.  |
 724 `----------------------------------------------------------------*/
 725
 726 static void
 727 handle_syncline (char *args, location loc)
 728 {
 729   char *after_num;
 730   unsigned long int lineno = strtoul (args, &after_num, 10);
 731   char *file = strchr (after_num, '"') + 1;
 732   *strchr (file, '"') = '\0';
 733   if (INT_MAX <= lineno)
 734     {
 735       warn_at (loc, _("line number overflow"));
 736       lineno = INT_MAX;
 737     }
 738   current_file = uniqstr_new (file);
 739   boundary_set (&scanner_cursor, current_file, lineno, 1);
 740 }
 741
 742
 743 /*----------------------------------------------------------------.
 744 | For a token or comment starting at START, report message MSGID, |
 745 | which should say that an end marker was found before            |
 746 | the expected TOKEN_END.                                         |
 747 `----------------------------------------------------------------*/
 748
 749 static void
 750 unexpected_end (boundary start, char const *msgid, char const *token_end)
 751 {
 752   location loc;
 753   loc.start = start;
 754   loc.end = scanner_cursor;
 755   complain_at (loc, _(msgid), token_end);
 756 }
 757
 758
 759 /*------------------------------------------------------------------------.
 760 | Report an unexpected EOF in a token or comment starting at START.       |
 761 | An end of file was encountered and the expected TOKEN_END was missing.  |
 762 `------------------------------------------------------------------------*/
 763
 764 static void
 765 unexpected_eof (boundary start, char const *token_end)
 766 {
 767   unexpected_end (start, N_("missing `%s' at end of file"), token_end);
 768 }
 769
 770
 771 /*----------------------------------------.
 772 | Likewise, but for unexpected newlines.  |
 773 `----------------------------------------*/
 774
 775 static void
 776 unexpected_newline (boundary start, char const *token_end)
 777 {
 778   unexpected_end (start, N_("missing `%s' at end of line"), token_end);
 779 }
 780
 781
 782 /*-------------------------.
 783 | Initialize the scanner.  |
 784 `-------------------------*/
 785
 786 void
 787 gram_scanner_initialize (void)
 788 {
 789   obstack_init (&obstack_for_string);
 790 }
 791
 792
 793 /*-----------------------------------------------.
 794 | Free all the memory allocated to the scanner.  |
 795 `-----------------------------------------------*/
 796
 797 void
 798 gram_scanner_free (void)
 799 {
 800   obstack_free (&obstack_for_string, 0);
 801   /* Reclaim Flex's buffers.  */
 802   yy_delete_buffer (YY_CURRENT_BUFFER);
 803 }