src/lex.c

   1 /* Token-reader for Bison's input parser,
   2    Copyright 1984, 1986, 1989, 1992, 2000, 2001 Free Software Foundation, Inc.
   3
   4    This file is part of Bison, the GNU Compiler Compiler.
   5
   6    Bison is free software; you can redistribute it and/or modify
   7    it under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 2, or (at your option)
   9    any later version.
  10
  11    Bison is distributed in the hope that it will be useful,
  12    but WITHOUT ANY WARRANTY; without even the implied warranty of
  13    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14    GNU General Public License for more details.
  15
  16    You should have received a copy of the GNU General Public License
  17    along with Bison; see the file COPYING.  If not, write to
  18    the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  19    Boston, MA 02111-1307, USA.  */
  20
  21 #include "system.h"
  22 #include "getargs.h"
  23 #include "files.h"
  24 #include "getopt.h"             /* for optarg */
  25 #include "symtab.h"
  26 #include "options.h"
  27 #include "lex.h"
  28 #include "complain.h"
  29 #include "gram.h"
  30 #include "quote.h"
  31
  32 /* Buffer for storing the current token.  */
  33 static struct obstack token_obstack;
  34 const char *token_buffer = NULL;
  35
  36 bucket *symval;
  37 int numval;
  38
  39 /* A token to be reread, see unlex and lex. */
  40 static token_t unlexed = tok_undef;
  41 static bucket *unlexed_symval = NULL;
  42 static const char *unlexed_token_buffer = NULL;
  43
  44 void
  45 lex_init (void)
  46 {
  47   obstack_init (&token_obstack);
  48   unlexed = tok_undef;
  49 }
  50
  51
  52 void
  53 lex_free (void)
  54 {
  55   obstack_free (&token_obstack, NULL);
  56 }
  57
  58
  59 int
  60 skip_white_space (void)
  61 {
  62   int c;
  63   int inside;
  64
  65   c = getc (finput);
  66
  67   for (;;)
  68     {
  69       int cplus_comment;
  70
  71       switch (c)
  72         {
  73         case '/':
  74           /* FIXME: Should probably be merged with copy_comment.  */
  75           c = getc (finput);
  76           if (c != '*' && c != '/')
  77             {
  78               complain (_("unexpected `/' found and ignored"));
  79               break;
  80             }
  81           cplus_comment = (c == '/');
  82
  83           c = getc (finput);
  84
  85           inside = 1;
  86           while (inside)
  87             {
  88               if (!cplus_comment && c == '*')
  89                 {
  90                   while (c == '*')
  91                     c = getc (finput);
  92
  93                   if (c == '/')
  94                     {
  95                       inside = 0;
  96                       c = getc (finput);
  97                     }
  98                 }
  99               else if (c == '\n')
 100                 {
 101                   lineno++;
 102                   if (cplus_comment)
 103                     inside = 0;
 104                   c = getc (finput);
 105                 }
 106               else if (c == EOF)
 107                 fatal (_("unterminated comment"));
 108               else
 109                 c = getc (finput);
 110             }
 111
 112           break;
 113
 114         case '\n':
 115           lineno++;
 116
 117         case ' ':
 118         case '\t':
 119         case '\f':
 120           c = getc (finput);
 121           break;
 122
 123         default:
 124           return c;
 125         }
 126     }
 127 }
 128
 129
 130 /*-----------------------------------------------------.
 131 | Do a getc, but give error message if EOF encountered |
 132 `-----------------------------------------------------*/
 133
 134 static int
 135 xgetc (FILE *f)
 136 {
 137   int c = getc (f);
 138   if (c == EOF)
 139     fatal (_("unexpected end of file"));
 140   return c;
 141 }
 142
 143
 144 /*------------------------------------------------------------------.
 145 | Read one literal character from finput.  Process \ escapes.       |
 146 | Append the normalized string version of the char to OUT.  Assign  |
 147 | the character code to *PCODE. Return 1 unless the character is an |
 148 | unescaped `term' or \n report error for \n.                       |
 149 `------------------------------------------------------------------*/
 150
 151 /* FIXME: We could directly work in the obstack, but that would make
 152    it more difficult to move to quotearg some day.  So for the time
 153    being, I prefer have literalchar behave like quotearg, and change
 154    my mind later if I was wrong.  */
 155
 156 static int
 157 literalchar (struct obstack *out, int *pcode, char term)
 158 {
 159   int c;
 160   char buf[4096];
 161   char *cp;
 162   int code;
 163   int wasquote = 0;
 164
 165   c = xgetc (finput);
 166   if (c == '\n')
 167     {
 168       complain (_("unescaped newline in constant"));
 169       ungetc (c, finput);
 170       code = '?';
 171       wasquote = 1;
 172     }
 173   else if (c != '\\')
 174     {
 175       code = c;
 176       if (c == term)
 177         wasquote = 1;
 178     }
 179   else
 180     {
 181       c = xgetc (finput);
 182       if (c == 't')
 183         code = '\t';
 184       else if (c == 'n')
 185         code = '\n';
 186       else if (c == 'a')
 187         code = '\007';
 188       else if (c == 'r')
 189         code = '\r';
 190       else if (c == 'f')
 191         code = '\f';
 192       else if (c == 'b')
 193         code = '\b';
 194       else if (c == 'v')
 195         code = '\013';
 196       else if (c == '\\')
 197         code = '\\';
 198       else if (c == '\'')
 199         code = '\'';
 200       else if (c == '\"')
 201         code = '\"';
 202       else if (c <= '7' && c >= '0')
 203         {
 204           code = 0;
 205           while (c <= '7' && c >= '0')
 206             {
 207               code = (code * 8) + (c - '0');
 208               if (code >= 256 || code < 0)
 209                 {
 210                   complain (_("octal value outside range 0...255: `\\%o'"),
 211                             code);
 212                   code &= 0xFF;
 213                   break;
 214                 }
 215               c = xgetc (finput);
 216             }
 217           ungetc (c, finput);
 218         }
 219       else if (c == 'x')
 220         {
 221           c = xgetc (finput);
 222           code = 0;
 223           while (1)
 224             {
 225               if (c >= '0' && c <= '9')
 226                 code *= 16, code += c - '0';
 227               else if (c >= 'a' && c <= 'f')
 228                 code *= 16, code += c - 'a' + 10;
 229               else if (c >= 'A' && c <= 'F')
 230                 code *= 16, code += c - 'A' + 10;
 231               else
 232                 break;
 233               if (code >= 256 || code < 0)
 234                 {
 235                   complain (_("hexadecimal value above 255: `\\x%x'"), code);
 236                   code &= 0xFF;
 237                   break;
 238                 }
 239               c = xgetc (finput);
 240             }
 241           ungetc (c, finput);
 242         }
 243       else
 244         {
 245           char badchar [] = "c";
 246           badchar[0] = c;
 247           complain (_("unknown escape sequence: `\\' followed by `%s'"),
 248                     quote (badchar));
 249           code = '?';
 250         }
 251     }                           /* has \ */
 252
 253   /* now fill BUF with the canonical name for this character as a
 254      literal token.  Do not use what the user typed, so that `\012'
 255      and `\n' can be interchangeable.  */
 256
 257   cp = buf;
 258   if (code == term && wasquote)
 259     *cp++ = code;
 260   else if (code == '\\')
 261     {
 262       *cp++ = '\\';
 263       *cp++ = '\\';
 264     }
 265   else if (code == '\'')
 266     {
 267       *cp++ = '\\';
 268       *cp++ = '\'';
 269     }
 270   else if (code == '\"')
 271     {
 272       *cp++ = '\\';
 273       *cp++ = '\"';
 274     }
 275   else if (code >= 040 && code < 0177)
 276     *cp++ = code;
 277   else if (code == '\t')
 278     {
 279       *cp++ = '\\';
 280       *cp++ = 't';
 281     }
 282   else if (code == '\n')
 283     {
 284       *cp++ = '\\';
 285       *cp++ = 'n';
 286     }
 287   else if (code == '\r')
 288     {
 289       *cp++ = '\\';
 290       *cp++ = 'r';
 291     }
 292   else if (code == '\v')
 293     {
 294       *cp++ = '\\';
 295       *cp++ = 'v';
 296     }
 297   else if (code == '\b')
 298     {
 299       *cp++ = '\\';
 300       *cp++ = 'b';
 301     }
 302   else if (code == '\f')
 303     {
 304       *cp++ = '\\';
 305       *cp++ = 'f';
 306     }
 307   else
 308     {
 309       *cp++ = '\\';
 310       *cp++ = code / 0100 + '0';
 311       *cp++ = ((code / 010) & 07) + '0';
 312       *cp++ = (code & 07) + '0';
 313     }
 314   *cp = '\0';
 315
 316   if (out)
 317     obstack_sgrow (out, buf);
 318   *pcode = code;
 319   return !wasquote;
 320 }
 321
 322
 323 void
 324 unlex (token_t token)
 325 {
 326   unlexed = token;
 327   unlexed_token_buffer = token_buffer;
 328   unlexed_symval = symval;
 329 }
 330
 331 /*-----------------------------------------------------------------.
 332 | We just read `<' from FIN.  Store in TOKEN_BUFFER, the type name |
 333 | specified between the `<...>'.                                   |
 334 `-----------------------------------------------------------------*/
 335
 336 void
 337 read_type_name (FILE *fin)
 338 {
 339   int c = getc (fin);
 340
 341   while (c != '>')
 342     {
 343       if (c == EOF)
 344         fatal (_("unterminated type name at end of file"));
 345       if (c == '\n')
 346         {
 347           complain (_("unterminated type name"));
 348           ungetc (c, fin);
 349           break;
 350         }
 351
 352       obstack_1grow (&token_obstack, c);
 353       c = getc (fin);
 354     }
 355   obstack_1grow (&token_obstack, '\0');
 356   token_buffer = obstack_finish (&token_obstack);
 357 }
 358
 359
 360 token_t
 361 lex (void)
 362 {
 363   int c;
 364
 365   /* Just to make sure. */
 366   token_buffer = NULL;
 367
 368   if (unlexed != tok_undef)
 369     {
 370       token_t res = unlexed;
 371       symval = unlexed_symval;
 372       token_buffer = unlexed_token_buffer;
 373       unlexed = tok_undef;
 374       return res;
 375     }
 376
 377   c = skip_white_space ();
 378
 379   switch (c)
 380     {
 381     case EOF:
 382       token_buffer = "EOF";
 383       return tok_eof;
 384
 385     case 'A':    case 'B':    case 'C':    case 'D':    case 'E':
 386     case 'F':    case 'G':    case 'H':    case 'I':    case 'J':
 387     case 'K':    case 'L':    case 'M':    case 'N':    case 'O':
 388     case 'P':    case 'Q':    case 'R':    case 'S':    case 'T':
 389     case 'U':    case 'V':    case 'W':    case 'X':    case 'Y':
 390     case 'Z':
 391     case 'a':    case 'b':    case 'c':    case 'd':    case 'e':
 392     case 'f':    case 'g':    case 'h':    case 'i':    case 'j':
 393     case 'k':    case 'l':    case 'm':    case 'n':    case 'o':
 394     case 'p':    case 'q':    case 'r':    case 's':    case 't':
 395     case 'u':    case 'v':    case 'w':    case 'x':    case 'y':
 396     case 'z':
 397     case '.':    case '_':
 398
 399       while (isalnum (c) || c == '_' || c == '.')
 400         {
 401           obstack_1grow (&token_obstack, c);
 402           c = getc (finput);
 403         }
 404       obstack_1grow (&token_obstack, '\0');
 405       token_buffer = obstack_finish (&token_obstack);
 406       ungetc (c, finput);
 407       symval = getsym (token_buffer);
 408       return tok_identifier;
 409
 410     case '0':    case '1':    case '2':    case '3':    case '4':
 411     case '5':    case '6':    case '7':    case '8':    case '9':
 412       {
 413         numval = 0;
 414
 415         while (isdigit (c))
 416           {
 417             obstack_1grow (&token_obstack, c);
 418             numval = numval * 10 + c - '0';
 419             c = getc (finput);
 420           }
 421         obstack_1grow (&token_obstack, '\0');
 422         token_buffer = obstack_finish (&token_obstack);
 423         ungetc (c, finput);
 424         return tok_number;
 425       }
 426
 427     case '\'':
 428       /* parse the literal token and compute character code in  code  */
 429
 430       {
 431         int code, discode;
 432
 433         obstack_1grow (&token_obstack, '\'');
 434         literalchar (&token_obstack, &code, '\'');
 435
 436         c = getc (finput);
 437         if (c != '\'')
 438           {
 439             complain (_("use \"...\" for multi-character literal tokens"));
 440             while (1)
 441               if (!literalchar (0, &discode, '\''))
 442                 break;
 443           }
 444         obstack_1grow (&token_obstack, '\'');
 445         obstack_1grow (&token_obstack, '\0');
 446         token_buffer = obstack_finish (&token_obstack);
 447         symval = getsym (token_buffer);
 448         symval->class = token_sym;
 449         if (symval->user_token_number == SUNDEF)
 450           symval->user_token_number = code;
 451         return tok_identifier;
 452       }
 453
 454     case '\"':
 455       /* parse the literal string token and treat as an identifier */
 456
 457       {
 458         int code;               /* ignored here */
 459
 460         obstack_1grow (&token_obstack, '\"');
 461         /* Read up to and including ".  */
 462         while (literalchar (&token_obstack, &code, '\"'))
 463           /* nothing */;
 464         obstack_1grow (&token_obstack, '\0');
 465         token_buffer = obstack_finish (&token_obstack);
 466
 467         symval = getsym (token_buffer);
 468         symval->class = token_sym;
 469
 470         return tok_identifier;
 471       }
 472
 473     case ',':
 474       token_buffer = ",";
 475       return tok_comma;
 476
 477     case ':':
 478       token_buffer = ":";
 479       return tok_colon;
 480
 481     case ';':
 482       token_buffer = ";";
 483       return tok_semicolon;
 484
 485     case '|':
 486       token_buffer = "|";
 487       return tok_bar;
 488
 489     case '{':
 490       token_buffer = "{";
 491       return tok_left_curly;
 492
 493     case '=':
 494       obstack_1grow (&token_obstack, c);
 495       do
 496         {
 497           c = getc (finput);
 498           obstack_1grow (&token_obstack, c);
 499           if (c == '\n')
 500             lineno++;
 501         }
 502       while (c == ' ' || c == '\n' || c == '\t');
 503       obstack_1grow (&token_obstack, '\0');
 504       token_buffer = obstack_finish (&token_obstack);
 505
 506       if (c == '{')
 507         {
 508           return tok_left_curly;
 509         }
 510       else
 511         {
 512           ungetc (c, finput);
 513           return tok_illegal;
 514         }
 515
 516     case '<':
 517       read_type_name (finput);
 518       return tok_typename;
 519
 520     case '%':
 521       return parse_percent_token ();
 522
 523     default:
 524       obstack_1grow (&token_obstack, c);
 525       obstack_1grow (&token_obstack, '\0');
 526       token_buffer = obstack_finish (&token_obstack);
 527       return tok_illegal;
 528     }
 529 }
 530
 531 /* This function is a strcmp, which doesn't differentiate `-' and `_'
 532    chars.  */
 533
 534 static int
 535 option_strcmp (const char *left, const char *right)
 536 {
 537   const unsigned char *l, *r;
 538   int c;
 539
 540   assert (left);
 541   assert (right);
 542   l = (const unsigned char *)left;
 543   r = (const unsigned char *)right;
 544   while (((c = *l - *r++) == 0 && *l != '\0')
 545          || ((*l == '-' || *l == '_') && (*r == '_' || *r == '-')))
 546     l++;
 547   return c;
 548 }
 549
 550 /* Parse a token which starts with %.
 551    Assumes the % has already been read and discarded.  */
 552
 553 token_t
 554 parse_percent_token (void)
 555 {
 556   const struct option_table_struct *tx;
 557
 558   int c = getc (finput);
 559
 560   switch (c)
 561     {
 562     case '%':
 563       return tok_two_percents;
 564
 565     case '{':
 566       return tok_percent_left_curly;
 567
 568     case '<':
 569       return tok_left;
 570
 571     case '>':
 572       return tok_right;
 573
 574     case '2':
 575       return tok_nonassoc;
 576
 577     case '0':
 578       return tok_token;
 579
 580     case '=':
 581       return tok_prec;
 582     }
 583
 584   if (!isalpha (c))
 585     return tok_illegal;
 586
 587   obstack_1grow (&token_obstack, '%');
 588   while (isalpha (c) || c == '_' || c == '-')
 589     {
 590       if (c == '_')
 591         c = '-';
 592       obstack_1grow (&token_obstack, c);
 593       c = getc (finput);
 594     }
 595
 596   ungetc (c, finput);
 597   obstack_1grow (&token_obstack, '\0');
 598   token_buffer = obstack_finish (&token_obstack);
 599
 600   /* table lookup % directive */
 601   for (tx = option_table; tx->name; tx++)
 602     if ((tx->access == opt_percent || tx->access == opt_both)
 603         && option_strcmp (token_buffer + 1, tx->name) == 0)
 604       break;
 605
 606   if (tx->set_flag)
 607     {
 608       *((int *) (tx->set_flag)) = 1;
 609       return tok_noop;
 610     }
 611
 612   switch (tx->ret_val)
 613     {
 614     case tok_setopt:
 615       *((char **) (tx->set_flag)) = optarg;
 616       return tok_noop;
 617       break;
 618
 619     case tok_obsolete:
 620       fatal (_("`%s' is no longer supported"), token_buffer);
 621       break;
 622
 623     default:
 624       /* Other cases do not apply here. */
 625       break;
 626     }
 627
 628   return tx->ret_val;
 629 }