Get rid of the ad hoc handling of token_buffer in the scanner: use

the obstacks.
* src/lex.c (token_obstack): New.
(init_lex): Initialize it.  No longer call...
(grow_token_buffer): this.  Remove it.
Adjust all the places which used it to use the obstack.

diff --git a/ChangeLog b/ChangeLog
index b006c3bbcfc4320d5d55933a082ed6453fc1b18c..069e4fa98379fbac2646dbabbe87628fa6de17c8 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2001-01-19  Akim Demaille  <akim@epita.fr>
+
+       Get rid of the ad hoc handling of token_buffer in the scanner: use
+       the obstacks.
+
+       * src/lex.c (token_obstack): New.
+       (init_lex): Initialize it.  No longer call...
+       (grow_token_buffer): this.  Remove it.
+       Adjust all the places which used it to use the obstack.
+
 2001-01-19  Akim Demaille  <akim@epita.fr>
 
        * src/lex.h: Rename all the tokens:

diff --git a/src/lex.c b/src/lex.c
index 408e30fcb8549fb664cf8cde73d953054aa26e7f..987777be5b39f7302beb5bf94251e3164277c3ca 100644 (file)
--- a/src/lex.c
+++ b/src/lex.c
@@ -30,10 +30,8 @@
 #include "quote.h"
 
 /* Buffer for storing the current token.  */
-char *token_buffer;
-
-/* Allocated size of token_buffer, not including space for terminator.  */
-static int maxtoken;
+struct obstack token_obstack;
+char *token_buffer = NULL;
 
 bucket *symval;
 int numval;
@@ -45,22 +43,11 @@ static bucket *unlexed_symval;      /* by the next call to lex */
 void
 init_lex (void)
 {
-  maxtoken = 100;
-  token_buffer = XCALLOC (char, maxtoken + 1);
+  obstack_init (&token_obstack);
   unlexed = -1;
 }
 
 
-static char *
-grow_token_buffer (char *p)
-{
-  int offset = p - token_buffer;
-  maxtoken *= 2;
-  token_buffer = XREALLOC (token_buffer, char, maxtoken + 1);
-  return token_buffer + offset;
-}
-
-
 int
 skip_white_space (void)
 {
@@ -148,16 +135,22 @@ xgetc (FILE *f)
 
 /*------------------------------------------------------------------.
 | Read one literal character from finput.  Process \ escapes.       |
-| Append the normalized string version of the char to *PP.  Assign  |
+| Append the normalized string version of the char to OUT.  Assign  |
 | the character code to *PCODE. Return 1 unless the character is an |
-| unescaped `term' or \n report error for \n                        |
+| unescaped `term' or \n report error for \n.                       |
 `------------------------------------------------------------------*/
 
+/* FIXME: We could directly work in the obstack, but that would make
+   it more difficult to move to quotearg some day.  So for the time
+   being, I prefer have literalchar behave like quotearg, and change
+   my mind later if I was wrong.  */
+
 static int
-literalchar (char **pp, int *pcode, char term)
+literalchar (struct obstack *out, int *pcode, char term)
 {
   int c;
-  char *p;
+  char buf[4096];
+  char *cp;
   int code;
   int wasquote = 0;
 
@@ -249,68 +242,71 @@ literalchar (char **pp, int *pcode, char term)
        }
     }                          /* has \ */
 
-  /* now fill token_buffer with the canonical name for this character
-     as a literal token.  Do not use what the user typed,
-     so that `\012' and `\n' can be interchangeable.  */
+  /* now fill BUF with the canonical name for this character as a
+     literal token.  Do not use what the user typed, so that `\012'
+     and `\n' can be interchangeable.  */
 
-  p = *pp;
+  cp = buf;
   if (code == term && wasquote)
-    *p++ = code;
+    *cp++ = code;
   else if (code == '\\')
     {
-      *p++ = '\\';
-      *p++ = '\\';
+      *cp++ = '\\';
+      *cp++ = '\\';
     }
   else if (code == '\'')
     {
-      *p++ = '\\';
-      *p++ = '\'';
+      *cp++ = '\\';
+      *cp++ = '\'';
     }
   else if (code == '\"')
     {
-      *p++ = '\\';
-      *p++ = '\"';
+      *cp++ = '\\';
+      *cp++ = '\"';
     }
   else if (code >= 040 && code < 0177)
-    *p++ = code;
+    *cp++ = code;
   else if (code == '\t')
     {
-      *p++ = '\\';
-      *p++ = 't';
+      *cp++ = '\\';
+      *cp++ = 't';
     }
   else if (code == '\n')
     {
-      *p++ = '\\';
-      *p++ = 'n';
+      *cp++ = '\\';
+      *cp++ = 'n';
     }
   else if (code == '\r')
     {
-      *p++ = '\\';
-      *p++ = 'r';
+      *cp++ = '\\';
+      *cp++ = 'r';
     }
   else if (code == '\v')
     {
-      *p++ = '\\';
-      *p++ = 'v';
+      *cp++ = '\\';
+      *cp++ = 'v';
     }
   else if (code == '\b')
     {
-      *p++ = '\\';
-      *p++ = 'b';
+      *cp++ = '\\';
+      *cp++ = 'b';
     }
   else if (code == '\f')
     {
-      *p++ = '\\';
-      *p++ = 'f';
+      *cp++ = '\\';
+      *cp++ = 'f';
     }
   else
     {
-      *p++ = '\\';
-      *p++ = code / 0100 + '0';
-      *p++ = ((code / 010) & 07) + '0';
-      *p++ = (code & 07) + '0';
+      *cp++ = '\\';
+      *cp++ = code / 0100 + '0';
+      *cp++ = ((code / 010) & 07) + '0';
+      *cp++ = (code & 07) + '0';
     }
-  *pp = p;
+  *cp = '\0';
+
+  if (out)
+    obstack_sgrow (out, buf);
   *pcode = code;
   return !wasquote;
 }
@@ -331,7 +327,6 @@ unlex (int token)
 void
 read_type_name (FILE *fin)
 {
-  char *p = token_buffer;
   int c = getc (fin);
 
   while (c != '>')
@@ -345,13 +340,11 @@ read_type_name (FILE *fin)
          break;
        }
 
-      if (p == token_buffer + maxtoken)
-       p = grow_token_buffer (p);
-
-      *p++ = c;
+      obstack_1grow (&token_obstack, c);
       c = getc (fin);
     }
-  *p = 0;
+  obstack_1grow (&token_obstack, '\0');
+  token_buffer = obstack_finish (&token_obstack);
 }
 
 
@@ -359,7 +352,9 @@ token_t
 lex (void)
 {
   int c;
-  char *p;
+
+  /* Just to make sure. */
+  token_buffer = NULL;
 
   if (unlexed >= 0)
     {
@@ -370,14 +365,11 @@ lex (void)
     }
 
   c = skip_white_space ();
-  /* for error messages (token buffer always valid) */
-  *token_buffer = c;
-  token_buffer[1] = 0;
 
   switch (c)
     {
     case EOF:
-      strcpy (token_buffer, "EOF");
+      token_buffer = "EOF";
       return tok_eof;
 
     case 'A':    case 'B':    case 'C':    case 'D':    case 'E':
@@ -394,17 +386,13 @@ lex (void)
     case 'z':
     case '.':    case '_':
 
-      p = token_buffer;
       while (isalnum (c) || c == '_' || c == '.')
        {
-         if (p == token_buffer + maxtoken)
-           p = grow_token_buffer (p);
-
-         *p++ = c;
+         obstack_1grow (&token_obstack, c);
          c = getc (finput);
        }
-
-      *p = 0;
+      obstack_1grow (&token_obstack, '\0');
+      token_buffer = obstack_finish (&token_obstack);
       ungetc (c, finput);
       symval = getsym (token_buffer);
       return tok_identifier;
@@ -414,17 +402,14 @@ lex (void)
       {
        numval = 0;
 
-       p = token_buffer;
        while (isdigit (c))
          {
-           if (p == token_buffer + maxtoken)
-             p = grow_token_buffer (p);
-
-           *p++ = c;
+           obstack_1grow (&token_obstack, c);
            numval = numval * 10 + c - '0';
            c = getc (finput);
          }
-       *p = 0;
+       obstack_1grow (&token_obstack, '\0');
+       token_buffer = obstack_finish (&token_obstack);
        ungetc (c, finput);
        return tok_number;
       }
@@ -435,25 +420,21 @@ lex (void)
       translations = -1;
       {
        int code, discode;
-       char discard[10], *dp;
 
-       p = token_buffer;
-       *p++ = '\'';
-       literalchar (&p, &code, '\'');
+       obstack_1grow (&token_obstack, '\'');
+       literalchar (&token_obstack, &code, '\'');
 
        c = getc (finput);
        if (c != '\'')
          {
            complain (_("use \"...\" for multi-character literal tokens"));
            while (1)
-             {
-               dp = discard;
-               if (!literalchar (&dp, &discode, '\''))
-                 break;
-             }
+             if (!literalchar (0, &discode, '\''))
+               break;
          }
-       *p++ = '\'';
-       *p = 0;
+       obstack_1grow (&token_obstack, '\'');
+       obstack_1grow (&token_obstack, '\0');
+       token_buffer = obstack_finish (&token_obstack);
        symval = getsym (token_buffer);
        symval->class = token_sym;
        if (!symval->user_token_number)
@@ -467,15 +448,13 @@ lex (void)
       translations = -1;
       {
        int code;               /* ignored here */
-       p = token_buffer;
-       *p++ = '\"';
+
+       obstack_1grow (&token_obstack, '\"');
        /* Read up to and including ".  */
-       while (literalchar (&p, &code, '\"'))
-         {
-           if (p >= token_buffer + maxtoken - 4)
-             p = grow_token_buffer (p);
-         }
-       *p = 0;
+       while (literalchar (&token_obstack, &code, '\"'))
+         /* nothing */;
+       obstack_1grow (&token_obstack, '\0');
+       token_buffer = obstack_finish (&token_obstack);
 
        symval = getsym (token_buffer);
        symval->class = token_sym;
@@ -509,7 +488,7 @@ lex (void)
 
       if (c == '{')
        {
-         strcpy (token_buffer, "={");
+         token_buffer = "={";
          return tok_left_curly;
        }
       else
@@ -589,14 +568,9 @@ int
 parse_percent_token (void)
 {
   int c;
-  char *p;
   struct percent_table_struct *tx;
 
-  p = token_buffer;
   c = getc (finput);
-  *p++ = '%';
-  *p++ = c;                    /* for error msg */
-  *p = 0;
 
   switch (c)
     {
@@ -621,25 +595,22 @@ parse_percent_token (void)
     case '=':
       return tok_prec;
     }
+
   if (!isalpha (c))
     return tok_illegal;
 
-  p = token_buffer;
-  *p++ = '%';
+  obstack_1grow (&token_obstack, '%');
   while (isalpha (c) || c == '_' || c == '-')
     {
-      if (p == token_buffer + maxtoken)
-       p = grow_token_buffer (p);
-
       if (c == '-')
        c = '_';
-      *p++ = c;
+      obstack_1grow (&token_obstack, c);
       c = getc (finput);
     }
 
   ungetc (c, finput);
-
-  *p = 0;
+  obstack_1grow (&token_obstack, '\0');
+  token_buffer = obstack_finish (&token_obstack);
 
   /* table lookup % directive */
   for (tx = percent_table; tx->name; tx++)

diff --git a/src/reader.c b/src/reader.c
index 2de754d04c7c4ac2870bbe8019b6d5b3c21a1eba..e073f028e5eb4f8f41662391e34b5c53fe1c9f21 100644 (file)
--- a/src/reader.c
+++ b/src/reader.c
@@ -466,7 +466,7 @@ copy_definition (void)
 static void
 parse_token_decl (symbol_class what_is, symbol_class what_is_not)
 {
-  int token = 0;
+  token_t token = 0;
   char *typename = 0;
 
   /* The symbol being defined.  */
@@ -593,7 +593,7 @@ parse_type_decl (void)
 
   for (;;)
     {
-      int t;
+      token_t t;
       int tmp_char = ungetc (skip_white_space (), finput);
 
       if (tmp_char == '%')
@@ -643,7 +643,7 @@ parse_assoc_decl (associativity assoc)
 
   for (;;)
     {
-      int t;
+      token_t t;
       int tmp_char = ungetc (skip_white_space (), finput);
 
       if (tmp_char == '%')
@@ -821,7 +821,7 @@ parse_expect_decl (void)
 static void
 parse_thong_decl (void)
 {
-  int token;
+  token_t token;
   struct bucket *symbol;
   char *typename = 0;
   int usrtoknum;
@@ -1198,12 +1198,12 @@ static int
 get_type (void)
 {
   int k;
-  int t;
+  token_t token;
   char *name;
 
-  t = lex ();
+  token = lex ();
 
-  if (t != tok_typename)
+  if (token != tok_typename)
     {
       complain (_("invalid %s declaration"), "%type");
       return t;
@@ -1213,9 +1213,9 @@ get_type (void)
 
   for (;;)
     {
-      t = lex ();
+      token = lex ();
 
-      switch (t)
+      switch (token)
        {
        case tok_semicolon:
          return lex ();
@@ -1232,7 +1232,7 @@ get_type (void)
          break;
 
        default:
-         return t;
+         return token;
        }
     }
 }
@@ -1253,7 +1253,7 @@ get_type (void)
 static void
 readgram (void)
 {
-  int t;
+  token_t t;
   bucket *lhs = NULL;
   symbol_list *p;
   symbol_list *p1;
@@ -1352,7 +1352,7 @@ readgram (void)
              if (t == tok_identifier)
                {
                  bucket *ssave;
-                 int t1;
+                 token_t t1;
 
                  ssave = symval;
                  t1 = lex ();