Get rid of the ad hoc handling of token_buffer in the scanner: use

the obstacks.
* src/lex.c (token_obstack): New.
(init_lex): Initialize it.  No longer call...
(grow_token_buffer): this.  Remove it.
Adjust all the places which used it to use the obstack.

diff --git a/ChangeLog b/ChangeLog
index b006c3bbcfc4320d5d55933a082ed6453fc1b18c..069e4fa98379fbac2646dbabbe87628fa6de17c8 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,13 @@
+2001-01-19  Akim Demaille  <akim@epita.fr>
+
+       Get rid of the ad hoc handling of token_buffer in the scanner: use
+       the obstacks.
+
+       * src/lex.c (token_obstack): New.
+       (init_lex): Initialize it.  No longer call...
+       (grow_token_buffer): this.  Remove it.
+       Adjust all the places which used it to use the obstack.
+

 2001-01-19  Akim Demaille  <akim@epita.fr>
 
        * src/lex.h: Rename all the tokens:
 2001-01-19  Akim Demaille  <akim@epita.fr>
 
        * src/lex.h: Rename all the tokens:

diff --git a/src/lex.c b/src/lex.c
index 408e30fcb8549fb664cf8cde73d953054aa26e7f..987777be5b39f7302beb5bf94251e3164277c3ca 100644 (file)
--- a/src/lex.c
+++ b/src/lex.c
@@ -30,10 +30,8 @@
 #include "quote.h"
 
 /* Buffer for storing the current token.  */
 #include "quote.h"
 
 /* Buffer for storing the current token.  */

-char *token_buffer;
-
-/* Allocated size of token_buffer, not including space for terminator.  */
-static int maxtoken;
+struct obstack token_obstack;
+char *token_buffer = NULL;

 
 bucket *symval;
 int numval;
 
 bucket *symval;
 int numval;


@@ -45,22 +43,11 @@ static bucket *unlexed_symval;      /* by the next call to lex */
 void
 init_lex (void)
 {
 void
 init_lex (void)
 {

-  maxtoken = 100;
-  token_buffer = XCALLOC (char, maxtoken + 1);
+  obstack_init (&token_obstack);

   unlexed = -1;
 }
 
 
   unlexed = -1;
 }
 
 

-static char *
-grow_token_buffer (char *p)
-{
-  int offset = p - token_buffer;
-  maxtoken *= 2;
-  token_buffer = XREALLOC (token_buffer, char, maxtoken + 1);
-  return token_buffer + offset;
-}
-
-

 int
 skip_white_space (void)
 {
 int
 skip_white_space (void)
 {


@@ -148,16 +135,22 @@ xgetc (FILE *f)
 
 /*------------------------------------------------------------------.
 | Read one literal character from finput.  Process \ escapes.       |
 
 /*------------------------------------------------------------------.
 | Read one literal character from finput.  Process \ escapes.       |

-| Append the normalized string version of the char to *PP.  Assign  |
+| Append the normalized string version of the char to OUT.  Assign  |

 | the character code to *PCODE. Return 1 unless the character is an |
 | the character code to *PCODE. Return 1 unless the character is an |

-| unescaped `term' or \n report error for \n                        |
+| unescaped `term' or \n report error for \n.                       |

 `------------------------------------------------------------------*/
 
 `------------------------------------------------------------------*/
 

+/* FIXME: We could directly work in the obstack, but that would make
+   it more difficult to move to quotearg some day.  So for the time
+   being, I prefer have literalchar behave like quotearg, and change
+   my mind later if I was wrong.  */
+

 static int
 static int

-literalchar (char **pp, int *pcode, char term)
+literalchar (struct obstack *out, int *pcode, char term)

 {
   int c;
 {
   int c;

-  char *p;
+  char buf[4096];
+  char *cp;

   int code;
   int wasquote = 0;
 
   int code;
   int wasquote = 0;
 


@@ -249,68 +242,71 @@ literalchar (char **pp, int *pcode, char term)
        }
     }                          /* has \ */
 
        }
     }                          /* has \ */
 

-  /* now fill token_buffer with the canonical name for this character
-     as a literal token.  Do not use what the user typed,
-     so that `\012' and `\n' can be interchangeable.  */
+  /* now fill BUF with the canonical name for this character as a
+     literal token.  Do not use what the user typed, so that `\012'
+     and `\n' can be interchangeable.  */

 
 

-  p = *pp;
+  cp = buf;

   if (code == term && wasquote)
   if (code == term && wasquote)

-    *p++ = code;
+    *cp++ = code;

   else if (code == '\\')
     {
   else if (code == '\\')
     {

-      *p++ = '\\';
-      *p++ = '\\';
+      *cp++ = '\\';
+      *cp++ = '\\';

     }
   else if (code == '\'')
     {
     }
   else if (code == '\'')
     {

-      *p++ = '\\';
-      *p++ = '\'';
+      *cp++ = '\\';
+      *cp++ = '\'';

     }
   else if (code == '\"')
     {
     }
   else if (code == '\"')
     {

-      *p++ = '\\';
-      *p++ = '\"';
+      *cp++ = '\\';
+      *cp++ = '\"';

     }
   else if (code >= 040 && code < 0177)
     }
   else if (code >= 040 && code < 0177)

-    *p++ = code;
+    *cp++ = code;

   else if (code == '\t')
     {
   else if (code == '\t')
     {

-      *p++ = '\\';
-      *p++ = 't';
+      *cp++ = '\\';
+      *cp++ = 't';

     }
   else if (code == '\n')
     {
     }
   else if (code == '\n')
     {

-      *p++ = '\\';
-      *p++ = 'n';
+      *cp++ = '\\';
+      *cp++ = 'n';

     }
   else if (code == '\r')
     {
     }
   else if (code == '\r')
     {

-      *p++ = '\\';
-      *p++ = 'r';
+      *cp++ = '\\';
+      *cp++ = 'r';

     }
   else if (code == '\v')
     {
     }
   else if (code == '\v')
     {

-      *p++ = '\\';
-      *p++ = 'v';
+      *cp++ = '\\';
+      *cp++ = 'v';

     }
   else if (code == '\b')
     {
     }
   else if (code == '\b')
     {

-      *p++ = '\\';
-      *p++ = 'b';
+      *cp++ = '\\';
+      *cp++ = 'b';

     }
   else if (code == '\f')
     {
     }
   else if (code == '\f')
     {

-      *p++ = '\\';
-      *p++ = 'f';
+      *cp++ = '\\';
+      *cp++ = 'f';

     }
   else
     {
     }
   else
     {

-      *p++ = '\\';
-      *p++ = code / 0100 + '0';
-      *p++ = ((code / 010) & 07) + '0';
-      *p++ = (code & 07) + '0';
+      *cp++ = '\\';
+      *cp++ = code / 0100 + '0';
+      *cp++ = ((code / 010) & 07) + '0';
+      *cp++ = (code & 07) + '0';

     }
     }

-  *pp = p;
+  *cp = '\0';
+
+  if (out)
+    obstack_sgrow (out, buf);

   *pcode = code;
   return !wasquote;
 }
   *pcode = code;
   return !wasquote;
 }


@@ -331,7 +327,6 @@ unlex (int token)
 void
 read_type_name (FILE *fin)
 {
 void
 read_type_name (FILE *fin)
 {

-  char *p = token_buffer;

   int c = getc (fin);
 
   while (c != '>')
   int c = getc (fin);
 
   while (c != '>')


@@ -345,13 +340,11 @@ read_type_name (FILE *fin)
          break;
        }
 
          break;
        }
 

-      if (p == token_buffer + maxtoken)
-       p = grow_token_buffer (p);
-
-      *p++ = c;
+      obstack_1grow (&token_obstack, c);

       c = getc (fin);
     }
       c = getc (fin);
     }

-  *p = 0;
+  obstack_1grow (&token_obstack, '\0');
+  token_buffer = obstack_finish (&token_obstack);

 }
 
 
 }
 
 


@@ -359,7 +352,9 @@ token_t
 lex (void)
 {
   int c;
 lex (void)
 {
   int c;

-  char *p;
+
+  /* Just to make sure. */
+  token_buffer = NULL;

 
   if (unlexed >= 0)
     {
 
   if (unlexed >= 0)
     {


@@ -370,14 +365,11 @@ lex (void)
     }
 
   c = skip_white_space ();
     }
 
   c = skip_white_space ();

-  /* for error messages (token buffer always valid) */
-  *token_buffer = c;
-  token_buffer[1] = 0;

 
   switch (c)
     {
     case EOF:
 
   switch (c)
     {
     case EOF:

-      strcpy (token_buffer, "EOF");
+      token_buffer = "EOF";

       return tok_eof;
 
     case 'A':    case 'B':    case 'C':    case 'D':    case 'E':
       return tok_eof;
 
     case 'A':    case 'B':    case 'C':    case 'D':    case 'E':


@@ -394,17 +386,13 @@ lex (void)
     case 'z':
     case '.':    case '_':
 
     case 'z':
     case '.':    case '_':
 

-      p = token_buffer;

       while (isalnum (c) || c == '_' || c == '.')
        {
       while (isalnum (c) || c == '_' || c == '.')
        {

-         if (p == token_buffer + maxtoken)
-           p = grow_token_buffer (p);
-
-         *p++ = c;
+         obstack_1grow (&token_obstack, c);

          c = getc (finput);
        }
          c = getc (finput);
        }

-
-      *p = 0;
+      obstack_1grow (&token_obstack, '\0');
+      token_buffer = obstack_finish (&token_obstack);

       ungetc (c, finput);
       symval = getsym (token_buffer);
       return tok_identifier;
       ungetc (c, finput);
       symval = getsym (token_buffer);
       return tok_identifier;


@@ -414,17 +402,14 @@ lex (void)
       {
        numval = 0;
 
       {
        numval = 0;
 

-       p = token_buffer;

        while (isdigit (c))
          {
        while (isdigit (c))
          {

-           if (p == token_buffer + maxtoken)
-             p = grow_token_buffer (p);
-
-           *p++ = c;
+           obstack_1grow (&token_obstack, c);

            numval = numval * 10 + c - '0';
            c = getc (finput);
          }
            numval = numval * 10 + c - '0';
            c = getc (finput);
          }

-       *p = 0;
+       obstack_1grow (&token_obstack, '\0');
+       token_buffer = obstack_finish (&token_obstack);

        ungetc (c, finput);
        return tok_number;
       }
        ungetc (c, finput);
        return tok_number;
       }


@@ -435,25 +420,21 @@ lex (void)
       translations = -1;
       {
        int code, discode;
       translations = -1;
       {
        int code, discode;

-       char discard[10], *dp;

 
 

-       p = token_buffer;
-       *p++ = '\'';
-       literalchar (&p, &code, '\'');
+       obstack_1grow (&token_obstack, '\'');
+       literalchar (&token_obstack, &code, '\'');

 
        c = getc (finput);
        if (c != '\'')
          {
            complain (_("use \"...\" for multi-character literal tokens"));
            while (1)
 
        c = getc (finput);
        if (c != '\'')
          {
            complain (_("use \"...\" for multi-character literal tokens"));
            while (1)

-             {
-               dp = discard;
-               if (!literalchar (&dp, &discode, '\''))
-                 break;
-             }
+             if (!literalchar (0, &discode, '\''))
+               break;

          }
          }

-       *p++ = '\'';
-       *p = 0;
+       obstack_1grow (&token_obstack, '\'');
+       obstack_1grow (&token_obstack, '\0');
+       token_buffer = obstack_finish (&token_obstack);

        symval = getsym (token_buffer);
        symval->class = token_sym;
        if (!symval->user_token_number)
        symval = getsym (token_buffer);
        symval->class = token_sym;
        if (!symval->user_token_number)


@@ -467,15 +448,13 @@ lex (void)
       translations = -1;
       {
        int code;               /* ignored here */
       translations = -1;
       {
        int code;               /* ignored here */

-       p = token_buffer;
-       *p++ = '\"';
+
+       obstack_1grow (&token_obstack, '\"');

        /* Read up to and including ".  */
        /* Read up to and including ".  */

-       while (literalchar (&p, &code, '\"'))
-         {
-           if (p >= token_buffer + maxtoken - 4)
-             p = grow_token_buffer (p);
-         }
-       *p = 0;
+       while (literalchar (&token_obstack, &code, '\"'))
+         /* nothing */;
+       obstack_1grow (&token_obstack, '\0');
+       token_buffer = obstack_finish (&token_obstack);

 
        symval = getsym (token_buffer);
        symval->class = token_sym;
 
        symval = getsym (token_buffer);
        symval->class = token_sym;


@@ -509,7 +488,7 @@ lex (void)
 
       if (c == '{')
        {
 
       if (c == '{')
        {

-         strcpy (token_buffer, "={");
+         token_buffer = "={";

          return tok_left_curly;
        }
       else
          return tok_left_curly;
        }
       else


@@ -589,14 +568,9 @@ int
 parse_percent_token (void)
 {
   int c;
 parse_percent_token (void)
 {
   int c;

-  char *p;

   struct percent_table_struct *tx;
 
   struct percent_table_struct *tx;
 

-  p = token_buffer;

   c = getc (finput);
   c = getc (finput);

-  *p++ = '%';
-  *p++ = c;                    /* for error msg */
-  *p = 0;

 
   switch (c)
     {
 
   switch (c)
     {


@@ -621,25 +595,22 @@ parse_percent_token (void)
     case '=':
       return tok_prec;
     }
     case '=':
       return tok_prec;
     }

+

   if (!isalpha (c))
     return tok_illegal;
 
   if (!isalpha (c))
     return tok_illegal;
 

-  p = token_buffer;
-  *p++ = '%';
+  obstack_1grow (&token_obstack, '%');

   while (isalpha (c) || c == '_' || c == '-')
     {
   while (isalpha (c) || c == '_' || c == '-')
     {

-      if (p == token_buffer + maxtoken)
-       p = grow_token_buffer (p);
-

       if (c == '-')
        c = '_';
       if (c == '-')
        c = '_';

-      *p++ = c;
+      obstack_1grow (&token_obstack, c);

       c = getc (finput);
     }
 
   ungetc (c, finput);
       c = getc (finput);
     }
 
   ungetc (c, finput);

-
-  *p = 0;
+  obstack_1grow (&token_obstack, '\0');
+  token_buffer = obstack_finish (&token_obstack);

 
   /* table lookup % directive */
   for (tx = percent_table; tx->name; tx++)
 
   /* table lookup % directive */
   for (tx = percent_table; tx->name; tx++)

diff --git a/src/reader.c b/src/reader.c
index 2de754d04c7c4ac2870bbe8019b6d5b3c21a1eba..e073f028e5eb4f8f41662391e34b5c53fe1c9f21 100644 (file)
--- a/src/reader.c
+++ b/src/reader.c
@@ -466,7 +466,7 @@ copy_definition (void)
 static void
 parse_token_decl (symbol_class what_is, symbol_class what_is_not)
 {
 static void
 parse_token_decl (symbol_class what_is, symbol_class what_is_not)
 {

-  int token = 0;
+  token_t token = 0;

   char *typename = 0;
 
   /* The symbol being defined.  */
   char *typename = 0;
 
   /* The symbol being defined.  */


@@ -593,7 +593,7 @@ parse_type_decl (void)
 
   for (;;)
     {
 
   for (;;)
     {

-      int t;
+      token_t t;

       int tmp_char = ungetc (skip_white_space (), finput);
 
       if (tmp_char == '%')
       int tmp_char = ungetc (skip_white_space (), finput);
 
       if (tmp_char == '%')


@@ -643,7 +643,7 @@ parse_assoc_decl (associativity assoc)
 
   for (;;)
     {
 
   for (;;)
     {

-      int t;
+      token_t t;

       int tmp_char = ungetc (skip_white_space (), finput);
 
       if (tmp_char == '%')
       int tmp_char = ungetc (skip_white_space (), finput);
 
       if (tmp_char == '%')


@@ -821,7 +821,7 @@ parse_expect_decl (void)
 static void
 parse_thong_decl (void)
 {
 static void
 parse_thong_decl (void)
 {

-  int token;
+  token_t token;

   struct bucket *symbol;
   char *typename = 0;
   int usrtoknum;
   struct bucket *symbol;
   char *typename = 0;
   int usrtoknum;


@@ -1198,12 +1198,12 @@ static int
 get_type (void)
 {
   int k;
 get_type (void)
 {
   int k;

-  int t;
+  token_t token;

   char *name;
 
   char *name;
 

-  t = lex ();
+  token = lex ();

 
 

-  if (t != tok_typename)
+  if (token != tok_typename)

     {
       complain (_("invalid %s declaration"), "%type");
       return t;
     {
       complain (_("invalid %s declaration"), "%type");
       return t;


@@ -1213,9 +1213,9 @@ get_type (void)
 
   for (;;)
     {
 
   for (;;)
     {

-      t = lex ();
+      token = lex ();

 
 

-      switch (t)
+      switch (token)

        {
        case tok_semicolon:
          return lex ();
        {
        case tok_semicolon:
          return lex ();


@@ -1232,7 +1232,7 @@ get_type (void)
          break;
 
        default:
          break;
 
        default:

-         return t;
+         return token;

        }
     }
 }
        }
     }
 }


@@ -1253,7 +1253,7 @@ get_type (void)
 static void
 readgram (void)
 {
 static void
 readgram (void)
 {

-  int t;
+  token_t t;

   bucket *lhs = NULL;
   symbol_list *p;
   symbol_list *p1;
   bucket *lhs = NULL;
   symbol_list *p;
   symbol_list *p1;


@@ -1352,7 +1352,7 @@ readgram (void)
              if (t == tok_identifier)
                {
                  bucket *ssave;
              if (t == tok_identifier)
                {
                  bucket *ssave;

-                 int t1;
+                 token_t t1;

 
                  ssave = symval;
                  t1 = lex ();
 
                  ssave = symval;
                  t1 = lex ();