[bison.git] / src / lex.c

/* Token-reader for Bison's input parser,
   Copyright (C) 1984, 1986, 1989, 1992 Free Software Foundation, Inc.

This file is part of Bison, the GNU Compiler Compiler.

Bison is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2, or (at your option)
any later version.

Bison is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with Bison; see the file COPYING.  If not, write to
the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
Boston, MA 02111-1307, USA.  */


/*
   lex is the entry point.  It is called from reader.c.
   It returns one of the token-type codes defined in lex.h.
   When an identifier is seen, the code IDENTIFIER is returned
   and the name is looked up in the symbol table using symtab.c;
   symval is set to a pointer to the entry found.  */

#include <stdio.h>
#include "system.h"
#include "files.h"
#include "getopt.h"		/* for optarg */
#include "symtab.h"
#include "lex.h"
#include "alloc.h"

/* flags set by % directives */
extern int definesflag;    	/* for -d */
extern int toknumflag;   	/* for -k */
extern int noparserflag;	/* for -n */
extern int fixed_outfiles;  	/* for -y */
extern int nolinesflag;    	/* for -l */
extern int rawtoknumflag;	/* for -r */
extern int verboseflag;	/* for -v */
extern int debugflag;  		/* for -t */
extern char *spec_name_prefix; 	/* for -p */
extern char *spec_file_prefix;	/* for -b */
/*spec_outfile is declared in files.h, for -o */

extern int lineno;
extern int translations;

void init_lex PARAMS((void));
char *grow_token_buffer PARAMS((char *));
int skip_white_space PARAMS((void));
int safegetc PARAMS((FILE *));
int literalchar PARAMS((char **, int *, char));
void unlex PARAMS((int));
int lex PARAMS((void));
int parse_percent_token PARAMS((void));

/* functions from main.c */
extern char *printable_version PARAMS((int));
extern void fatal PARAMS((char *));
extern void warn PARAMS((char *));
extern void warni PARAMS((char *, int));
extern void warns PARAMS((char *, char *));

/* Buffer for storing the current token.  */
char *token_buffer;

/* Allocated size of token_buffer, not including space for terminator.  */
int maxtoken;

bucket *symval;
int numval;

static int unlexed;		/* these two describe a token to be reread */
static bucket *unlexed_symval;	/* by the next call to lex */


void
init_lex (void)
{
  maxtoken = 100;
  token_buffer = NEW2 (maxtoken + 1, char);
  unlexed = -1;
}


char *
grow_token_buffer (char *p)
{
  int offset = p - token_buffer;
  maxtoken *= 2;
  token_buffer = (char *) xrealloc(token_buffer, maxtoken + 1);
  return token_buffer + offset;
}


int
skip_white_space (void)
{
  register int c;
  register int inside;

  c = getc(finput);

  for (;;)
    {
      int cplus_comment;

      switch (c)
	{
	case '/':
	  c = getc(finput);
	  if (c != '*' && c != '/')
	    {
	      warn(_("unexpected `/' found and ignored"));
	      break;
	    }
	  cplus_comment = (c == '/');

	  c = getc(finput);

	  inside = 1;
	  while (inside)
	    {
	      if (!cplus_comment && c == '*')
		{
		  while (c == '*')
		    c = getc(finput);

		  if (c == '/')
		    {
		      inside = 0;
		      c = getc(finput);
		    }
		}
	      else if (c == '\n')
		{
		  lineno++;
		  if (cplus_comment)
		    inside = 0;
		  c = getc(finput);
		}
	      else if (c == EOF)
		fatal(_("unterminated comment"));
	      else
		c = getc(finput);
	    }

	  break;

	case '\n':
	  lineno++;

	case ' ':
	case '\t':
	case '\f':
	  c = getc(finput);
	  break;

	default:
	  return (c);
	}
    }
}

/* do a getc, but give error message if EOF encountered */
int
safegetc (FILE *f)
{
  register int c = getc(f);
  if (c == EOF)
    fatal(_("Unexpected end of file"));
  return c;
}

/* read one literal character from finput.  process \ escapes.
   append the normalized string version of the char to *pp.
   assign the character code to *pcode
   return 1 unless the character is an unescaped `term' or \n
	report error for \n
*/
int
literalchar (char **pp, int *pcode, char term)
{
  register int c;
  register char *p;
  register int code;
  int wasquote = 0;

  c = safegetc(finput);
  if (c == '\n')
    {
      warn(_("unescaped newline in constant"));
      ungetc(c, finput);
      code = '?';
      wasquote = 1;
    }
  else if (c != '\\')
    {
      code = c;
      if (c == term)
	wasquote = 1;
    }
  else
    {
      c = safegetc(finput);
      if (c == 't')  code = '\t';
      else if (c == 'n')  code = '\n';
      else if (c == 'a')  code = '\007';
      else if (c == 'r')  code = '\r';
      else if (c == 'f')  code = '\f';
      else if (c == 'b')  code = '\b';
      else if (c == 'v')  code = '\013';
      else if (c == '\\')  code = '\\';
      else if (c == '\'')  code = '\'';
      else if (c == '\"')  code = '\"';
      else if (c <= '7' && c >= '0')
	{
	  code = 0;
	  while (c <= '7' && c >= '0')
	    {
	      code = (code * 8) + (c - '0');
	      if (code >= 256 || code < 0)
		{
		  warni(_("octal value outside range 0...255: `\\%o'"), code);
		  code &= 0xFF;
		  break;
		}
	      c = safegetc(finput);
	    }
	  ungetc(c, finput);
	}
      else if (c == 'x')
	{
	  c = safegetc(finput);
	  code = 0;
	  while (1)
	    {
	      if (c >= '0' && c <= '9')
		code *= 16,  code += c - '0';
	      else if (c >= 'a' && c <= 'f')
		code *= 16,  code += c - 'a' + 10;
	      else if (c >= 'A' && c <= 'F')
		code *= 16,  code += c - 'A' + 10;
	      else
		break;
	      if (code >= 256 || code<0)
		{
		  warni(_("hexadecimal value above 255: `\\x%x'"), code);
		  code &= 0xFF;
		  break;
		}
	      c = safegetc(finput);
	    }
	  ungetc(c, finput);
	}
      else
	{
	  warns (_("unknown escape sequence: `\\' followed by `%s'"),
		 printable_version(c));
	  code = '?';
	}
    } /* has \ */

  /* now fill token_buffer with the canonical name for this character
     as a literal token.  Do not use what the user typed,
     so that `\012' and `\n' can be interchangeable.  */

  p = *pp;
  if (code == '\\')  {*p++ = '\\'; *p++ = '\\';}
  else if (code == '\'')  {*p++ = '\\'; *p++ = '\'';}
  else if (code == '\"')  {*p++ = '\\'; *p++ = '\"';}
  else if (code >= 040 && code < 0177)
    *p++ = code;
  else if (code == '\t')  {*p++ = '\\'; *p++ = 't';}
  else if (code == '\n')  {*p++ = '\\'; *p++ = 'n';}
  else if (code == '\r')  {*p++ = '\\'; *p++ = 'r';}
  else if (code == '\v')  {*p++ = '\\'; *p++ = 'v';}
  else if (code == '\b')  {*p++ = '\\'; *p++ = 'b';}
  else if (code == '\f')  {*p++ = '\\'; *p++ = 'f';}
  else
    {
      *p++ = '\\';
      *p++ = code / 0100 + '0';
      *p++ = ((code / 010) & 07) + '0';
      *p++ = (code & 07) + '0';
    }
  *pp = p;
  *pcode = code;
  return  ! wasquote;
}


void
unlex (int token)
{
  unlexed = token;
  unlexed_symval = symval;
}


int
lex (void)
{
  register int c;
  char *p;

  if (unlexed >= 0)
    {
      symval = unlexed_symval;
      c = unlexed;
      unlexed = -1;
      return (c);
    }

  c = skip_white_space();
  *token_buffer = c;	/* for error messages (token buffer always valid) */
  token_buffer[1] = 0;

  switch (c)
    {
    case EOF:
      strcpy(token_buffer, "EOF");
      return (ENDFILE);

    case 'A':  case 'B':  case 'C':  case 'D':  case 'E':
    case 'F':  case 'G':  case 'H':  case 'I':  case 'J':
    case 'K':  case 'L':  case 'M':  case 'N':  case 'O':
    case 'P':  case 'Q':  case 'R':  case 'S':  case 'T':
    case 'U':  case 'V':  case 'W':  case 'X':  case 'Y':
    case 'Z':
    case 'a':  case 'b':  case 'c':  case 'd':  case 'e':
    case 'f':  case 'g':  case 'h':  case 'i':  case 'j':
    case 'k':  case 'l':  case 'm':  case 'n':  case 'o':
    case 'p':  case 'q':  case 'r':  case 's':  case 't':
    case 'u':  case 'v':  case 'w':  case 'x':  case 'y':
    case 'z':
    case '.':  case '_':
      p = token_buffer;
      while (isalnum(c) || c == '_' || c == '.')
	{
	  if (p == token_buffer + maxtoken)
	    p = grow_token_buffer(p);

	  *p++ = c;
	  c = getc(finput);
	}

      *p = 0;
      ungetc(c, finput);
      symval = getsym(token_buffer);
      return (IDENTIFIER);

    case '0':  case '1':  case '2':  case '3':  case '4':
    case '5':  case '6':  case '7':  case '8':  case '9':
      {
	numval = 0;

	p = token_buffer;
	while (isdigit(c))
	  {
	    if (p == token_buffer + maxtoken)
	      p = grow_token_buffer(p);

	    *p++ = c;
	    numval = numval*10 + c - '0';
	    c = getc(finput);
	  }
	*p = 0;
	ungetc(c, finput);
	return (NUMBER);
      }

    case '\'':

      /* parse the literal token and compute character code in  code  */

      translations = -1;
      {
	int code, discode;
	char discard[10], *dp;

	p = token_buffer;
	*p++ = '\'';
	literalchar(&p, &code, '\'');

	c = getc(finput);
	if (c != '\'')
	  {
	    warn(_("use \"...\" for multi-character literal tokens"));
	    while (1)
	      {
		dp = discard;
		if (! literalchar(&dp, &discode, '\''))
		  break;
	      }
	  }
	*p++ = '\'';
	*p = 0;
	symval = getsym(token_buffer);
	symval->class = STOKEN;
	if (! symval->user_token_number)
	  symval->user_token_number = code;
	return (IDENTIFIER);
      }

    case '\"':

      /* parse the literal string token and treat as an identifier */

      translations = -1;
      {
	int code;	/* ignored here */
	p = token_buffer;
	*p++ = '\"';
	while (literalchar(&p, &code, '\"'))  /* read up to and including " */
	  {
	    if (p >= token_buffer + maxtoken - 4)
	      p = grow_token_buffer(p);
	  }
	*p = 0;

	symval = getsym(token_buffer);
	symval->class = STOKEN;

	return (IDENTIFIER);
      }

    case ',':
      return (COMMA);

    case ':':
      return (COLON);

    case ';':
      return (SEMICOLON);

    case '|':
      return (BAR);

    case '{':
      return (LEFT_CURLY);

    case '=':
      do
	{
	  c = getc(finput);
	  if (c == '\n') lineno++;
	}
      while(c==' ' || c=='\n' || c=='\t');

      if (c == '{')
	{
	  strcpy(token_buffer, "={");
	  return(LEFT_CURLY);
	}
      else
	{
	  ungetc(c, finput);
	  return(ILLEGAL);
	}

    case '<':
      p = token_buffer;
      c = getc(finput);
      while (c != '>')
	{
	  if (c == EOF)
	    fatal(_("unterminated type name at end of file"));
	  if (c == '\n')
	    {
	      warn(_("unterminated type name"));
	      ungetc(c, finput);
	      break;
	    }

	  if (p == token_buffer + maxtoken)
	    p = grow_token_buffer(p);

	  *p++ = c;
	  c = getc(finput);
	}
      *p = 0;
      return (TYPENAME);


    case '%':
      return (parse_percent_token());

    default:
      return (ILLEGAL);
    }
}

/* the following table dictates the action taken for the various
	% directives.  A setflag value causes the named flag to be
	set.  A retval action returns the code.
*/
struct percent_table_struct {
	char *name;
	void *setflag;
	int retval;
} percent_table[] =
{
  {"token", NULL, TOKEN},
  {"term", NULL, TOKEN},
  {"nterm", NULL, NTERM},
  {"type", NULL, TYPE},
  {"guard", NULL, GUARD},
  {"union", NULL, UNION},
  {"expect", NULL, EXPECT},
  {"thong", NULL, THONG},
  {"start", NULL, START},
  {"left", NULL, LEFT},
  {"right", NULL, RIGHT},
  {"nonassoc", NULL, NONASSOC},
  {"binary", NULL, NONASSOC},
  {"semantic_parser", NULL, SEMANTIC_PARSER},
  {"pure_parser", NULL, PURE_PARSER},
  {"prec", NULL, PREC},

  {"no_lines", &nolinesflag, NOOP}, /* -l */
  {"raw", &rawtoknumflag, NOOP}, /* -r */
  {"token_table", &toknumflag, NOOP}, /* -k */

#if 0
  /* These can be utilized after main is reoganized so
     open_files() is deferred 'til after read_declarations().
     But %{ and %union both put information into files
     that have to be opened before read_declarations().
     */
  {"yacc", &fixed_outfiles, NOOP}, /* -y */
  {"fixed_output_files", &fixed_outfiles, NOOP}, /* -y */
  {"defines", &definesflag, NOOP}, /* -d */
  {"no_parser", &noparserflag, NOOP}, /* -n */
  {"output_file", &spec_outfile, SETOPT}, /* -o */
  {"file_prefix", &spec_file_prefix, SETOPT}, /* -b */
  {"name_prefix", &spec_name_prefix, SETOPT}, /* -p */

  /* These would be acceptable, but they do not affect processing */
  {"verbose", &verboseflag, NOOP}, /* -v */
  {"debug", &debugflag, NOOP},	/* -t */
  /*	{"help", <print usage stmt>, NOOP},*/	/* -h */
  /*	{"version", <print version number> ,  NOOP},*/	/* -V */
#endif

  {NULL, NULL, ILLEGAL}
};

/* Parse a token which starts with %.
   Assumes the % has already been read and discarded.  */

int
parse_percent_token (void)
{
  register int c;
  register char *p;
  register struct percent_table_struct *tx;

  p = token_buffer;
  c = getc(finput);
  *p++ = '%';
  *p++ = c;	/* for error msg */
  *p = 0;

  switch (c)
    {
    case '%':
      return (TWO_PERCENTS);

    case '{':
      return (PERCENT_LEFT_CURLY);

    case '<':
      return (LEFT);

    case '>':
      return (RIGHT);

    case '2':
      return (NONASSOC);

    case '0':
      return (TOKEN);

    case '=':
      return (PREC);
    }
  if (!isalpha(c))
    return (ILLEGAL);

  p = token_buffer;
  *p++ = '%';
  while (isalpha(c) || c == '_' || c == '-')
    {
      if (p == token_buffer + maxtoken)
	p = grow_token_buffer(p);

      if (c == '-') c = '_';
      *p++ = c;
      c = getc(finput);
    }

  ungetc(c, finput);

  *p = 0;

  /* table lookup % directive */
  for (tx = percent_table; tx->name; tx++)
    if (strcmp(token_buffer+1, tx->name) == 0)
      break;
  if (tx->retval == SETOPT)
    {
      *((char **)(tx->setflag)) = optarg;
      return NOOP;
    }
  if (tx->setflag)
    {
      *((int *)(tx->setflag)) = 1;
      return NOOP;
    }
  return tx->retval;
}
Commit	Line	Data
40675e7c	1	/* Token-reader for Bison's input parser,
a44c2277	2	Copyright (C) 1984, 1986, 1989, 1992 Free Software Foundation, Inc.
40675e7c DM	3
	4	This file is part of Bison, the GNU Compiler Compiler.
	5
	6	Bison is free software; you can redistribute it and/or modify
	7	it under the terms of the GNU General Public License as published by
	8	the Free Software Foundation; either version 2, or (at your option)
	9	any later version.
	10
	11	Bison is distributed in the hope that it will be useful,
	12	but WITHOUT ANY WARRANTY; without even the implied warranty of
	13	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	14	GNU General Public License for more details.
	15
	16	You should have received a copy of the GNU General Public License
	17	along with Bison; see the file COPYING. If not, write to
c49a8e71 JT	18	the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
c49a8e71 JT	19	Boston, MA 02111-1307, USA. */
40675e7c DM	20
40675e7c DM	21
a083fbbf	22	/*
a44c2277	23	lex is the entry point. It is called from reader.c.
40675e7c DM	24	It returns one of the token-type codes defined in lex.h.
	25	When an identifier is seen, the code IDENTIFIER is returned
	26	and the name is looked up in the symbol table using symtab.c;
	27	symval is set to a pointer to the entry found. */
	28
	29	#include <stdio.h>
40675e7c DM	30	#include "system.h"
40675e7c DM	31	#include "files.h"
a44c2277	32	#include "getopt.h" /* for optarg */
40675e7c DM	33	#include "symtab.h"
40675e7c DM	34	#include "lex.h"
7612000c	35	#include "alloc.h"
40675e7c	36
a44c2277 RS	37	/* flags set by % directives */
	38	extern int definesflag; /* for -d */
	39	extern int toknumflag; /* for -k */
	40	extern int noparserflag; /* for -n */
	41	extern int fixed_outfiles; /* for -y */
	42	extern int nolinesflag; /* for -l */
	43	extern int rawtoknumflag; /* for -r */
	44	extern int verboseflag; /* for -v */
	45	extern int debugflag; /* for -t */
	46	extern char spec_name_prefix; / for -p */
	47	extern char spec_file_prefix; / for -b */
	48	/spec_outfile is declared in files.h, for -o /
40675e7c DM	49
	50	extern int lineno;
	51	extern int translations;
	52
d2729d44 JT	53	void init_lex PARAMS((void));
	54	char grow_token_buffer PARAMS((char ));
	55	int skip_white_space PARAMS((void));
	56	int safegetc PARAMS((FILE *));
	57	int literalchar PARAMS((char *, int , char));
	58	void unlex PARAMS((int));
	59	int lex PARAMS((void));
	60	int parse_percent_token PARAMS((void));
40675e7c	61
a44c2277	62	/* functions from main.c */
d2729d44 JT	63	extern char *printable_version PARAMS((int));
	64	extern void fatal PARAMS((char *));
	65	extern void warn PARAMS((char *));
	66	extern void warni PARAMS((char *, int));
	67	extern void warns PARAMS((char , char ));
40675e7c DM	68
	69	/* Buffer for storing the current token. */
	70	char *token_buffer;
	71
	72	/* Allocated size of token_buffer, not including space for terminator. */
d2729d44	73	int maxtoken;
40675e7c DM	74
	75	bucket *symval;
	76	int numval;
	77
	78	static int unlexed; /* these two describe a token to be reread */
	79	static bucket unlexed_symval; / by the next call to lex */
	80
	81
	82	void
d2729d44	83	init_lex (void)
40675e7c DM	84	{
	85	maxtoken = 100;
	86	token_buffer = NEW2 (maxtoken + 1, char);
	87	unlexed = -1;
	88	}
	89
	90
d2729d44 JT	91	char *
d2729d44 JT	92	grow_token_buffer (char *p)
40675e7c DM	93	{
	94	int offset = p - token_buffer;
	95	maxtoken *= 2;
	96	token_buffer = (char *) xrealloc(token_buffer, maxtoken + 1);
	97	return token_buffer + offset;
	98	}
	99
	100
	101	int
d2729d44	102	skip_white_space (void)
40675e7c DM	103	{
	104	register int c;
	105	register int inside;
	106
	107	c = getc(finput);
	108
	109	for (;;)
	110	{
	111	int cplus_comment;
	112
	113	switch (c)
	114	{
	115	case '/':
	116	c = getc(finput);
a083fbbf	117	if (c != '*' && c != '/')
a44c2277	118	{
a083fbbf	119	warn(_("unexpected `/' found and ignored"));
a44c2277 RS	120	break;
a44c2277 RS	121	}
40675e7c DM	122	cplus_comment = (c == '/');
	123
	124	c = getc(finput);
	125
	126	inside = 1;
	127	while (inside)
	128	{
	129	if (!cplus_comment && c == '*')
	130	{
	131	while (c == '*')
	132	c = getc(finput);
	133
	134	if (c == '/')
	135	{
	136	inside = 0;
	137	c = getc(finput);
	138	}
	139	}
	140	else if (c == '\n')
	141	{
	142	lineno++;
	143	if (cplus_comment)
	144	inside = 0;
	145	c = getc(finput);
	146	}
	147	else if (c == EOF)
a083fbbf	148	fatal(_("unterminated comment"));
40675e7c DM	149	else
	150	c = getc(finput);
	151	}
	152
	153	break;
	154
	155	case '\n':
	156	lineno++;
	157
	158	case ' ':
	159	case '\t':
	160	case '\f':
	161	c = getc(finput);
	162	break;
	163
	164	default:
	165	return (c);
	166	}
	167	}
	168	}
	169
a44c2277 RS	170	/* do a getc, but give error message if EOF encountered */
a44c2277 RS	171	int
d2729d44	172	safegetc (FILE *f)
a44c2277 RS	173	{
	174	register int c = getc(f);
	175	if (c == EOF)
a083fbbf	176	fatal(_("Unexpected end of file"));
a44c2277 RS	177	return c;
	178	}
	179
	180	/* read one literal character from finput. process \ escapes.
	181	append the normalized string version of the char to *pp.
	182	assign the character code to *pcode
	183	return 1 unless the character is an unescaped `term' or \n
	184	report error for \n
	185	*/
	186	int
d2729d44	187	literalchar (char *pp, int pcode, char term)
a44c2277 RS	188	{
	189	register int c;
	190	register char *p;
	191	register int code;
	192	int wasquote = 0;
	193
	194	c = safegetc(finput);
a083fbbf	195	if (c == '\n')
a44c2277	196	{
a083fbbf	197	warn(_("unescaped newline in constant"));
a44c2277 RS	198	ungetc(c, finput);
	199	code = '?';
	200	wasquote = 1;
	201	}
	202	else if (c != '\\')
	203	{
	204	code = c;
a083fbbf	205	if (c == term)
a44c2277 RS	206	wasquote = 1;
	207	}
	208	else
	209	{
	210	c = safegetc(finput);
	211	if (c == 't') code = '\t';
	212	else if (c == 'n') code = '\n';
	213	else if (c == 'a') code = '\007';
	214	else if (c == 'r') code = '\r';
	215	else if (c == 'f') code = '\f';
	216	else if (c == 'b') code = '\b';
d2729d44	217	else if (c == 'v') code = '\013';
a44c2277 RS	218	else if (c == '\\') code = '\\';
	219	else if (c == '\'') code = '\'';
	220	else if (c == '\"') code = '\"';
	221	else if (c <= '7' && c >= '0')
	222	{
	223	code = 0;
	224	while (c <= '7' && c >= '0')
	225	{
	226	code = (code * 8) + (c - '0');
	227	if (code >= 256 \|\| code < 0)
	228	{
a083fbbf	229	warni(_("octal value outside range 0...255: `\\%o'"), code);
a44c2277 RS	230	code &= 0xFF;
	231	break;
	232	}
	233	c = safegetc(finput);
	234	}
	235	ungetc(c, finput);
	236	}
	237	else if (c == 'x')
	238	{
	239	c = safegetc(finput);
	240	code = 0;
	241	while (1)
	242	{
	243	if (c >= '0' && c <= '9')
	244	code *= 16, code += c - '0';
	245	else if (c >= 'a' && c <= 'f')
	246	code *= 16, code += c - 'a' + 10;
	247	else if (c >= 'A' && c <= 'F')
	248	code *= 16, code += c - 'A' + 10;
a083fbbf	249	else
a44c2277 RS	250	break;
	251	if (code >= 256 \|\| code<0)
	252	{
a083fbbf	253	warni(_("hexadecimal value above 255: `\\x%x'"), code);
a44c2277 RS	254	code &= 0xFF;
	255	break;
	256	}
	257	c = safegetc(finput);
	258	}
	259	ungetc(c, finput);
	260	}
	261	else
	262	{
b0180c64	263	warns (_("unknown escape sequence: `\\' followed by `%s'"),
a44c2277 RS	264	printable_version(c));
	265	code = '?';
	266	}
	267	} /* has \ */
	268
	269	/* now fill token_buffer with the canonical name for this character
	270	as a literal token. Do not use what the user typed,
	271	so that `\012' and `\n' can be interchangeable. */
	272
	273	p = *pp;
5ce94c29	274	if (code == '\\') {p++ = '\\'; p++ = '\\';}
a44c2277 RS	275	else if (code == '\'') {p++ = '\\'; p++ = '\'';}
a44c2277 RS	276	else if (code == '\"') {p++ = '\\'; p++ = '\"';}
5ce94c29 RS	277	else if (code >= 040 && code < 0177)
5ce94c29 RS	278	*p++ = code;
a44c2277 RS	279	else if (code == '\t') {p++ = '\\'; p++ = 't';}
	280	else if (code == '\n') {p++ = '\\'; p++ = 'n';}
	281	else if (code == '\r') {p++ = '\\'; p++ = 'r';}
	282	else if (code == '\v') {p++ = '\\'; p++ = 'v';}
	283	else if (code == '\b') {p++ = '\\'; p++ = 'b';}
	284	else if (code == '\f') {p++ = '\\'; p++ = 'f';}
	285	else
	286	{
	287	*p++ = '\\';
	288	*p++ = code / 0100 + '0';
	289	*p++ = ((code / 010) & 07) + '0';
	290	*p++ = (code & 07) + '0';
	291	}
	292	*pp = p;
	293	*pcode = code;
	294	return ! wasquote;
	295	}
	296
40675e7c DM	297
40675e7c DM	298	void
d2729d44	299	unlex (int token)
40675e7c DM	300	{
	301	unlexed = token;
	302	unlexed_symval = symval;
	303	}
	304
	305
40675e7c	306	int
d2729d44	307	lex (void)
40675e7c DM	308	{
40675e7c DM	309	register int c;
a44c2277	310	char *p;
40675e7c DM	311
	312	if (unlexed >= 0)
	313	{
	314	symval = unlexed_symval;
	315	c = unlexed;
	316	unlexed = -1;
	317	return (c);
	318	}
	319
	320	c = skip_white_space();
a44c2277 RS	321	token_buffer = c; / for error messages (token buffer always valid) */
a44c2277 RS	322	token_buffer[1] = 0;
40675e7c DM	323
	324	switch (c)
	325	{
	326	case EOF:
a44c2277	327	strcpy(token_buffer, "EOF");
40675e7c DM	328	return (ENDFILE);
	329
	330	case 'A': case 'B': case 'C': case 'D': case 'E':
	331	case 'F': case 'G': case 'H': case 'I': case 'J':
	332	case 'K': case 'L': case 'M': case 'N': case 'O':
	333	case 'P': case 'Q': case 'R': case 'S': case 'T':
	334	case 'U': case 'V': case 'W': case 'X': case 'Y':
	335	case 'Z':
	336	case 'a': case 'b': case 'c': case 'd': case 'e':
	337	case 'f': case 'g': case 'h': case 'i': case 'j':
	338	case 'k': case 'l': case 'm': case 'n': case 'o':
	339	case 'p': case 'q': case 'r': case 's': case 't':
	340	case 'u': case 'v': case 'w': case 'x': case 'y':
	341	case 'z':
	342	case '.': case '_':
	343	p = token_buffer;
	344	while (isalnum(c) \|\| c == '_' \|\| c == '.')
	345	{
	346	if (p == token_buffer + maxtoken)
	347	p = grow_token_buffer(p);
	348
	349	*p++ = c;
	350	c = getc(finput);
	351	}
	352
	353	*p = 0;
	354	ungetc(c, finput);
	355	symval = getsym(token_buffer);
	356	return (IDENTIFIER);
	357
	358	case '0': case '1': case '2': case '3': case '4':
	359	case '5': case '6': case '7': case '8': case '9':
	360	{
	361	numval = 0;
	362
a44c2277	363	p = token_buffer;
40675e7c DM	364	while (isdigit(c))
40675e7c DM	365	{
a44c2277 RS	366	if (p == token_buffer + maxtoken)
	367	p = grow_token_buffer(p);
	368
	369	*p++ = c;
40675e7c DM	370	numval = numval*10 + c - '0';
	371	c = getc(finput);
	372	}
a44c2277	373	*p = 0;
40675e7c DM	374	ungetc(c, finput);
	375	return (NUMBER);
	376	}
	377
	378	case '\'':
40675e7c DM	379
	380	/* parse the literal token and compute character code in code */
	381
a44c2277	382	translations = -1;
40675e7c	383	{
a44c2277 RS	384	int code, discode;
a44c2277 RS	385	char discard[10], *dp;
5ce94c29	386
a44c2277 RS	387	p = token_buffer;
	388	*p++ = '\'';
	389	literalchar(&p, &code, '\'');
40675e7c	390
a44c2277 RS	391	c = getc(finput);
a44c2277 RS	392	if (c != '\'')
40675e7c	393	{
a083fbbf	394	warn(_("use \"...\" for multi-character literal tokens"));
5ce94c29 RS	395	while (1)
	396	{
	397	dp = discard;
	398	if (! literalchar(&dp, &discode, '\''))
	399	break;
	400	}
40675e7c	401	}
a44c2277 RS	402	*p++ = '\'';
	403	*p = 0;
	404	symval = getsym(token_buffer);
	405	symval->class = STOKEN;
	406	if (! symval->user_token_number)
	407	symval->user_token_number = code;
	408	return (IDENTIFIER);
	409	}
40675e7c	410
a44c2277	411	case '\"':
40675e7c	412
a44c2277 RS	413	/* parse the literal string token and treat as an identifier */
	414
	415	translations = -1;
	416	{
	417	int code; /* ignored here */
40675e7c	418	p = token_buffer;
a44c2277 RS	419	*p++ = '\"';
a44c2277 RS	420	while (literalchar(&p, &code, '\"')) /* read up to and including " */
40675e7c	421	{
a44c2277 RS	422	if (p >= token_buffer + maxtoken - 4)
a44c2277 RS	423	p = grow_token_buffer(p);
40675e7c	424	}
40675e7c	425	*p = 0;
a44c2277	426
40675e7c DM	427	symval = getsym(token_buffer);
40675e7c DM	428	symval->class = STOKEN;
a44c2277	429
40675e7c DM	430	return (IDENTIFIER);
	431	}
	432
	433	case ',':
	434	return (COMMA);
	435
	436	case ':':
	437	return (COLON);
	438
	439	case ';':
	440	return (SEMICOLON);
	441
	442	case '\|':
	443	return (BAR);
	444
	445	case '{':
	446	return (LEFT_CURLY);
	447
	448	case '=':
	449	do
	450	{
	451	c = getc(finput);
	452	if (c == '\n') lineno++;
	453	}
	454	while(c==' ' \|\| c=='\n' \|\| c=='\t');
	455
	456	if (c == '{')
a44c2277 RS	457	{
	458	strcpy(token_buffer, "={");
	459	return(LEFT_CURLY);
	460	}
40675e7c DM	461	else
	462	{
	463	ungetc(c, finput);
	464	return(ILLEGAL);
	465	}
	466
	467	case '<':
	468	p = token_buffer;
	469	c = getc(finput);
	470	while (c != '>')
	471	{
a44c2277	472	if (c == EOF)
a083fbbf RS	473	fatal(_("unterminated type name at end of file"));
a083fbbf RS	474	if (c == '\n')
a44c2277	475	{
a083fbbf	476	warn(_("unterminated type name"));
a44c2277 RS	477	ungetc(c, finput);
	478	break;
	479	}
40675e7c DM	480
	481	if (p == token_buffer + maxtoken)
	482	p = grow_token_buffer(p);
	483
	484	*p++ = c;
	485	c = getc(finput);
	486	}
	487	*p = 0;
	488	return (TYPENAME);
a083fbbf	489
40675e7c DM	490
	491	case '%':
	492	return (parse_percent_token());
	493
	494	default:
	495	return (ILLEGAL);
	496	}
	497	}
	498
a083fbbf	499	/* the following table dictates the action taken for the various
a44c2277 RS	500	% directives. A setflag value causes the named flag to be
	501	set. A retval action returns the code.
	502	*/
	503	struct percent_table_struct {
	504	char *name;
a083fbbf	505	void *setflag;
a44c2277 RS	506	int retval;
	507	} percent_table[] =
	508	{
	509	{"token", NULL, TOKEN},
	510	{"term", NULL, TOKEN},
	511	{"nterm", NULL, NTERM},
	512	{"type", NULL, TYPE},
	513	{"guard", NULL, GUARD},
	514	{"union", NULL, UNION},
	515	{"expect", NULL, EXPECT},
	516	{"thong", NULL, THONG},
	517	{"start", NULL, START},
	518	{"left", NULL, LEFT},
	519	{"right", NULL, RIGHT},
	520	{"nonassoc", NULL, NONASSOC},
	521	{"binary", NULL, NONASSOC},
	522	{"semantic_parser", NULL, SEMANTIC_PARSER},
	523	{"pure_parser", NULL, PURE_PARSER},
	524	{"prec", NULL, PREC},
	525
	526	{"no_lines", &nolinesflag, NOOP}, /* -l */
	527	{"raw", &rawtoknumflag, NOOP}, /* -r */
	528	{"token_table", &toknumflag, NOOP}, /* -k */
	529
	530	#if 0
	531	/* These can be utilized after main is reoganized so
	532	open_files() is deferred 'til after read_declarations().
	533	But %{ and %union both put information into files
	534	that have to be opened before read_declarations().
	535	*/
	536	{"yacc", &fixed_outfiles, NOOP}, /* -y */
	537	{"fixed_output_files", &fixed_outfiles, NOOP}, /* -y */
	538	{"defines", &definesflag, NOOP}, /* -d */
	539	{"no_parser", &noparserflag, NOOP}, /* -n */
	540	{"output_file", &spec_outfile, SETOPT}, /* -o */
	541	{"file_prefix", &spec_file_prefix, SETOPT}, /* -b */
	542	{"name_prefix", &spec_name_prefix, SETOPT}, /* -p */
	543
	544	/* These would be acceptable, but they do not affect processing */
	545	{"verbose", &verboseflag, NOOP}, /* -v */
	546	{"debug", &debugflag, NOOP}, /* -t */
d2729d44 JT	547	/* {"help", <print usage stmt>, NOOP},/ / -h */
d2729d44 JT	548	/* {"version", <print version number> , NOOP},/ / -V */
a44c2277 RS	549	#endif
	550
	551	{NULL, NULL, ILLEGAL}
	552	};
	553
	554	/* Parse a token which starts with %.
	555	Assumes the % has already been read and discarded. */
40675e7c DM	556
40675e7c DM	557	int
d2729d44	558	parse_percent_token (void)
40675e7c DM	559	{
	560	register int c;
	561	register char *p;
a44c2277	562	register struct percent_table_struct *tx;
40675e7c DM	563
	564	p = token_buffer;
	565	c = getc(finput);
a44c2277 RS	566	*p++ = '%';
	567	p++ = c; / for error msg */
	568	*p = 0;
40675e7c DM	569
	570	switch (c)
	571	{
	572	case '%':
	573	return (TWO_PERCENTS);
	574
	575	case '{':
	576	return (PERCENT_LEFT_CURLY);
	577
	578	case '<':
	579	return (LEFT);
	580
	581	case '>':
	582	return (RIGHT);
	583
	584	case '2':
	585	return (NONASSOC);
	586
	587	case '0':
	588	return (TOKEN);
	589
	590	case '=':
	591	return (PREC);
	592	}
a083fbbf	593	if (!isalpha(c))
40675e7c DM	594	return (ILLEGAL);
40675e7c DM	595
a44c2277 RS	596	p = token_buffer;
	597	*p++ = '%';
	598	while (isalpha(c) \|\| c == '_' \|\| c == '-')
40675e7c DM	599	{
	600	if (p == token_buffer + maxtoken)
	601	p = grow_token_buffer(p);
	602
a44c2277	603	if (c == '-') c = '_';
40675e7c DM	604	*p++ = c;
	605	c = getc(finput);
	606	}
	607
	608	ungetc(c, finput);
	609
	610	*p = 0;
	611
a44c2277 RS	612	/* table lookup % directive */
	613	for (tx = percent_table; tx->name; tx++)
	614	if (strcmp(token_buffer+1, tx->name) == 0)
	615	break;
	616	if (tx->retval == SETOPT)
	617	{
	618	((char *)(tx->setflag)) = optarg;
	619	return NOOP;
	620	}
	621	if (tx->setflag)
	622	{
	623	((int )(tx->setflag)) = 1;
	624	return NOOP;
	625	}
	626	return tx->retval;
40675e7c	627	}