From aa94def12d5256d2bd3d5ed49d905f00372a0111 Mon Sep 17 00:00:00 2001 From: Dennis Heimbigner Date: Thu, 13 Jun 2013 10:08:19 +0200 Subject: [PATCH] java: add push-parser support * data/lalr1.java: Capture the declarations as m4 macros to avoid duplication. When push parsing, the declarations occur at the class instance level rather than within the parse() function. Change the way that the parser state is initialized. For push-parsing, the parse state declarations are moved to "push_parse_initialize()", which is called on the first invocation of "push_parse()". The %initial-action code is also inserted after the invocation of "push_parse_initialize()". The body of the parse loop is modified to return values at appropriate points when doing push parsing. In order to make push parsing work, it is necessary to divide YYNEWSTATE into two states: YYNEWSTATE and YYGETTOKEN. On the first call to push_parse(), the state is YYNEWSTATE. On all later entries, the state is set to YYGETTOKEN. The YYNEWSTATE switch arm falls through into YYGETTOKEN. YYGETTOKEN indicates that a new token is potentially needed. Normally, with a pull parser, this new token would be obtained by calling "yylex()". In the push parser, the value YYMORE is returned to the caller. On the next call to push_parse(), the parser will return to the YYGETTOKEN state and continue operation. * tests/javapush.at: New test file for java push parsing. * tests/testsuite.at: Use it. * tests/local.mk: Adjust. * doc/bison.texi (Java Push Parser Interface): New. Signed-off-by: Akim Demaille --- NEWS | 6 +- data/lalr1.java | 262 +++++++++++--- doc/bison.texi | 70 +++- tests/javapush.at | 850 +++++++++++++++++++++++++++++++++++++++++++++ tests/local.mk | 1 + tests/testsuite.at | 7 +- 6 files changed, 1142 insertions(+), 54 deletions(-) create mode 100644 tests/javapush.at diff --git a/NEWS b/NEWS index 977498a5..c6b6cfbe 100644 --- a/NEWS +++ b/NEWS @@ -519,11 +519,13 @@ GNU Bison NEWS ** Java skeleton improvements - Contributed by Paolo Bonzini. - The constants for token names were moved to the Lexer interface. Also, it is possible to add code to the parser's constructors using "%code init" and "%define init_throws". + Contributed by Paolo Bonzini. + + The Java skeleton now supports push parsing. + Contributed by Dennis Heimbigner. ** C++ skeletons improvements diff --git a/data/lalr1.java b/data/lalr1.java index 5c0235f0..d226ce3c 100644 --- a/data/lalr1.java +++ b/data/lalr1.java @@ -20,7 +20,8 @@ m4_include(b4_pkgdatadir/[java.m4]) b4_defines_if([b4_fatal([%s: %%defines does not make sense in Java], [b4_skeleton])]) -# We don't depend on %debug in Java, but pacify warnings about non-used flags. +# We do not depend on %debug in Java, but pacify warnings about +# non-used flags. b4_parse_trace_if([0], [0]) m4_define([b4_symbol_no_destructor_assert], @@ -30,6 +31,57 @@ m4_define([b4_symbol_no_destructor_assert], [b4_symbol_action_location([$1], [destructor])])])]) b4_symbol_foreach([b4_symbol_no_destructor_assert]) +# Setup some macros for api.push-pull. +b4_percent_define_default([[api.push-pull]], [[pull]]) +b4_percent_define_check_values([[[[api.push-pull]], + [[pull]], [[push]], [[both]]]]) + +# Define m4 conditional macros that encode the value +# of the api.push-pull flag. +b4_define_flag_if([pull]) m4_define([b4_pull_flag], [[1]]) +b4_define_flag_if([push]) m4_define([b4_push_flag], [[1]]) +m4_case(b4_percent_define_get([[api.push-pull]]), + [pull], [m4_define([b4_push_flag], [[0]])], + [push], [m4_define([b4_pull_flag], [[0]])]) + +# Define a macro to be true when api.push-pull has the value "both". +m4_define([b4_both_if],[b4_push_if([b4_pull_if([$1],[$2])],[$2])]) + +# Handle BISON_USE_PUSH_FOR_PULL for the test suite. So that push parsing +# tests function as written, do not let BISON_USE_PUSH_FOR_PULL modify the +# behavior of Bison at all when push parsing is already requested. +b4_define_flag_if([use_push_for_pull]) +b4_use_push_for_pull_if([ + b4_push_if([m4_define([b4_use_push_for_pull_flag], [[0]])], + [m4_define([b4_push_flag], [[1]])])]) + +# Define a macro to encapsulate the parse state variables. +# This allows them to be defined either in parse() when doing +# pull parsing, or as class instance variable when doing push parsing. +m4_define([b4_define_state],[[ + /* Lookahead and lookahead in internal form. */ + int yychar = yyempty_; + int yytoken = 0; + + /* State. */ + int yyn = 0; + int yylen = 0; + int yystate = 0; + YYStack yystack = new YYStack (); + int label = YYNEWSTATE; + + /* Error handling. */ + int yynerrs_ = 0; + ]b4_locations_if([/* The location where the error started. */ + b4_location_type yyerrloc = null; + + /* Location. */ + b4_location_type yylloc = new b4_location_type (null, null);])[ + + /* Semantic value of the lookahead. */ + ]b4_yystype[ yylval = null; +]]) + b4_output_begin([b4_parser_file_name]) b4_copyright([Skeleton implementation for Bison LALR(1) parsers in Java], [2007-2013]) @@ -363,6 +415,12 @@ b4_lexer_if([[ */ public static final int YYABORT = 1; +]b4_push_if([ + /** + * Returned by a Bison action in order to request a new token. + */ + public static final int YYMORE = 4;])[ + /** * Returned by a Bison action in order to start error recovery without * printing an error message. @@ -379,9 +437,12 @@ b4_lexer_if([[ private static final int YYREDUCE = 6; private static final int YYERRLAB1 = 7; private static final int YYRETURN = 8; +]b4_push_if([[ private static final int YYGETTOKEN = 9; /* Signify that a new token is expected when doing push-parsing. */]])[ private int yyerrstatus_ = 0; +]b4_push_if([dnl +b4_define_state])[ /** * Return whether error recovery is being done. In this state, the parser * reads token until it reaches a known state, and then restarts normal @@ -486,6 +547,7 @@ b4_lexer_if([[ + (yyvaluep == null ? "(null)" : yyvaluep.toString ()) + ")"); } +]b4_push_if([],[[ /** * Parse input from the scanner that was specified at object construction * time. Return whether the end of the input was reached successfully. @@ -493,46 +555,53 @@ b4_lexer_if([[ * @@return true if the parsing succeeds. Note that this does not * imply that there were no syntax errors. */ - public boolean parse () ]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[ + public boolean parse () ]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[]])[ +]b4_push_if([ + /** + * Push Parse input from external lexer + * + * @@param yylextoken current token + * @@param yylexval current lval +]b4_locations_if([ * @@param yylexloc current position])[ + * + * @@return YYACCEPT, YYABORT, YYMORE + */ + public int push_parse (int yylextoken, b4_yystype yylexval[]b4_locations_if([, b4_location_type yylexloc])) + b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])])[ { - /// Lookahead and lookahead in internal form. - int yychar = yyempty_; - int yytoken = 0; - - /* State. */ - int yyn = 0; - int yylen = 0; - int yystate = 0; - - YYStack yystack = new YYStack (); - - /* Error handling. */ - int yynerrs_ = 0; - ]b4_locations_if([/// The location where the error started. - ]b4_location_type[ yyerrloc = null; - - /// ]b4_location_type[ of the lookahead. - ]b4_location_type[ yylloc = new ]b4_location_type[ (null, null); - - /// @@$. - ]b4_location_type[ yyloc;]) - - /// Semantic value of the lookahead. - b4_yystype[ yylval = null; - + ]b4_locations_if([/* @@$. */ + b4_location_type yyloc;])[ +]b4_push_if([],[[ +]b4_define_state[ yycdebug ("Starting parse\n"); yyerrstatus_ = 0; + /* Initialize the stack. */ + yystack.push (yystate, yylval ]b4_locations_if([, yylloc])[); ]m4_ifdef([b4_initial_action], [ b4_dollar_pushdef([yylval], [], [yylloc])dnl /* User initialization code. */ b4_user_initial_action -b4_dollar_popdef])[]dnl - - [ /* Initialize the stack. */ - yystack.push (yystate, yylval]b4_locations_if([, yylloc])[); - - int label = YYNEWSTATE; +b4_dollar_popdef[]dnl +])[ +]])[ +]b4_push_if([[ + if (!this.push_parse_initialized) + { + push_parse_initialize (); +]m4_ifdef([b4_initial_action], [ +b4_dollar_pushdef([yylval], [], [yylloc])dnl + /* User initialization code. */ + b4_user_initial_action +b4_dollar_popdef[]dnl +])[ + yycdebug ("Starting parse\n"); + yyerrstatus_ = 0; + } else + label = YYGETTOKEN; + + boolean push_token_consumed = true; +]])[ for (;;) switch (label) { @@ -545,7 +614,8 @@ b4_dollar_popdef])[]dnl /* Accept? */ if (yystate == yyfinal_) - return true; + ]b4_push_if([{label = YYACCEPT; break;}], + [return true;])[ /* Take a decision. First try without lookahead. */ yyn = yypact_[yystate]; @@ -554,16 +624,27 @@ b4_dollar_popdef])[]dnl label = YYDEFAULT; break; } +]b4_push_if([ /* Fall Through */ + case YYGETTOKEN:])[ /* Read a lookahead token. */ if (yychar == yyempty_) { +]b4_push_if([[ + if (!push_token_consumed) + return YYMORE; + yycdebug ("Reading a token: "); + yychar = yylextoken; + yylval = yylexval;]b4_locations_if([ + yylloc = yylexloc;])[ + push_token_consumed = false;]])[ +]b4_push_if([],[[ yycdebug ("Reading a token: "); - yychar = yylexer.yylex ();] - b4_locations_if([[ - yylloc = new ]b4_location_type[(yylexer.getStartPos (), - yylexer.getEndPos ());]]) - yylval = yylexer.getLVal ();[ + yychar = yylexer.yylex (); + yylval = yylexer.getLVal ();]b4_locations_if([ + yylloc = new b4_location_type (yylexer.getStartPos (), + yylexer.getEndPos ());])[ +]])[ } /* Convert token to internal form. */ @@ -660,10 +741,10 @@ b4_dollar_popdef])[]dnl { /* Return failure if at end of input. */ if (yychar == Lexer.EOF) - return false; + ]b4_push_if([{label = YYABORT; break;}],[return false;])[ } else - yychar = yyempty_; + yychar = yyempty_; } /* Else will try to reuse lookahead token after shifting the error @@ -671,9 +752,9 @@ b4_dollar_popdef])[]dnl label = YYERRLAB1; break; - /*---------------------------------------------------. + /*-------------------------------------------------. | errorlab -- error raised explicitly by YYERROR. | - `---------------------------------------------------*/ + `-------------------------------------------------*/ case YYERROR: ]b4_locations_if([yyerrloc = yystack.locationAt (yylen - 1);])[ @@ -705,9 +786,10 @@ b4_dollar_popdef])[]dnl } } - /* Pop the current state because it cannot handle the error token. */ + /* Pop the current state because it cannot handle the + * error token. */ if (yystack.height == 0) - return false; + ]b4_push_if([{label = YYABORT; break;}],[return false;])[ ]b4_locations_if([yyerrloc = yystack.locationAt (0);])[ yystack.pop (); @@ -716,7 +798,11 @@ b4_dollar_popdef])[]dnl yystack.print (yyDebugStream); } - ]b4_locations_if([ + if (label == YYABORT) + /* Leave the switch. */ + break; + +]b4_locations_if([ /* Muck with the stack to setup for yylloc. */ yystack.push (0, null, yylloc); yystack.push (0, null, yyerrloc); @@ -734,13 +820,92 @@ b4_dollar_popdef])[]dnl /* Accept. */ case YYACCEPT: - return true; + ]b4_push_if([this.push_parse_initialized = false; return YYACCEPT;], + [return true;])[ /* Abort. */ case YYABORT: - return false; + ]b4_push_if([this.push_parse_initialized = false; return YYABORT;], + [return false;])[ } +} +]b4_push_if([[ + boolean push_parse_initialized = false; + + /** + * (Re-)Initialize the state of the push parser. + */ + public void push_parse_initialize() + { + /* Lookahead and lookahead in internal form. */ + this.yychar = yyempty_; + this.yytoken = 0; + + /* State. */ + this.yyn = 0; + this.yylen = 0; + this.yystate = 0; + this.yystack = new YYStack (); + this.label = YYNEWSTATE; + + /* Error handling. */ + this.yynerrs_ = 0; + ]b4_locations_if([/* The location where the error started. */ + this.yyerrloc = null; + this.yylloc = new b4_location_type (null, null);])[ + + /* Semantic value of the lookahead. */ + this.yylval = null; + + yystack.push (this.yystate, this.yylval]b4_locations_if([, this.yylloc])[); + + this.push_parse_initialized = true; + } +]b4_locations_if([ + /** + * Push parse given input from an external lexer. + * + * @@param yylextoken current token + * @@param yylexval current lval + * @@param yyylexpos current position + * + * @@return YYACCEPT, YYABORT, YYMORE + */ + public int push_parse (int yylextoken, b4_yystype yylexval, b4_position_type yylexpos) + b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])]) + { + return push_parse (yylextoken, yylexval, new b4_location_type (yylexpos)); + } +])[]]) + +b4_both_if([[ + /** + * Parse input from the scanner that was specified at object construction + * time. Return whether the end of the input was reached successfully. + * This version of parse () is defined only when api.push-push=both. + * + * @@return true if the parsing succeeds. Note that this does not + * imply that there were no syntax errors. + */ + public boolean parse () ]b4_maybe_throws([b4_list2([b4_lex_throws], [b4_throws])])[ + { + if (yylexer == null) + throw new NullPointerException("Null Lexer"); + int status; + do { + int token = yylexer.yylex(); + ]b4_yystype[ lval = yylexer.getLVal(); +]b4_locations_if([dnl + b4_location_type yyloc = new b4_location_type (yylexer.getStartPos (), + yylexer.getEndPos ());])[ + this.yyerrstatus_ = 0; + ]b4_locations_if([status = push_parse(token,lval,yyloc);],[ + status = push_parse(token,lval);])[ + } while (status == YYMORE); + return (status == YYACCEPT); + } +]])[ // Generate an error message. private String yysyntax_error (int yystate, int tok) @@ -849,6 +1014,7 @@ b4_dollar_popdef])[]dnl ]b4_integral_parser_table_define([rline], [b4_rline], [[YYRLINE[YYN] -- Source line where rule number YYN was defined.]])[ + // Report on the debug stream that the rule yyrule is going to be reduced. private void yy_reduce_print (int yyrule, YYStack yystack) { diff --git a/doc/bison.texi b/doc/bison.texi index 4a4bd600..bd023edf 100644 --- a/doc/bison.texi +++ b/doc/bison.texi @@ -366,6 +366,7 @@ Java Parsers * Java Parser Interface:: Instantiating and running the parser * Java Scanner Interface:: Specifying the scanner for the parser * Java Action Features:: Special features for use in actions +* Java Push Parser Interface:: Instantiating and running the a push parser * Java Differences:: Differences between C/C++ and Java Grammars * Java Declarations Summary:: List of Bison declarations used with Java @@ -11500,6 +11501,7 @@ main (int argc, char *argv[]) * Java Parser Interface:: Instantiating and running the parser * Java Scanner Interface:: Specifying the scanner for the parser * Java Action Features:: Special features for use in actions +* Java Push Parser Interface:: Instantiating and running the a push parser * Java Differences:: Differences between C/C++ and Java Grammars * Java Declarations Summary:: List of Bison declarations used with Java @end menu @@ -11811,7 +11813,6 @@ The return type can be changed using @samp{%define api.value.type @{@var{class-name}@}}. @end deftypemethod - @node Java Action Features @subsection Special Features for Use in Java Actions @@ -11890,6 +11891,73 @@ instance in use. The @code{Location} and @code{Position} parameters are available only if location tracking is active. @end deftypefn +@node Java Push Parser Interface +@subsection Java Push Parser Interface +@c - define push_parse +@findex %define api.push-pull + +(The current push parsing interface is experimental and may evolve. More +user feedback will help to stabilize it.) + +Normally, Bison generates a pull parser for Java. +The following Bison declaration says that you want the parser to be a push +parser (@pxref{%define Summary,,api.push-pull}): + +@example +%define api.push-pull push +@end example + +Most of the discussion about the Java pull Parser Interface, (@pxref{Java +Parser Interface}) applies to the push parser interface as well. + +When generating a push parser, the method @code{push_parse} is created with +the following signature (depending on if locations are enabled). + +@deftypemethod {YYParser} {void} push_parse ({int} @var{token}, {Object} @var{yylval}) +@deftypemethodx {YYParser} {void} push_parse ({int} @var{token}, {Object} @var{yylval}, {Location} @var{yyloc}) +@deftypemethodx {YYParser} {void} push_parse ({int} @var{token}, {Object} @var{yylval}, {Position} @var{yypos}) +@end deftypemethod + +The primary difference with respect to a pull parser is that the parser +method @code{push_parse} is invoked repeatedly to parse each token. This +function is available if either the "%define api.push-pull push" or "%define +api.push-pull both" declaration is used (@pxref{%define +Summary,,api.push-pull}). The @code{Location} and @code{Position} +parameters are available only if location tracking is active. + +The value returned by the @code{push_parse} method is one of the following +four constants: @code{YYABORT}, @code{YYACCEPT}, @code{YYERROR}, or +@code{YYMORE}. This new value, @code{YYMORE}, may be returned if more input +is required to finish parsing the grammar. + +If api.push-pull is declared as @code{both}, then the generated parser class +will also implement the @code{parse} method. This method's body is a loop +that repeatedly invokes the scanner and then passes the values obtained from +the scanner to the @code{push_parse} method. + +There is one additional complication. Technically, the push parser does not +need to know about the scanner (i.e. an object implementing the +@code{YYParser.Lexer} interface), but it does need access to the +@code{yyerror} method. Currently, the @code{yyerror} method is defined in +the @code{YYParser.Lexer} interface. Hence, an implementation of that +interface is still required in order to provide an implementation of +@code{yyerror}. The current approach (and subject to change) is to require +the @code{YYParser} constructor to be given an object implementing the +@code{YYParser.Lexer} interface. This object need only implement the +@code{yyerror} method; the other methods can be stubbed since they will +never be invoked. The simplest way to do this is to add a trivial scanner +implementation to your grammar file using whatever implementation of +@code{yyerror} is desired. The following code sample shows a simple way to +accomplish this. + +@example +%code lexer +@{ + public Object getLVal () @{return null;@} + public int yylex () @{return 0;@} + public void yyerror (String s) @{System.err.println(s);@} +@} +@end example @node Java Differences @subsection Differences between C/C++ and Java Grammars diff --git a/tests/javapush.at b/tests/javapush.at new file mode 100644 index 00000000..65027c31 --- /dev/null +++ b/tests/javapush.at @@ -0,0 +1,850 @@ +# Checking Java Push Parsing. -*- Autotest -*- + +# Copyright (C) 2013 Free Software Foundation, Inc. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# The Java push parser tests are intended primarily +# to verify that the sequence of states that the parser +# traverses is the same as a pull parser would traverse. + +################################################## +# Provide a way to generate data with and without push parsing +# so it is possible to capture the output for comparison +# (except the "trivial" tests). +# Use "both" rather than "push" so we can also set it to "pull" to +# get the "experr" data. + +m4_define([PUSHPULLFLAG],[-Dapi.push-pull=both]) + +# Modify the grep tester from java.at +# to ignore status so we can test for zero matching lines +m4_define([AT_CHECK_JAVA_GREP], + [AT_CHECK([grep -c '^$2$' $1], [ignore], [m4_default([$3], [1]) +]) +]) + +################################################## + +AT_BANNER([[Java Push Parsing Tests]]) + +# Define a single copy of the trivial parser grammar. +# This is missing main(), so two versions +# are instantiated with different main() procedures. +m4_define([AT_TRIVIAL_GRAMMAR],[ +%define parser_class_name {YYParser} +%error-verbose + +%code imports { +import java.io.*; +import java.util.*; +} + +%% + +start: 'a' 'b' 'c' ; + +%% +]) + +# Define comon code across to be includede in +# class Main for the trivial parser tests. +m4_define([AT_TRIVIAL_COMMON],[ + static class YYerror implements YYParser.Lexer + { + public Object getLVal() {return null;} + public int yylex () throws java.io.IOException { return 0; } + public void yyerror (String msg) { System.err.println(msg); } + } + + static YYParser parser = null; + static YYerror yyerror = null; + static int teststate = -1; + + static void setup() + throws IOException + { + yyerror = new YYerror(); + parser = new YYParser(yyerror); + parser.setDebugLevel(1); + teststate = -1; + } + + static String[[]] teststatename + = new String[[]]{"YYACCEPT","YYABORT","YYERROR","UNKNOWN","YYMORE"}; + + static void check(int teststate, int expected, String msg) + { + System.err.println("teststate="+teststatename[[teststate]] + +"; expected="+teststatename[[expected]]); + if (teststate == expected) + return; + System.err.println("unexpected state: "+msg); + System.exit(1); + } +]) + +m4_define([AT_TRIVIAL_PARSER],[ + AT_TRIVIAL_GRAMMAR + + public class Main + { + + AT_TRIVIAL_COMMON + + static public void main (String[[]] argv) + throws IOException + { + setup(); + + teststate = parser.push_parse('a', null); + check(teststate,YYParser.YYMORE,"push_parse('a', null)"); + + setup(); + + teststate = parser.push_parse('a', null); + check(teststate,YYParser.YYMORE,"push_parse('a', null)"); + teststate = parser.push_parse('b', null); + check(teststate,YYParser.YYMORE,"push_parse('b', null)"); + teststate = parser.push_parse('c', null); + check(teststate,YYParser.YYMORE,"push_parse('c', null)"); + teststate = parser.push_parse('\0', null); + check(teststate,YYParser.YYACCEPT,"push_parse('\\0', null)"); + + /* Reuse the parser instance and cause a failure */ + teststate = parser.push_parse('b', null); + check(teststate,YYParser.YYABORT,"push_parse('b', null)"); + + System.exit(0); + } + +} +]) + +m4_define([AT_TRIVIAL_PARSER_INITIAL_ACTION],[ + AT_TRIVIAL_GRAMMAR + + public class Main + { + + AT_TRIVIAL_COMMON + + static public void main (String[[]] argv) + throws IOException + { + setup(); + + teststate = parser.push_parse('a', null); + check(teststate,YYParser.YYMORE,"push_parse('a', null)"); + teststate = parser.push_parse('b', null); + check(teststate,YYParser.YYMORE,"push_parse('b', null)"); + teststate = parser.push_parse('c', null); + check(teststate,YYParser.YYMORE,"push_parse('c', null)"); + teststate = parser.push_parse('\0', null); + check(teststate,YYParser.YYACCEPT,"push_parse('\\0', null)"); + + System.exit(0); + } + +} +]) + +AT_SETUP([Trivial Push Parser with api.push-pull verification]) +AT_BISON_OPTION_PUSHDEFS +AT_DATA([[input.y]],[[%language "Java" +]AT_TRIVIAL_PARSER[ +]]) +AT_BISON_OPTION_POPDEFS + +# Verify that the proper procedure(s) are generated for each case. +AT_BISON_CHECK([[-Dapi.push-pull=pull -o Main.java input.y]]) +AT_CHECK_JAVA_GREP([[Main.java]], + [[.*public boolean parse ().*]], + [1]) +AT_CHECK_JAVA_GREP( + [[Main.java]], + [[.*public int push_parse (int yylextoken, Object yylexval).*]], + [0]) +AT_BISON_CHECK([[-Dapi.push-pull=both -o Main.java input.y]]) +AT_CHECK_JAVA_GREP([[Main.java]], + [[.*public boolean parse ().*]], + [1]) +AT_CHECK_JAVA_GREP( + [[Main.java]], + [[.*public int push_parse (int yylextoken, Object yylexval).*]], + [1]) +AT_BISON_CHECK([[-Dapi.push-pull=push -o Main.java input.y]]) +AT_CHECK_JAVA_GREP([[Main.java]], + [[.*public boolean parse ().*]], + [0]) +AT_CHECK_JAVA_GREP( + [[Main.java]], + [[.*public int push_parse (int yylextoken, Object yylexval).*]], + [1]) + +AT_JAVA_COMPILE([[Main.java]]) +AT_JAVA_PARSER_CHECK([Main], 0, [], [stderr-nolog]) +AT_CLEANUP + +AT_SETUP([Trivial Push Parser with %initial-action]) +AT_BISON_OPTION_PUSHDEFS +AT_DATA([[input.y]],[[%language "Java" +%initial-action { +System.err.println("Initial action invoked"); +} +]AT_TRIVIAL_PARSER_INITIAL_ACTION[ +]]) +AT_BISON_OPTION_POPDEFS +AT_BISON_CHECK([[-Dapi.push-pull=push -o Main.java input.y]]) +AT_CHECK_JAVA_GREP([[Main.java]], + [[System.err.println("Initial action invoked");]]) +AT_JAVA_COMPILE([[Main.java]]) +AT_JAVA_PARSER_CHECK([Main], 0, [], [stderr-nolog]) +# Verify that initial action is called exactly once. +AT_CHECK_JAVA_GREP( + [[stderr]], + [[Initial action invoked]], + [1]) +AT_CLEANUP + +# Define a single copy of the Calculator grammar. +m4_define([AT_CALC_BODY],[ +%code imports { + import java.io.*; +} + +%code { + static StringReader + getinput(String filename) throws IOException + { + StringBuilder buf = new StringBuilder(); + FileReader file = new FileReader(filename); + int c; + while ((c=file.read()) > 0) + buf.append((char)c); + file.close(); + return new StringReader(buf.toString()); + } +} + +/* Bison Declarations */ +%token NUM "number" +%type exp + +%nonassoc '=' /* comparison */ +%left '-' '+' +%left '*' '/' +%left NEG /* negation--unary minus */ +%right '^' /* exponentiation */ + +/* Grammar follows */ +%% +input: + line +| input line +; + +line: + '\n' +| exp '\n' + {System.out.println("total = "+$[]1);} +| error '\n' +; + +exp: + NUM { $[]$ = $[]1;} +| exp '=' exp + { + if ($[]1.intValue() != $[]3.intValue()) + yyerror (]AT_LOCATION_IF([[@$,]])[ "calc: error: " + $[]1 + " != " + $[]3); + } +| exp '+' exp + { $[]$ = new Integer ($[]1.intValue () + $[]3.intValue ()); } +| exp '-' exp + { $[]$ = new Integer ($[]1.intValue () - $[]3.intValue ()); } +| exp '*' exp + { $[]$ = new Integer ($[]1.intValue () * $[]3.intValue ()); } +| exp '/' exp + { $[]$ = new Integer ($[]1.intValue () / $[]3.intValue ()); } +| '-' exp %prec NEG + { $[]$ = new Integer (-$[]2.intValue ()); } +| exp '^' exp + { $[]$ = new Integer ((int)Math.pow ($[]1.intValue (), + $[]3.intValue ())); } +| '(' exp ')' { $[]$ = $[]2;} +| '(' error ')' { $[]$ = new Integer (1111);} +| '!' { $[]$ = new Integer (0); return YYERROR;} +| '-' error { $[]$ = new Integer (0); return YYERROR;} +; +]) + +# Test that the states transitioned by the push parser are the +# same as for the pull parser. This test is assumed to work +# if it produces the same partial trace of stack states as is +# produced when using pull parsing. The output is verbose, +# but seems essential for verifying push parsing. + +AT_SETUP([Calc parser with api.push-pull both]) +AT_BISON_OPTION_PUSHDEFS + +# Define the calculator input. +# Warning: if you changes the input file +# then the locations test file position numbers +# may be incorrect and you will have +# to modify that file as well. + +AT_DATA([input],[[1 + 2 * 3 = 7 +1 + 2 * -3 = -5 + +-1^2 = -1 +(-1)^2 = 1 + +---1 = -1 + +1 - 2 - 3 = -4 +1 - (2 - 3) = 2 + +2^2^3 = 256 +(2^2)^3 = 64 +]]) + +# Compose pieces to build the actual .y file. +AT_DATA([Calc.y],[[/* Infix notation calculator--calc */ +%language "Java" +%name-prefix "Calc" +%define parser_class_name {Calc} + +%code { +static class UserLexer implements Calc.Lexer +{ + StreamTokenizer st; + StringReader rdr; + + public UserLexer(StringReader reader) + { + rdr = reader; + st = new StreamTokenizer(rdr); + st.resetSyntax(); + st.eolIsSignificant(true); + st.whitespaceChars(9, 9); + st.whitespaceChars(32, 32); + st.wordChars(48, 57); + } + + Integer yylval; + + public Object getLVal() { return yylval; } + + public void yyerror(String msg) { System.err.println(msg); } + + public int yylex () throws IOException + { + switch (st.nextToken()) { + case StreamTokenizer.TT_EOF: return EOF; + case StreamTokenizer.TT_EOL: return (int) '\n'; + case StreamTokenizer.TT_WORD: + yylval = new Integer (st.sval); + return NUM; + default: return st.ttype; + } + } +} + +} + +%code { +public static void main (String[] argv) + throws IOException +{ + StringReader reader = getinput(argv[0]); + UserLexer lexer = new UserLexer(reader); + Calc calc = new Calc(lexer); + calc.setDebugLevel(1); + calc.parse(); +}//main + +} + +]AT_CALC_BODY[ + +]]) + +# This data was captured from running a pull parser. +AT_DATA([[expout]],[[Stack now 0 +Stack now 0 2 +Stack now 0 9 +Stack now 0 9 19 +Stack now 0 9 19 2 +Stack now 0 9 19 28 +Stack now 0 9 19 28 20 +Stack now 0 9 19 28 20 2 +Stack now 0 9 19 28 20 29 +Stack now 0 9 19 28 +Stack now 0 9 +Stack now 0 9 17 +Stack now 0 9 17 2 +Stack now 0 9 17 26 +Stack now 0 9 +Stack now 0 9 23 +Stack now 0 8 +Stack now 0 7 +Stack now 0 7 2 +Stack now 0 7 9 +Stack now 0 7 9 19 +Stack now 0 7 9 19 2 +Stack now 0 7 9 19 28 +Stack now 0 7 9 19 28 20 +Stack now 0 7 9 19 28 20 3 +Stack now 0 7 9 19 28 20 3 2 +Stack now 0 7 9 19 28 20 3 12 +Stack now 0 7 9 19 28 20 29 +Stack now 0 7 9 19 28 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 3 +Stack now 0 7 9 17 3 2 +Stack now 0 7 9 17 3 12 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 4 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 3 +Stack now 0 7 3 2 +Stack now 0 7 3 12 +Stack now 0 7 3 12 22 +Stack now 0 7 3 12 22 2 +Stack now 0 7 3 12 22 31 +Stack now 0 7 3 12 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 3 +Stack now 0 7 9 17 3 2 +Stack now 0 7 9 17 3 12 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 5 +Stack now 0 7 5 3 +Stack now 0 7 5 3 2 +Stack now 0 7 5 3 12 +Stack now 0 7 5 14 +Stack now 0 7 5 14 25 +Stack now 0 7 9 +Stack now 0 7 9 22 +Stack now 0 7 9 22 2 +Stack now 0 7 9 22 31 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 2 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 4 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 3 +Stack now 0 7 3 3 +Stack now 0 7 3 3 3 +Stack now 0 7 3 3 3 2 +Stack now 0 7 3 3 3 12 +Stack now 0 7 3 3 12 +Stack now 0 7 3 12 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 3 +Stack now 0 7 9 17 3 2 +Stack now 0 7 9 17 3 12 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 4 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 2 +Stack now 0 7 9 +Stack now 0 7 9 18 +Stack now 0 7 9 18 2 +Stack now 0 7 9 18 27 +Stack now 0 7 9 +Stack now 0 7 9 18 +Stack now 0 7 9 18 2 +Stack now 0 7 9 18 27 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 3 +Stack now 0 7 9 17 3 2 +Stack now 0 7 9 17 3 12 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 2 +Stack now 0 7 9 +Stack now 0 7 9 18 +Stack now 0 7 9 18 5 +Stack now 0 7 9 18 5 2 +Stack now 0 7 9 18 5 14 +Stack now 0 7 9 18 5 14 18 +Stack now 0 7 9 18 5 14 18 2 +Stack now 0 7 9 18 5 14 18 27 +Stack now 0 7 9 18 5 14 +Stack now 0 7 9 18 5 14 25 +Stack now 0 7 9 18 27 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 2 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 4 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 2 +Stack now 0 7 9 +Stack now 0 7 9 22 +Stack now 0 7 9 22 2 +Stack now 0 7 9 22 31 +Stack now 0 7 9 22 31 22 +Stack now 0 7 9 22 31 22 2 +Stack now 0 7 9 22 31 22 31 +Stack now 0 7 9 22 31 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 2 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 5 +Stack now 0 7 5 2 +Stack now 0 7 5 14 +Stack now 0 7 5 14 22 +Stack now 0 7 5 14 22 2 +Stack now 0 7 5 14 22 31 +Stack now 0 7 5 14 +Stack now 0 7 5 14 25 +Stack now 0 7 9 +Stack now 0 7 9 22 +Stack now 0 7 9 22 2 +Stack now 0 7 9 22 31 +Stack now 0 7 9 +Stack now 0 7 9 17 +Stack now 0 7 9 17 2 +Stack now 0 7 9 17 26 +Stack now 0 7 9 +Stack now 0 7 9 23 +Stack now 0 7 16 +Stack now 0 7 +Stack now 0 7 15 +]]) + +AT_BISON_CHECK([PUSHPULLFLAG [-o Calc.java Calc.y]]) +AT_JAVA_COMPILE([[Calc.java]]) +#Verify that this is a push parser. +AT_CHECK_JAVA_GREP([[Calc.java]], + [[.*public void push_parse_initialize().*]]) +# Capture stderr output for comparison purposes. +AT_JAVA_PARSER_CHECK([Calc input], 0, [ignore-nolog], [stderr-nolog]) +# Extract the "Stack Now" lines from the error output, +# send them to stdout (via the sed command) and compare to expout. +# NOTE: because the target is "expout", this macro automatically +# compares the output of the sed command with the contents of +# the file "expout" (defined above). +AT_CHECK([[sed -e '/^Stack now.*$/p' -e d ./stderr|tee jj]],[ignore],[expout],[ignore-nolog]) +AT_BISON_OPTION_POPDEFS +AT_CLEANUP + +# This test looks for location reporting by looking +# at the lexer output with locations enabled. +# It defines a lexer that reports location info. +AT_SETUP([Calc parser with %locations %code lexer and api.push-pull both]) +AT_BISON_OPTION_PUSHDEFS + +AT_DATA([Calc.y],[[/* Infix notation calculator--calc. */ +%language "Java" +%name-prefix "Calc" +%define parser_class_name {Calc} +%lex-param { Reader rdr } +%locations + +%code imports { + import java.io.*; +} + +%code lexer { + StreamTokenizer st; + Integer yylval; + + public YYLexer(Reader rdr) + { + st = new StreamTokenizer(rdr); + st.resetSyntax(); + st.eolIsSignificant(true); + st.whitespaceChars(9, 9); + st.whitespaceChars(32, 32); + st.wordChars(48, 57); + } + + Position yypos = new Position (1, 0); + + public Position getStartPos() { return yypos; } + + public Position getEndPos() { return yypos; } + + public Object getLVal() { return yylval; } + + public void yyerror(Location loc, String msg) + { + System.err.println(loc+":"+msg); + } + + public int yylex () throws IOException + { + yypos = new Position (yypos.lineno (),yypos.token () + 1); + switch (st.nextToken()) { + case StreamTokenizer.TT_EOF: + return EOF; + case StreamTokenizer.TT_EOL: + yypos = new Position (yypos.lineno () + 1, 0); + return (int) '\n'; + case StreamTokenizer.TT_WORD: + yylval = new Integer (st.sval); + return NUM; + default: + return st.ttype; + } + } +} + +%code { +class Position { + public int line; + public int token; + + public Position () { line = 0; token = 0; } + + public Position (int l, int t) { line = l; token = t; } + + public boolean equals (Position l) + { + return l.line == line && l.token == token; + } + + public String toString () + { + return Integer.toString(line) + "." + Integer.toString(token); + } + + public int lineno () { return line; } + + public int token () { return token; } +}//Class Position +} + +%code { +public static void main (String[] argv) + throws IOException +{ + StringReader reader = getinput(argv[0]); + Calc calc = new Calc(reader); + calc.setDebugLevel(1); + calc.parse(); +} +} + +]AT_CALC_BODY[ + +]]) + +# Define the expected calculator output. +# This should match the output from a pull parser. +AT_DATA([output],[[total = 7 +total = -5 +total = -1 +total = 1 +total = -1 +total = -4 +total = 2 +total = 256 +total = 64 +]]) + +AT_DATA([locations],[[Next token is token "number" (1.1: 1) +Next token is token '+' (1.2: 1) +Next token is token "number" (1.3: 2) +Next token is token '*' (1.4: 2) +Next token is token "number" (1.5: 3) +Next token is token '=' (1.6: 3) +Next token is token '=' (1.6: 3) +Next token is token '=' (1.6: 3) +Next token is token "number" (1.7: 7) +Next token is token '\n' (2.0: 7) +Next token is token '\n' (2.0: 7) +Next token is token "number" (2.1: 1) +Next token is token '+' (2.2: 1) +Next token is token "number" (2.3: 2) +Next token is token '*' (2.4: 2) +Next token is token '-' (2.5: 2) +Next token is token "number" (2.6: 3) +Next token is token '=' (2.7: 3) +Next token is token '=' (2.7: 3) +Next token is token '=' (2.7: 3) +Next token is token '=' (2.7: 3) +Next token is token '-' (2.8: 3) +Next token is token "number" (2.9: 5) +Next token is token '\n' (3.0: 5) +Next token is token '\n' (3.0: 5) +Next token is token '\n' (3.0: 5) +Next token is token '\n' (4.0: 5) +Next token is token '-' (4.1: 5) +Next token is token "number" (4.2: 1) +Next token is token '^' (4.3: 1) +Next token is token "number" (4.4: 2) +Next token is token '=' (4.5: 2) +Next token is token '=' (4.5: 2) +Next token is token '=' (4.5: 2) +Next token is token '-' (4.6: 2) +Next token is token "number" (4.7: 1) +Next token is token '\n' (5.0: 1) +Next token is token '\n' (5.0: 1) +Next token is token '\n' (5.0: 1) +Next token is token '(' (5.1: 1) +Next token is token '-' (5.2: 1) +Next token is token "number" (5.3: 1) +Next token is token ')' (5.4: 1) +Next token is token ')' (5.4: 1) +Next token is token '^' (5.5: 1) +Next token is token "number" (5.6: 2) +Next token is token '=' (5.7: 2) +Next token is token '=' (5.7: 2) +Next token is token "number" (5.8: 1) +Next token is token '\n' (6.0: 1) +Next token is token '\n' (6.0: 1) +Next token is token '\n' (7.0: 1) +Next token is token '-' (7.1: 1) +Next token is token '-' (7.2: 1) +Next token is token '-' (7.3: 1) +Next token is token "number" (7.4: 1) +Next token is token '=' (7.5: 1) +Next token is token '=' (7.5: 1) +Next token is token '=' (7.5: 1) +Next token is token '=' (7.5: 1) +Next token is token '-' (7.6: 1) +Next token is token "number" (7.7: 1) +Next token is token '\n' (8.0: 1) +Next token is token '\n' (8.0: 1) +Next token is token '\n' (8.0: 1) +Next token is token '\n' (9.0: 1) +Next token is token "number" (9.1: 1) +Next token is token '-' (9.2: 1) +Next token is token "number" (9.3: 2) +Next token is token '-' (9.4: 2) +Next token is token '-' (9.4: 2) +Next token is token "number" (9.5: 3) +Next token is token '=' (9.6: 3) +Next token is token '=' (9.6: 3) +Next token is token '-' (9.7: 3) +Next token is token "number" (9.8: 4) +Next token is token '\n' (10.0: 4) +Next token is token '\n' (10.0: 4) +Next token is token '\n' (10.0: 4) +Next token is token "number" (10.1: 1) +Next token is token '-' (10.2: 1) +Next token is token '(' (10.3: 1) +Next token is token "number" (10.4: 2) +Next token is token '-' (10.5: 2) +Next token is token "number" (10.6: 3) +Next token is token ')' (10.7: 3) +Next token is token ')' (10.7: 3) +Next token is token '=' (10.8: 3) +Next token is token '=' (10.8: 3) +Next token is token "number" (10.9: 2) +Next token is token '\n' (11.0: 2) +Next token is token '\n' (11.0: 2) +Next token is token '\n' (12.0: 2) +Next token is token "number" (12.1: 2) +Next token is token '^' (12.2: 2) +Next token is token "number" (12.3: 2) +Next token is token '^' (12.4: 2) +Next token is token "number" (12.5: 3) +Next token is token '=' (12.6: 3) +Next token is token '=' (12.6: 3) +Next token is token '=' (12.6: 3) +Next token is token "number" (12.7: 256) +Next token is token '\n' (13.0: 256) +Next token is token '\n' (13.0: 256) +Next token is token '(' (13.1: 256) +Next token is token "number" (13.2: 2) +Next token is token '^' (13.3: 2) +Next token is token "number" (13.4: 2) +Next token is token ')' (13.5: 2) +Next token is token ')' (13.5: 2) +Next token is token '^' (13.6: 2) +Next token is token "number" (13.7: 3) +Next token is token '=' (13.8: 3) +Next token is token '=' (13.8: 3) +Next token is token "number" (13.9: 64) +Next token is token '\n' (14.0: 64) +Next token is token '\n' (14.0: 64) +]]) + +# Define the calculator input. +# Warning: if you changes the input file +# then the locations test file position numbers +# may be incorrect and you will have +# to modify that file as well. + +AT_DATA([input],[[1 + 2 * 3 = 7 +1 + 2 * -3 = -5 + +-1^2 = -1 +(-1)^2 = 1 + +---1 = -1 + +1 - 2 - 3 = -4 +1 - (2 - 3) = 2 + +2^2^3 = 256 +(2^2)^3 = 64 +]]) + +AT_BISON_CHECK([PUSHPULLFLAG [-o Calc.java Calc.y]]) +AT_JAVA_COMPILE([[Calc.java]]) +# Verify that this is a push parser +AT_CHECK_JAVA_GREP([[Calc.java]], + [[.*public void push_parse_initialize().*]]) +# Capture the stdout and stderr output for comparison purposes. +AT_JAVA_PARSER_CHECK([Calc input], 0, [stdout-nolog], [stderr-nolog]) +# 1. Check that the token locations are correct +AT_CHECK([[cp -f ./locations ./expout]],[ignore],[ignore-nolog],[ignore-nolog]) +AT_CHECK([[sed -e '/^Next token.*$/p' -e d ./stderr]],[ignore],[expout],[ignore-nolog]) +# 2. Check that the calculator output matches that of a pull parser +AT_CHECK([[rm -f ./expout; cp -f ./output ./expout]],[ignore],[ignore-nolog],[ignore-nolog]) +AT_CHECK([[cat ./stdout]],[ignore],[expout],[ignore-nolog]) +AT_CLEANUP diff --git a/tests/local.mk b/tests/local.mk index 5299c2e4..5f7fa452 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -53,6 +53,7 @@ TESTSUITE_AT = \ tests/headers.at \ tests/input.at \ tests/java.at \ + tests/javapush.at \ tests/local.at \ tests/named-refs.at \ tests/output.at \ diff --git a/tests/testsuite.at b/tests/testsuite.at index 4c995132..47913add 100644 --- a/tests/testsuite.at +++ b/tests/testsuite.at @@ -63,17 +63,18 @@ m4_include([existing.at]) # Some old bugs. m4_include([regression.at]) +# Push parsing specific tests. +m4_include([push.at]) + # Some C++ specific tests. m4_include([c++.at]) # And some Java specific tests. m4_include([java.at]) +m4_include([javapush.at]) # GLR tests: # C++ types, simplified m4_include([cxx-type.at]) # Regression tests m4_include([glr-regression.at]) - -# Push parsing specific tests. -m4_include([push.at]) -- 2.45.2