1 #include <antlr/config.hpp>
13 #include <antlr/TokenStream.hpp>
14 #include <antlr/TokenWithIndex.hpp>
15 #include <antlr/BitSet.hpp>
16 #include <antlr/TokenStreamRewriteEngine.hpp>
18 #ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
22 #ifndef NO_STATIC_CONSTS
23 const size_t TokenStreamRewriteEngine::MIN_TOKEN_INDEX
= 0;
24 const int TokenStreamRewriteEngine::PROGRAM_INIT_SIZE
= 100;
27 const char* TokenStreamRewriteEngine::DEFAULT_PROGRAM_NAME
= "default";
31 struct compareOperationIndex
{
32 typedef TokenStreamRewriteEngine::RewriteOperation RewriteOperation
;
33 bool operator() ( const RewriteOperation
* a
, const RewriteOperation
* b
) const
35 return a
->getIndex() < b
->getIndex();
38 struct dumpTokenWithIndex
{
39 dumpTokenWithIndex( ANTLR_USE_NAMESPACE(std
)ostream
& o
) : out(o
) {}
40 void operator() ( const RefTokenWithIndex
& t
) {
41 out
<< "[txt='" << t
->getText() << "' tp=" << t
->getType() << " idx=" << t
->getIndex() << "]\n";
43 ANTLR_USE_NAMESPACE(std
)ostream
& out
;
47 TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream
& upstream
)
49 , index(MIN_TOKEN_INDEX
)
56 TokenStreamRewriteEngine::TokenStreamRewriteEngine(TokenStream
& upstream
, size_t initialSize
)
58 , index(MIN_TOKEN_INDEX
)
65 RefToken
TokenStreamRewriteEngine::nextToken( void )
68 // suck tokens until end of stream or we find a non-discarded token
70 t
= RefTokenWithIndex(stream
.nextToken());
73 t
->setIndex(index
); // what is t's index in list?
74 if ( t
->getType() != Token::EOF_TYPE
) {
75 tokens
.push_back(t
); // track all tokens except EOF
77 index
++; // move to next position
79 } while ( t
&& discardMask
.member(t
->getType()) );
83 void TokenStreamRewriteEngine::rollback( const std::string
& programName
,
84 size_t instructionIndex
)
86 program_map::iterator rewrite
= programs
.find(programName
);
87 if( rewrite
!= programs
.end() )
89 operation_list
& prog
= rewrite
->second
;
90 operation_list::iterator
94 std::advance(j
,instructionIndex
);
100 void TokenStreamRewriteEngine::originalToStream( std::ostream
& out
,
104 token_list::const_iterator s
= tokens
.begin();
105 std::advance( s
, start
);
106 token_list::const_iterator e
= s
;
107 std::advance( e
, end
-start
);
108 std::for_each( s
, e
, tokenToStream(out
) );
111 void TokenStreamRewriteEngine::toStream( std::ostream
& out
,
112 const std::string
& programName
,
114 size_t lastToken
) const
116 if( tokens
.size() == 0 )
119 program_map::const_iterator rewriter
= programs
.find(programName
);
121 if ( rewriter
== programs
.end() )
124 // get the prog and some iterators in it...
125 const operation_list
& prog
= rewriter
->second
;
126 operation_list::const_iterator
127 rewriteOpIndex
= prog
.begin(),
128 rewriteOpEnd
= prog
.end();
130 size_t tokenCursor
= firstToken
;
131 // make sure we don't run out of the tokens we have...
132 if( lastToken
> (tokens
.size() - 1) ) {
133 lastToken
= tokens
.size() - 1;
136 while ( tokenCursor
<= lastToken
)
138 // std::cout << "tokenCursor = " << tokenCursor << " first prog index = " << (*rewriteOpIndex)->getIndex() << std::endl;
140 if( rewriteOpIndex
!= rewriteOpEnd
)
142 size_t up_to_here
= std::min(lastToken
,(*rewriteOpIndex
)->getIndex());
143 while( tokenCursor
< up_to_here
)
144 out
<< tokens
[tokenCursor
++]->getText();
146 while ( rewriteOpIndex
!= rewriteOpEnd
&&
147 tokenCursor
== (*rewriteOpIndex
)->getIndex() &&
148 tokenCursor
<= lastToken
)
150 tokenCursor
= (*rewriteOpIndex
)->execute(out
);
153 if( tokenCursor
<= lastToken
)
154 out
<< tokens
[tokenCursor
++]->getText();
156 // std::cout << "Handling tail operations # left = " << std::distance(rewriteOpIndex,rewriteOpEnd) << std::endl;
157 // now see if there are operations (append) beyond last token index
158 std::for_each( rewriteOpIndex
, rewriteOpEnd
, executeOperation(out
) );
159 rewriteOpIndex
= rewriteOpEnd
;
162 void TokenStreamRewriteEngine::toDebugStream( std::ostream
& out
,
166 token_list::const_iterator s
= tokens
.begin();
167 std::advance( s
, start
);
168 token_list::const_iterator e
= s
;
169 std::advance( e
, end
-start
);
170 std::for_each( s
, e
, dumpTokenWithIndex(out
) );
173 void TokenStreamRewriteEngine::addToSortedRewriteList( const std::string
& programName
,
174 RewriteOperation
* op
)
176 program_map::iterator rewrites
= programs
.find(programName
);
177 // check if we got the program already..
178 if ( rewrites
== programs
.end() )
180 // no prog make a new one...
183 programs
.insert(std::make_pair(programName
,ops
));
186 operation_list
& prog
= rewrites
->second
;
194 operation_list::iterator i
, end
= prog
.end();
197 // if at or beyond last op's index, just append
198 if ( op
->getIndex() >= (*i
)->getIndex() ) {
199 prog
.push_back(op
); // append to list of operations
206 operation_list::iterator pos
= std::upper_bound( i
, end
, op
, compareOperationIndex() );
213 #ifdef ANTLR_CXX_SUPPORTS_NAMESPACE