+++ /dev/null
-#ifndef INC_TokenStreamRewriteEngine_hpp__
-#define INC_TokenStreamRewriteEngine_hpp__
-
-/* ANTLR Translator Generator
- * Project led by Terence Parr at http://www.jGuru.com
- * Software rights: http://www.antlr.org/license.html
- */
-
-#include <string>
-#include <list>
-#include <vector>
-#include <map>
-#include <utility>
-#include <ostream>
-#include <iterator>
-#include <cassert>
-#include <algorithm>
-
-#include <antlr/config.hpp>
-
-#include <antlr/TokenStream.hpp>
-#include <antlr/TokenWithIndex.hpp>
-#include <antlr/BitSet.hpp>
-
-#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
-namespace antlr {
-#endif
-
-/** This token stream tracks the *entire* token stream coming from
- * a lexer, but does not pass on the whitespace (or whatever else
- * you want to discard) to the parser.
- *
- * This class can then be asked for the ith token in the input stream.
- * Useful for dumping out the input stream exactly after doing some
- * augmentation or other manipulations. Tokens are index from 0..n-1
- *
- * You can insert stuff, replace, and delete chunks. Note that the
- * operations are done lazily--only if you convert the buffer to a
- * String. This is very efficient because you are not moving data around
- * all the time. As the buffer of tokens is converted to strings, the
- * toString() method(s) check to see if there is an operation at the
- * current index. If so, the operation is done and then normal String
- * rendering continues on the buffer. This is like having multiple Turing
- * machine instruction streams (programs) operating on a single input tape. :)
- *
- * Since the operations are done lazily at toString-time, operations do not
- * screw up the token index values. That is, an insert operation at token
- * index i does not change the index values for tokens i+1..n-1.
- *
- * Because operations never actually alter the buffer, you may always get
- * the original token stream back without undoing anything. Since
- * the instructions are queued up, you can easily simulate transactions and
- * roll back any changes if there is an error just by removing instructions.
- * For example,
- *
- * TokenStreamRewriteEngine rewriteEngine =
- * new TokenStreamRewriteEngine(lexer);
- * JavaRecognizer parser = new JavaRecognizer(rewriteEngine);
- * ...
- * rewriteEngine.insertAfter("pass1", t, "foobar");}
- * rewriteEngine.insertAfter("pass2", u, "start");}
- * System.out.println(rewriteEngine.toString("pass1"));
- * System.out.println(rewriteEngine.toString("pass2"));
- *
- * You can also have multiple "instruction streams" and get multiple
- * rewrites from a single pass over the input. Just name the instruction
- * streams and use that name again when printing the buffer. This could be
- * useful for generating a C file and also its header file--all from the
- * same buffer.
- *
- * If you don't use named rewrite streams, a "default" stream is used.
- *
- * Terence Parr, parrt@cs.usfca.edu
- * University of San Francisco
- * February 2004
- */
-class TokenStreamRewriteEngine : public TokenStream
-{
-public:
- typedef ANTLR_USE_NAMESPACE(std)vector<antlr::RefTokenWithIndex> token_list;
- static const char* DEFAULT_PROGRAM_NAME;
-#ifndef NO_STATIC_CONSTS
- static const size_t MIN_TOKEN_INDEX;
- static const int PROGRAM_INIT_SIZE;
-#else
- enum {
- MIN_TOKEN_INDEX = 0,
- PROGRAM_INIT_SIZE = 100
- };
-#endif
-
- struct tokenToStream {
- tokenToStream( ANTLR_USE_NAMESPACE(std)ostream& o ) : out(o) {}
- template <typename T> void operator() ( const T& t ) {
- out << t->getText();
- }
- ANTLR_USE_NAMESPACE(std)ostream& out;
- };
-
- class RewriteOperation {
- protected:
- RewriteOperation( size_t idx, const ANTLR_USE_NAMESPACE(std)string& txt )
- : index(idx), text(txt)
- {
- }
- public:
- virtual ~RewriteOperation()
- {
- }
- /** Execute the rewrite operation by possibly adding to the buffer.
- * Return the index of the next token to operate on.
- */
- virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& /* out */ ) {
- return index;
- }
- virtual size_t getIndex() const {
- return index;
- }
- virtual const char* type() const {
- return "RewriteOperation";
- }
- protected:
- size_t index;
- ANTLR_USE_NAMESPACE(std)string text;
- };
-
- struct executeOperation {
- ANTLR_USE_NAMESPACE(std)ostream& out;
- executeOperation( ANTLR_USE_NAMESPACE(std)ostream& s ) : out(s) {}
- void operator () ( RewriteOperation* t ) {
- t->execute(out);
- }
- };
-
- /// list of rewrite operations
- typedef ANTLR_USE_NAMESPACE(std)list<RewriteOperation*> operation_list;
- /// map program name to <program counter,program> tuple
- typedef ANTLR_USE_NAMESPACE(std)map<ANTLR_USE_NAMESPACE(std)string,operation_list> program_map;
-
- class InsertBeforeOp : public RewriteOperation
- {
- public:
- InsertBeforeOp( size_t index, const ANTLR_USE_NAMESPACE(std)string& text )
- : RewriteOperation(index, text)
- {
- }
- virtual ~InsertBeforeOp() {}
- virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out )
- {
- out << text;
- return index;
- }
- virtual const char* type() const {
- return "InsertBeforeOp";
- }
- };
-
- class ReplaceOp : public RewriteOperation
- {
- public:
- ReplaceOp(size_t from, size_t to, ANTLR_USE_NAMESPACE(std)string text)
- : RewriteOperation(from,text)
- , lastIndex(to)
- {
- }
- virtual ~ReplaceOp() {}
- virtual size_t execute( ANTLR_USE_NAMESPACE(std)ostream& out ) {
- out << text;
- return lastIndex+1;
- }
- virtual const char* type() const {
- return "ReplaceOp";
- }
- protected:
- size_t lastIndex;
- };
-
- class DeleteOp : public ReplaceOp {
- public:
- DeleteOp(size_t from, size_t to)
- : ReplaceOp(from,to,"")
- {
- }
- virtual const char* type() const {
- return "DeleteOp";
- }
- };
-
- TokenStreamRewriteEngine(TokenStream& upstream);
-
- TokenStreamRewriteEngine(TokenStream& upstream, size_t initialSize);
-
- RefToken nextToken( void );
-
- void rollback(size_t instructionIndex) {
- rollback(DEFAULT_PROGRAM_NAME, instructionIndex);
- }
-
- /** Rollback the instruction stream for a program so that
- * the indicated instruction (via instructionIndex) is no
- * longer in the stream. UNTESTED!
- */
- void rollback(const ANTLR_USE_NAMESPACE(std)string& programName,
- size_t instructionIndex );
-
- void deleteProgram() {
- deleteProgram(DEFAULT_PROGRAM_NAME);
- }
-
- /** Reset the program so that no instructions exist */
- void deleteProgram(const ANTLR_USE_NAMESPACE(std)string& programName) {
- rollback(programName, MIN_TOKEN_INDEX);
- }
-
- void insertAfter( RefTokenWithIndex t,
- const ANTLR_USE_NAMESPACE(std)string& text )
- {
- insertAfter(DEFAULT_PROGRAM_NAME, t, text);
- }
-
- void insertAfter(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
- insertAfter(DEFAULT_PROGRAM_NAME, index, text);
- }
-
- void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
- RefTokenWithIndex t,
- const ANTLR_USE_NAMESPACE(std)string& text )
- {
- insertAfter(programName, t->getIndex(), text);
- }
-
- void insertAfter( const ANTLR_USE_NAMESPACE(std)string& programName,
- size_t index,
- const ANTLR_USE_NAMESPACE(std)string& text )
- {
- // to insert after, just insert before next index (even if past end)
- insertBefore(programName,index+1, text);
- }
-
- void insertBefore( RefTokenWithIndex t,
- const ANTLR_USE_NAMESPACE(std)string& text )
- {
- // std::cout << "insertBefore index " << t->getIndex() << " " << text << std::endl;
- insertBefore(DEFAULT_PROGRAM_NAME, t, text);
- }
-
- void insertBefore(size_t index, const ANTLR_USE_NAMESPACE(std)string& text) {
- insertBefore(DEFAULT_PROGRAM_NAME, index, text);
- }
-
- void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
- RefTokenWithIndex t,
- const ANTLR_USE_NAMESPACE(std)string& text )
- {
- insertBefore(programName, t->getIndex(), text);
- }
-
- void insertBefore( const ANTLR_USE_NAMESPACE(std)string& programName,
- size_t index,
- const ANTLR_USE_NAMESPACE(std)string& text )
- {
- addToSortedRewriteList(programName, new InsertBeforeOp(index,text));
- }
-
- void replace(size_t index, const ANTLR_USE_NAMESPACE(std)string& text)
- {
- replace(DEFAULT_PROGRAM_NAME, index, index, text);
- }
-
- void replace( size_t from, size_t to,
- const ANTLR_USE_NAMESPACE(std)string& text)
- {
- replace(DEFAULT_PROGRAM_NAME, from, to, text);
- }
-
- void replace( RefTokenWithIndex indexT,
- const ANTLR_USE_NAMESPACE(std)string& text )
- {
- replace(DEFAULT_PROGRAM_NAME, indexT->getIndex(), indexT->getIndex(), text);
- }
-
- void replace( RefTokenWithIndex from,
- RefTokenWithIndex to,
- const ANTLR_USE_NAMESPACE(std)string& text )
- {
- replace(DEFAULT_PROGRAM_NAME, from, to, text);
- }
-
- void replace(const ANTLR_USE_NAMESPACE(std)string& programName,
- size_t from, size_t to,
- const ANTLR_USE_NAMESPACE(std)string& text )
- {
- addToSortedRewriteList(programName,new ReplaceOp(from, to, text));
- }
-
- void replace( const ANTLR_USE_NAMESPACE(std)string& programName,
- RefTokenWithIndex from,
- RefTokenWithIndex to,
- const ANTLR_USE_NAMESPACE(std)string& text )
- {
- replace(programName,
- from->getIndex(),
- to->getIndex(),
- text);
- }
-
- void remove(size_t index) {
- remove(DEFAULT_PROGRAM_NAME, index, index);
- }
-
- void remove(size_t from, size_t to) {
- remove(DEFAULT_PROGRAM_NAME, from, to);
- }
-
- void remove(RefTokenWithIndex indexT) {
- remove(DEFAULT_PROGRAM_NAME, indexT, indexT);
- }
-
- void remove(RefTokenWithIndex from, RefTokenWithIndex to) {
- remove(DEFAULT_PROGRAM_NAME, from, to);
- }
-
- void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
- size_t from, size_t to)
- {
- replace(programName,from,to,"");
- }
-
- void remove( const ANTLR_USE_NAMESPACE(std)string& programName,
- RefTokenWithIndex from, RefTokenWithIndex to )
- {
- replace(programName,from,to,"");
- }
-
- void discard(int ttype) {
- discardMask.add(ttype);
- }
-
- RefToken getToken( size_t i )
- {
- return RefToken(tokens.at(i));
- }
-
- size_t getTokenStreamSize() const {
- return tokens.size();
- }
-
- void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
- ANTLR_USE_NAMESPACE(std)for_each( tokens.begin(), tokens.end(), tokenToStream(out) );
- }
-
- void originalToStream( ANTLR_USE_NAMESPACE(std)ostream& out,
- size_t start, size_t end ) const;
-
- void toStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
- toStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
- }
-
- void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
- const ANTLR_USE_NAMESPACE(std)string& programName ) const
- {
- toStream( out, programName, MIN_TOKEN_INDEX, getTokenStreamSize());
- }
-
- void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
- size_t start, size_t end ) const
- {
- toStream(out, DEFAULT_PROGRAM_NAME, start, end);
- }
-
- void toStream( ANTLR_USE_NAMESPACE(std)ostream& out,
- const ANTLR_USE_NAMESPACE(std)string& programName,
- size_t firstToken, size_t lastToken ) const;
-
- void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out ) const {
- toDebugStream( out, MIN_TOKEN_INDEX, getTokenStreamSize());
- }
-
- void toDebugStream( ANTLR_USE_NAMESPACE(std)ostream& out,
- size_t start, size_t end ) const;
-
- size_t getLastRewriteTokenIndex() const {
- return getLastRewriteTokenIndex(DEFAULT_PROGRAM_NAME);
- }
-
- /** Return the last index for the program named programName
- * return 0 if the program does not exist or the program is empty.
- * (Note this is different from the java implementation that returns -1)
- */
- size_t getLastRewriteTokenIndex(const ANTLR_USE_NAMESPACE(std)string& programName) const {
- program_map::const_iterator rewrites = programs.find(programName);
-
- if( rewrites == programs.end() )
- return 0;
-
- const operation_list& prog = rewrites->second;
- if( !prog.empty() )
- {
- operation_list::const_iterator last = prog.end();
- --last;
- return (*last)->getIndex();
- }
- return 0;
- }
-
-protected:
- /** If op.index > lastRewriteTokenIndexes, just add to the end.
- * Otherwise, do linear */
- void addToSortedRewriteList(RewriteOperation* op) {
- addToSortedRewriteList(DEFAULT_PROGRAM_NAME, op);
- }
-
- void addToSortedRewriteList( const ANTLR_USE_NAMESPACE(std)string& programName,
- RewriteOperation* op );
-
-protected:
- /** Who do we suck tokens from? */
- TokenStream& stream;
- /** track index of tokens */
- size_t index;
-
- /** Track the incoming list of tokens */
- token_list tokens;
-
- /** You may have multiple, named streams of rewrite operations.
- * I'm calling these things "programs."
- * Maps String (name) -> rewrite (List)
- */
- program_map programs;
-
- /** Which (whitespace) token(s) to throw out */
- BitSet discardMask;
-};
-
-#ifdef ANTLR_CXX_SUPPORTS_NAMESPACE
-}
-#endif
-
-#endif
projects / apple / security.git / blobdiff