Move antlr. Add `make test` to Makefile.

1 files changed, 0 insertions, 2235 deletions

diff --git a/impl/antlr/libantlr3c-3.4/src/antlr3baserecognizer.c b/impl/antlr/libantlr3c-3.4/src/antlr3baserecognizer.c
deleted file mode 100644
index e2eccc6..0000000
--- a/impl/antlr/libantlr3c-3.4/src/antlr3baserecognizer.c
+++ /dev/null
@@ -1,2235 +0,0 @@
-/** \file
- * Contains the base functions that all recognizers require.
- * Any function can be overridden by a lexer/parser/tree parser or by the
- * ANTLR3 programmer.
- * 
- * \addtogroup pANTLR3_BASE_RECOGNIZER
- * @{
- */
-#include    <antlr3baserecognizer.h>
-
-// [The "BSD licence"]
-// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
-// http://www.temporal-wave.com
-// http://www.linkedin.com/in/jimidle
-//
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions
-// are met:
-// 1. Redistributions of source code must retain the above copyright
-//    notice, this list of conditions and the following disclaimer.
-// 2. Redistributions in binary form must reproduce the above copyright
-//    notice, this list of conditions and the following disclaimer in the
-//    documentation and/or other materials provided with the distribution.
-// 3. The name of the author may not be used to endorse or promote products
-//    derived from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
-// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
-// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
-// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
-// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
-// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#ifdef	ANTLR3_WINDOWS
-#pragma warning( disable : 4100 )
-#endif
-
-/* Interface functions -standard implementations cover parser and treeparser
- * almost completely but are overridden by the parser or tree parser as needed. Lexer overrides
- * most of these functions.
- */
-static void					beginResync					(pANTLR3_BASE_RECOGNIZER recognizer);
-static pANTLR3_BITSET		computeErrorRecoverySet	    (pANTLR3_BASE_RECOGNIZER recognizer);
-static void					endResync					(pANTLR3_BASE_RECOGNIZER recognizer);
-static void					beginBacktrack				(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 level);
-static void					endBacktrack				(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 level, ANTLR3_BOOLEAN successful);
-
-static void *				match						(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 ttype, pANTLR3_BITSET_LIST follow);
-static void					matchAny					(pANTLR3_BASE_RECOGNIZER recognizer);
-static void					mismatch					(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 ttype, pANTLR3_BITSET_LIST follow);
-static ANTLR3_BOOLEAN		mismatchIsUnwantedToken		(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM is, ANTLR3_UINT32 ttype);
-static ANTLR3_BOOLEAN		mismatchIsMissingToken		(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM is, pANTLR3_BITSET_LIST follow);
-static void					reportError					(pANTLR3_BASE_RECOGNIZER recognizer);
-static pANTLR3_BITSET		computeCSRuleFollow			(pANTLR3_BASE_RECOGNIZER recognizer);
-static pANTLR3_BITSET		combineFollows				(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_BOOLEAN exact);
-static void					displayRecognitionError	    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 * tokenNames);
-static void					recover						(pANTLR3_BASE_RECOGNIZER recognizer);
-static void	*				recoverFromMismatchedToken  (pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 ttype, pANTLR3_BITSET_LIST follow);
-static void	*				recoverFromMismatchedSet    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_BITSET_LIST follow);
-static ANTLR3_BOOLEAN		recoverFromMismatchedElement(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_BITSET_LIST follow);
-static void					consumeUntil				(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 tokenType);
-static void					consumeUntilSet				(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_BITSET set);
-static pANTLR3_STACK		getRuleInvocationStack	    (pANTLR3_BASE_RECOGNIZER recognizer);
-static pANTLR3_STACK		getRuleInvocationStackNamed (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 name);
-static pANTLR3_HASH_TABLE	toStrings					(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_HASH_TABLE);
-static ANTLR3_MARKER		getRuleMemoization			(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_INTKEY ruleIndex, ANTLR3_MARKER ruleParseStart);
-static ANTLR3_BOOLEAN		alreadyParsedRule			(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_MARKER ruleIndex);
-static void					memoize						(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_MARKER ruleIndex, ANTLR3_MARKER ruleParseStart);
-static ANTLR3_BOOLEAN		synpred						(pANTLR3_BASE_RECOGNIZER recognizer, void * ctx, void (*predicate)(void * ctx));
-static void					reset						(pANTLR3_BASE_RECOGNIZER recognizer);
-static void					freeBR						(pANTLR3_BASE_RECOGNIZER recognizer);
-static void *				getCurrentInputSymbol		(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream);
-static void *				getMissingSymbol			(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM	istream, pANTLR3_EXCEPTION	e,
-															ANTLR3_UINT32 expectedTokenType, pANTLR3_BITSET_LIST follow);
-static ANTLR3_UINT32		getNumberOfSyntaxErrors		(pANTLR3_BASE_RECOGNIZER recognizer);
-
-ANTLR3_API pANTLR3_BASE_RECOGNIZER
-antlr3BaseRecognizerNew(ANTLR3_UINT32 type, ANTLR3_UINT32 sizeHint, pANTLR3_RECOGNIZER_SHARED_STATE state)
-{
-    pANTLR3_BASE_RECOGNIZER recognizer;
-
-    // Allocate memory for the structure
-    //
-    recognizer	    = (pANTLR3_BASE_RECOGNIZER) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_BASE_RECOGNIZER));
-
-    if	(recognizer == NULL)
-    {
-		// Allocation failed
-		//
-		return	NULL;
-    }
-
-	
-	// If we have been supplied with a pre-existing recognizer state
-	// then we just install it, otherwise we must create one from scratch
-	//
-	if	(state == NULL)
-	{
-		recognizer->state = (pANTLR3_RECOGNIZER_SHARED_STATE) ANTLR3_CALLOC(1, (size_t)sizeof(ANTLR3_RECOGNIZER_SHARED_STATE));
-
-		if	(recognizer->state == NULL)
-		{
-			ANTLR3_FREE(recognizer);
-			return	NULL;
-		}
-
-		// Initialize any new recognizer state
-		//
-		recognizer->state->errorRecovery	= ANTLR3_FALSE;
-		recognizer->state->lastErrorIndex	= -1;
-		recognizer->state->failed		= ANTLR3_FALSE;
-		recognizer->state->errorCount		= 0;
-		recognizer->state->backtracking		= 0;
-		recognizer->state->following		= NULL;
-		recognizer->state->ruleMemo		= NULL;
-		recognizer->state->tokenNames		= NULL;
-		recognizer->state->sizeHint             = sizeHint;
-		recognizer->state->tokSource		= NULL;
-                recognizer->state->tokFactory           = NULL;
-
-		// Rather than check to see if we must initialize
-		// the stack every time we are asked for an new rewrite stream
-		// we just always create an empty stack and then just
-		// free it when the base recognizer is freed.
-		//
-		recognizer->state->rStreams		= antlr3VectorNew(0);  // We don't know the size.
-
-		if	(recognizer->state->rStreams == NULL)
-		{
-			// Out of memory
-			//
-			ANTLR3_FREE(recognizer->state);
-			ANTLR3_FREE(recognizer);
-			return	NULL;
-		}
-	}
-	else
-	{
-		// Install the one we were given, and do not reset it here
-		// as it will either already have been initialized or will
-		// be in a state that needs to be preserved.
-		//
-		recognizer->state = state;
-	}
-		
-    // Install the BR API
-    //
-    recognizer->alreadyParsedRule           = alreadyParsedRule;
-    recognizer->beginResync                 = beginResync;
-    recognizer->combineFollows              = combineFollows;
-    recognizer->beginBacktrack              = beginBacktrack;
-    recognizer->endBacktrack                = endBacktrack;
-    recognizer->computeCSRuleFollow         = computeCSRuleFollow;
-    recognizer->computeErrorRecoverySet     = computeErrorRecoverySet;
-    recognizer->consumeUntil                = consumeUntil;
-    recognizer->consumeUntilSet             = consumeUntilSet;
-    recognizer->displayRecognitionError     = displayRecognitionError;
-    recognizer->endResync                   = endResync;
-    recognizer->exConstruct                 = antlr3MTExceptionNew;
-    recognizer->getRuleInvocationStack      = getRuleInvocationStack;
-    recognizer->getRuleInvocationStackNamed = getRuleInvocationStackNamed;
-    recognizer->getRuleMemoization          = getRuleMemoization;
-    recognizer->match                       = match;
-    recognizer->matchAny                    = matchAny;
-    recognizer->memoize                     = memoize;
-    recognizer->mismatch                    = mismatch;
-    recognizer->mismatchIsUnwantedToken     = mismatchIsUnwantedToken;
-    recognizer->mismatchIsMissingToken      = mismatchIsMissingToken;
-    recognizer->recover                     = recover;
-    recognizer->recoverFromMismatchedElement= recoverFromMismatchedElement;
-    recognizer->recoverFromMismatchedSet    = recoverFromMismatchedSet;
-    recognizer->recoverFromMismatchedToken  = recoverFromMismatchedToken;
-    recognizer->getNumberOfSyntaxErrors     = getNumberOfSyntaxErrors;
-    recognizer->reportError                 = reportError;
-    recognizer->reset                       = reset;
-    recognizer->synpred                     = synpred;
-    recognizer->toStrings                   = toStrings;
-    recognizer->getCurrentInputSymbol       = getCurrentInputSymbol;
-    recognizer->getMissingSymbol            = getMissingSymbol;
-    recognizer->debugger                    = NULL;
-
-    recognizer->free = freeBR;
-
-    /* Initialize variables
-     */
-    recognizer->type			= type;
-
-
-    return  recognizer;
-}
-static void	
-freeBR	    (pANTLR3_BASE_RECOGNIZER recognizer)
-{
-    pANTLR3_EXCEPTION thisE;
-
-	// Did we have a state allocated?
-	//
-	if	(recognizer->state != NULL)
-	{
-		// Free any rule memoization we set up
-		//
-		if	(recognizer->state->ruleMemo != NULL)
-		{
-			recognizer->state->ruleMemo->free(recognizer->state->ruleMemo);
-			recognizer->state->ruleMemo = NULL;
-		}
-
-		// Free any exception space we have left around
-		//
-		thisE = recognizer->state->exception;
-		if	(thisE != NULL)
-		{
-			thisE->freeEx(thisE);
-		}
-
-		// Free any rewrite streams we have allocated
-		//
-		if	(recognizer->state->rStreams != NULL)
-		{
-			recognizer->state->rStreams->free(recognizer->state->rStreams);
-		}
-
-		// Free up any token factory we created (error recovery for instance)
-		//
-		if	(recognizer->state->tokFactory != NULL)
-		{
-			recognizer->state->tokFactory->close(recognizer->state->tokFactory);
-		}
-		// Free the shared state memory
-		//
-		ANTLR3_FREE(recognizer->state);
-	}
-
-	// Free the actual recognizer space
-	//
-    ANTLR3_FREE(recognizer);
-}
-
-/**
- * Creates a new Mismatched Token Exception and inserts in the recognizer
- * exception stack.
- * 
- * \param recognizer
- * Context pointer for this recognizer
- * 
- */
-ANTLR3_API	void
-antlr3MTExceptionNew(pANTLR3_BASE_RECOGNIZER recognizer)
-{
-    /* Create a basic recognition exception structure
-     */
-    antlr3RecognitionExceptionNew(recognizer);
-
-    /* Now update it to indicate this is a Mismatched token exception
-     */
-    recognizer->state->exception->name		= ANTLR3_MISMATCHED_EX_NAME;
-    recognizer->state->exception->type		= ANTLR3_MISMATCHED_TOKEN_EXCEPTION;
-
-    return;
-}
-
-ANTLR3_API	void
-antlr3RecognitionExceptionNew(pANTLR3_BASE_RECOGNIZER recognizer)
-{
-	pANTLR3_EXCEPTION				ex;
-	pANTLR3_LEXER					lexer;
-	pANTLR3_PARSER					parser;
-	pANTLR3_TREE_PARSER				tparser;
-
-	pANTLR3_INPUT_STREAM			ins;
-	pANTLR3_INT_STREAM				is;
-	pANTLR3_COMMON_TOKEN_STREAM	    cts;
-	pANTLR3_TREE_NODE_STREAM	    tns;
-
-	ins	    = NULL;
-	cts	    = NULL;
-	tns	    = NULL;
-	is	    = NULL;
-	lexer   = NULL;
-	parser  = NULL;
-	tparser = NULL;
-
-	switch	(recognizer->type)
-	{
-	case	ANTLR3_TYPE_LEXER:
-
-		lexer	= (pANTLR3_LEXER) (recognizer->super);
-		ins	= lexer->input;
-		is	= ins->istream;
-
-		break;
-
-	case	ANTLR3_TYPE_PARSER:
-
-		parser  = (pANTLR3_PARSER) (recognizer->super);
-		cts	= (pANTLR3_COMMON_TOKEN_STREAM)(parser->tstream->super);
-		is	= parser->tstream->istream;
-
-		break;
-
-	case	ANTLR3_TYPE_TREE_PARSER:
-
-		tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
-		tns	= tparser->ctnstream->tnstream;
-		is	= tns->istream;
-
-		break;
-
-	default:
-
-		ANTLR3_FPRINTF(stderr, "Base recognizer function antlr3RecognitionExceptionNew called by unknown parser type - provide override for this function\n");
-		return;
-
-		break;
-	}
-
-	/* Create a basic exception structure
-	 */
-	ex = antlr3ExceptionNew(ANTLR3_RECOGNITION_EXCEPTION,
-		(void *)ANTLR3_RECOGNITION_EX_NAME,
-		NULL,
-		ANTLR3_FALSE);
-
-	/* Rest of information depends on the base type of the 
-	 * input stream.
-	 */
-	switch  (is->type & ANTLR3_INPUT_MASK)
-	{
-	case    ANTLR3_CHARSTREAM:
-
-		ex->c			= is->_LA		    	(is, 1);					/* Current input character			*/
-		ex->line		= ins->getLine			(ins);						/* Line number comes from stream		*/
-		ex->charPositionInLine	= ins->getCharPositionInLine	(ins);	    /* Line offset also comes from the stream   */
-		ex->index		= is->index			(is);
-		ex->streamName		= ins->fileName;
-		ex->message		= "Unexpected character";
-		break;
-
-	case    ANTLR3_TOKENSTREAM:
-
-		ex->token		= cts->tstream->_LT						(cts->tstream, 1);	    /* Current input token			    */
-		ex->line		= ((pANTLR3_COMMON_TOKEN)(ex->token))->getLine			(ex->token);
-		ex->charPositionInLine	= ((pANTLR3_COMMON_TOKEN)(ex->token))->getCharPositionInLine	(ex->token);
-		ex->index		= cts->tstream->istream->index					(cts->tstream->istream);
-		if	(((pANTLR3_COMMON_TOKEN)(ex->token))->type == ANTLR3_TOKEN_EOF)
-		{
-			ex->streamName		= NULL;
-		}
-		else
-		{
-			ex->streamName		= ((pANTLR3_COMMON_TOKEN)(ex->token))->input->fileName;
-		}
-		ex->message		= "Unexpected token";
-		break;
-
-	case    ANTLR3_COMMONTREENODE:
-
-		ex->token		= tns->_LT						    (tns, 1);	    /* Current input tree node			    */
-		ex->line		= ((pANTLR3_BASE_TREE)(ex->token))->getLine		    (ex->token);
-		ex->charPositionInLine	= ((pANTLR3_BASE_TREE)(ex->token))->getCharPositionInLine   (ex->token);
-		ex->index		= tns->istream->index					    (tns->istream);
-
-		// Are you ready for this? Deep breath now...
-		//
-		{
-			pANTLR3_COMMON_TREE tnode;
-
-			tnode		= ((pANTLR3_COMMON_TREE)(((pANTLR3_BASE_TREE)(ex->token))->super));
-
-			if	(tnode->token    == NULL)
-			{
-				ex->streamName = ((pANTLR3_BASE_TREE)(ex->token))->strFactory->newStr(((pANTLR3_BASE_TREE)(ex->token))->strFactory, (pANTLR3_UINT8)"-unknown source-");
-			}
-			else
-			{
-				if	(tnode->token->input == NULL)
-				{
-					ex->streamName		= NULL;
-				}
-				else
-				{
-					ex->streamName		= tnode->token->input->fileName;
-				}
-			}
-			ex->message		= "Unexpected node";
-		}
-		break;
-	}
-
-	ex->input						= is;
-	ex->nextException				= recognizer->state->exception;	/* So we don't leak the memory */
-	recognizer->state->exception	= ex;
-	recognizer->state->error	    = ANTLR3_TRUE;	    /* Exception is outstanding	*/
-
-	return;
-}
-
-
-/// Match current input symbol against ttype.  Upon error, do one token
-/// insertion or deletion if possible.  
-/// To turn off single token insertion or deletion error
-/// recovery, override mismatchRecover() and have it call
-/// plain mismatch(), which does not recover.  Then any error
-/// in a rule will cause an exception and immediate exit from
-/// rule.  Rule would recover by resynchronizing to the set of
-/// symbols that can follow rule ref.
-///
-static void *
-match(	pANTLR3_BASE_RECOGNIZER recognizer,
-		ANTLR3_UINT32 ttype, pANTLR3_BITSET_LIST follow)
-{
-    pANTLR3_PARSER			parser;
-    pANTLR3_TREE_PARSER	    tparser;
-    pANTLR3_INT_STREAM	    is;
-	void					* matchedSymbol;
-
-    switch	(recognizer->type)
-    {
-		case	ANTLR3_TYPE_PARSER:
-
-			parser  = (pANTLR3_PARSER) (recognizer->super);
-			tparser	= NULL;
-			is	= parser->tstream->istream;
-
-			break;
-
-		case	ANTLR3_TYPE_TREE_PARSER:
-
-			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
-			parser	= NULL;
-			is	= tparser->ctnstream->tnstream->istream;
-
-			break;
-
-		default:
-		    
-			ANTLR3_FPRINTF(stderr, "Base recognizer function 'match' called by unknown parser type - provide override for this function\n");
-			return ANTLR3_FALSE;
-
-			break;
-    }
-
-	// Pick up the current input token/node for assignment to labels
-	//
-	matchedSymbol = recognizer->getCurrentInputSymbol(recognizer, is);
-
-    if	(is->_LA(is, 1) == ttype)
-    {
-		// The token was the one we were told to expect
-		//
-		is->consume(is);									// Consume that token from the stream
-		recognizer->state->errorRecovery	= ANTLR3_FALSE;	// Not in error recovery now (if we were)
-		recognizer->state->failed			= ANTLR3_FALSE;	// The match was a success
-		return matchedSymbol;								// We are done
-    }
-
-    // We did not find the expected token type, if we are backtracking then
-    // we just set the failed flag and return.
-    //
-    if	(recognizer->state->backtracking > 0)
-    {
-		// Backtracking is going on
-		//
-		recognizer->state->failed  = ANTLR3_TRUE;
-		return matchedSymbol;
-	}
-
-    // We did not find the expected token and there is no backtracking
-    // going on, so we mismatch, which creates an exception in the recognizer exception
-    // stack.
-    //
-	matchedSymbol = recognizer->recoverFromMismatchedToken(recognizer, ttype, follow);
-    return matchedSymbol;
-}
-
-/// Consumes the next token, whatever it is, and resets the recognizer state
-/// so that it is not in error.
-///
-/// \param recognizer
-/// Recognizer context pointer
-///
-static void
-matchAny(pANTLR3_BASE_RECOGNIZER recognizer)
-{
-    pANTLR3_PARSER	    parser;
-    pANTLR3_TREE_PARSER	    tparser;
-    pANTLR3_INT_STREAM	    is;
-
-    switch	(recognizer->type)
-    {
-		case	ANTLR3_TYPE_PARSER:
-
-			parser  = (pANTLR3_PARSER) (recognizer->super);
-			tparser	= NULL;
-			is	= parser->tstream->istream;
-
-			break;
-
-		case	ANTLR3_TYPE_TREE_PARSER:
-
-			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
-			parser	= NULL;
-			is	= tparser->ctnstream->tnstream->istream;
-
-			break;
-
-		default:
-		    
-			ANTLR3_FPRINTF(stderr, "Base recognizer function 'matchAny' called by unknown parser type - provide override for this function\n");
-			return;
-
-		break;
-    }
-    recognizer->state->errorRecovery	= ANTLR3_FALSE;
-    recognizer->state->failed		    = ANTLR3_FALSE;
-    is->consume(is);
-
-    return;
-}
-///
-///
-static ANTLR3_BOOLEAN
-mismatchIsUnwantedToken(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM is, ANTLR3_UINT32 ttype)
-{
-	ANTLR3_UINT32 nextt;
-
-	nextt = is->_LA(is, 2);
-
-	if	(nextt == ttype)
-	{
-		if	(recognizer->state->exception != NULL)
-		{
-			recognizer->state->exception->expecting = nextt;
-		}
-		return ANTLR3_TRUE;		// This token is unknown, but the next one is the one we wanted
-	}
-	else
-	{
-		return ANTLR3_FALSE;	// Neither this token, nor the one following is the one we wanted
-	}
-}
-
-///
-///
-static ANTLR3_BOOLEAN
-mismatchIsMissingToken(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM is, pANTLR3_BITSET_LIST follow)
-{
-	ANTLR3_BOOLEAN	retcode;
-	pANTLR3_BITSET	followClone;
-	pANTLR3_BITSET	viableTokensFollowingThisRule;
-
-	if	(follow == NULL)
-	{
-		// There is no information about the tokens that can follow the last one
-		// hence we must say that the current one we found is not a member of the 
-		// follow set and does not indicate a missing token. We will just consume this
-		// single token and see if the parser works it out from there.
-		//
-		return	ANTLR3_FALSE;
-	}
-
-	followClone						= NULL;
-	viableTokensFollowingThisRule	= NULL;
-
-	// The C bitset maps are laid down at compile time by the
-	// C code generation. Hence we cannot remove things from them
-	// and so on. So, in order to remove EOR (if we need to) then
-	// we clone the static bitset.
-	//
-	followClone = antlr3BitsetLoad(follow);
-	if	(followClone == NULL)
-	{
-		return ANTLR3_FALSE;
-	}
-
-	// Compute what can follow this grammar reference
-	//
-	if	(followClone->isMember(followClone, ANTLR3_EOR_TOKEN_TYPE))
-	{
-		// EOR can follow, but if we are not the start symbol, we
-		// need to remove it.
-		//
-		if	(recognizer->state->following->vector->count >= 0)
-		{
-			followClone->remove(followClone, ANTLR3_EOR_TOKEN_TYPE);
-		}
-
-		// Now compute the visiable tokens that can follow this rule, according to context
-		// and make them part of the follow set.
-		//
-		viableTokensFollowingThisRule = recognizer->computeCSRuleFollow(recognizer);
-		followClone->borInPlace(followClone, viableTokensFollowingThisRule);
-	}
-
-	/// if current token is consistent with what could come after set
-	/// then we know we're missing a token; error recovery is free to
-	/// "insert" the missing token
-	///
-	/// BitSet cannot handle negative numbers like -1 (EOF) so I leave EOR
-	/// in follow set to indicate that the fall of the start symbol is
-	/// in the set (EOF can follow).
-	///
-	if	(		followClone->isMember(followClone, is->_LA(is, 1))
-			||	followClone->isMember(followClone, ANTLR3_EOR_TOKEN_TYPE)
-		)
-	{
-		retcode = ANTLR3_TRUE;
-	}
-	else
-	{
-		retcode	= ANTLR3_FALSE;
-	}
-
-	if	(viableTokensFollowingThisRule != NULL)
-	{
-		viableTokensFollowingThisRule->free(viableTokensFollowingThisRule);
-	}
-	if	(followClone != NULL)
-	{
-		followClone->free(followClone);
-	}
-
-	return retcode;
-
-}
-
-/// Factor out what to do upon token mismatch so tree parsers can behave
-/// differently.  Override and call mismatchRecover(input, ttype, follow)
-/// to get single token insertion and deletion.  Use this to turn off
-/// single token insertion and deletion. Override mismatchRecover
-/// to call this instead.
-///
-/// \remark mismatch only works for parsers and must be overridden for anything else.
-///
-static	void
-mismatch(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 ttype, pANTLR3_BITSET_LIST follow)
-{
-    pANTLR3_PARSER	    parser;
-    pANTLR3_TREE_PARSER	    tparser;
-    pANTLR3_INT_STREAM	    is;
-
-    // Install a mismatched token exception in the exception stack
-    //
-    antlr3MTExceptionNew(recognizer);
-    recognizer->state->exception->expecting    = ttype;
-
-    switch	(recognizer->type)
-    {
-		case	ANTLR3_TYPE_PARSER:
-
-			parser  = (pANTLR3_PARSER) (recognizer->super);
-			tparser	= NULL;
-			is	= parser->tstream->istream;
-
-			break;
-
-		default:
-		    
-			ANTLR3_FPRINTF(stderr, "Base recognizer function 'mismatch' called by unknown parser type - provide override for this function\n");
-			return;
-
-			break;
-    }
-
-	if	(mismatchIsUnwantedToken(recognizer, is, ttype))
-	{
-		// Create a basic recognition exception structure
-		//
-	    antlr3RecognitionExceptionNew(recognizer);
-		
-		// Now update it to indicate this is an unwanted token exception
-		//
-		recognizer->state->exception->name		= ANTLR3_UNWANTED_TOKEN_EXCEPTION_NAME;
-		recognizer->state->exception->type		= ANTLR3_UNWANTED_TOKEN_EXCEPTION;
-
-		return;
-	}
-	
-	if	(mismatchIsMissingToken(recognizer, is, follow))
-	{
-		// Create a basic recognition exception structure
-		//
-	    antlr3RecognitionExceptionNew(recognizer);
-		
-		// Now update it to indicate this is an unwanted token exception
-		//
-		recognizer->state->exception->name		= ANTLR3_MISSING_TOKEN_EXCEPTION_NAME;
-		recognizer->state->exception->type		= ANTLR3_MISSING_TOKEN_EXCEPTION;
-
-		return;
-	}
-
-	// Just a mismatched token is all we can dtermine
-	//
-	antlr3MTExceptionNew(recognizer);
-
-	return;
-}
-/// Report a recognition problem.
-///
-/// This method sets errorRecovery to indicate the parser is recovering
-/// not parsing.  Once in recovery mode, no errors are generated.
-/// To get out of recovery mode, the parser must successfully match
-/// a token (after a resync).  So it will go:
-///
-///		1. error occurs
-///		2. enter recovery mode, report error
-///		3. consume until token found in resynch set
-///		4. try to resume parsing
-///		5. next match() will reset errorRecovery mode
-///
-/// If you override, make sure to update errorCount if you care about that.
-///
-static void			
-reportError		    (pANTLR3_BASE_RECOGNIZER recognizer)
-{
-    	// Invoke the debugger event if there is a debugger listening to us
-	//
-	if	(recognizer->debugger != NULL)
-	{
-		recognizer->debugger->recognitionException(recognizer->debugger, recognizer->state->exception);
-	}
-
-    if	(recognizer->state->errorRecovery == ANTLR3_TRUE)
-    {
-		// Already in error recovery so don't display another error while doing so
-		//
-		return;
-    }
-
-    // Signal we are in error recovery now
-    //
-    recognizer->state->errorRecovery = ANTLR3_TRUE;
-	
-	// Indicate this recognizer had an error while processing.
-	//
-	recognizer->state->errorCount++;
-
-	// Call the error display routine
-	//
-    recognizer->displayRecognitionError(recognizer, recognizer->state->tokenNames);
-}
-
-static void
-beginBacktrack		(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 level)
-{
-	if	(recognizer->debugger != NULL)
-	{
-		recognizer->debugger->beginBacktrack(recognizer->debugger, level);
-	}
-}
-
-static void
-endBacktrack		(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 level, ANTLR3_BOOLEAN successful)
-{
-	if	(recognizer->debugger != NULL)
-	{
-		recognizer->debugger->endBacktrack(recognizer->debugger, level, successful);
-	}
-}
-static void			
-beginResync		    (pANTLR3_BASE_RECOGNIZER recognizer)
-{
-	if	(recognizer->debugger != NULL)
-	{
-		recognizer->debugger->beginResync(recognizer->debugger);
-	}
-}
-
-static void			
-endResync		    (pANTLR3_BASE_RECOGNIZER recognizer)
-{
-	if	(recognizer->debugger != NULL)
-	{
-		recognizer->debugger->endResync(recognizer->debugger);
-	}
-}
-
-/// Compute the error recovery set for the current rule.
-/// Documentation below is from the Java implementation.
-///
-/// During rule invocation, the parser pushes the set of tokens that can
-/// follow that rule reference on the stack; this amounts to
-/// computing FIRST of what follows the rule reference in the
-/// enclosing rule. This local follow set only includes tokens
-/// from within the rule; i.e., the FIRST computation done by
-/// ANTLR stops at the end of a rule.
-//
-/// EXAMPLE
-//
-/// When you find a "no viable alt exception", the input is not
-/// consistent with any of the alternatives for rule r.  The best
-/// thing to do is to consume tokens until you see something that
-/// can legally follow a call to r *or* any rule that called r.
-/// You don't want the exact set of viable next tokens because the
-/// input might just be missing a token--you might consume the
-/// rest of the input looking for one of the missing tokens.
-///
-/// Consider grammar:
-///
-/// a : '[' b ']'
-///   | '(' b ')'
-///   ;
-/// b : c '^' INT ;
-/// c : ID
-///   | INT
-///   ;
-///
-/// At each rule invocation, the set of tokens that could follow
-/// that rule is pushed on a stack.  Here are the various "local"
-/// follow sets:
-///
-/// FOLLOW(b1_in_a) = FIRST(']') = ']'
-/// FOLLOW(b2_in_a) = FIRST(')') = ')'
-/// FOLLOW(c_in_b) = FIRST('^') = '^'
-///
-/// Upon erroneous input "[]", the call chain is
-///
-/// a -> b -> c
-///
-/// and, hence, the follow context stack is:
-///
-/// depth  local follow set     after call to rule
-///   0         <EOF>                    a (from main())
-///   1          ']'                     b
-///   3          '^'                     c
-///
-/// Notice that ')' is not included, because b would have to have
-/// been called from a different context in rule a for ')' to be
-/// included.
-///
-/// For error recovery, we cannot consider FOLLOW(c)
-/// (context-sensitive or otherwise).  We need the combined set of
-/// all context-sensitive FOLLOW sets--the set of all tokens that
-/// could follow any reference in the call chain.  We need to
-/// resync to one of those tokens.  Note that FOLLOW(c)='^' and if
-/// we resync'd to that token, we'd consume until EOF.  We need to
-/// sync to context-sensitive FOLLOWs for a, b, and c: {']','^'}.
-/// In this case, for input "[]", LA(1) is in this set so we would
-/// not consume anything and after printing an error rule c would
-/// return normally.  It would not find the required '^' though.
-/// At this point, it gets a mismatched token error and throws an
-/// exception (since LA(1) is not in the viable following token
-/// set).  The rule exception handler tries to recover, but finds
-/// the same recovery set and doesn't consume anything.  Rule b
-/// exits normally returning to rule a.  Now it finds the ']' (and
-/// with the successful match exits errorRecovery mode).
-///
-/// So, you can see that the parser walks up call chain looking
-/// for the token that was a member of the recovery set.
-///
-/// Errors are not generated in errorRecovery mode.
-///
-/// ANTLR's error recovery mechanism is based upon original ideas:
-///
-/// "Algorithms + Data Structures = Programs" by Niklaus Wirth
-///
-/// and
-///
-/// "A note on error recovery in recursive descent parsers":
-/// http://portal.acm.org/citation.cfm?id=947902.947905
-///
-/// Later, Josef Grosch had some good ideas:
-///
-/// "Efficient and Comfortable Error Recovery in Recursive Descent
-/// Parsers":
-/// ftp://www.cocolab.com/products/cocktail/doca4.ps/ell.ps.zip
-///
-/// Like Grosch I implemented local FOLLOW sets that are combined
-/// at run-time upon error to avoid overhead during parsing.
-///
-static pANTLR3_BITSET		
-computeErrorRecoverySet	    (pANTLR3_BASE_RECOGNIZER recognizer)
-{
-    return   recognizer->combineFollows(recognizer, ANTLR3_FALSE);
-}
-
-/// Compute the context-sensitive FOLLOW set for current rule.
-/// Documentation below is from the Java runtime.
-///
-/// This is the set of token types that can follow a specific rule
-/// reference given a specific call chain.  You get the set of
-/// viable tokens that can possibly come next (look ahead depth 1)
-/// given the current call chain.  Contrast this with the
-/// definition of plain FOLLOW for rule r:
-///
-///  FOLLOW(r)={x | S=>*alpha r beta in G and x in FIRST(beta)}
-///
-/// where x in T* and alpha, beta in V*; T is set of terminals and
-/// V is the set of terminals and non terminals.  In other words,
-/// FOLLOW(r) is the set of all tokens that can possibly follow
-/// references to r in///any* sentential form (context).  At
-/// runtime, however, we know precisely which context applies as
-/// we have the call chain.  We may compute the exact (rather
-/// than covering superset) set of following tokens.
-///
-/// For example, consider grammar:
-///
-/// stat : ID '=' expr ';'      // FOLLOW(stat)=={EOF}
-///      | "return" expr '.'
-///      ;
-/// expr : atom ('+' atom)* ;   // FOLLOW(expr)=={';','.',')'}
-/// atom : INT                  // FOLLOW(atom)=={'+',')',';','.'}
-///      | '(' expr ')'
-///      ;
-///
-/// The FOLLOW sets are all inclusive whereas context-sensitive
-/// FOLLOW sets are precisely what could follow a rule reference.
-/// For input input "i=(3);", here is the derivation:
-///
-/// stat => ID '=' expr ';'
-///      => ID '=' atom ('+' atom)* ';'
-///      => ID '=' '(' expr ')' ('+' atom)* ';'
-///      => ID '=' '(' atom ')' ('+' atom)* ';'
-///      => ID '=' '(' INT ')' ('+' atom)* ';'
-///      => ID '=' '(' INT ')' ';'
-///
-/// At the "3" token, you'd have a call chain of
-///
-///   stat -> expr -> atom -> expr -> atom
-///
-/// What can follow that specific nested ref to atom?  Exactly ')'
-/// as you can see by looking at the derivation of this specific
-/// input.  Contrast this with the FOLLOW(atom)={'+',')',';','.'}.
-///
-/// You want the exact viable token set when recovering from a
-/// token mismatch.  Upon token mismatch, if LA(1) is member of
-/// the viable next token set, then you know there is most likely
-/// a missing token in the input stream.  "Insert" one by just not
-/// throwing an exception.
-///
-static pANTLR3_BITSET		
-computeCSRuleFollow	    (pANTLR3_BASE_RECOGNIZER recognizer)
-{
-    return   recognizer->combineFollows(recognizer, ANTLR3_FALSE);
-}
-
-/// Compute the current followset for the input stream.
-///
-static pANTLR3_BITSET		
-combineFollows		    (pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_BOOLEAN exact)
-{
-    pANTLR3_BITSET	followSet;
-    pANTLR3_BITSET	localFollowSet;
-    ANTLR3_UINT32	top;
-    ANTLR3_UINT32	i;
-
-    top	= recognizer->state->following->size(recognizer->state->following);
-
-    followSet	    = antlr3BitsetNew(0);
-	localFollowSet	= NULL;
-
-    for (i = top; i>0; i--)
-    {
-		localFollowSet = antlr3BitsetLoad((pANTLR3_BITSET_LIST) recognizer->state->following->get(recognizer->state->following, i-1));
-
-		if  (localFollowSet != NULL)
-		{
-			followSet->borInPlace(followSet, localFollowSet);
-
-			if	(exact == ANTLR3_TRUE)
-			{
-				if	(localFollowSet->isMember(localFollowSet, ANTLR3_EOR_TOKEN_TYPE) == ANTLR3_FALSE)
-				{
-					// Only leave EOR in the set if at top (start rule); this lets us know
-					// if we have to include the follow(start rule); I.E., EOF
-					//
-					if	(i>1)
-					{
-						followSet->remove(followSet, ANTLR3_EOR_TOKEN_TYPE);
-					}
-				}
-				else
-				{
-					break;	// Cannot see End Of Rule from here, just drop out
-				}
-			}
-			localFollowSet->free(localFollowSet);
-			localFollowSet = NULL;
-		}
-    }
-
-	if	(localFollowSet != NULL)
-	{
-		localFollowSet->free(localFollowSet);
-	}
-    return  followSet;
-}
-
-/// Standard/Example error display method.
-/// No generic error message display funciton coudl possibly do everything correctly
-/// for all possible parsers. Hence you are provided with this example routine, which
-/// you should override in your parser/tree parser to do as you will.
-///
-/// Here we depart somewhat from the Java runtime as that has now split up a lot
-/// of the error display routines into spearate units. However, ther is little advantage
-/// to this in the C version as you will probably implement all such routines as a 
-/// separate translation unit, rather than install them all as pointers to functions
-/// in the base recognizer.
-///
-static void			
-displayRecognitionError	    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 * tokenNames)
-{
-	pANTLR3_PARSER			parser;
-	pANTLR3_TREE_PARSER	    tparser;
-	pANTLR3_INT_STREAM	    is;
-	pANTLR3_STRING			ttext;
-	pANTLR3_STRING			ftext;
-	pANTLR3_EXCEPTION	    ex;
-	pANTLR3_COMMON_TOKEN    theToken;
-	pANTLR3_BASE_TREE	    theBaseTree;
-	pANTLR3_COMMON_TREE	    theCommonTree;
-
-	// Retrieve some info for easy reading.
-	//
-	ex	    =		recognizer->state->exception;
-	ttext   =		NULL;
-
-	// See if there is a 'filename' we can use
-	//
-	if	(ex->streamName == NULL)
-	{
-		if	(((pANTLR3_COMMON_TOKEN)(ex->token))->type == ANTLR3_TOKEN_EOF)
-		{
-			ANTLR3_FPRINTF(stderr, "-end of input-(");
-		}
-		else
-		{
-			ANTLR3_FPRINTF(stderr, "-unknown source-(");
-		}
-	}
-	else
-	{
-		ftext = ex->streamName->to8(ex->streamName);
-		ANTLR3_FPRINTF(stderr, "%s(", ftext->chars);
-	}
-
-	// Next comes the line number
-	//
-
-	ANTLR3_FPRINTF(stderr, "%d) ", recognizer->state->exception->line);
-	ANTLR3_FPRINTF(stderr, " : error %d : %s", 
-										recognizer->state->exception->type,
-					(pANTLR3_UINT8)	   (recognizer->state->exception->message));
-
-
-	// How we determine the next piece is dependent on which thing raised the
-	// error.
-	//
-	switch	(recognizer->type)
-	{
-	case	ANTLR3_TYPE_PARSER:
-
-		// Prepare the knowledge we know we have
-		//
-		parser	    = (pANTLR3_PARSER) (recognizer->super);
-		tparser	    = NULL;
-		is			= parser->tstream->istream;
-		theToken    = (pANTLR3_COMMON_TOKEN)(recognizer->state->exception->token);
-		ttext	    = theToken->toString(theToken);
-
-		ANTLR3_FPRINTF(stderr, ", at offset %d", recognizer->state->exception->charPositionInLine);
-		if  (theToken != NULL)
-		{
-			if (theToken->type == ANTLR3_TOKEN_EOF)
-			{
-				ANTLR3_FPRINTF(stderr, ", at <EOF>");
-			}
-			else
-			{
-				// Guard against null text in a token
-				//
-				ANTLR3_FPRINTF(stderr, "\n    near %s\n    ", ttext == NULL ? (pANTLR3_UINT8)"<no text for the token>" : ttext->chars);
-			}
-		}
-		break;
-
-	case	ANTLR3_TYPE_TREE_PARSER:
-
-		tparser		= (pANTLR3_TREE_PARSER) (recognizer->super);
-		parser		= NULL;
-		is			= tparser->ctnstream->tnstream->istream;
-		theBaseTree	= (pANTLR3_BASE_TREE)(recognizer->state->exception->token);
-		ttext		= theBaseTree->toStringTree(theBaseTree);
-
-		if  (theBaseTree != NULL)
-		{
-			theCommonTree	= (pANTLR3_COMMON_TREE)	    theBaseTree->super;
-
-			if	(theCommonTree != NULL)
-			{
-				theToken	= (pANTLR3_COMMON_TOKEN)    theBaseTree->getToken(theBaseTree);
-			}
-			ANTLR3_FPRINTF(stderr, ", at offset %d", theBaseTree->getCharPositionInLine(theBaseTree));
-			ANTLR3_FPRINTF(stderr, ", near %s", ttext->chars);
-		}
-		break;
-
-	default:
-
-		ANTLR3_FPRINTF(stderr, "Base recognizer function displayRecognitionError called by unknown parser type - provide override for this function\n");
-		return;
-		break;
-	}
-
-	// Although this function should generally be provided by the implementation, this one
-	// should be as helpful as possible for grammar developers and serve as an example
-	// of what you can do with each exception type. In general, when you make up your
-	// 'real' handler, you should debug the routine with all possible errors you expect
-	// which will then let you be as specific as possible about all circumstances.
-	//
-	// Note that in the general case, errors thrown by tree parsers indicate a problem
-	// with the output of the parser or with the tree grammar itself. The job of the parser
-	// is to produce a perfect (in traversal terms) syntactically correct tree, so errors
-	// at that stage should really be semantic errors that your own code determines and handles
-	// in whatever way is appropriate.
-	//
-	switch  (ex->type)
-	{
-	case	ANTLR3_UNWANTED_TOKEN_EXCEPTION:
-
-		// Indicates that the recognizer was fed a token which seesm to be
-		// spurious input. We can detect this when the token that follows
-		// this unwanted token would normally be part of the syntactically
-		// correct stream. Then we can see that the token we are looking at
-		// is just something that should not be there and throw this exception.
-		//
-		if	(tokenNames == NULL)
-		{
-			ANTLR3_FPRINTF(stderr, " : Extraneous input...");
-		}
-		else
-		{
-			if	(ex->expecting == ANTLR3_TOKEN_EOF)
-			{
-				ANTLR3_FPRINTF(stderr, " : Extraneous input - expected <EOF>\n");
-			}
-			else
-			{
-				ANTLR3_FPRINTF(stderr, " : Extraneous input - expected %s ...\n", tokenNames[ex->expecting]);
-			}
-		}
-		break;
-
-	case	ANTLR3_MISSING_TOKEN_EXCEPTION:
-
-		// Indicates that the recognizer detected that the token we just
-		// hit would be valid syntactically if preceeded by a particular 
-		// token. Perhaps a missing ';' at line end or a missing ',' in an
-		// expression list, and such like.
-		//
-		if	(tokenNames == NULL)
-		{
-			ANTLR3_FPRINTF(stderr, " : Missing token (%d)...\n", ex->expecting);
-		}
-		else
-		{
-			if	(ex->expecting == ANTLR3_TOKEN_EOF)
-			{
-				ANTLR3_FPRINTF(stderr, " : Missing <EOF>\n");
-			}
-			else
-			{
-				ANTLR3_FPRINTF(stderr, " : Missing %s \n", tokenNames[ex->expecting]);
-			}
-		}
-		break;
-
-	case	ANTLR3_RECOGNITION_EXCEPTION:
-
-		// Indicates that the recognizer received a token
-		// in the input that was not predicted. This is the basic exception type 
-		// from which all others are derived. So we assume it was a syntax error.
-		// You may get this if there are not more tokens and more are needed
-		// to complete a parse for instance.
-		//
-		ANTLR3_FPRINTF(stderr, " : syntax error...\n");    
-		break;
-
-	case    ANTLR3_MISMATCHED_TOKEN_EXCEPTION:
-
-		// We were expecting to see one thing and got another. This is the
-		// most common error if we coudl not detect a missing or unwanted token.
-		// Here you can spend your efforts to
-		// derive more useful error messages based on the expected
-		// token set and the last token and so on. The error following
-		// bitmaps do a good job of reducing the set that we were looking
-		// for down to something small. Knowing what you are parsing may be
-		// able to allow you to be even more specific about an error.
-		//
-		if	(tokenNames == NULL)
-		{
-			ANTLR3_FPRINTF(stderr, " : syntax error...\n");
-		}
-		else
-		{
-			if	(ex->expecting == ANTLR3_TOKEN_EOF)
-			{
-				ANTLR3_FPRINTF(stderr, " : expected <EOF>\n");
-			}
-			else
-			{
-				ANTLR3_FPRINTF(stderr, " : expected %s ...\n", tokenNames[ex->expecting]);
-			}
-		}
-		break;
-
-	case	ANTLR3_NO_VIABLE_ALT_EXCEPTION:
-
-		// We could not pick any alt decision from the input given
-		// so god knows what happened - however when you examine your grammar,
-		// you should. It means that at the point where the current token occurred
-		// that the DFA indicates nowhere to go from here.
-		//
-		ANTLR3_FPRINTF(stderr, " : cannot match to any predicted input...\n");
-
-		break;
-
-	case	ANTLR3_MISMATCHED_SET_EXCEPTION:
-
-		{
-			ANTLR3_UINT32	  count;
-			ANTLR3_UINT32	  bit;
-			ANTLR3_UINT32	  size;
-			ANTLR3_UINT32	  numbits;
-			pANTLR3_BITSET	  errBits;
-
-			// This means we were able to deal with one of a set of
-			// possible tokens at this point, but we did not see any
-			// member of that set.
-			//
-			ANTLR3_FPRINTF(stderr, " : unexpected input...\n  expected one of : ");
-
-			// What tokens could we have accepted at this point in the
-			// parse?
-			//
-			count   = 0;
-			errBits = antlr3BitsetLoad		(ex->expectingSet);
-			numbits = errBits->numBits		(errBits);
-			size    = errBits->size			(errBits);
-
-			if  (size > 0)
-			{
-				// However many tokens we could have dealt with here, it is usually
-				// not useful to print ALL of the set here. I arbitrarily chose 8
-				// here, but you should do whatever makes sense for you of course.
-				// No token number 0, so look for bit 1 and on.
-				//
-				for	(bit = 1; bit < numbits && count < 8 && count < size; bit++)
-				{
-					// TODO: This doesn;t look right - should be asking if the bit is set!!
-					//
-					if  (tokenNames[bit])
-					{
-						ANTLR3_FPRINTF(stderr, "%s%s", count > 0 ? ", " : "", tokenNames[bit]); 
-						count++;
-					}
-				}
-				ANTLR3_FPRINTF(stderr, "\n");
-			}
-			else
-			{
-				ANTLR3_FPRINTF(stderr, "Actually dude, we didn't seem to be expecting anything here, or at least\n");
-				ANTLR3_FPRINTF(stderr, "I could not work out what I was expecting, like so many of us these days!\n");
-			}
-		}
-		break;
-
-	case	ANTLR3_EARLY_EXIT_EXCEPTION:
-
-		// We entered a loop requiring a number of token sequences
-		// but found a token that ended that sequence earlier than
-		// we should have done.
-		//
-		ANTLR3_FPRINTF(stderr, " : missing elements...\n");
-		break;
-
-	default:
-
-		// We don't handle any other exceptions here, but you can
-		// if you wish. If we get an exception that hits this point
-		// then we are just going to report what we know about the
-		// token.
-		//
-		ANTLR3_FPRINTF(stderr, " : syntax not recognized...\n");
-		break;
-	}
-
-	// Here you have the token that was in error which if this is
-	// the standard implementation will tell you the line and offset
-	// and also record the address of the start of the line in the
-	// input stream. You could therefore print the source line and so on.
-	// Generally though, I would expect that your lexer/parser will keep
-	// its own map of lines and source pointers or whatever as there
-	// are a lot of specific things you need to know about the input
-	// to do something like that.
-	// Here is where you do it though :-).
-	//
-}
-
-/// Return how many syntax errors were detected by this recognizer
-///
-static ANTLR3_UINT32
-getNumberOfSyntaxErrors(pANTLR3_BASE_RECOGNIZER recognizer)
-{
-	return	recognizer->state->errorCount;
-}
-
-/// Recover from an error found on the input stream.  Mostly this is
-/// NoViableAlt exceptions, but could be a mismatched token that
-/// the match() routine could not recover from.
-///
-static void			
-recover			    (pANTLR3_BASE_RECOGNIZER recognizer)
-{
-    // Used to compute the follow set of tokens
-    //
-    pANTLR3_BITSET			followSet;
-    pANTLR3_PARSER			parser;
-    pANTLR3_TREE_PARSER	    tparser;
-    pANTLR3_INT_STREAM	    is;
-
-    switch	(recognizer->type)
-    {
-		case	ANTLR3_TYPE_PARSER:
-
-		parser  = (pANTLR3_PARSER) (recognizer->super);
-		tparser	= NULL;
-		is		= parser->tstream->istream;
-
-	break;
-
-    case	ANTLR3_TYPE_TREE_PARSER:
-
-		tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
-		parser	= NULL;
-		is		= tparser->ctnstream->tnstream->istream;
-
-	break;
-
-    default:
-	    
-		ANTLR3_FPRINTF(stderr, "Base recognizer function recover called by unknown parser type - provide override for this function\n");
-		return;
-
-	break;
-    }
-
-	// Are we about to repeat the same error?
-	//
-    if	(recognizer->state->lastErrorIndex == is->index(is))
-    {
-		// The last error was at the same token index point. This must be a case
-		// where LT(1) is in the recovery token set so nothing is
-		// consumed. Consume a single token so at least to prevent
-		// an infinite loop; this is a failsafe.
-		//
-		is->consume(is);
-    }
-
-    // Record error index position
-    //
-    recognizer->state->lastErrorIndex	 = is->index(is);
-    
-    // Work out the follows set for error recovery
-    //
-    followSet	= recognizer->computeErrorRecoverySet(recognizer);
-
-    // Call resync hook (for debuggers and so on)
-    //
-    recognizer->beginResync(recognizer);
-
-    // Consume tokens until we have resynced to something in the follows set
-    //
-    recognizer->consumeUntilSet(recognizer, followSet);
-
-    // End resync hook 
-    //
-    recognizer->endResync(recognizer);
-
-    // Destroy the temporary bitset we produced.
-    //
-    followSet->free(followSet);
-
-    // Reset the inError flag so we don't re-report the exception
-    //
-    recognizer->state->error	= ANTLR3_FALSE;
-    recognizer->state->failed	= ANTLR3_FALSE;
-}
-
-
-/// Attempt to recover from a single missing or extra token.
-///
-/// EXTRA TOKEN
-///
-/// LA(1) is not what we are looking for.  If LA(2) has the right token,
-/// however, then assume LA(1) is some extra spurious token.  Delete it
-/// and LA(2) as if we were doing a normal match(), which advances the
-/// input.
-///
-/// MISSING TOKEN
-///
-/// If current token is consistent with what could come after
-/// ttype then it is ok to "insert" the missing token, else throw
-/// exception For example, Input "i=(3;" is clearly missing the
-/// ')'.  When the parser returns from the nested call to expr, it
-/// will have call chain:
-///
-///    stat -> expr -> atom
-///
-/// and it will be trying to match the ')' at this point in the
-/// derivation:
-///
-///       => ID '=' '(' INT ')' ('+' atom)* ';'
-///                          ^
-/// match() will see that ';' doesn't match ')' and report a
-/// mismatched token error.  To recover, it sees that LA(1)==';'
-/// is in the set of tokens that can follow the ')' token
-/// reference in rule atom.  It can assume that you forgot the ')'.
-///
-/// The exception that was passed in, in the java implementation is
-/// sorted in the recognizer exception stack in the C version. To 'throw' it we set the
-/// error flag and rules cascade back when this is set.
-///
-static void *	
-recoverFromMismatchedToken  (pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 ttype, pANTLR3_BITSET_LIST follow)
-{
-	pANTLR3_PARSER			  parser;
-	pANTLR3_TREE_PARSER	      tparser;
-	pANTLR3_INT_STREAM	      is;
-	void					* matchedSymbol;
-
-
-
-	switch	(recognizer->type)
-	{
-	case	ANTLR3_TYPE_PARSER:
-
-		parser  = (pANTLR3_PARSER) (recognizer->super);
-		tparser	= NULL;
-		is	= parser->tstream->istream;
-
-		break;
-
-	case	ANTLR3_TYPE_TREE_PARSER:
-
-		tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
-		parser	= NULL;
-		is	= tparser->ctnstream->tnstream->istream;
-
-		break;
-
-	default:
-
-		ANTLR3_FPRINTF(stderr, "Base recognizer function recoverFromMismatchedToken called by unknown parser type - provide override for this function\n");
-		return NULL;
-
-		break;
-	}
-
-	// Create an exception if we need one
-	//
-	if	(recognizer->state->exception == NULL)
-	{
-		antlr3RecognitionExceptionNew(recognizer);
-	}
-
-	// If the next token after the one we are looking at in the input stream
-	// is what we are looking for then we remove the one we have discovered
-	// from the stream by consuming it, then consume this next one along too as
-	// if nothing had happened.
-	//
-	if	( recognizer->mismatchIsUnwantedToken(recognizer, is, ttype) == ANTLR3_TRUE)
-	{
-		recognizer->state->exception->type		= ANTLR3_UNWANTED_TOKEN_EXCEPTION;
-		recognizer->state->exception->message	= ANTLR3_UNWANTED_TOKEN_EXCEPTION_NAME;
-
-		// Call resync hook (for debuggers and so on)
-		//
-		if	(recognizer->debugger != NULL)
-		{
-			recognizer->debugger->beginResync(recognizer->debugger);
-		}
-
-		// "delete" the extra token
-		//
-		recognizer->beginResync(recognizer);
-		is->consume(is);
-		recognizer->endResync(recognizer);
-		// End resync hook 
-		//
-		if	(recognizer->debugger != NULL)
-		{
-			recognizer->debugger->endResync(recognizer->debugger);
-		}
-
-		// Print out the error after we consume so that ANTLRWorks sees the
-		// token in the exception.
-		//
-		recognizer->reportError(recognizer);
-
-		// Return the token we are actually matching
-		//
-		matchedSymbol = recognizer->getCurrentInputSymbol(recognizer, is);
-
-		// Consume the token that the rule actually expected to get as if everything
-		// was hunky dory.
-		//
-		is->consume(is);
-
-		recognizer->state->error  = ANTLR3_FALSE;	// Exception is not outstanding any more
-
-		return	matchedSymbol;
-	}
-
-	// Single token deletion (Unwanted above) did not work
-	// so we see if we can insert a token instead by calculating which
-	// token would be missing
-	//
-	if	(mismatchIsMissingToken(recognizer, is, follow))
-	{
-		// We can fake the missing token and proceed
-		//
-		matchedSymbol = recognizer->getMissingSymbol(recognizer, is, recognizer->state->exception, ttype, follow);
-		recognizer->state->exception->type		= ANTLR3_MISSING_TOKEN_EXCEPTION;
-		recognizer->state->exception->message	= ANTLR3_MISSING_TOKEN_EXCEPTION_NAME;
-		recognizer->state->exception->token		= matchedSymbol;
-		recognizer->state->exception->expecting	= ttype;
-
-		// Print out the error after we insert so that ANTLRWorks sees the
-		// token in the exception.
-		//
-		recognizer->reportError(recognizer);
-
-		recognizer->state->error  = ANTLR3_FALSE;	// Exception is not outstanding any more
-
-		return	matchedSymbol;
-	}
-
-
-	// Neither deleting nor inserting tokens allows recovery
-	// must just report the exception.
-	//
-	recognizer->state->error	    = ANTLR3_TRUE;
-	return NULL;
-}
-
-static void *
-recoverFromMismatchedSet	    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_BITSET_LIST follow)
-{
-    pANTLR3_PARSER			parser;
-    pANTLR3_TREE_PARSER	    tparser;
-    pANTLR3_INT_STREAM	    is;
-	pANTLR3_COMMON_TOKEN	matchedSymbol;
-
-    switch	(recognizer->type)
-    {
-    case	ANTLR3_TYPE_PARSER:
-
-		parser  = (pANTLR3_PARSER) (recognizer->super);
-		tparser	= NULL;
-		is	= parser->tstream->istream;
-
-	break;
-
-    case	ANTLR3_TYPE_TREE_PARSER:
-
-		tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
-		parser	= NULL;
-		is	= tparser->ctnstream->tnstream->istream;
-
-	break;
-
-    default:
-	    
-		ANTLR3_FPRINTF(stderr, "Base recognizer function recoverFromMismatchedSet called by unknown parser type - provide override for this function\n");
-		return NULL;
-
-	break;
-    }
-
-	if	(recognizer->mismatchIsMissingToken(recognizer, is, follow) == ANTLR3_TRUE)
-	{
-		// We can fake the missing token and proceed
-		//
-		matchedSymbol = recognizer->getMissingSymbol(recognizer, is, recognizer->state->exception, ANTLR3_TOKEN_INVALID, follow);
-		recognizer->state->exception->type	= ANTLR3_MISSING_TOKEN_EXCEPTION;
-		recognizer->state->exception->token	= matchedSymbol;
-
-		// Print out the error after we insert so that ANTLRWorks sees the
-		// token in the exception.
-		//
-		recognizer->reportError(recognizer);
-
-		recognizer->state->error  = ANTLR3_FALSE;	// Exception is not outstanding any more
-
-		return	matchedSymbol;
-	}
-
-    // TODO - Single token deletion like in recoverFromMismatchedToken()
-    //
-    recognizer->state->error	= ANTLR3_TRUE;
-	recognizer->state->failed	= ANTLR3_TRUE;
-	return NULL;
-}
-
-/// This code is factored out from mismatched token and mismatched set
-///  recovery.  It handles "single token insertion" error recovery for
-/// both.  No tokens are consumed to recover from insertions.  Return
-/// true if recovery was possible else return false.
-///
-static ANTLR3_BOOLEAN	
-recoverFromMismatchedElement	    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_BITSET_LIST followBits)
-{
-    pANTLR3_BITSET	    viableToksFollowingRule;
-    pANTLR3_BITSET	    follow;
-    pANTLR3_PARSER	    parser;
-    pANTLR3_TREE_PARSER	    tparser;
-    pANTLR3_INT_STREAM	    is;
-
-    switch	(recognizer->type)
-    {
-    case	ANTLR3_TYPE_PARSER:
-
-		parser  = (pANTLR3_PARSER) (recognizer->super);
-		tparser	= NULL;
-		is	= parser->tstream->istream;
-
-	break;
-
-    case	ANTLR3_TYPE_TREE_PARSER:
-
-		tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
-		parser	= NULL;
-		is	= tparser->ctnstream->tnstream->istream;
-
-	break;
-
-    default:
-	    
-		ANTLR3_FPRINTF(stderr, "Base recognizer function recover called by unknown parser type - provide override for this function\n");
-		return ANTLR3_FALSE;
-
-	break;
-    }
-
-    follow	= antlr3BitsetLoad(followBits);
-
-    if	(follow == NULL)
-    {
-		/* The follow set is NULL, which means we don't know what can come 
-		 * next, so we "hit and hope" by just signifying that we cannot
-		 * recover, which will just cause the next token to be consumed,
-		 * which might dig us out.
-		 */
-		return	ANTLR3_FALSE;
-    }
-
-    /* We have a bitmap for the follow set, hence we can compute 
-     * what can follow this grammar element reference.
-     */
-    if	(follow->isMember(follow, ANTLR3_EOR_TOKEN_TYPE) == ANTLR3_TRUE)
-    {
-		/* First we need to know which of the available tokens are viable
-		 * to follow this reference.
-		 */
-		viableToksFollowingRule	= recognizer->computeCSRuleFollow(recognizer);
-
-		/* Remove the EOR token, which we do not wish to compute with
-		 */
-		follow->remove(follow, ANTLR3_EOR_TOKEN_TYPE);
-		viableToksFollowingRule->free(viableToksFollowingRule);
-		/* We now have the computed set of what can follow the current token
-		 */
-    }
-
-    /* We can now see if the current token works with the set of tokens
-     * that could follow the current grammar reference. If it looks like it
-     * is consistent, then we can "insert" that token by not throwing
-     * an exception and assuming that we saw it. 
-     */
-    if	( follow->isMember(follow, is->_LA(is, 1)) == ANTLR3_TRUE)
-    {
-		/* report the error, but don't cause any rules to abort and stuff
-		 */
-		recognizer->reportError(recognizer);
-		if	(follow != NULL)
-		{
-			follow->free(follow);
-		}
-		recognizer->state->error			= ANTLR3_FALSE;
-		recognizer->state->failed			= ANTLR3_FALSE;
-		return ANTLR3_TRUE;	/* Success in recovery	*/
-    }
-
-    if	(follow != NULL)
-    {
-		follow->free(follow);
-    }
-
-    /* We could not find anything viable to do, so this is going to 
-     * cause an exception.
-     */
-    return  ANTLR3_FALSE;
-}
-
-/// Eat tokens from the input stream until we get one of JUST the right type
-///
-static void		
-consumeUntil	(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_UINT32 tokenType)
-{
-    ANTLR3_UINT32			ttype;
-    pANTLR3_PARSER			parser;
-    pANTLR3_TREE_PARSER	    tparser;
-    pANTLR3_INT_STREAM	    is;
-
-    switch	(recognizer->type)
-    {
-		case	ANTLR3_TYPE_PARSER:
-
-			parser  = (pANTLR3_PARSER) (recognizer->super);
-			tparser	= NULL;
-			is	= parser->tstream->istream;
-
-			break;
-
-		case	ANTLR3_TYPE_TREE_PARSER:
-
-			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
-			parser	= NULL;
-			is	= tparser->ctnstream->tnstream->istream;
-
-			break;
-
-		default:
-		    
-			ANTLR3_FPRINTF(stderr, "Base recognizer function 'consumeUntil' called by unknown parser type - provide override for this function\n");
-			return;
-
-			break;
-    }
-
-    // What do have at the moment?
-    //
-    ttype	= is->_LA(is, 1);
-
-    // Start eating tokens until we get to the one we want.
-    //
-    while   (ttype != ANTLR3_TOKEN_EOF && ttype != tokenType)
-    {
-		is->consume(is);
-		ttype	= is->_LA(is, 1);
-    }
-}
-
-/// Eat tokens from the input stream until we find one that
-/// belongs to the supplied set.
-///
-static void		
-consumeUntilSet			    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_BITSET set)
-{
-    ANTLR3_UINT32	    ttype;
-    pANTLR3_PARSER	    parser;
-    pANTLR3_TREE_PARSER	    tparser;
-    pANTLR3_INT_STREAM	    is;
-
-    switch	(recognizer->type)
-    {
-		case	ANTLR3_TYPE_PARSER:
-
-			parser  = (pANTLR3_PARSER) (recognizer->super);
-			tparser	= NULL;
-			is	= parser->tstream->istream;
-
-			break;
-
-		case	ANTLR3_TYPE_TREE_PARSER:
-
-			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
-			parser	= NULL;
-			is	= tparser->ctnstream->tnstream->istream;
-
-			break;
-
-		default:
-		    
-			ANTLR3_FPRINTF(stderr, "Base recognizer function 'consumeUntilSet' called by unknown parser type - provide override for this function\n");
-			return;
-
-			break;
-    }
-
-    // What do have at the moment?
-    //
-    ttype	= is->_LA(is, 1);
-
-    // Start eating tokens until we get to one we want.
-    //
-    while   (ttype != ANTLR3_TOKEN_EOF && set->isMember(set, ttype) == ANTLR3_FALSE)
-    {
-		is->consume(is);
-		ttype	= is->_LA(is, 1);
-    }
-}
-
-/** Return the rule invocation stack (how we got here in the parse.
- *  In the java version Ter just asks the JVM for all the information
- *  but in C we don't get this information, so I am going to do nothing 
- *  right now.
- */
-static pANTLR3_STACK	
-getRuleInvocationStack		    (pANTLR3_BASE_RECOGNIZER recognizer)
-{
-    return NULL;
-}
-
-static pANTLR3_STACK	
-getRuleInvocationStackNamed	    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 name)
-{
-    return NULL;
-}
-
-/** Convenience method for template rewrites - NYI.
- */
-static pANTLR3_HASH_TABLE	
-toStrings			    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_HASH_TABLE tokens)
-{
-    return NULL;
-}
-
-static	void ANTLR3_CDECL
-freeIntTrie    (void * trie)
-{
-    ((pANTLR3_INT_TRIE)trie)->free((pANTLR3_INT_TRIE)trie);
-}
-
-
-/** Pointer to a function to return whether the rule has parsed input starting at the supplied 
- *  start index before. If the rule has not parsed input starting from the supplied start index,
- *  then it will return ANTLR3_MEMO_RULE_UNKNOWN. If it has parsed from the suppled start point
- *  then it will return the point where it last stopped parsing after that start point.
- *
- * \remark
- * The rule memos are an ANTLR3_LIST of ANTLR3_LISTS, however if this becomes any kind of performance
- * issue (it probably won't, the hash tables are pretty quick) then we could make a special int only
- * version of the table.
- */
-static ANTLR3_MARKER	
-getRuleMemoization		    (pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_INTKEY ruleIndex, ANTLR3_MARKER ruleParseStart)
-{
-    /* The rule memos are an ANTLR3_LIST of ANTLR3_LIST.
-     */
-    pANTLR3_INT_TRIE	ruleList;
-    ANTLR3_MARKER	stopIndex;
-    pANTLR3_TRIE_ENTRY	entry;
-
-    /* See if we have a list in the ruleMemos for this rule, and if not, then create one
-     * as we will need it eventually if we are being asked for the memo here.
-     */
-    entry	= recognizer->state->ruleMemo->get(recognizer->state->ruleMemo, (ANTLR3_INTKEY)ruleIndex);
-
-    if	(entry == NULL)
-    {
-		/* Did not find it, so create a new one for it, with a bit depth based on the 
-		 * size of the input stream. We need the bit depth to incorporate the number if
-		 * bits required to represent the largest possible stop index in the input, which is the
-		 * last character. An int stream is free to return the largest 64 bit offset if it has
-		 * no idea of the size, but you should remember that this will cause the leftmost
-		 * bit match algorithm to run to 63 bits, which will be the whole time spent in the trie ;-)
-		 */
-		ruleList    = antlr3IntTrieNew(63);	/* Depth is theoretically 64 bits, but probably not ;-)	*/
-
-		if (ruleList != NULL)
-		{
-			recognizer->state->ruleMemo->add(recognizer->state->ruleMemo, (ANTLR3_INTKEY)ruleIndex, ANTLR3_HASH_TYPE_STR, 0, ANTLR3_FUNC_PTR(ruleList), freeIntTrie);
-		}
-
-		/* We cannot have a stopIndex in a trie we have just created of course
-		 */
-		return	MEMO_RULE_UNKNOWN;
-    }
-
-    ruleList	= (pANTLR3_INT_TRIE) (entry->data.ptr);
-
-    /* See if there is a stop index associated with the supplied start index.
-     */
-    stopIndex	= 0;
-
-    entry = ruleList->get(ruleList, ruleParseStart);
-    if (entry != NULL)
-    {
-		stopIndex = (ANTLR3_MARKER)(entry->data.intVal);
-    }
-
-    if	(stopIndex == 0)
-    {
-		return MEMO_RULE_UNKNOWN;
-    }
-
-    return  stopIndex;
-}
-
-/** Has this rule already parsed input at the current index in the
- *  input stream?  Return ANTLR3_TRUE if we have and ANTLR3_FALSE
- *  if we have not.
- *
- *  This method has a side-effect: if we have seen this input for
- *  this rule and successfully parsed before, then seek ahead to
- *  1 past the stop token matched for this rule last time.
- */
-static ANTLR3_BOOLEAN	
-alreadyParsedRule		    (pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_MARKER ruleIndex)
-{
-    ANTLR3_MARKER			stopIndex;
-    pANTLR3_LEXER			lexer;
-    pANTLR3_PARSER			parser;
-    pANTLR3_TREE_PARSER	    tparser;
-    pANTLR3_INT_STREAM	    is;
-
-    switch	(recognizer->type)
-    {
-		case	ANTLR3_TYPE_PARSER:
-
-			parser  = (pANTLR3_PARSER) (recognizer->super);
-			tparser	= NULL;
-			lexer	= NULL;
-			is	= parser->tstream->istream;
-
-			break;
-
-		case	ANTLR3_TYPE_TREE_PARSER:
-
-			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
-			parser	= NULL;
-			lexer	= NULL;
-			is	= tparser->ctnstream->tnstream->istream;
-
-			break;
-
-		case	ANTLR3_TYPE_LEXER:
-
-			lexer	= (pANTLR3_LEXER)   (recognizer->super);
-			parser	= NULL;
-			tparser	= NULL;
-			is	= lexer->input->istream;
-			break;
-
-		default:
-		    
-			ANTLR3_FPRINTF(stderr, "Base recognizer function 'alreadyParsedRule' called by unknown parser type - provide override for this function\n");
-			return ANTLR3_FALSE;
-
-			break;
-    }
-
-    /* See if we have a memo marker for this.
-     */
-    stopIndex	    = recognizer->getRuleMemoization(recognizer, ruleIndex, is->index(is));
-
-    if	(stopIndex  == MEMO_RULE_UNKNOWN)
-    {
-		return ANTLR3_FALSE;
-    }
-
-    if	(stopIndex == MEMO_RULE_FAILED)
-    {
-		recognizer->state->failed = ANTLR3_TRUE;
-    }
-    else
-    {
-		is->seek(is, stopIndex+1);
-    }
-
-    /* If here then the rule was executed for this input already
-     */
-    return  ANTLR3_TRUE;
-}
-
-/** Record whether or not this rule parsed the input at this position
- *  successfully.
- */
-static void		
-memoize	(pANTLR3_BASE_RECOGNIZER recognizer, ANTLR3_MARKER ruleIndex, ANTLR3_MARKER ruleParseStart)
-{
-    /* The rule memos are an ANTLR3_LIST of ANTLR3_LIST.
-     */
-    pANTLR3_INT_TRIE	    ruleList;
-    pANTLR3_TRIE_ENTRY	    entry;
-    ANTLR3_MARKER	    stopIndex;
-    pANTLR3_LEXER	    lexer;
-    pANTLR3_PARSER	    parser;
-    pANTLR3_TREE_PARSER	    tparser;
-    pANTLR3_INT_STREAM	    is;
-
-    switch	(recognizer->type)
-    {
-		case	ANTLR3_TYPE_PARSER:
-
-			parser  = (pANTLR3_PARSER) (recognizer->super);
-			tparser	= NULL;
-			is	= parser->tstream->istream;
-
-			break;
-
-		case	ANTLR3_TYPE_TREE_PARSER:
-
-			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
-			parser	= NULL;
-			is	= tparser->ctnstream->tnstream->istream;
-
-			break;
-
-		case	ANTLR3_TYPE_LEXER:
-
-			lexer	= (pANTLR3_LEXER)   (recognizer->super);
-			parser	= NULL;
-			tparser	= NULL;
-			is		= lexer->input->istream;
-			break;
-
-		default:
-		    
-			ANTLR3_FPRINTF(stderr, "Base recognizer function consumeUntilSet called by unknown parser type - provide override for this function\n");
-			return;
-
-			break;
-    }
-    
-    stopIndex	= recognizer->state->failed == ANTLR3_TRUE ? MEMO_RULE_FAILED : is->index(is) - 1;
-
-    entry	= recognizer->state->ruleMemo->get(recognizer->state->ruleMemo, (ANTLR3_INTKEY)ruleIndex);
-
-    if	(entry != NULL)
-    {
-		ruleList = (pANTLR3_INT_TRIE)(entry->data.ptr);
-
-		/* If we don't already have this entry, append it. The memoize trie does not
-		 * accept duplicates so it won't add it if already there and we just ignore the
-		 * return code as we don't care if it is there already.
-		 */
-		ruleList->add(ruleList, ruleParseStart, ANTLR3_HASH_TYPE_INT, stopIndex, NULL, NULL);
-    }
-}
-/** A syntactic predicate.  Returns true/false depending on whether
- *  the specified grammar fragment matches the current input stream.
- *  This resets the failed instance var afterwards.
- */
-static ANTLR3_BOOLEAN	
-synpred	(pANTLR3_BASE_RECOGNIZER recognizer, void * ctx, void (*predicate)(void * ctx))
-{
-    ANTLR3_MARKER   start;
-    pANTLR3_PARSER	    parser;
-    pANTLR3_TREE_PARSER	    tparser;
-    pANTLR3_INT_STREAM	    is;
-
-    switch	(recognizer->type)
-    {
-		case	ANTLR3_TYPE_PARSER:
-
-			parser  = (pANTLR3_PARSER) (recognizer->super);
-			tparser	= NULL;
-			is	= parser->tstream->istream;
-
-			break;
-
-		case	ANTLR3_TYPE_TREE_PARSER:
-
-			tparser = (pANTLR3_TREE_PARSER) (recognizer->super);
-			parser	= NULL;
-			is	= tparser->ctnstream->tnstream->istream;
-
-			break;
-
-		default:
-		    
-			ANTLR3_FPRINTF(stderr, "Base recognizer function 'synPred' called by unknown parser type - provide override for this function\n");
-			return ANTLR3_FALSE;
-
-			break;
-    }
-
-    /* Begin backtracking so we can get back to where we started after trying out
-     * the syntactic predicate.
-     */
-    start   = is->mark(is);
-    recognizer->state->backtracking++;
-
-    /* Try the syntactical predicate
-     */
-    predicate(ctx);
-
-    /* Reset
-     */
-    is->rewind(is, start);
-    recognizer->state->backtracking--;
-
-    if	(recognizer->state->failed == ANTLR3_TRUE)
-    {
-		/* Predicate failed
-		 */
-		recognizer->state->failed = ANTLR3_FALSE;
-		return	ANTLR3_FALSE;
-    }
-    else
-    {
-		/* Predicate was successful
-		 */
-		recognizer->state->failed	= ANTLR3_FALSE;
-		return	ANTLR3_TRUE;
-    }
-}
-
-static void
-reset(pANTLR3_BASE_RECOGNIZER recognizer)
-{
-    if	(recognizer->state->following != NULL)
-    {
-		recognizer->state->following->free(recognizer->state->following);
-    }
-
-	// Reset the state flags
-	//
-	recognizer->state->errorRecovery	= ANTLR3_FALSE;
-	recognizer->state->lastErrorIndex	= -1;
-	recognizer->state->failed			= ANTLR3_FALSE;
-	recognizer->state->errorCount		= 0;
-	recognizer->state->backtracking		= 0;
-	recognizer->state->following		= NULL;
-
-	if	(recognizer->state != NULL)
-	{
-		if	(recognizer->state->ruleMemo != NULL)
-		{
-			recognizer->state->ruleMemo->free(recognizer->state->ruleMemo);
-			recognizer->state->ruleMemo = antlr3IntTrieNew(15);	/* 16 bit depth is enough for 32768 rules! */
-		}
-	}
-	
-
-    // Install a new following set
-    //
-    recognizer->state->following   = antlr3StackNew(8);
-
-}
-
-// Default implementation is for parser and assumes a token stream as supplied by the runtime.
-// You MAY need override this function if the standard TOKEN_STREAM is not what you are using.
-//
-static void *				
-getCurrentInputSymbol		(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream)
-{
-	return ((pANTLR3_TOKEN_STREAM)istream->super)->_LT((pANTLR3_TOKEN_STREAM)istream->super, 1);
-}
-
-// Default implementation is for parser and assumes a token stream as supplied by the runtime.
-// You MAY need override this function if the standard COMMON_TOKEN_STREAM is not what you are using.
-//
-static void *				
-getMissingSymbol			(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM	istream, pANTLR3_EXCEPTION	e,
-									ANTLR3_UINT32 expectedTokenType, pANTLR3_BITSET_LIST follow)
-{
-	pANTLR3_TOKEN_STREAM			ts;
-	pANTLR3_COMMON_TOKEN_STREAM		cts;
-	pANTLR3_COMMON_TOKEN			token;
-	pANTLR3_COMMON_TOKEN			current;
-	pANTLR3_STRING					text;
-
-	// Dereference the standard pointers
-	//
-	ts		= (pANTLR3_TOKEN_STREAM)istream->super;
-	cts		= (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
-	
-	// Work out what to use as the current symbol to make a line and offset etc
-	// If we are at EOF, we use the token before EOF
-	//
-	current	= ts->_LT(ts, 1);
-	if	(current->getType(current) == ANTLR3_TOKEN_EOF)
-	{
-		current = ts->_LT(ts, -1);
-	}
-
-	// Create a new empty token
-	//
-	if	(recognizer->state->tokFactory == NULL)
-	{
-		// We don't yet have a token factory for making tokens
-		// we just need a fake one using the input stream of the current
-		// token.
-		//
-		recognizer->state->tokFactory = antlr3TokenFactoryNew(current->input);
-	}
-	token	= recognizer->state->tokFactory->newToken(recognizer->state->tokFactory);
-
-	// Set some of the token properties based on the current token
-	//
-	token->setLine					(token, current->getLine(current));
-	token->setCharPositionInLine	(token, current->getCharPositionInLine(current));
-	token->setChannel				(token, ANTLR3_TOKEN_DEFAULT_CHANNEL);
-	token->setType					(token, expectedTokenType);
-    token->user1                    = current->user1;
-    token->user2                    = current->user2;
-    token->user3                    = current->user3;
-    token->custom                   = current->custom;
-    token->lineStart                = current->lineStart;
-    
-	// Create the token text that shows it has been inserted
-	//
-	token->setText8(token, (pANTLR3_UINT8)"<missing ");
-	text = token->getText(token);
-
-	if	(text != NULL)
-	{
-		text->append8(text, (const char *)recognizer->state->tokenNames[expectedTokenType]);
-		text->append8(text, (const char *)">");
-	}
-	
-	// Finally return the pointer to our new token
-	//
-	return	token;
-}
-
-
-#ifdef	ANTLR3_WINDOWS
-#pragma warning( default : 4100 )
-#endif
-
-/// @}
-///
-