Move antlr. Add `make test` to Makefile.

1 files changed, 904 insertions, 0 deletions

diff --git a/antlr/libantlr3c-3.4/src/antlr3lexer.c b/antlr/libantlr3c-3.4/src/antlr3lexer.c
new file mode 100644
index 0000000..d981ab7
--- /dev/null
+++ b/antlr/libantlr3c-3.4/src/antlr3lexer.c
@@ -0,0 +1,904 @@
+/** \file
+ *
+ * Base implementation of an antlr 3 lexer.
+ *
+ * An ANTLR3 lexer implements a base recongizer, a token source and
+ * a lexer interface. It constructs a base recognizer with default
+ * functions, then overrides any of these that are parser specific (usual
+ * default implementation of base recognizer.
+ */
+
+// [The "BSD licence"]
+// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
+// http://www.temporal-wave.com
+// http://www.linkedin.com/in/jimidle
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+// 3. The name of the author may not be used to endorse or promote products
+//    derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include    <antlr3lexer.h>
+
+static void					mTokens						(pANTLR3_LEXER lexer);
+static void					setCharStream				(pANTLR3_LEXER lexer,  pANTLR3_INPUT_STREAM input);
+static void					pushCharStream				(pANTLR3_LEXER lexer,  pANTLR3_INPUT_STREAM input);
+static void					popCharStream				(pANTLR3_LEXER lexer);
+
+static void					emitNew						(pANTLR3_LEXER lexer,  pANTLR3_COMMON_TOKEN token);
+static pANTLR3_COMMON_TOKEN emit						(pANTLR3_LEXER lexer);
+static ANTLR3_BOOLEAN	    matchs						(pANTLR3_LEXER lexer, ANTLR3_UCHAR * string);
+static ANTLR3_BOOLEAN	    matchc						(pANTLR3_LEXER lexer, ANTLR3_UCHAR c);
+static ANTLR3_BOOLEAN	    matchRange					(pANTLR3_LEXER lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high);
+static void					matchAny					(pANTLR3_LEXER lexer);
+static void					recover						(pANTLR3_LEXER lexer);
+static ANTLR3_UINT32	    getLine						(pANTLR3_LEXER lexer);
+static ANTLR3_MARKER	    getCharIndex				(pANTLR3_LEXER lexer);
+static ANTLR3_UINT32	    getCharPositionInLine		(pANTLR3_LEXER lexer);
+static pANTLR3_STRING	    getText						(pANTLR3_LEXER lexer);
+static pANTLR3_COMMON_TOKEN nextToken					(pANTLR3_TOKEN_SOURCE toksource);
+
+static void					displayRecognitionError	    (pANTLR3_BASE_RECOGNIZER rec, pANTLR3_UINT8 * tokenNames);
+static void					reportError					(pANTLR3_BASE_RECOGNIZER rec);
+static void *				getCurrentInputSymbol		(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream);
+static void *				getMissingSymbol			(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM	istream, pANTLR3_EXCEPTION	e,
+															ANTLR3_UINT32 expectedTokenType, pANTLR3_BITSET_LIST follow);
+
+static void					reset						(pANTLR3_BASE_RECOGNIZER rec);
+
+static void					freeLexer					(pANTLR3_LEXER lexer);
+
+
+ANTLR3_API pANTLR3_LEXER
+antlr3LexerNew(ANTLR3_UINT32 sizeHint, pANTLR3_RECOGNIZER_SHARED_STATE state)
+{
+    pANTLR3_LEXER   lexer;
+    pANTLR3_COMMON_TOKEN	specialT;
+
+	/* Allocate memory
+	*/
+	lexer   = (pANTLR3_LEXER) ANTLR3_MALLOC(sizeof(ANTLR3_LEXER));
+
+	if	(lexer == NULL)
+	{
+		return	NULL;
+	}
+
+	/* Now we need to create the base recognizer
+	*/
+	lexer->rec	    =  antlr3BaseRecognizerNew(ANTLR3_TYPE_LEXER, sizeHint, state);
+
+	if	(lexer->rec == NULL)
+	{
+		lexer->free(lexer);
+		return	NULL;
+	}
+	lexer->rec->super  =  lexer;
+
+	lexer->rec->displayRecognitionError	    = displayRecognitionError;
+	lexer->rec->reportError					= reportError;
+	lexer->rec->reset						= reset;
+	lexer->rec->getCurrentInputSymbol		= getCurrentInputSymbol;
+	lexer->rec->getMissingSymbol			= getMissingSymbol;
+
+	/* Now install the token source interface
+	*/
+	if	(lexer->rec->state->tokSource == NULL) 
+	{
+		lexer->rec->state->tokSource	= (pANTLR3_TOKEN_SOURCE)ANTLR3_CALLOC(1, sizeof(ANTLR3_TOKEN_SOURCE));
+
+		if	(lexer->rec->state->tokSource == NULL) 
+		{
+			lexer->rec->free(lexer->rec);
+			lexer->free(lexer);
+
+			return	NULL;
+		}
+		lexer->rec->state->tokSource->super    =  lexer;
+
+		/* Install the default nextToken() method, which may be overridden
+		 * by generated code, or by anything else in fact.
+		 */
+		lexer->rec->state->tokSource->nextToken	    =  nextToken;
+		lexer->rec->state->tokSource->strFactory    = NULL;
+
+		lexer->rec->state->tokFactory				= NULL;
+	}
+
+    /* Install the lexer API
+     */
+    lexer->setCharStream			=  setCharStream;
+    lexer->mTokens					= (void (*)(void *))(mTokens);
+    lexer->setCharStream			=  setCharStream;
+    lexer->pushCharStream			=  pushCharStream;
+    lexer->popCharStream			=  popCharStream;
+    lexer->emit						=  emit;
+    lexer->emitNew					=  emitNew;
+    lexer->matchs					=  matchs;
+    lexer->matchc					=  matchc;
+    lexer->matchRange				=  matchRange;
+    lexer->matchAny					=  matchAny;
+    lexer->recover					=  recover;
+    lexer->getLine					=  getLine;
+    lexer->getCharIndex				=  getCharIndex;
+    lexer->getCharPositionInLine    =  getCharPositionInLine;
+    lexer->getText					=  getText;
+    lexer->free						=  freeLexer;
+    
+    /* Initialise the eof token
+     */
+    specialT					= &(lexer->rec->state->tokSource->eofToken);
+    antlr3SetTokenAPI	  (specialT);
+    specialT->setType	  (specialT, ANTLR3_TOKEN_EOF);
+    specialT->factoryMade		= ANTLR3_TRUE;					// Prevent things trying to free() it
+    specialT->strFactory        = NULL;
+	specialT->textState			= ANTLR3_TEXT_NONE;
+	specialT->custom			= NULL;
+	specialT->user1				= 0;
+	specialT->user2				= 0;
+	specialT->user3				= 0;
+
+	// Initialize the skip token.
+	//
+    specialT					= &(lexer->rec->state->tokSource->skipToken);
+    antlr3SetTokenAPI	  (specialT);
+    specialT->setType	  (specialT, ANTLR3_TOKEN_INVALID);
+    specialT->factoryMade		= ANTLR3_TRUE;					// Prevent things trying to free() it
+    specialT->strFactory        = NULL;
+	specialT->custom			= NULL;
+	specialT->user1				= 0;
+	specialT->user2				= 0;
+	specialT->user3				= 0;
+    return  lexer;
+}
+
+static void
+reset	(pANTLR3_BASE_RECOGNIZER rec)
+{
+    pANTLR3_LEXER   lexer;
+
+    lexer   = rec->super;
+
+    lexer->rec->state->token			    = NULL;
+    lexer->rec->state->type			    = ANTLR3_TOKEN_INVALID;
+    lexer->rec->state->channel			    = ANTLR3_TOKEN_DEFAULT_CHANNEL;
+    lexer->rec->state->tokenStartCharIndex	    = -1;
+    lexer->rec->state->tokenStartCharPositionInLine = -1;
+    lexer->rec->state->tokenStartLine		    = -1;
+
+    lexer->rec->state->text	                    = NULL;
+
+    // OK - that's all hunky dory, but we may well have had
+    // a token factory that needs a reset. Do that here
+    //
+    if  (lexer->rec->state->tokFactory != NULL)
+    {
+        lexer->rec->state->tokFactory->reset(lexer->rec->state->tokFactory);
+    }
+}
+
+///
+/// \brief
+/// Returns the next available token from the current input stream.
+/// 
+/// \param toksource
+/// Points to the implementation of a token source. The lexer is 
+/// addressed by the super structure pointer.
+/// 
+/// \returns
+/// The next token in the current input stream or the EOF token
+/// if there are no more tokens.
+/// 
+/// \remarks
+/// Write remarks for nextToken here.
+/// 
+/// \see nextToken
+///
+ANTLR3_INLINE static pANTLR3_COMMON_TOKEN
+nextTokenStr	    (pANTLR3_TOKEN_SOURCE toksource)
+{
+    pANTLR3_LEXER                   lexer;
+    pANTLR3_RECOGNIZER_SHARED_STATE state;
+    pANTLR3_INPUT_STREAM            input;
+    pANTLR3_INT_STREAM              istream;
+
+    lexer   = (pANTLR3_LEXER)(toksource->super);
+    state   = lexer->rec->state;
+    input   = lexer->input;
+    istream = input->istream;
+
+    /// Loop until we get a non skipped token or EOF
+    ///
+    for	(;;)
+    {
+        // Get rid of any previous token (token factory takes care of
+        // any de-allocation when this token is finally used up.
+        //
+        state->token		    = NULL;
+        state->error		    = ANTLR3_FALSE;	    // Start out without an exception
+        state->failed		    = ANTLR3_FALSE;
+
+        // Now call the matching rules and see if we can generate a new token
+        //
+        for	(;;)
+        {
+            // Record the start of the token in our input stream.
+            //
+            state->channel			    = ANTLR3_TOKEN_DEFAULT_CHANNEL;
+            state->tokenStartCharIndex	            = (ANTLR3_MARKER)(((pANTLR3_UINT8)input->nextChar));
+            state->tokenStartCharPositionInLine     = input->charPositionInLine;
+            state->tokenStartLine		    = input->line;
+            state->text			            = NULL;
+            state->custom                           = NULL;
+            state->user1                            = 0;
+            state->user2                            = 0;
+            state->user3                            = 0;
+
+            if  (istream->_LA(istream, 1) == ANTLR3_CHARSTREAM_EOF)
+            {
+                // Reached the end of the current stream, nothing more to do if this is
+                // the last in the stack.
+                //
+                pANTLR3_COMMON_TOKEN    teof = &(toksource->eofToken);
+
+                teof->setStartIndex (teof, lexer->getCharIndex(lexer));
+                teof->setStopIndex  (teof, lexer->getCharIndex(lexer));
+                teof->setLine	    (teof, lexer->getLine(lexer));
+                teof->factoryMade = ANTLR3_TRUE;	// This isn't really manufactured but it stops things from trying to free it
+                return  teof;
+            }
+
+            state->token		= NULL;
+            state->error		= ANTLR3_FALSE;	    // Start out without an exception
+            state->failed		= ANTLR3_FALSE;
+
+            // Call the generated lexer, see if it can get a new token together.
+            //
+            lexer->mTokens(lexer->ctx);
+
+            if  (state->error  == ANTLR3_TRUE)
+            {
+                // Recognition exception, report it and try to recover.
+                //
+                state->failed	    = ANTLR3_TRUE;
+                lexer->rec->reportError(lexer->rec);
+                lexer->recover(lexer); 
+            }
+            else
+            {
+                if (state->token == NULL)
+                {
+                    // Emit the real token, which adds it in to the token stream basically
+                    //
+                    emit(lexer);
+                }
+                else if	(state->token ==  &(toksource->skipToken))
+                {
+                    // A real token could have been generated, but "Computer say's naaaaah" and it
+                    // it is just something we need to skip altogether.
+                    //
+                    continue;
+                }
+
+                // Good token, not skipped, not EOF token
+                //
+                return  state->token;
+            }
+        }
+    }
+}
+
+/**
+ * \brief
+ * Default implementation of the nextToken() call for a lexer.
+ * 
+ * \param toksource
+ * Points to the implementation of a token source. The lexer is 
+ * addressed by the super structure pointer.
+ * 
+ * \returns
+ * The next token in the current input stream or the EOF token
+ * if there are no more tokens in any input stream in the stack.
+ * 
+ * Write detailed description for nextToken here.
+ * 
+ * \remarks
+ * Write remarks for nextToken here.
+ * 
+ * \see nextTokenStr
+ */
+static pANTLR3_COMMON_TOKEN
+nextToken	    (pANTLR3_TOKEN_SOURCE toksource)
+{
+	pANTLR3_COMMON_TOKEN tok;
+
+	// Find the next token in the current stream
+	//
+	tok = nextTokenStr(toksource);
+
+	// If we got to the EOF token then switch to the previous
+	// input stream if there were any and just return the
+	// EOF if there are none. We must check the next token
+	// in any outstanding input stream we pop into the active
+	// role to see if it was sitting at EOF after PUSHing the
+	// stream we just consumed, otherwise we will return EOF
+	// on the reinstalled input stream, when in actual fact
+	// there might be more input streams to POP before the
+	// real EOF of the whole logical inptu stream. Hence we
+	// use a while loop here until we find somethign in the stream
+	// that isn't EOF or we reach the actual end of the last input
+	// stream on the stack.
+	//
+	while	(tok->type == ANTLR3_TOKEN_EOF)
+	{
+		pANTLR3_LEXER   lexer;
+
+		lexer   = (pANTLR3_LEXER)(toksource->super);
+
+		if  (lexer->rec->state->streams != NULL && lexer->rec->state->streams->size(lexer->rec->state->streams) > 0)
+		{
+			// We have another input stream in the stack so we
+			// need to revert to it, then resume the loop to check
+			// it wasn't sitting at EOF itself.
+			//
+			lexer->popCharStream(lexer);
+			tok = nextTokenStr(toksource);
+		}
+		else
+		{
+			// There were no more streams on the input stack
+			// so this EOF is the 'real' logical EOF for
+			// the input stream. So we just exit the loop and 
+			// return the EOF we have found.
+			//
+			break;
+		}
+		
+	}
+
+	// return whatever token we have, which may be EOF
+	//
+	return  tok;
+}
+
+ANTLR3_API pANTLR3_LEXER
+antlr3LexerNewStream(ANTLR3_UINT32 sizeHint, pANTLR3_INPUT_STREAM input, pANTLR3_RECOGNIZER_SHARED_STATE state)
+{
+    pANTLR3_LEXER   lexer;
+
+    // Create a basic lexer first
+    //
+    lexer   = antlr3LexerNew(sizeHint, state);
+
+    if	(lexer != NULL) 
+    {
+		// Install the input stream and reset the lexer
+		//
+		setCharStream(lexer, input);
+    }
+
+    return  lexer;
+}
+
+static void mTokens	    (pANTLR3_LEXER lexer)
+{
+    if	(lexer)	    // Fool compiler, avoid pragmas
+    {
+		ANTLR3_FPRINTF(stderr, "lexer->mTokens(): Error: No lexer rules were added to the lexer yet!\n");
+    }
+}
+
+static void			
+reportError		    (pANTLR3_BASE_RECOGNIZER rec)
+{
+    // Indicate this recognizer had an error while processing.
+	//
+	rec->state->errorCount++;
+
+    rec->displayRecognitionError(rec, rec->state->tokenNames);
+}
+
+#ifdef	ANTLR3_WINDOWS
+#pragma warning( disable : 4100 )
+#endif
+
+/** Default lexer error handler (works for 8 bit streams only!!!)
+ */
+static void			
+displayRecognitionError	    (pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_UINT8 * tokenNames)
+{
+    pANTLR3_LEXER			lexer;
+	pANTLR3_EXCEPTION	    ex;
+	pANTLR3_STRING			ftext;
+
+    lexer   = (pANTLR3_LEXER)(recognizer->super);
+	ex		= lexer->rec->state->exception;
+
+	// See if there is a 'filename' we can use
+    //
+    if	(ex->name == NULL)
+    {
+		ANTLR3_FPRINTF(stderr, "-unknown source-(");
+    }
+    else
+    {
+		ftext = ex->streamName->to8(ex->streamName);
+		ANTLR3_FPRINTF(stderr, "%s(", ftext->chars);
+    }
+
+    ANTLR3_FPRINTF(stderr, "%d) ", recognizer->state->exception->line);
+    ANTLR3_FPRINTF(stderr, ": lexer error %d :\n\t%s at offset %d, ", 
+						ex->type,
+						(pANTLR3_UINT8)	   (ex->message),
+					    ex->charPositionInLine+1
+		    );
+	{
+		ANTLR3_INT32	width;
+
+		width	= ANTLR3_UINT32_CAST(( (pANTLR3_UINT8)(lexer->input->data) + (lexer->input->size(lexer->input) )) - (pANTLR3_UINT8)(ex->index));
+
+		if	(width >= 1)
+		{			
+			if	(isprint(ex->c))
+			{
+				ANTLR3_FPRINTF(stderr, "near '%c' :\n", ex->c);
+			}
+			else
+			{
+				ANTLR3_FPRINTF(stderr, "near char(%#02X) :\n", (ANTLR3_UINT8)(ex->c));
+			}
+			ANTLR3_FPRINTF(stderr, "\t%.*s\n", width > 20 ? 20 : width ,((pANTLR3_UINT8)ex->index));
+		}
+		else
+		{
+			ANTLR3_FPRINTF(stderr, "(end of input).\n\t This indicates a poorly specified lexer RULE\n\t or unterminated input element such as: \"STRING[\"]\n");
+			ANTLR3_FPRINTF(stderr, "\t The lexer was matching from line %d, offset %d, which\n\t ", 
+								(ANTLR3_UINT32)(lexer->rec->state->tokenStartLine),
+								(ANTLR3_UINT32)(lexer->rec->state->tokenStartCharPositionInLine)
+								);
+			width = ANTLR3_UINT32_CAST(((pANTLR3_UINT8)(lexer->input->data)+(lexer->input->size(lexer->input))) - (pANTLR3_UINT8)(lexer->rec->state->tokenStartCharIndex));
+
+			if	(width >= 1)
+			{
+				ANTLR3_FPRINTF(stderr, "looks like this:\n\t\t%.*s\n", width > 20 ? 20 : width ,(pANTLR3_UINT8)(lexer->rec->state->tokenStartCharIndex));
+			}
+			else
+			{
+				ANTLR3_FPRINTF(stderr, "is also the end of the line, so you must check your lexer rules\n");
+			}
+		}
+	}
+}
+
+static void setCharStream   (pANTLR3_LEXER lexer,  pANTLR3_INPUT_STREAM input)
+{
+    /* Install the input interface
+     */
+    lexer->input	= input;
+
+    /* We may need a token factory for the lexer; we don't destroy any existing factory
+     * until the lexer is destroyed, as people may still be using the tokens it produced.
+     * TODO: Later I will provide a dup() method for a token so that it can extract itself
+     * out of the factory. 
+     */
+    if	(lexer->rec->state->tokFactory == NULL)
+    {
+	lexer->rec->state->tokFactory	= antlr3TokenFactoryNew(input);
+    }
+    else
+    {
+	/* When the input stream is being changed on the fly, rather than
+	 * at the start of a new lexer, then we must tell the tokenFactory
+	 * which input stream to adorn the tokens with so that when they
+	 * are asked to provide their original input strings they can
+	 * do so from the correct text stream.
+	 */
+	lexer->rec->state->tokFactory->setInputStream(lexer->rec->state->tokFactory, input);
+    }
+
+    /* Propagate the string factory so that we preserve the encoding form from
+     * the input stream.
+     */
+    if	(lexer->rec->state->tokSource->strFactory == NULL)
+    {
+        lexer->rec->state->tokSource->strFactory	= input->strFactory;
+
+        // Set the newly acquired string factory up for our pre-made tokens
+        // for EOF.
+        //
+        if (lexer->rec->state->tokSource->eofToken.strFactory == NULL)
+        {
+            lexer->rec->state->tokSource->eofToken.strFactory = input->strFactory;
+        }
+    }
+
+    /* This is a lexer, install the appropriate exception creator
+     */
+    lexer->rec->exConstruct = antlr3RecognitionExceptionNew;
+
+    /* Set the current token to nothing
+     */
+    lexer->rec->state->token		= NULL;
+    lexer->rec->state->text			= NULL;
+    lexer->rec->state->tokenStartCharIndex	= -1;
+
+    /* Copy the name of the char stream to the token source
+     */
+    lexer->rec->state->tokSource->fileName = input->fileName;
+}
+
+/*!
+ * \brief
+ * Change to a new input stream, remembering the old one.
+ * 
+ * \param lexer
+ * Pointer to the lexer instance to switch input streams for.
+ * 
+ * \param input
+ * New input stream to install as the current one.
+ * 
+ * Switches the current character input stream to 
+ * a new one, saving the old one, which we will revert to at the end of this 
+ * new one.
+ */
+static void
+pushCharStream  (pANTLR3_LEXER lexer,  pANTLR3_INPUT_STREAM input)
+{
+	// Do we need a new input stream stack?
+	//
+	if	(lexer->rec->state->streams == NULL)
+	{
+		// This is the first call to stack a new
+		// stream and so we must create the stack first.
+		//
+		lexer->rec->state->streams = antlr3StackNew(0);
+
+		if  (lexer->rec->state->streams == NULL)
+		{
+			// Could not do this, we just fail to push it.
+			// TODO: Consider if this is what we want to do, but then
+			//       any programmer can override this method to do something else.
+			return;
+		}
+	}
+
+	// We have a stack, so we can save the current input stream
+	// into it.
+	//
+	lexer->input->istream->mark(lexer->input->istream);
+	lexer->rec->state->streams->push(lexer->rec->state->streams, lexer->input, NULL);
+
+	// And now we can install this new one
+	//
+	lexer->setCharStream(lexer, input);
+}
+
+/*!
+ * \brief
+ * Stops using the current input stream and reverts to any prior
+ * input stream on the stack.
+ * 
+ * \param lexer
+ * Description of parameter lexer.
+ * 
+ * Pointer to a function that abandons the current input stream, whether it
+ * is empty or not and reverts to the previous stacked input stream.
+ *
+ * \remark
+ * The function fails silently if there are no prior input streams.
+ */
+static void
+popCharStream   (pANTLR3_LEXER lexer)
+{
+    pANTLR3_INPUT_STREAM input;
+
+    // If we do not have a stream stack or we are already at the
+    // stack bottom, then do nothing.
+    //
+    if	(lexer->rec->state->streams != NULL && lexer->rec->state->streams->size(lexer->rec->state->streams) > 0)
+    {
+	// We just leave the current stream to its fate, we do not close
+	// it or anything as we do not know what the programmer intended
+	// for it. This method can always be overridden of course.
+	// So just find out what was currently saved on the stack and use
+	// that now, then pop it from the stack.
+	//
+	input	= (pANTLR3_INPUT_STREAM)(lexer->rec->state->streams->top);
+	lexer->rec->state->streams->pop(lexer->rec->state->streams);
+
+	// Now install the stream as the current one.
+	//
+	lexer->setCharStream(lexer, input);
+	lexer->input->istream->rewindLast(lexer->input->istream);
+    }
+    return;
+}
+
+static void emitNew	    (pANTLR3_LEXER lexer,  pANTLR3_COMMON_TOKEN token)
+{
+    lexer->rec->state->token    = token;	/* Voila!   */
+}
+
+static pANTLR3_COMMON_TOKEN
+emit	    (pANTLR3_LEXER lexer)
+{
+    pANTLR3_COMMON_TOKEN	token;
+
+    /* We could check pointers to token factories and so on, but
+    * we are in code that we want to run as fast as possible
+    * so we are not checking any errors. So make sure you have installed an input stream before
+    * trying to emit a new token.
+    */
+    token   = lexer->rec->state->tokFactory->newToken(lexer->rec->state->tokFactory);
+
+    /* Install the supplied information, and some other bits we already know
+    * get added automatically, such as the input stream it is associated with
+    * (though it can all be overridden of course)
+    */
+    token->type		    = lexer->rec->state->type;
+    token->channel	    = lexer->rec->state->channel;
+    token->start	    = lexer->rec->state->tokenStartCharIndex;
+    token->stop		    = lexer->getCharIndex(lexer) - 1;
+    token->line		    = lexer->rec->state->tokenStartLine;
+    token->charPosition	= lexer->rec->state->tokenStartCharPositionInLine;
+
+    if	(lexer->rec->state->text != NULL)
+    {
+        token->textState	    = ANTLR3_TEXT_STRING;
+        token->tokText.text	    = lexer->rec->state->text;
+    }
+    else
+    {
+        token->textState	= ANTLR3_TEXT_NONE;
+    }
+    token->lineStart	= lexer->input->currentLine;
+    token->user1	= lexer->rec->state->user1;
+    token->user2	= lexer->rec->state->user2;
+    token->user3	= lexer->rec->state->user3;
+    token->custom	= lexer->rec->state->custom;
+
+    lexer->rec->state->token	    = token;
+
+    return  token;
+}
+
+/**
+ * Free the resources allocated by a lexer
+ */
+static void 
+freeLexer    (pANTLR3_LEXER lexer)
+{
+	// This may have ben a delegate or delegator lexer, in which case the
+	// state may already have been freed (and set to NULL therefore)
+	// so we ignore the state if we don't have it.
+	//
+	if	(lexer->rec->state != NULL)
+	{
+		if	(lexer->rec->state->streams != NULL)
+		{
+			lexer->rec->state->streams->free(lexer->rec->state->streams);
+		}
+		if	(lexer->rec->state->tokFactory != NULL)
+		{
+			lexer->rec->state->tokFactory->close(lexer->rec->state->tokFactory);
+			lexer->rec->state->tokFactory = NULL;
+		}
+		if	(lexer->rec->state->tokSource != NULL)
+		{
+			ANTLR3_FREE(lexer->rec->state->tokSource);
+			lexer->rec->state->tokSource = NULL;
+		}
+	}
+	if	(lexer->rec != NULL)
+	{
+		lexer->rec->free(lexer->rec);
+		lexer->rec = NULL;
+	}
+	ANTLR3_FREE(lexer);
+}
+
+/** Implementation of matchs for the lexer, overrides any
+ *  base implementation in the base recognizer. 
+ *
+ *  \remark
+ *  Note that the generated code lays down arrays of ints for constant
+ *  strings so that they are int UTF32 form!
+ */
+static ANTLR3_BOOLEAN
+matchs(pANTLR3_LEXER lexer, ANTLR3_UCHAR * string)
+{
+	while   (*string != ANTLR3_STRING_TERMINATOR)
+	{
+		if  (lexer->input->istream->_LA(lexer->input->istream, 1) != (*string))
+		{
+			if	(lexer->rec->state->backtracking > 0)
+			{
+				lexer->rec->state->failed = ANTLR3_TRUE;
+				return ANTLR3_FALSE;
+			}
+
+			lexer->rec->exConstruct(lexer->rec);
+			lexer->rec->state->failed	 = ANTLR3_TRUE;
+
+			/* TODO: Implement exception creation more fully perhaps
+			 */
+			lexer->recover(lexer);
+			return  ANTLR3_FALSE;
+		}
+
+		/* Matched correctly, do consume it
+		 */
+		lexer->input->istream->consume(lexer->input->istream);
+		string++;
+
+		/* Reset any failed indicator
+		 */
+		lexer->rec->state->failed = ANTLR3_FALSE;
+	}
+
+
+	return  ANTLR3_TRUE;
+}
+
+/** Implementation of matchc for the lexer, overrides any
+ *  base implementation in the base recognizer. 
+ *
+ *  \remark
+ *  Note that the generated code lays down arrays of ints for constant
+ *  strings so that they are int UTF32 form!
+ */
+static ANTLR3_BOOLEAN
+matchc(pANTLR3_LEXER lexer, ANTLR3_UCHAR c)
+{
+	if	(lexer->input->istream->_LA(lexer->input->istream, 1) == c)
+	{
+		/* Matched correctly, do consume it
+		 */
+		lexer->input->istream->consume(lexer->input->istream);
+
+		/* Reset any failed indicator
+		 */
+		lexer->rec->state->failed = ANTLR3_FALSE;
+
+		return	ANTLR3_TRUE;
+	}
+
+	/* Failed to match, exception and recovery time.
+	 */
+	if	(lexer->rec->state->backtracking > 0)
+	{
+		lexer->rec->state->failed  = ANTLR3_TRUE;
+		return	ANTLR3_FALSE;
+	}
+
+	lexer->rec->exConstruct(lexer->rec);
+
+	/* TODO: Implement exception creation more fully perhaps
+	 */
+	lexer->recover(lexer);
+
+	return  ANTLR3_FALSE;
+}
+
+/** Implementation of match range for the lexer, overrides any
+ *  base implementation in the base recognizer. 
+ *
+ *  \remark
+ *  Note that the generated code lays down arrays of ints for constant
+ *  strings so that they are int UTF32 form!
+ */
+static ANTLR3_BOOLEAN
+matchRange(pANTLR3_LEXER lexer, ANTLR3_UCHAR low, ANTLR3_UCHAR high)
+{
+    ANTLR3_UCHAR    c;
+
+    /* What is in the stream at the moment?
+     */
+    c	= lexer->input->istream->_LA(lexer->input->istream, 1);
+    if	( c >= low && c <= high)
+    {
+	/* Matched correctly, consume it
+	 */
+	lexer->input->istream->consume(lexer->input->istream);
+
+	/* Reset any failed indicator
+	 */
+	lexer->rec->state->failed = ANTLR3_FALSE;
+
+	return	ANTLR3_TRUE;
+    }
+    
+    /* Failed to match, execption and recovery time.
+     */
+
+    if	(lexer->rec->state->backtracking > 0)
+    {
+	lexer->rec->state->failed  = ANTLR3_TRUE;
+	return	ANTLR3_FALSE;
+    }
+
+    lexer->rec->exConstruct(lexer->rec);
+
+    /* TODO: Implement exception creation more fully
+     */
+    lexer->recover(lexer);
+
+    return  ANTLR3_FALSE;
+}
+
+static void
+matchAny	    (pANTLR3_LEXER lexer)
+{
+    lexer->input->istream->consume(lexer->input->istream);
+}
+
+static void
+recover	    (pANTLR3_LEXER lexer)
+{
+    lexer->input->istream->consume(lexer->input->istream);
+}
+
+static ANTLR3_UINT32
+getLine	    (pANTLR3_LEXER lexer)
+{
+    return  lexer->input->getLine(lexer->input);
+}
+
+static ANTLR3_UINT32
+getCharPositionInLine	(pANTLR3_LEXER lexer)
+{
+    return  lexer->input->charPositionInLine;
+}
+
+static ANTLR3_MARKER	getCharIndex	    (pANTLR3_LEXER lexer)
+{
+    return lexer->input->istream->index(lexer->input->istream);
+}
+
+static pANTLR3_STRING
+getText	    (pANTLR3_LEXER lexer)
+{
+	if (lexer->rec->state->text)
+	{
+		return	lexer->rec->state->text;
+
+	}
+	return  lexer->input->substr(
+									lexer->input, 
+									lexer->rec->state->tokenStartCharIndex,
+									lexer->getCharIndex(lexer) - lexer->input->charByteSize
+							);
+
+}
+
+static void *				
+getCurrentInputSymbol		(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM istream)
+{
+	return NULL;
+}
+
+static void *				
+getMissingSymbol			(pANTLR3_BASE_RECOGNIZER recognizer, pANTLR3_INT_STREAM	istream, pANTLR3_EXCEPTION	e,
+									ANTLR3_UINT32 expectedTokenType, pANTLR3_BITSET_LIST follow)
+{
+	return NULL;
+}