Move antlr. Add `make test` to Makefile.

1 files changed, 0 insertions, 1091 deletions

diff --git a/impl/antlr/libantlr3c-3.4/src/antlr3tokenstream.c b/impl/antlr/libantlr3c-3.4/src/antlr3tokenstream.c
deleted file mode 100644
index ec06f15..0000000
--- a/impl/antlr/libantlr3c-3.4/src/antlr3tokenstream.c
+++ /dev/null
@@ -1,1091 +0,0 @@
-/// \file 
-/// Default implementation of CommonTokenStream
-///
-
-// [The "BSD licence"]
-// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
-// http://www.temporal-wave.com
-// http://www.linkedin.com/in/jimidle
-//
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without
-// modification, are permitted provided that the following conditions
-// are met:
-// 1. Redistributions of source code must retain the above copyright
-//    notice, this list of conditions and the following disclaimer.
-// 2. Redistributions in binary form must reproduce the above copyright
-//    notice, this list of conditions and the following disclaimer in the
-//    documentation and/or other materials provided with the distribution.
-// 3. The name of the author may not be used to endorse or promote products
-//    derived from this software without specific prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
-// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
-// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
-// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
-// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
-// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-#include    <antlr3tokenstream.h>
-
-#ifdef	ANTLR3_WINDOWS
-#pragma warning( disable : 4100 )
-#endif
-
-// COMMON_TOKEN_STREAM API
-//
-static void		setTokenTypeChannel     (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);
-static void		discardTokenType        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype);
-static void		discardOffChannel       (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard);
-static pANTLR3_VECTOR	getTokens	        (pANTLR3_COMMON_TOKEN_STREAM cts);
-static pANTLR3_LIST	getTokenRange	        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
-static pANTLR3_LIST	getTokensSet	        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);
-static pANTLR3_LIST	getTokensList	        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);
-static pANTLR3_LIST	getTokensType	        (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);
-static void             reset                   (pANTLR3_COMMON_TOKEN_STREAM cts);
-
-// TOKEN_STREAM API 
-//
-static pANTLR3_COMMON_TOKEN tokLT	        (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
-static pANTLR3_COMMON_TOKEN dbgTokLT		(pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
-static pANTLR3_COMMON_TOKEN get			(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);
-static pANTLR3_TOKEN_SOURCE getTokenSource	(pANTLR3_TOKEN_STREAM ts);
-static void		    setTokenSource	(pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);
-static pANTLR3_STRING	    toString		(pANTLR3_TOKEN_STREAM ts);
-static pANTLR3_STRING	    toStringSS		(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
-static pANTLR3_STRING	    toStringTT		(pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);
-static void		    setDebugListener	(pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);
-
-// INT STREAM API
-//
-static void		    consume			(pANTLR3_INT_STREAM is);
-static void		    dbgConsume			(pANTLR3_INT_STREAM is);
-static ANTLR3_UINT32	    _LA				(pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
-static ANTLR3_UINT32	    dbgLA			(pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
-static ANTLR3_MARKER	    mark			(pANTLR3_INT_STREAM is);
-static ANTLR3_MARKER	    dbgMark			(pANTLR3_INT_STREAM is);
-static void		    release			(pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);
-static ANTLR3_UINT32	    size			(pANTLR3_INT_STREAM is);
-static ANTLR3_MARKER	    tindex			(pANTLR3_INT_STREAM is);
-static void		    rewindStream		(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
-static void		    dbgRewindStream		(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
-static void		    rewindLast			(pANTLR3_INT_STREAM is);
-static void		    dbgRewindLast		(pANTLR3_INT_STREAM is);
-static void		    seek			(pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
-static void		    dbgSeek			(pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
-static pANTLR3_STRING	    getSourceName		(pANTLR3_INT_STREAM is);
-static void		    antlr3TokenStreamFree	(pANTLR3_TOKEN_STREAM	    stream);
-static void		    antlr3CTSFree		(pANTLR3_COMMON_TOKEN_STREAM    stream);
-
-// Helpers
-//
-static void		    fillBuffer			(pANTLR3_COMMON_TOKEN_STREAM tokenStream);
-static ANTLR3_UINT32	    skipOffTokenChannels	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
-static ANTLR3_UINT32	    skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
-static pANTLR3_COMMON_TOKEN LB				(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
-
-ANTLR3_API pANTLR3_TOKEN_STREAM
-antlr3TokenStreamNew()
-{
-    pANTLR3_TOKEN_STREAM stream;
-
-    // Memory for the interface structure
-    //
-    stream  = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));
-
-    if	(stream == NULL)
-    {
-		return	NULL;
-    }
-
-    // Install basic API 
-    //
-    stream->free    =  antlr3TokenStreamFree;
-
-    
-    return stream;
-}
-
-static void
-antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)
-{   
-    ANTLR3_FREE(stream);
-}
-
-static void		    
-antlr3CTSFree	    (pANTLR3_COMMON_TOKEN_STREAM stream)
-{
-	// We only free up our subordinate interfaces if they belong
-	// to us, otherwise we let whoever owns them deal with them.
-	//
-	if	(stream->tstream->super == stream)
-	{
-		if	(stream->tstream->istream->super == stream->tstream)
-		{
-			stream->tstream->istream->free(stream->tstream->istream);
-			stream->tstream->istream = NULL;
-		}
-		stream->tstream->free(stream->tstream);
-	}
-
-	// Now we free our own resources
-	//
-	if	(stream->tokens != NULL)
-	{
-		stream->tokens->free(stream->tokens);
-		stream->tokens	= NULL;
-	}
-	if	(stream->discardSet != NULL)
-	{
-		stream->discardSet->free(stream->discardSet);
-		stream->discardSet  = NULL;
-	}
-	if	(stream->channelOverrides != NULL)
-	{
-		stream->channelOverrides->free(stream->channelOverrides);
-		stream->channelOverrides = NULL;
-	}
-
-	// Free our memory now
-	//
-	ANTLR3_FREE(stream);
-}
-
-// Reset a token stream so it can be used again and can reuse it's
-// resources.
-//
-static void             
-reset   (pANTLR3_COMMON_TOKEN_STREAM cts)
-{
-
-    // Free any resources that ar most like specifc to the
-    // run we just did.
-    //
-    if	(cts->discardSet != NULL)
-    {
-        cts->discardSet->free(cts->discardSet);
-        cts->discardSet  = NULL;
-    }
-    if	(cts->channelOverrides != NULL)
-    {
-        cts->channelOverrides->free(cts->channelOverrides);
-        cts->channelOverrides = NULL;
-    }
-
-    // Now, if there were any existing tokens in the stream,
-    // then we just reset the vector count so that it starts
-    // again. We must traverse the entries unfortunately as
-    // there may be free pointers for custom token types and
-    // so on. However that is just a quick NULL check on the
-    // vector entries.
-    //
-    if	(cts->tokens != NULL)
-    {
-        cts->tokens->clear(cts->tokens);
-    }
-    else
-    {
-        /* Install the token tracking tables
-         */
-        cts->tokens  = antlr3VectorNew(0);
-    }
-
-    // Reset to defaults
-    //
-    cts->discardOffChannel  = ANTLR3_FALSE;
-    cts->channel            = ANTLR3_TOKEN_DEFAULT_CHANNEL;
-    cts->p	            = -1;
-}
-
-ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
-antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)
-{
-    pANTLR3_COMMON_TOKEN_STREAM	stream;
-
-	// Create a standard token stream
-	//
-	stream = antlr3CommonTokenStreamSourceNew(hint, source);
-
-	// Install the debugger object
-	//
-	stream->tstream->debugger = debugger;
-
-	// Override standard token stream methods with debugging versions
-	//
-	stream->tstream->initialStreamState	= ANTLR3_FALSE;
-
-	stream->tstream->_LT				= dbgTokLT;
-
-	stream->tstream->istream->consume		= dbgConsume;
-	stream->tstream->istream->_LA			= dbgLA;
-	stream->tstream->istream->mark			= dbgMark;
-	stream->tstream->istream->rewind		= dbgRewindStream;
-	stream->tstream->istream->rewindLast	= dbgRewindLast;
-	stream->tstream->istream->seek			= dbgSeek;
-
-	return stream;
-}
-
-ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
-antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)
-{
-    pANTLR3_COMMON_TOKEN_STREAM	stream;
-
-    stream = antlr3CommonTokenStreamNew(hint);
-
-    stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
-    
-    stream->channelOverrides	= NULL;
-    stream->discardSet		= NULL;
-    stream->discardOffChannel	= ANTLR3_FALSE;
-
-    stream->tstream->setTokenSource(stream->tstream, source);
-
-    stream->free		=  antlr3CTSFree;
-    return  stream;
-}
-
-ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
-antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)
-{
-    pANTLR3_COMMON_TOKEN_STREAM stream;
-
-    /* Memory for the interface structure
-     */
-    stream  = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));
-
-    if	(stream == NULL)
-    {
-	return	NULL;
-    }
-
-    /* Create space for the token stream interface
-     */
-    stream->tstream	    = antlr3TokenStreamNew();
-    stream->tstream->super  =  stream;
-
-    /* Create space for the INT_STREAM interfacce
-     */
-    stream->tstream->istream		    =  antlr3IntStreamNew();
-    stream->tstream->istream->super	    =  (stream->tstream);
-    stream->tstream->istream->type	    = ANTLR3_TOKENSTREAM;
-
-    /* Install the token tracking tables
-     */
-    stream->tokens  = antlr3VectorNew(0);
-
-    /* Defaults
-     */
-    stream->p	    = -1;
-
-    /* Install the common token stream API
-     */
-    stream->setTokenTypeChannel	    = setTokenTypeChannel;
-    stream->discardTokenType	    = discardTokenType;
-    stream->discardOffChannelToks   = discardOffChannel;
-    stream->getTokens		    = getTokens;
-    stream->getTokenRange	    = getTokenRange;
-    stream->getTokensSet	    = getTokensSet;
-    stream->getTokensList	    = getTokensList;
-    stream->getTokensType	    = getTokensType;
-    stream->reset                   = reset;
-
-    /* Install the token stream API
-     */
-    stream->tstream->_LT			=  tokLT;
-    stream->tstream->get			=  get;
-    stream->tstream->getTokenSource	        =  getTokenSource;
-    stream->tstream->setTokenSource	        =  setTokenSource;
-    stream->tstream->toString		        =  toString;
-    stream->tstream->toStringSS		        =  toStringSS;
-    stream->tstream->toStringTT		        =  toStringTT;
-    stream->tstream->setDebugListener           =  setDebugListener;
-
-    /* Install INT_STREAM interface
-     */
-    stream->tstream->istream->_LA	=  _LA;
-    stream->tstream->istream->mark	=  mark;
-    stream->tstream->istream->release	=  release;
-    stream->tstream->istream->size	=  size;
-    stream->tstream->istream->index	=  tindex;
-    stream->tstream->istream->rewind	=  rewindStream;
-    stream->tstream->istream->rewindLast=  rewindLast;
-    stream->tstream->istream->seek	=  seek;
-    stream->tstream->istream->consume	=  consume;
-    stream->tstream->istream->getSourceName = getSourceName;
-
-    return  stream;
-}
-
-// Install a debug listener adn switch to debug mode methods
-//
-static void					
-setDebugListener	(pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)
-{
-		// Install the debugger object
-	//
-	ts->debugger = debugger;
-
-	// Override standard token stream methods with debugging versions
-	//
-	ts->initialStreamState	= ANTLR3_FALSE;
-
-	ts->_LT				= dbgTokLT;
-
-	ts->istream->consume		= dbgConsume;
-	ts->istream->_LA			= dbgLA;
-	ts->istream->mark			= dbgMark;
-	ts->istream->rewind			= dbgRewindStream;
-	ts->istream->rewindLast		= dbgRewindLast;
-	ts->istream->seek			= dbgSeek;
-}
-
-/** Get the ith token from the current position 1..n where k=1 is the
-*  first symbol of lookahead.
-*/
-static pANTLR3_COMMON_TOKEN 
-tokLT  (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
-{
-	ANTLR3_INT32    i;
-	ANTLR3_INT32    n;
-	pANTLR3_COMMON_TOKEN_STREAM cts;
-
-	cts	    = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
-
-        if	(k < 0)
-	{
-		return LB(cts, -k);
-	}
-
-	if	(cts->p == -1)
-	{
-		fillBuffer(cts);
-	}
-
-        // Here we used to check for k == 0 and return 0, but this seems
-        // a superfluous check to me. LT(k=0) is therefore just undefined
-        // and we won't waste the clock cycles on the check
-        //
-
-	if	((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)
-	{
-		pANTLR3_COMMON_TOKEN    teof = &(ts->tokenSource->eofToken);
-
-		teof->setStartIndex (teof, ts->istream->index	    (ts->istream));
-		teof->setStopIndex  (teof, ts->istream->index	    (ts->istream));
-		return  teof;
-	}
-
-	i	= cts->p;
-	n	= 1;
-
-	/* Need to find k good tokens, skipping ones that are off channel
-	*/
-	while   ( n < k)
-	{
-		/* Skip off-channel tokens */
-		i = skipOffTokenChannels(cts, i+1); /* leave p on valid token    */
-		n++;
-	}
-	if	( (ANTLR3_UINT32) i >= ts->istream->cachedSize)
-	{
-		pANTLR3_COMMON_TOKEN    teof = &(ts->tokenSource->eofToken);
-
-		teof->setStartIndex (teof, ts->istream->index(ts->istream));
-		teof->setStopIndex  (teof, ts->istream->index(ts->istream));
-		return  teof;
-	}
-
-	// Here the token must be in the input vector. Rather then incur
-	// function call penalty, we just return the pointer directly
-	// from the vector
-	//
-	return  (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
-	//return  (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);
-}
-
-/// Debug only method to flag consumption of initial off-channel
-/// tokens in the input stream
-///
-static void
-consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)
-{
-	ANTLR3_MARKER	first;
-	ANTLR3_INT32	i;
-	pANTLR3_TOKEN_STREAM	ts;
-
-	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
-	first	= is->index(is);
-
-	for	(i=0; i<first; i++)
-	{
-		ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i));
-	}
-
-	ts->initialStreamState = ANTLR3_FALSE;
-
-}
-
-/// As per the normal tokLT but sends information to the debugger
-///
-static pANTLR3_COMMON_TOKEN 
-dbgTokLT  (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
-{
-	if	(ts->initialStreamState == ANTLR3_TRUE)
-	{
-		consumeInitialHiddenTokens(ts->istream);
-	}
-	return tokLT(ts, k);
-}
-
-#ifdef	ANTLR3_WINDOWS
-	/* When fully optimized VC7 complains about non reachable code.
-	 * Not yet sure if this is an optimizer bug, or a bug in the flow analysis
-	 */
-#pragma warning( disable : 4702 )
-#endif
-
-static pANTLR3_COMMON_TOKEN
-LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k)
-{
-    ANTLR3_INT32 i;
-    ANTLR3_INT32 n;
-
-    if (cts->p == -1)
-    {
-        fillBuffer(cts);
-    }
-    if (k == 0)
-    {
-        return NULL;
-    }
-    if ((cts->p - k) < 0)
-    {
-        return NULL;
-    }
-
-    i = cts->p;
-    n = 1;
-
-    /* Need to find k good tokens, going backwards, skipping ones that are off channel
-     */
-    while (n <= (ANTLR3_INT32) k)
-    {
-        /* Skip off-channel tokens
-         */
-
-        i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token    */
-        n++;
-    }
-    if (i < 0)
-    {
-        return NULL;
-    }
-	// Here the token must be in the input vector. Rather then incut
-	// function call penalty, we jsut return the pointer directly
-	// from the vector
-	//
-	return  (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
-}
-
-static pANTLR3_COMMON_TOKEN 
-get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)
-{
-    pANTLR3_COMMON_TOKEN_STREAM cts;
-
-    cts	    = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
-
-    return  (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i));  /* Token index is zero based but vectors are 1 based */
-}
-
-static pANTLR3_TOKEN_SOURCE 
-getTokenSource	(pANTLR3_TOKEN_STREAM ts)
-{
-    return  ts->tokenSource;
-}
-
-static void
-setTokenSource	(   pANTLR3_TOKEN_STREAM ts,
-		    pANTLR3_TOKEN_SOURCE tokenSource)
-{
-    ts->tokenSource	= tokenSource;
-}
-
-static pANTLR3_STRING	    
-toString    (pANTLR3_TOKEN_STREAM ts)
-{
-    pANTLR3_COMMON_TOKEN_STREAM cts;
-
-    cts	    = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
-
-    if	(cts->p == -1)
-    {
-	fillBuffer(cts);
-    }
-
-    return  ts->toStringSS(ts, 0, ts->istream->size(ts->istream));
-}
-
-static pANTLR3_STRING
-toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
-{
-    pANTLR3_STRING string;
-    pANTLR3_TOKEN_SOURCE tsource;
-    pANTLR3_COMMON_TOKEN tok;
-    ANTLR3_UINT32 i;
-    pANTLR3_COMMON_TOKEN_STREAM cts;
-
-    cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
-
-    if (cts->p == -1)
-    {
-        fillBuffer(cts);
-    }
-    if (stop >= ts->istream->size(ts->istream))
-    {
-        stop = ts->istream->size(ts->istream) - 1;
-    }
-
-    /* Who is giving us these tokens?
-     */
-    tsource = ts->getTokenSource(ts);
-
-    if (tsource != NULL && cts->tokens != NULL)
-    {
-        /* Finally, let's get a string
-         */
-        string = tsource->strFactory->newRaw(tsource->strFactory);
-
-        for (i = start; i <= stop; i++)
-        {
-            tok = ts->get(ts, i);
-            if (tok != NULL)
-            {
-                string->appendS(string, tok->getText(tok));
-            }
-        }
-
-        return string;
-    }
-    return NULL;
-
-}
-
-static pANTLR3_STRING	    
-toStringTT  (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
-{
-	if	(start != NULL && stop != NULL)
-	{
-		return	ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));
-	}
-	else
-	{
-		return	NULL;
-	}
-}
-
-/** Move the input pointer to the next incoming token.  The stream
- *  must become active with LT(1) available.  consume() simply
- *  moves the input pointer so that LT(1) points at the next
- *  input symbol. Consume at least one token.
- *
- *  Walk past any token not on the channel the parser is listening to.
- */
-static void		    
-consume	(pANTLR3_INT_STREAM is)
-{
-	pANTLR3_COMMON_TOKEN_STREAM cts;
-	pANTLR3_TOKEN_STREAM	ts;
-
-	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
-	cts	    = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
-
-        if	((ANTLR3_UINT32)cts->p < cts->tokens->count)
-	{
-		cts->p++;
-		cts->p	= skipOffTokenChannels(cts, cts->p);
-	}
-}
-
-
-/// As per ordinary consume but notifies the debugger about hidden
-/// tokens and so on.
-///
-static void
-dbgConsume	(pANTLR3_INT_STREAM is)
-{
-	pANTLR3_TOKEN_STREAM	ts;
-	ANTLR3_MARKER			a;
-	ANTLR3_MARKER			b;
-	pANTLR3_COMMON_TOKEN	t;
-
-	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
-
-	if	(ts->initialStreamState == ANTLR3_TRUE)
-	{
-		consumeInitialHiddenTokens(is);
-	}
-	
-	a = is->index(is);		// Where are we right now?
-	t = ts->_LT(ts, 1);		// Current token from stream
-
-	consume(is);			// Standard consumer
-
-	b = is->index(is);		// Where are we after consuming 1 on channel token?
-
-	ts->debugger->consumeToken(ts->debugger, t);	// Tell the debugger that we consumed the first token
-
-	if	(b>a+1)
-	{
-		// The standard consume caused the index to advance by more than 1,
-		// which can only happen if it skipped some off-channel tokens.
-		// we need to tell the debugger about those tokens.
-		//
-		ANTLR3_MARKER	i;
-
-		for	(i = a+1; i<b; i++)
-		{
-			ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i));
-		}
-
-	}
-}
-
-/** A simple filter mechanism whereby you can tell this token stream
- *  to force all tokens of type ttype to be on channel.  For example,
- *  when interpreting, we cannot execute actions so we need to tell
- *  the stream to force all WS and NEWLINE to be a different, ignored,
- *  channel.
- */
-static void		    
-setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)
-{
-    if	(tokenStream->channelOverrides == NULL)
-    {
-	tokenStream->channelOverrides	= antlr3ListNew(10);
-    }
-
-    /* We add one to the channel so we can distinguish NULL as being no entry in the
-     * table for a particular token type.
-     */
-    tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL);
-}
-
-static void		    
-discardTokenType    (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype)
-{
-    if	(tokenStream->discardSet == NULL)
-    {
-	tokenStream->discardSet	= antlr3ListNew(31);
-    }
-
-    /* We add one to the channel so we can distinguish NULL as being no entry in the
-     * table for a particular token type. We could use bitsets for this I suppose too.
-     */
-    tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL);
-}
-
-static void		    
-discardOffChannel   (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard)
-{
-    tokenStream->discardOffChannel  = discard;
-}
-
-static pANTLR3_VECTOR	    
-getTokens   (pANTLR3_COMMON_TOKEN_STREAM tokenStream)
-{
-    if	(tokenStream->p == -1)
-    {
-	fillBuffer(tokenStream);
-    }
-
-    return  tokenStream->tokens;
-}
-
-static pANTLR3_LIST	    
-getTokenRange	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
-{
-    return tokenStream->getTokensSet(tokenStream, start, stop, NULL);
-}                                                   
-/** Given a start and stop index, return a List of all tokens in
- *  the token type BitSet.  Return null if no tokens were found.  This
- *  method looks at both on and off channel tokens.
- */
-static pANTLR3_LIST	    
-getTokensSet	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)
-{
-    pANTLR3_LIST	    filteredList;
-    ANTLR3_UINT32	    i;
-    ANTLR3_UINT32	    n;
-    pANTLR3_COMMON_TOKEN    tok;
-
-    if	(tokenStream->p == -1)
-    {
-	fillBuffer(tokenStream);
-    }
-    if	(stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream))
-    {
-	stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream);
-    }
-    if	(start > stop)
-    {
-	return NULL;
-    }
-
-    /* We have the range set, now we need to iterate through the
-     * installed tokens and create a new list with just the ones we want
-     * in it. We are just moving pointers about really.
-     */
-    filteredList    = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream));
-
-    for	(i = start, n = 0; i<= stop; i++)
-    {
-	tok = tokenStream->tstream->get(tokenStream->tstream, i);
-
-	if  (	   types == NULL
-		|| types->isMember(types, tok->getType(tok) == ANTLR3_TRUE)
-	    )
-	{
-	    filteredList->put(filteredList, n++, (void *)tok, NULL);
-	}
-    }
-    
-    /* Did we get any then?
-     */
-    if	(filteredList->size(filteredList) == 0)
-    {
-	filteredList->free(filteredList);
-	filteredList	= NULL;
-    }
-
-    return  filteredList;
-}
-
-static pANTLR3_LIST	    
-getTokensList	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)
-{
-    pANTLR3_BITSET  bitSet;
-    pANTLR3_LIST    newlist;
-
-    bitSet  = antlr3BitsetList(list->table);
-
-    newlist    = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
-
-    bitSet->free(bitSet);
-
-    return  newlist;
-
-}
-
-static pANTLR3_LIST	    
-getTokensType	(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)
-{
-    pANTLR3_BITSET  bitSet;
-    pANTLR3_LIST    newlist;
-
-    bitSet  = antlr3BitsetOf(type, -1);
-    newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
-
-    bitSet->free(bitSet);
-
-    return  newlist;
-}
-
-static ANTLR3_UINT32	    
-_LA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
-{
-	pANTLR3_TOKEN_STREAM    ts;
-	pANTLR3_COMMON_TOKEN    tok;
-
-	ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
-
-	tok	    =  ts->_LT(ts, i);
-
-	if	(tok != NULL)
-	{
-		return	tok->getType(tok);
-	}
-	else
-	{
-		return	ANTLR3_TOKEN_INVALID;
-	}
-}
-
-/// As per _LA() but for debug mode.
-///
-static ANTLR3_UINT32	    
-dbgLA  (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
-{
-    pANTLR3_TOKEN_STREAM    ts;
-   
-    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
-
-	if	(ts->initialStreamState == ANTLR3_TRUE)
-	{
-		consumeInitialHiddenTokens(is);
-	}
-	ts->debugger->LT(ts->debugger, i, tokLT(ts, i));
-	return	_LA(is, i);
-}
-
-static ANTLR3_MARKER
-mark	(pANTLR3_INT_STREAM is)
-{
-    is->lastMarker = is->index(is);
-    return  is->lastMarker;
-}
-
-/// As per mark() but with a call to tell the debugger we are doing this
-///
-static ANTLR3_MARKER
-dbgMark	(pANTLR3_INT_STREAM is)
-{
-    pANTLR3_TOKEN_STREAM    ts;
-   
-    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
-	
-	is->lastMarker = is->index(is);
-	ts->debugger->mark(ts->debugger, is->lastMarker);
-
-    return  is->lastMarker;
-}
-
-static void		    
-release	(pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)
-{
-    return;
-}
-
-static ANTLR3_UINT32	    
-size	(pANTLR3_INT_STREAM is)
-{
-    pANTLR3_COMMON_TOKEN_STREAM cts;
-    pANTLR3_TOKEN_STREAM	ts;
-
-    if (is->cachedSize > 0)
-    {
-	return  is->cachedSize;
-    }
-    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
-    cts	    = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
-
-    is->cachedSize =  cts->tokens->count;
-    return  is->cachedSize;
-}
-
-static ANTLR3_MARKER   
-tindex	(pANTLR3_INT_STREAM is)
-{
-    pANTLR3_COMMON_TOKEN_STREAM cts;
-    pANTLR3_TOKEN_STREAM	ts;
-
-    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
-    cts	    = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
-
-    return  cts->p;
-}
-
-static void		    
-dbgRewindLast	(pANTLR3_INT_STREAM is)
-{
-	pANTLR3_TOKEN_STREAM	ts;
-
-    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
-
-	ts->debugger->rewindLast(ts->debugger);
-
-    is->rewind(is, is->lastMarker);
-}
-static void		    
-rewindLast	(pANTLR3_INT_STREAM is)
-{
-    is->rewind(is, is->lastMarker);
-}
-static void		    
-rewindStream	(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
-{
-    is->seek(is, (ANTLR3_UINT32)(marker));
-}
-static void		    
-dbgRewindStream	(pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
-{
-    pANTLR3_TOKEN_STREAM	ts;
-
-    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
-
-	ts->debugger->rewind(ts->debugger, marker);
-
-    is->seek(is, (ANTLR3_UINT32)(marker));
-}
-
-static void		    
-seek	(pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
-{
-    pANTLR3_COMMON_TOKEN_STREAM cts;
-    pANTLR3_TOKEN_STREAM	ts;
-
-    ts	    = (pANTLR3_TOKEN_STREAM)	    is->super;
-    cts	    = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
-
-    cts->p  = (ANTLR3_UINT32)index;
-}
-static void		    
-dbgSeek	(pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
-{
-	// TODO: Implement seek in debugger when Ter adds it to Java
-	//
-	seek(is, index);
-}
-ANTLR3_API void
-fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)
-{
-    fillBuffer(tokenStream);
-}
-static void
-fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {
-    ANTLR3_UINT32 index;
-    pANTLR3_COMMON_TOKEN tok;
-    ANTLR3_BOOLEAN discard;
-    void * channelI;
-
-    /* Start at index 0 of course
-     */
-    index = 0;
-
-    /* Pick out the next token from the token source
-     * Remember we just get a pointer (reference if you like) here
-     * and so if we store it anywhere, we don't set any pointers to auto free it.
-     */
-    tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
-
-    while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)
-    {
-        discard = ANTLR3_FALSE; /* Assume we are not discarding	*/
-
-        /* I employ a bit of a trick, or perhaps hack here. Rather than
-         * store a pointer to a structure in the override map and discard set
-         * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
-         * we can distinguish "not being there" from "being channel or type 0"
-         */
-
-        if (tokenStream->discardSet != NULL
-            && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL)
-        {
-            discard = ANTLR3_TRUE;
-        }
-        else if (   tokenStream->discardOffChannel == ANTLR3_TRUE
-                 && tok->getChannel(tok) != tokenStream->channel
-                 )
-        {
-            discard = ANTLR3_TRUE;
-        }
-        else if (tokenStream->channelOverrides != NULL)
-        {
-            /* See if this type is in the override map
-             */
-            channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1);
-
-            if (channelI != NULL)
-            {
-                /* Override found
-                 */
-                tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1);
-            }
-        }
-
-        /* If not discarding it, add it to the list at the current index
-         */
-        if (discard == ANTLR3_FALSE)
-        {
-            /* Add it, indicating that we will delete it and the table should not
-             */
-            tok->setTokenIndex(tok, index);
-            tokenStream->p++;
-            tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL);
-            index++;
-        }
-
-        tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
-    }
-
-    /* Cache the size so we don't keep doing indirect method calls. We do this as
-     * early as possible so that anything after this may utilize the cached value.
-     */
-    tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;
-
-    /* Set the consume pointer to the first token that is on our channel
-     */
-    tokenStream->p = 0;
-    tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);
-
-}
-
-/// Given a starting index, return the index of the first on-channel
-///  token.
-///
-static ANTLR3_UINT32
-skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) {
-    ANTLR3_INT32 n;
-    pANTLR3_COMMON_TOKEN tok;
-
-    n = tokenStream->tstream->istream->cachedSize;
-
-    while (i < n)
-    {
-        tok =  (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element;
-
-        if (tok->channel!= tokenStream->channel)
-        {
-            i++;
-        }
-        else
-        {
-            return i;
-        }
-    }
-    return i;
-}
-
-static ANTLR3_UINT32
-skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x)
-{
-    pANTLR3_COMMON_TOKEN tok;
-
-    while (x >= 0)
-    {
-        tok =  (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element;
-        
-        if ((tok->channel != tokenStream->channel))
-        {
-            x--;
-        }
-        else
-        {
-            return x;
-        }
-    }
-    return x;
-}
-
-/// Return a string that represents the name assoicated with the input source
-///
-/// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.
-///
-/// /returns 
-/// /implements ANTLR3_INT_STREAM_struct::getSourceName()
-///
-static pANTLR3_STRING		
-getSourceName				(pANTLR3_INT_STREAM is)
-{
-	// Slightly convoluted as we must trace back to the lexer's input source
-	// via the token source. The streamName that is here is not initialized
-	// because this is a token stream, not a file or string stream, which are the
-	// only things that have a context for a source name.
-	//
-	return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;
-}