diff options
Diffstat (limited to 'impl/antlr/libantlr3c-3.4/src/antlr3tokenstream.c')
| -rw-r--r-- | impl/antlr/libantlr3c-3.4/src/antlr3tokenstream.c | 1091 |
1 files changed, 1091 insertions, 0 deletions
diff --git a/impl/antlr/libantlr3c-3.4/src/antlr3tokenstream.c b/impl/antlr/libantlr3c-3.4/src/antlr3tokenstream.c new file mode 100644 index 0000000..ec06f15 --- /dev/null +++ b/impl/antlr/libantlr3c-3.4/src/antlr3tokenstream.c @@ -0,0 +1,1091 @@ +/// \file +/// Default implementation of CommonTokenStream +/// + +// [The "BSD licence"] +// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC +// http://www.temporal-wave.com +// http://www.linkedin.com/in/jimidle +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions +// are met: +// 1. Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// 3. The name of the author may not be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include <antlr3tokenstream.h> + +#ifdef ANTLR3_WINDOWS +#pragma warning( disable : 4100 ) +#endif + +// COMMON_TOKEN_STREAM API +// +static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel); +static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype); +static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard); +static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts); +static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); +static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types); +static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list); +static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type); +static void reset (pANTLR3_COMMON_TOKEN_STREAM cts); + +// TOKEN_STREAM API +// +static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); +static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); +static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i); +static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts); +static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource); +static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts); +static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); +static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop); +static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger); + +// INT STREAM API +// +static void consume (pANTLR3_INT_STREAM is); +static void dbgConsume (pANTLR3_INT_STREAM is); +static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); +static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); +static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is); +static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is); +static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark); +static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is); +static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is); +static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); +static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); +static void rewindLast (pANTLR3_INT_STREAM is); +static void dbgRewindLast (pANTLR3_INT_STREAM is); +static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); +static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); +static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is); +static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream); +static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream); + +// Helpers +// +static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream); +static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); +static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); +static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); + +ANTLR3_API pANTLR3_TOKEN_STREAM +antlr3TokenStreamNew() +{ + pANTLR3_TOKEN_STREAM stream; + + // Memory for the interface structure + // + stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM)); + + if (stream == NULL) + { + return NULL; + } + + // Install basic API + // + stream->free = antlr3TokenStreamFree; + + + return stream; +} + +static void +antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream) +{ + ANTLR3_FREE(stream); +} + +static void +antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream) +{ + // We only free up our subordinate interfaces if they belong + // to us, otherwise we let whoever owns them deal with them. + // + if (stream->tstream->super == stream) + { + if (stream->tstream->istream->super == stream->tstream) + { + stream->tstream->istream->free(stream->tstream->istream); + stream->tstream->istream = NULL; + } + stream->tstream->free(stream->tstream); + } + + // Now we free our own resources + // + if (stream->tokens != NULL) + { + stream->tokens->free(stream->tokens); + stream->tokens = NULL; + } + if (stream->discardSet != NULL) + { + stream->discardSet->free(stream->discardSet); + stream->discardSet = NULL; + } + if (stream->channelOverrides != NULL) + { + stream->channelOverrides->free(stream->channelOverrides); + stream->channelOverrides = NULL; + } + + // Free our memory now + // + ANTLR3_FREE(stream); +} + +// Reset a token stream so it can be used again and can reuse it's +// resources. +// +static void +reset (pANTLR3_COMMON_TOKEN_STREAM cts) +{ + + // Free any resources that ar most like specifc to the + // run we just did. + // + if (cts->discardSet != NULL) + { + cts->discardSet->free(cts->discardSet); + cts->discardSet = NULL; + } + if (cts->channelOverrides != NULL) + { + cts->channelOverrides->free(cts->channelOverrides); + cts->channelOverrides = NULL; + } + + // Now, if there were any existing tokens in the stream, + // then we just reset the vector count so that it starts + // again. We must traverse the entries unfortunately as + // there may be free pointers for custom token types and + // so on. However that is just a quick NULL check on the + // vector entries. + // + if (cts->tokens != NULL) + { + cts->tokens->clear(cts->tokens); + } + else + { + /* Install the token tracking tables + */ + cts->tokens = antlr3VectorNew(0); + } + + // Reset to defaults + // + cts->discardOffChannel = ANTLR3_FALSE; + cts->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; + cts->p = -1; +} + +ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM +antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger) +{ + pANTLR3_COMMON_TOKEN_STREAM stream; + + // Create a standard token stream + // + stream = antlr3CommonTokenStreamSourceNew(hint, source); + + // Install the debugger object + // + stream->tstream->debugger = debugger; + + // Override standard token stream methods with debugging versions + // + stream->tstream->initialStreamState = ANTLR3_FALSE; + + stream->tstream->_LT = dbgTokLT; + + stream->tstream->istream->consume = dbgConsume; + stream->tstream->istream->_LA = dbgLA; + stream->tstream->istream->mark = dbgMark; + stream->tstream->istream->rewind = dbgRewindStream; + stream->tstream->istream->rewindLast = dbgRewindLast; + stream->tstream->istream->seek = dbgSeek; + + return stream; +} + +ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM +antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source) +{ + pANTLR3_COMMON_TOKEN_STREAM stream; + + stream = antlr3CommonTokenStreamNew(hint); + + stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; + + stream->channelOverrides = NULL; + stream->discardSet = NULL; + stream->discardOffChannel = ANTLR3_FALSE; + + stream->tstream->setTokenSource(stream->tstream, source); + + stream->free = antlr3CTSFree; + return stream; +} + +ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM +antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint) +{ + pANTLR3_COMMON_TOKEN_STREAM stream; + + /* Memory for the interface structure + */ + stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM)); + + if (stream == NULL) + { + return NULL; + } + + /* Create space for the token stream interface + */ + stream->tstream = antlr3TokenStreamNew(); + stream->tstream->super = stream; + + /* Create space for the INT_STREAM interfacce + */ + stream->tstream->istream = antlr3IntStreamNew(); + stream->tstream->istream->super = (stream->tstream); + stream->tstream->istream->type = ANTLR3_TOKENSTREAM; + + /* Install the token tracking tables + */ + stream->tokens = antlr3VectorNew(0); + + /* Defaults + */ + stream->p = -1; + + /* Install the common token stream API + */ + stream->setTokenTypeChannel = setTokenTypeChannel; + stream->discardTokenType = discardTokenType; + stream->discardOffChannelToks = discardOffChannel; + stream->getTokens = getTokens; + stream->getTokenRange = getTokenRange; + stream->getTokensSet = getTokensSet; + stream->getTokensList = getTokensList; + stream->getTokensType = getTokensType; + stream->reset = reset; + + /* Install the token stream API + */ + stream->tstream->_LT = tokLT; + stream->tstream->get = get; + stream->tstream->getTokenSource = getTokenSource; + stream->tstream->setTokenSource = setTokenSource; + stream->tstream->toString = toString; + stream->tstream->toStringSS = toStringSS; + stream->tstream->toStringTT = toStringTT; + stream->tstream->setDebugListener = setDebugListener; + + /* Install INT_STREAM interface + */ + stream->tstream->istream->_LA = _LA; + stream->tstream->istream->mark = mark; + stream->tstream->istream->release = release; + stream->tstream->istream->size = size; + stream->tstream->istream->index = tindex; + stream->tstream->istream->rewind = rewindStream; + stream->tstream->istream->rewindLast= rewindLast; + stream->tstream->istream->seek = seek; + stream->tstream->istream->consume = consume; + stream->tstream->istream->getSourceName = getSourceName; + + return stream; +} + +// Install a debug listener adn switch to debug mode methods +// +static void +setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger) +{ + // Install the debugger object + // + ts->debugger = debugger; + + // Override standard token stream methods with debugging versions + // + ts->initialStreamState = ANTLR3_FALSE; + + ts->_LT = dbgTokLT; + + ts->istream->consume = dbgConsume; + ts->istream->_LA = dbgLA; + ts->istream->mark = dbgMark; + ts->istream->rewind = dbgRewindStream; + ts->istream->rewindLast = dbgRewindLast; + ts->istream->seek = dbgSeek; +} + +/** Get the ith token from the current position 1..n where k=1 is the +* first symbol of lookahead. +*/ +static pANTLR3_COMMON_TOKEN +tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) +{ + ANTLR3_INT32 i; + ANTLR3_INT32 n; + pANTLR3_COMMON_TOKEN_STREAM cts; + + cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; + + if (k < 0) + { + return LB(cts, -k); + } + + if (cts->p == -1) + { + fillBuffer(cts); + } + + // Here we used to check for k == 0 and return 0, but this seems + // a superfluous check to me. LT(k=0) is therefore just undefined + // and we won't waste the clock cycles on the check + // + + if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize) + { + pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); + + teof->setStartIndex (teof, ts->istream->index (ts->istream)); + teof->setStopIndex (teof, ts->istream->index (ts->istream)); + return teof; + } + + i = cts->p; + n = 1; + + /* Need to find k good tokens, skipping ones that are off channel + */ + while ( n < k) + { + /* Skip off-channel tokens */ + i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */ + n++; + } + if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize) + { + pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); + + teof->setStartIndex (teof, ts->istream->index(ts->istream)); + teof->setStopIndex (teof, ts->istream->index(ts->istream)); + return teof; + } + + // Here the token must be in the input vector. Rather then incur + // function call penalty, we just return the pointer directly + // from the vector + // + return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; + //return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i); +} + +/// Debug only method to flag consumption of initial off-channel +/// tokens in the input stream +/// +static void +consumeInitialHiddenTokens(pANTLR3_INT_STREAM is) +{ + ANTLR3_MARKER first; + ANTLR3_INT32 i; + pANTLR3_TOKEN_STREAM ts; + + ts = (pANTLR3_TOKEN_STREAM) is->super; + first = is->index(is); + + for (i=0; i<first; i++) + { + ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i)); + } + + ts->initialStreamState = ANTLR3_FALSE; + +} + +/// As per the normal tokLT but sends information to the debugger +/// +static pANTLR3_COMMON_TOKEN +dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) +{ + if (ts->initialStreamState == ANTLR3_TRUE) + { + consumeInitialHiddenTokens(ts->istream); + } + return tokLT(ts, k); +} + +#ifdef ANTLR3_WINDOWS + /* When fully optimized VC7 complains about non reachable code. + * Not yet sure if this is an optimizer bug, or a bug in the flow analysis + */ +#pragma warning( disable : 4702 ) +#endif + +static pANTLR3_COMMON_TOKEN +LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k) +{ + ANTLR3_INT32 i; + ANTLR3_INT32 n; + + if (cts->p == -1) + { + fillBuffer(cts); + } + if (k == 0) + { + return NULL; + } + if ((cts->p - k) < 0) + { + return NULL; + } + + i = cts->p; + n = 1; + + /* Need to find k good tokens, going backwards, skipping ones that are off channel + */ + while (n <= (ANTLR3_INT32) k) + { + /* Skip off-channel tokens + */ + + i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */ + n++; + } + if (i < 0) + { + return NULL; + } + // Here the token must be in the input vector. Rather then incut + // function call penalty, we jsut return the pointer directly + // from the vector + // + return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; +} + +static pANTLR3_COMMON_TOKEN +get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i) +{ + pANTLR3_COMMON_TOKEN_STREAM cts; + + cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; + + return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */ +} + +static pANTLR3_TOKEN_SOURCE +getTokenSource (pANTLR3_TOKEN_STREAM ts) +{ + return ts->tokenSource; +} + +static void +setTokenSource ( pANTLR3_TOKEN_STREAM ts, + pANTLR3_TOKEN_SOURCE tokenSource) +{ + ts->tokenSource = tokenSource; +} + +static pANTLR3_STRING +toString (pANTLR3_TOKEN_STREAM ts) +{ + pANTLR3_COMMON_TOKEN_STREAM cts; + + cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; + + if (cts->p == -1) + { + fillBuffer(cts); + } + + return ts->toStringSS(ts, 0, ts->istream->size(ts->istream)); +} + +static pANTLR3_STRING +toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) +{ + pANTLR3_STRING string; + pANTLR3_TOKEN_SOURCE tsource; + pANTLR3_COMMON_TOKEN tok; + ANTLR3_UINT32 i; + pANTLR3_COMMON_TOKEN_STREAM cts; + + cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; + + if (cts->p == -1) + { + fillBuffer(cts); + } + if (stop >= ts->istream->size(ts->istream)) + { + stop = ts->istream->size(ts->istream) - 1; + } + + /* Who is giving us these tokens? + */ + tsource = ts->getTokenSource(ts); + + if (tsource != NULL && cts->tokens != NULL) + { + /* Finally, let's get a string + */ + string = tsource->strFactory->newRaw(tsource->strFactory); + + for (i = start; i <= stop; i++) + { + tok = ts->get(ts, i); + if (tok != NULL) + { + string->appendS(string, tok->getText(tok)); + } + } + + return string; + } + return NULL; + +} + +static pANTLR3_STRING +toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop) +{ + if (start != NULL && stop != NULL) + { + return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop)); + } + else + { + return NULL; + } +} + +/** Move the input pointer to the next incoming token. The stream + * must become active with LT(1) available. consume() simply + * moves the input pointer so that LT(1) points at the next + * input symbol. Consume at least one token. + * + * Walk past any token not on the channel the parser is listening to. + */ +static void +consume (pANTLR3_INT_STREAM is) +{ + pANTLR3_COMMON_TOKEN_STREAM cts; + pANTLR3_TOKEN_STREAM ts; + + ts = (pANTLR3_TOKEN_STREAM) is->super; + cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; + + if ((ANTLR3_UINT32)cts->p < cts->tokens->count) + { + cts->p++; + cts->p = skipOffTokenChannels(cts, cts->p); + } +} + + +/// As per ordinary consume but notifies the debugger about hidden +/// tokens and so on. +/// +static void +dbgConsume (pANTLR3_INT_STREAM is) +{ + pANTLR3_TOKEN_STREAM ts; + ANTLR3_MARKER a; + ANTLR3_MARKER b; + pANTLR3_COMMON_TOKEN t; + + ts = (pANTLR3_TOKEN_STREAM) is->super; + + if (ts->initialStreamState == ANTLR3_TRUE) + { + consumeInitialHiddenTokens(is); + } + + a = is->index(is); // Where are we right now? + t = ts->_LT(ts, 1); // Current token from stream + + consume(is); // Standard consumer + + b = is->index(is); // Where are we after consuming 1 on channel token? + + ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token + + if (b>a+1) + { + // The standard consume caused the index to advance by more than 1, + // which can only happen if it skipped some off-channel tokens. + // we need to tell the debugger about those tokens. + // + ANTLR3_MARKER i; + + for (i = a+1; i<b; i++) + { + ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i)); + } + + } +} + +/** A simple filter mechanism whereby you can tell this token stream + * to force all tokens of type ttype to be on channel. For example, + * when interpreting, we cannot execute actions so we need to tell + * the stream to force all WS and NEWLINE to be a different, ignored, + * channel. + */ +static void +setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel) +{ + if (tokenStream->channelOverrides == NULL) + { + tokenStream->channelOverrides = antlr3ListNew(10); + } + + /* We add one to the channel so we can distinguish NULL as being no entry in the + * table for a particular token type. + */ + tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL); +} + +static void +discardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype) +{ + if (tokenStream->discardSet == NULL) + { + tokenStream->discardSet = antlr3ListNew(31); + } + + /* We add one to the channel so we can distinguish NULL as being no entry in the + * table for a particular token type. We could use bitsets for this I suppose too. + */ + tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL); +} + +static void +discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard) +{ + tokenStream->discardOffChannel = discard; +} + +static pANTLR3_VECTOR +getTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream) +{ + if (tokenStream->p == -1) + { + fillBuffer(tokenStream); + } + + return tokenStream->tokens; +} + +static pANTLR3_LIST +getTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) +{ + return tokenStream->getTokensSet(tokenStream, start, stop, NULL); +} +/** Given a start and stop index, return a List of all tokens in + * the token type BitSet. Return null if no tokens were found. This + * method looks at both on and off channel tokens. + */ +static pANTLR3_LIST +getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types) +{ + pANTLR3_LIST filteredList; + ANTLR3_UINT32 i; + ANTLR3_UINT32 n; + pANTLR3_COMMON_TOKEN tok; + + if (tokenStream->p == -1) + { + fillBuffer(tokenStream); + } + if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream)) + { + stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream); + } + if (start > stop) + { + return NULL; + } + + /* We have the range set, now we need to iterate through the + * installed tokens and create a new list with just the ones we want + * in it. We are just moving pointers about really. + */ + filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream)); + + for (i = start, n = 0; i<= stop; i++) + { + tok = tokenStream->tstream->get(tokenStream->tstream, i); + + if ( types == NULL + || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE) + ) + { + filteredList->put(filteredList, n++, (void *)tok, NULL); + } + } + + /* Did we get any then? + */ + if (filteredList->size(filteredList) == 0) + { + filteredList->free(filteredList); + filteredList = NULL; + } + + return filteredList; +} + +static pANTLR3_LIST +getTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list) +{ + pANTLR3_BITSET bitSet; + pANTLR3_LIST newlist; + + bitSet = antlr3BitsetList(list->table); + + newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); + + bitSet->free(bitSet); + + return newlist; + +} + +static pANTLR3_LIST +getTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type) +{ + pANTLR3_BITSET bitSet; + pANTLR3_LIST newlist; + + bitSet = antlr3BitsetOf(type, -1); + newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); + + bitSet->free(bitSet); + + return newlist; +} + +static ANTLR3_UINT32 +_LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) +{ + pANTLR3_TOKEN_STREAM ts; + pANTLR3_COMMON_TOKEN tok; + + ts = (pANTLR3_TOKEN_STREAM) is->super; + + tok = ts->_LT(ts, i); + + if (tok != NULL) + { + return tok->getType(tok); + } + else + { + return ANTLR3_TOKEN_INVALID; + } +} + +/// As per _LA() but for debug mode. +/// +static ANTLR3_UINT32 +dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) +{ + pANTLR3_TOKEN_STREAM ts; + + ts = (pANTLR3_TOKEN_STREAM) is->super; + + if (ts->initialStreamState == ANTLR3_TRUE) + { + consumeInitialHiddenTokens(is); + } + ts->debugger->LT(ts->debugger, i, tokLT(ts, i)); + return _LA(is, i); +} + +static ANTLR3_MARKER +mark (pANTLR3_INT_STREAM is) +{ + is->lastMarker = is->index(is); + return is->lastMarker; +} + +/// As per mark() but with a call to tell the debugger we are doing this +/// +static ANTLR3_MARKER +dbgMark (pANTLR3_INT_STREAM is) +{ + pANTLR3_TOKEN_STREAM ts; + + ts = (pANTLR3_TOKEN_STREAM) is->super; + + is->lastMarker = is->index(is); + ts->debugger->mark(ts->debugger, is->lastMarker); + + return is->lastMarker; +} + +static void +release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark) +{ + return; +} + +static ANTLR3_UINT32 +size (pANTLR3_INT_STREAM is) +{ + pANTLR3_COMMON_TOKEN_STREAM cts; + pANTLR3_TOKEN_STREAM ts; + + if (is->cachedSize > 0) + { + return is->cachedSize; + } + ts = (pANTLR3_TOKEN_STREAM) is->super; + cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; + + is->cachedSize = cts->tokens->count; + return is->cachedSize; +} + +static ANTLR3_MARKER +tindex (pANTLR3_INT_STREAM is) +{ + pANTLR3_COMMON_TOKEN_STREAM cts; + pANTLR3_TOKEN_STREAM ts; + + ts = (pANTLR3_TOKEN_STREAM) is->super; + cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; + + return cts->p; +} + +static void +dbgRewindLast (pANTLR3_INT_STREAM is) +{ + pANTLR3_TOKEN_STREAM ts; + + ts = (pANTLR3_TOKEN_STREAM) is->super; + + ts->debugger->rewindLast(ts->debugger); + + is->rewind(is, is->lastMarker); +} +static void +rewindLast (pANTLR3_INT_STREAM is) +{ + is->rewind(is, is->lastMarker); +} +static void +rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) +{ + is->seek(is, (ANTLR3_UINT32)(marker)); +} +static void +dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) +{ + pANTLR3_TOKEN_STREAM ts; + + ts = (pANTLR3_TOKEN_STREAM) is->super; + + ts->debugger->rewind(ts->debugger, marker); + + is->seek(is, (ANTLR3_UINT32)(marker)); +} + +static void +seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) +{ + pANTLR3_COMMON_TOKEN_STREAM cts; + pANTLR3_TOKEN_STREAM ts; + + ts = (pANTLR3_TOKEN_STREAM) is->super; + cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; + + cts->p = (ANTLR3_UINT32)index; +} +static void +dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) +{ + // TODO: Implement seek in debugger when Ter adds it to Java + // + seek(is, index); +} +ANTLR3_API void +fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream) +{ + fillBuffer(tokenStream); +} +static void +fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) { + ANTLR3_UINT32 index; + pANTLR3_COMMON_TOKEN tok; + ANTLR3_BOOLEAN discard; + void * channelI; + + /* Start at index 0 of course + */ + index = 0; + + /* Pick out the next token from the token source + * Remember we just get a pointer (reference if you like) here + * and so if we store it anywhere, we don't set any pointers to auto free it. + */ + tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); + + while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF) + { + discard = ANTLR3_FALSE; /* Assume we are not discarding */ + + /* I employ a bit of a trick, or perhaps hack here. Rather than + * store a pointer to a structure in the override map and discard set + * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0 + * we can distinguish "not being there" from "being channel or type 0" + */ + + if (tokenStream->discardSet != NULL + && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL) + { + discard = ANTLR3_TRUE; + } + else if ( tokenStream->discardOffChannel == ANTLR3_TRUE + && tok->getChannel(tok) != tokenStream->channel + ) + { + discard = ANTLR3_TRUE; + } + else if (tokenStream->channelOverrides != NULL) + { + /* See if this type is in the override map + */ + channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1); + + if (channelI != NULL) + { + /* Override found + */ + tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1); + } + } + + /* If not discarding it, add it to the list at the current index + */ + if (discard == ANTLR3_FALSE) + { + /* Add it, indicating that we will delete it and the table should not + */ + tok->setTokenIndex(tok, index); + tokenStream->p++; + tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL); + index++; + } + + tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); + } + + /* Cache the size so we don't keep doing indirect method calls. We do this as + * early as possible so that anything after this may utilize the cached value. + */ + tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count; + + /* Set the consume pointer to the first token that is on our channel + */ + tokenStream->p = 0; + tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p); + +} + +/// Given a starting index, return the index of the first on-channel +/// token. +/// +static ANTLR3_UINT32 +skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) { + ANTLR3_INT32 n; + pANTLR3_COMMON_TOKEN tok; + + n = tokenStream->tstream->istream->cachedSize; + + while (i < n) + { + tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element; + + if (tok->channel!= tokenStream->channel) + { + i++; + } + else + { + return i; + } + } + return i; +} + +static ANTLR3_UINT32 +skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x) +{ + pANTLR3_COMMON_TOKEN tok; + + while (x >= 0) + { + tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element; + + if ((tok->channel != tokenStream->channel)) + { + x--; + } + else + { + return x; + } + } + return x; +} + +/// Return a string that represents the name assoicated with the input source +/// +/// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream. +/// +/// /returns +/// /implements ANTLR3_INT_STREAM_struct::getSourceName() +/// +static pANTLR3_STRING +getSourceName (pANTLR3_INT_STREAM is) +{ + // Slightly convoluted as we must trace back to the lexer's input source + // via the token source. The streamName that is here is not initialized + // because this is a token stream, not a file or string stream, which are the + // only things that have a context for a source name. + // + return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName; +} |