diff options
Diffstat (limited to 'impl/antlr/libantlr3c-3.4/src/antlr3tokenstream.c')
| -rw-r--r-- | impl/antlr/libantlr3c-3.4/src/antlr3tokenstream.c | 1091 |
1 files changed, 0 insertions, 1091 deletions
diff --git a/impl/antlr/libantlr3c-3.4/src/antlr3tokenstream.c b/impl/antlr/libantlr3c-3.4/src/antlr3tokenstream.c deleted file mode 100644 index ec06f15..0000000 --- a/impl/antlr/libantlr3c-3.4/src/antlr3tokenstream.c +++ /dev/null @@ -1,1091 +0,0 @@ -/// \file -/// Default implementation of CommonTokenStream -/// - -// [The "BSD licence"] -// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC -// http://www.temporal-wave.com -// http://www.linkedin.com/in/jimidle -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions -// are met: -// 1. Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// 3. The name of the author may not be used to endorse or promote products -// derived from this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR -// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES -// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. -// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, -// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -#include <antlr3tokenstream.h> - -#ifdef ANTLR3_WINDOWS -#pragma warning( disable : 4100 ) -#endif - -// COMMON_TOKEN_STREAM API -// -static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel); -static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype); -static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard); -static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts); -static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); -static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types); -static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list); -static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type); -static void reset (pANTLR3_COMMON_TOKEN_STREAM cts); - -// TOKEN_STREAM API -// -static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); -static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k); -static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i); -static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts); -static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource); -static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts); -static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop); -static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop); -static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger); - -// INT STREAM API -// -static void consume (pANTLR3_INT_STREAM is); -static void dbgConsume (pANTLR3_INT_STREAM is); -static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); -static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i); -static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is); -static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is); -static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark); -static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is); -static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is); -static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); -static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker); -static void rewindLast (pANTLR3_INT_STREAM is); -static void dbgRewindLast (pANTLR3_INT_STREAM is); -static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); -static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index); -static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is); -static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream); -static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream); - -// Helpers -// -static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream); -static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); -static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); -static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i); - -ANTLR3_API pANTLR3_TOKEN_STREAM -antlr3TokenStreamNew() -{ - pANTLR3_TOKEN_STREAM stream; - - // Memory for the interface structure - // - stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM)); - - if (stream == NULL) - { - return NULL; - } - - // Install basic API - // - stream->free = antlr3TokenStreamFree; - - - return stream; -} - -static void -antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream) -{ - ANTLR3_FREE(stream); -} - -static void -antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream) -{ - // We only free up our subordinate interfaces if they belong - // to us, otherwise we let whoever owns them deal with them. - // - if (stream->tstream->super == stream) - { - if (stream->tstream->istream->super == stream->tstream) - { - stream->tstream->istream->free(stream->tstream->istream); - stream->tstream->istream = NULL; - } - stream->tstream->free(stream->tstream); - } - - // Now we free our own resources - // - if (stream->tokens != NULL) - { - stream->tokens->free(stream->tokens); - stream->tokens = NULL; - } - if (stream->discardSet != NULL) - { - stream->discardSet->free(stream->discardSet); - stream->discardSet = NULL; - } - if (stream->channelOverrides != NULL) - { - stream->channelOverrides->free(stream->channelOverrides); - stream->channelOverrides = NULL; - } - - // Free our memory now - // - ANTLR3_FREE(stream); -} - -// Reset a token stream so it can be used again and can reuse it's -// resources. -// -static void -reset (pANTLR3_COMMON_TOKEN_STREAM cts) -{ - - // Free any resources that ar most like specifc to the - // run we just did. - // - if (cts->discardSet != NULL) - { - cts->discardSet->free(cts->discardSet); - cts->discardSet = NULL; - } - if (cts->channelOverrides != NULL) - { - cts->channelOverrides->free(cts->channelOverrides); - cts->channelOverrides = NULL; - } - - // Now, if there were any existing tokens in the stream, - // then we just reset the vector count so that it starts - // again. We must traverse the entries unfortunately as - // there may be free pointers for custom token types and - // so on. However that is just a quick NULL check on the - // vector entries. - // - if (cts->tokens != NULL) - { - cts->tokens->clear(cts->tokens); - } - else - { - /* Install the token tracking tables - */ - cts->tokens = antlr3VectorNew(0); - } - - // Reset to defaults - // - cts->discardOffChannel = ANTLR3_FALSE; - cts->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; - cts->p = -1; -} - -ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM -antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger) -{ - pANTLR3_COMMON_TOKEN_STREAM stream; - - // Create a standard token stream - // - stream = antlr3CommonTokenStreamSourceNew(hint, source); - - // Install the debugger object - // - stream->tstream->debugger = debugger; - - // Override standard token stream methods with debugging versions - // - stream->tstream->initialStreamState = ANTLR3_FALSE; - - stream->tstream->_LT = dbgTokLT; - - stream->tstream->istream->consume = dbgConsume; - stream->tstream->istream->_LA = dbgLA; - stream->tstream->istream->mark = dbgMark; - stream->tstream->istream->rewind = dbgRewindStream; - stream->tstream->istream->rewindLast = dbgRewindLast; - stream->tstream->istream->seek = dbgSeek; - - return stream; -} - -ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM -antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source) -{ - pANTLR3_COMMON_TOKEN_STREAM stream; - - stream = antlr3CommonTokenStreamNew(hint); - - stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL; - - stream->channelOverrides = NULL; - stream->discardSet = NULL; - stream->discardOffChannel = ANTLR3_FALSE; - - stream->tstream->setTokenSource(stream->tstream, source); - - stream->free = antlr3CTSFree; - return stream; -} - -ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM -antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint) -{ - pANTLR3_COMMON_TOKEN_STREAM stream; - - /* Memory for the interface structure - */ - stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM)); - - if (stream == NULL) - { - return NULL; - } - - /* Create space for the token stream interface - */ - stream->tstream = antlr3TokenStreamNew(); - stream->tstream->super = stream; - - /* Create space for the INT_STREAM interfacce - */ - stream->tstream->istream = antlr3IntStreamNew(); - stream->tstream->istream->super = (stream->tstream); - stream->tstream->istream->type = ANTLR3_TOKENSTREAM; - - /* Install the token tracking tables - */ - stream->tokens = antlr3VectorNew(0); - - /* Defaults - */ - stream->p = -1; - - /* Install the common token stream API - */ - stream->setTokenTypeChannel = setTokenTypeChannel; - stream->discardTokenType = discardTokenType; - stream->discardOffChannelToks = discardOffChannel; - stream->getTokens = getTokens; - stream->getTokenRange = getTokenRange; - stream->getTokensSet = getTokensSet; - stream->getTokensList = getTokensList; - stream->getTokensType = getTokensType; - stream->reset = reset; - - /* Install the token stream API - */ - stream->tstream->_LT = tokLT; - stream->tstream->get = get; - stream->tstream->getTokenSource = getTokenSource; - stream->tstream->setTokenSource = setTokenSource; - stream->tstream->toString = toString; - stream->tstream->toStringSS = toStringSS; - stream->tstream->toStringTT = toStringTT; - stream->tstream->setDebugListener = setDebugListener; - - /* Install INT_STREAM interface - */ - stream->tstream->istream->_LA = _LA; - stream->tstream->istream->mark = mark; - stream->tstream->istream->release = release; - stream->tstream->istream->size = size; - stream->tstream->istream->index = tindex; - stream->tstream->istream->rewind = rewindStream; - stream->tstream->istream->rewindLast= rewindLast; - stream->tstream->istream->seek = seek; - stream->tstream->istream->consume = consume; - stream->tstream->istream->getSourceName = getSourceName; - - return stream; -} - -// Install a debug listener adn switch to debug mode methods -// -static void -setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger) -{ - // Install the debugger object - // - ts->debugger = debugger; - - // Override standard token stream methods with debugging versions - // - ts->initialStreamState = ANTLR3_FALSE; - - ts->_LT = dbgTokLT; - - ts->istream->consume = dbgConsume; - ts->istream->_LA = dbgLA; - ts->istream->mark = dbgMark; - ts->istream->rewind = dbgRewindStream; - ts->istream->rewindLast = dbgRewindLast; - ts->istream->seek = dbgSeek; -} - -/** Get the ith token from the current position 1..n where k=1 is the -* first symbol of lookahead. -*/ -static pANTLR3_COMMON_TOKEN -tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) -{ - ANTLR3_INT32 i; - ANTLR3_INT32 n; - pANTLR3_COMMON_TOKEN_STREAM cts; - - cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; - - if (k < 0) - { - return LB(cts, -k); - } - - if (cts->p == -1) - { - fillBuffer(cts); - } - - // Here we used to check for k == 0 and return 0, but this seems - // a superfluous check to me. LT(k=0) is therefore just undefined - // and we won't waste the clock cycles on the check - // - - if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize) - { - pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); - - teof->setStartIndex (teof, ts->istream->index (ts->istream)); - teof->setStopIndex (teof, ts->istream->index (ts->istream)); - return teof; - } - - i = cts->p; - n = 1; - - /* Need to find k good tokens, skipping ones that are off channel - */ - while ( n < k) - { - /* Skip off-channel tokens */ - i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */ - n++; - } - if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize) - { - pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken); - - teof->setStartIndex (teof, ts->istream->index(ts->istream)); - teof->setStopIndex (teof, ts->istream->index(ts->istream)); - return teof; - } - - // Here the token must be in the input vector. Rather then incur - // function call penalty, we just return the pointer directly - // from the vector - // - return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; - //return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i); -} - -/// Debug only method to flag consumption of initial off-channel -/// tokens in the input stream -/// -static void -consumeInitialHiddenTokens(pANTLR3_INT_STREAM is) -{ - ANTLR3_MARKER first; - ANTLR3_INT32 i; - pANTLR3_TOKEN_STREAM ts; - - ts = (pANTLR3_TOKEN_STREAM) is->super; - first = is->index(is); - - for (i=0; i<first; i++) - { - ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i)); - } - - ts->initialStreamState = ANTLR3_FALSE; - -} - -/// As per the normal tokLT but sends information to the debugger -/// -static pANTLR3_COMMON_TOKEN -dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k) -{ - if (ts->initialStreamState == ANTLR3_TRUE) - { - consumeInitialHiddenTokens(ts->istream); - } - return tokLT(ts, k); -} - -#ifdef ANTLR3_WINDOWS - /* When fully optimized VC7 complains about non reachable code. - * Not yet sure if this is an optimizer bug, or a bug in the flow analysis - */ -#pragma warning( disable : 4702 ) -#endif - -static pANTLR3_COMMON_TOKEN -LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k) -{ - ANTLR3_INT32 i; - ANTLR3_INT32 n; - - if (cts->p == -1) - { - fillBuffer(cts); - } - if (k == 0) - { - return NULL; - } - if ((cts->p - k) < 0) - { - return NULL; - } - - i = cts->p; - n = 1; - - /* Need to find k good tokens, going backwards, skipping ones that are off channel - */ - while (n <= (ANTLR3_INT32) k) - { - /* Skip off-channel tokens - */ - - i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */ - n++; - } - if (i < 0) - { - return NULL; - } - // Here the token must be in the input vector. Rather then incut - // function call penalty, we jsut return the pointer directly - // from the vector - // - return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element; -} - -static pANTLR3_COMMON_TOKEN -get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i) -{ - pANTLR3_COMMON_TOKEN_STREAM cts; - - cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; - - return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */ -} - -static pANTLR3_TOKEN_SOURCE -getTokenSource (pANTLR3_TOKEN_STREAM ts) -{ - return ts->tokenSource; -} - -static void -setTokenSource ( pANTLR3_TOKEN_STREAM ts, - pANTLR3_TOKEN_SOURCE tokenSource) -{ - ts->tokenSource = tokenSource; -} - -static pANTLR3_STRING -toString (pANTLR3_TOKEN_STREAM ts) -{ - pANTLR3_COMMON_TOKEN_STREAM cts; - - cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super; - - if (cts->p == -1) - { - fillBuffer(cts); - } - - return ts->toStringSS(ts, 0, ts->istream->size(ts->istream)); -} - -static pANTLR3_STRING -toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) -{ - pANTLR3_STRING string; - pANTLR3_TOKEN_SOURCE tsource; - pANTLR3_COMMON_TOKEN tok; - ANTLR3_UINT32 i; - pANTLR3_COMMON_TOKEN_STREAM cts; - - cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; - - if (cts->p == -1) - { - fillBuffer(cts); - } - if (stop >= ts->istream->size(ts->istream)) - { - stop = ts->istream->size(ts->istream) - 1; - } - - /* Who is giving us these tokens? - */ - tsource = ts->getTokenSource(ts); - - if (tsource != NULL && cts->tokens != NULL) - { - /* Finally, let's get a string - */ - string = tsource->strFactory->newRaw(tsource->strFactory); - - for (i = start; i <= stop; i++) - { - tok = ts->get(ts, i); - if (tok != NULL) - { - string->appendS(string, tok->getText(tok)); - } - } - - return string; - } - return NULL; - -} - -static pANTLR3_STRING -toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop) -{ - if (start != NULL && stop != NULL) - { - return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop)); - } - else - { - return NULL; - } -} - -/** Move the input pointer to the next incoming token. The stream - * must become active with LT(1) available. consume() simply - * moves the input pointer so that LT(1) points at the next - * input symbol. Consume at least one token. - * - * Walk past any token not on the channel the parser is listening to. - */ -static void -consume (pANTLR3_INT_STREAM is) -{ - pANTLR3_COMMON_TOKEN_STREAM cts; - pANTLR3_TOKEN_STREAM ts; - - ts = (pANTLR3_TOKEN_STREAM) is->super; - cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; - - if ((ANTLR3_UINT32)cts->p < cts->tokens->count) - { - cts->p++; - cts->p = skipOffTokenChannels(cts, cts->p); - } -} - - -/// As per ordinary consume but notifies the debugger about hidden -/// tokens and so on. -/// -static void -dbgConsume (pANTLR3_INT_STREAM is) -{ - pANTLR3_TOKEN_STREAM ts; - ANTLR3_MARKER a; - ANTLR3_MARKER b; - pANTLR3_COMMON_TOKEN t; - - ts = (pANTLR3_TOKEN_STREAM) is->super; - - if (ts->initialStreamState == ANTLR3_TRUE) - { - consumeInitialHiddenTokens(is); - } - - a = is->index(is); // Where are we right now? - t = ts->_LT(ts, 1); // Current token from stream - - consume(is); // Standard consumer - - b = is->index(is); // Where are we after consuming 1 on channel token? - - ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token - - if (b>a+1) - { - // The standard consume caused the index to advance by more than 1, - // which can only happen if it skipped some off-channel tokens. - // we need to tell the debugger about those tokens. - // - ANTLR3_MARKER i; - - for (i = a+1; i<b; i++) - { - ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i)); - } - - } -} - -/** A simple filter mechanism whereby you can tell this token stream - * to force all tokens of type ttype to be on channel. For example, - * when interpreting, we cannot execute actions so we need to tell - * the stream to force all WS and NEWLINE to be a different, ignored, - * channel. - */ -static void -setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel) -{ - if (tokenStream->channelOverrides == NULL) - { - tokenStream->channelOverrides = antlr3ListNew(10); - } - - /* We add one to the channel so we can distinguish NULL as being no entry in the - * table for a particular token type. - */ - tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL); -} - -static void -discardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype) -{ - if (tokenStream->discardSet == NULL) - { - tokenStream->discardSet = antlr3ListNew(31); - } - - /* We add one to the channel so we can distinguish NULL as being no entry in the - * table for a particular token type. We could use bitsets for this I suppose too. - */ - tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL); -} - -static void -discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard) -{ - tokenStream->discardOffChannel = discard; -} - -static pANTLR3_VECTOR -getTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream) -{ - if (tokenStream->p == -1) - { - fillBuffer(tokenStream); - } - - return tokenStream->tokens; -} - -static pANTLR3_LIST -getTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop) -{ - return tokenStream->getTokensSet(tokenStream, start, stop, NULL); -} -/** Given a start and stop index, return a List of all tokens in - * the token type BitSet. Return null if no tokens were found. This - * method looks at both on and off channel tokens. - */ -static pANTLR3_LIST -getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types) -{ - pANTLR3_LIST filteredList; - ANTLR3_UINT32 i; - ANTLR3_UINT32 n; - pANTLR3_COMMON_TOKEN tok; - - if (tokenStream->p == -1) - { - fillBuffer(tokenStream); - } - if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream)) - { - stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream); - } - if (start > stop) - { - return NULL; - } - - /* We have the range set, now we need to iterate through the - * installed tokens and create a new list with just the ones we want - * in it. We are just moving pointers about really. - */ - filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream)); - - for (i = start, n = 0; i<= stop; i++) - { - tok = tokenStream->tstream->get(tokenStream->tstream, i); - - if ( types == NULL - || types->isMember(types, tok->getType(tok) == ANTLR3_TRUE) - ) - { - filteredList->put(filteredList, n++, (void *)tok, NULL); - } - } - - /* Did we get any then? - */ - if (filteredList->size(filteredList) == 0) - { - filteredList->free(filteredList); - filteredList = NULL; - } - - return filteredList; -} - -static pANTLR3_LIST -getTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list) -{ - pANTLR3_BITSET bitSet; - pANTLR3_LIST newlist; - - bitSet = antlr3BitsetList(list->table); - - newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); - - bitSet->free(bitSet); - - return newlist; - -} - -static pANTLR3_LIST -getTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type) -{ - pANTLR3_BITSET bitSet; - pANTLR3_LIST newlist; - - bitSet = antlr3BitsetOf(type, -1); - newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet); - - bitSet->free(bitSet); - - return newlist; -} - -static ANTLR3_UINT32 -_LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) -{ - pANTLR3_TOKEN_STREAM ts; - pANTLR3_COMMON_TOKEN tok; - - ts = (pANTLR3_TOKEN_STREAM) is->super; - - tok = ts->_LT(ts, i); - - if (tok != NULL) - { - return tok->getType(tok); - } - else - { - return ANTLR3_TOKEN_INVALID; - } -} - -/// As per _LA() but for debug mode. -/// -static ANTLR3_UINT32 -dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i) -{ - pANTLR3_TOKEN_STREAM ts; - - ts = (pANTLR3_TOKEN_STREAM) is->super; - - if (ts->initialStreamState == ANTLR3_TRUE) - { - consumeInitialHiddenTokens(is); - } - ts->debugger->LT(ts->debugger, i, tokLT(ts, i)); - return _LA(is, i); -} - -static ANTLR3_MARKER -mark (pANTLR3_INT_STREAM is) -{ - is->lastMarker = is->index(is); - return is->lastMarker; -} - -/// As per mark() but with a call to tell the debugger we are doing this -/// -static ANTLR3_MARKER -dbgMark (pANTLR3_INT_STREAM is) -{ - pANTLR3_TOKEN_STREAM ts; - - ts = (pANTLR3_TOKEN_STREAM) is->super; - - is->lastMarker = is->index(is); - ts->debugger->mark(ts->debugger, is->lastMarker); - - return is->lastMarker; -} - -static void -release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark) -{ - return; -} - -static ANTLR3_UINT32 -size (pANTLR3_INT_STREAM is) -{ - pANTLR3_COMMON_TOKEN_STREAM cts; - pANTLR3_TOKEN_STREAM ts; - - if (is->cachedSize > 0) - { - return is->cachedSize; - } - ts = (pANTLR3_TOKEN_STREAM) is->super; - cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; - - is->cachedSize = cts->tokens->count; - return is->cachedSize; -} - -static ANTLR3_MARKER -tindex (pANTLR3_INT_STREAM is) -{ - pANTLR3_COMMON_TOKEN_STREAM cts; - pANTLR3_TOKEN_STREAM ts; - - ts = (pANTLR3_TOKEN_STREAM) is->super; - cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; - - return cts->p; -} - -static void -dbgRewindLast (pANTLR3_INT_STREAM is) -{ - pANTLR3_TOKEN_STREAM ts; - - ts = (pANTLR3_TOKEN_STREAM) is->super; - - ts->debugger->rewindLast(ts->debugger); - - is->rewind(is, is->lastMarker); -} -static void -rewindLast (pANTLR3_INT_STREAM is) -{ - is->rewind(is, is->lastMarker); -} -static void -rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) -{ - is->seek(is, (ANTLR3_UINT32)(marker)); -} -static void -dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker) -{ - pANTLR3_TOKEN_STREAM ts; - - ts = (pANTLR3_TOKEN_STREAM) is->super; - - ts->debugger->rewind(ts->debugger, marker); - - is->seek(is, (ANTLR3_UINT32)(marker)); -} - -static void -seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) -{ - pANTLR3_COMMON_TOKEN_STREAM cts; - pANTLR3_TOKEN_STREAM ts; - - ts = (pANTLR3_TOKEN_STREAM) is->super; - cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super; - - cts->p = (ANTLR3_UINT32)index; -} -static void -dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index) -{ - // TODO: Implement seek in debugger when Ter adds it to Java - // - seek(is, index); -} -ANTLR3_API void -fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream) -{ - fillBuffer(tokenStream); -} -static void -fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) { - ANTLR3_UINT32 index; - pANTLR3_COMMON_TOKEN tok; - ANTLR3_BOOLEAN discard; - void * channelI; - - /* Start at index 0 of course - */ - index = 0; - - /* Pick out the next token from the token source - * Remember we just get a pointer (reference if you like) here - * and so if we store it anywhere, we don't set any pointers to auto free it. - */ - tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); - - while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF) - { - discard = ANTLR3_FALSE; /* Assume we are not discarding */ - - /* I employ a bit of a trick, or perhaps hack here. Rather than - * store a pointer to a structure in the override map and discard set - * we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0 - * we can distinguish "not being there" from "being channel or type 0" - */ - - if (tokenStream->discardSet != NULL - && tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL) - { - discard = ANTLR3_TRUE; - } - else if ( tokenStream->discardOffChannel == ANTLR3_TRUE - && tok->getChannel(tok) != tokenStream->channel - ) - { - discard = ANTLR3_TRUE; - } - else if (tokenStream->channelOverrides != NULL) - { - /* See if this type is in the override map - */ - channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1); - - if (channelI != NULL) - { - /* Override found - */ - tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1); - } - } - - /* If not discarding it, add it to the list at the current index - */ - if (discard == ANTLR3_FALSE) - { - /* Add it, indicating that we will delete it and the table should not - */ - tok->setTokenIndex(tok, index); - tokenStream->p++; - tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL); - index++; - } - - tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource); - } - - /* Cache the size so we don't keep doing indirect method calls. We do this as - * early as possible so that anything after this may utilize the cached value. - */ - tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count; - - /* Set the consume pointer to the first token that is on our channel - */ - tokenStream->p = 0; - tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p); - -} - -/// Given a starting index, return the index of the first on-channel -/// token. -/// -static ANTLR3_UINT32 -skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) { - ANTLR3_INT32 n; - pANTLR3_COMMON_TOKEN tok; - - n = tokenStream->tstream->istream->cachedSize; - - while (i < n) - { - tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element; - - if (tok->channel!= tokenStream->channel) - { - i++; - } - else - { - return i; - } - } - return i; -} - -static ANTLR3_UINT32 -skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x) -{ - pANTLR3_COMMON_TOKEN tok; - - while (x >= 0) - { - tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element; - - if ((tok->channel != tokenStream->channel)) - { - x--; - } - else - { - return x; - } - } - return x; -} - -/// Return a string that represents the name assoicated with the input source -/// -/// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream. -/// -/// /returns -/// /implements ANTLR3_INT_STREAM_struct::getSourceName() -/// -static pANTLR3_STRING -getSourceName (pANTLR3_INT_STREAM is) -{ - // Slightly convoluted as we must trace back to the lexer's input source - // via the token source. The streamName that is here is not initialized - // because this is a token stream, not a file or string stream, which are the - // only things that have a context for a source name. - // - return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName; -} |