summaryrefslogtreecommitdiff
path: root/impl/antlr/libantlr3c-3.4/include/antlr3tokenstream.h
blob: a5137ad4f11239ec711717c77ddc667de4e3a1ae (about) (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
/** \file
 * Defines the interface for an ANTLR3 common token stream. Custom token streams should create
 * one of these and then override any functions by installing their own pointers
 * to implement the various functions.
 */
#ifndef	_ANTLR3_TOKENSTREAM_H
#define	_ANTLR3_TOKENSTREAM_H

// [The "BSD licence"]
// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
// http://www.temporal-wave.com
// http://www.linkedin.com/in/jimidle
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
// are met:
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
// 3. The name of the author may not be used to endorse or promote products
//    derived from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#include    <antlr3defs.h>
#include    <antlr3string.h>
#include    <antlr3collections.h>
#include    <antlr3input.h>
#include    <antlr3commontoken.h>
#include    <antlr3bitset.h>
#include	<antlr3debugeventlistener.h>

#ifdef __cplusplus
extern "C" {
#endif

/** Definition of a token source, which has a pointer to a function that 
 *  returns the next token (using a token factory if it is going to be
 *  efficient) and a pointer to an ANTLR3_INPUT_STREAM. This is slightly
 *  different to the Java interface because we have no way to implement
 *  multiple interfaces without defining them in the interface structure
 *  or casting (void *), which is too convoluted.
 */
typedef struct ANTLR3_TOKEN_SOURCE_struct
{
    /** Pointer to a function that returns the next token in the stream. 
     */
    pANTLR3_COMMON_TOKEN    (*nextToken)(struct ANTLR3_TOKEN_SOURCE_struct * tokenSource);

    /** Whoever is providing tokens, needs to provide a string factory too
     */
    pANTLR3_STRING_FACTORY  strFactory;

    /** A special pre-allocated token, which signifies End Of Tokens. Because this must
     *  be set up with the current input index and so on, we embed the structure and 
     *  return the address of it. It is marked as factoryMade, so that it is never
     *  attempted to be freed.
     */
    ANTLR3_COMMON_TOKEN	    eofToken;

	/// A special pre-allocated token, which is returned by mTokens() if the
	/// lexer rule said to just skip the generated token altogether.
	/// Having this single token stops us wasting memory by have the token factory
	/// actually create something that we are going to SKIP(); anyway.
	///
	ANTLR3_COMMON_TOKEN		skipToken;

    /** Whatever is supplying the token source interface, needs a pointer to 
     *  itself so that this pointer can be passed to it when the nextToken
     *  function is called.
     */
    void		    * super;

    /** When the token source is constructed, it is populated with the file
     *  name from whence the tokens were produced by the lexer. This pointer is a
     *  copy of the one supplied by the CharStream (and may be NULL) so should
     *  not be manipulated other than to copy or print it.
     */
    pANTLR3_STRING	    fileName;
}
    ANTLR3_TOKEN_SOURCE;

/** Definition of the ANTLR3 common token stream interface.
 * \remark
 * Much of the documentation for this interface is stolen from Ter's Java implementation.
 */
typedef	struct ANTLR3_TOKEN_STREAM_struct
{
    /** Pointer to the token source for this stream
     */
    pANTLR3_TOKEN_SOURCE    tokenSource;

    /** Whatever is providing this interface needs a pointer to itself
     *  so that this can be passed back to it whenever the api functions
     *  are called.
     */
    void	      * super;

    /** All input streams implement the ANTLR3_INT_STREAM interface...
     */
    pANTLR3_INT_STREAM	    istream;

	/// Debugger interface, is this is a debugging token stream
	///
	pANTLR3_DEBUG_EVENT_LISTENER		debugger;

	/// Indicates the initial stream state for dbgConsume()
	///
	ANTLR3_BOOLEAN			initialStreamState;

    /** Get Token at current input pointer + i ahead where i=1 is next Token.
     *  i<0 indicates tokens in the past.  So -1 is previous token and -2 is
     *  two tokens ago. LT(0) is undefined.  For i>=n, return Token.EOFToken.
     *  Return null for LT(0) and any index that results in an absolute address
     *  that is negative.
     */
    pANTLR3_COMMON_TOKEN    (*_LT)		(struct ANTLR3_TOKEN_STREAM_struct * tokenStream, ANTLR3_INT32 k);

    /** Get a token at an absolute index i; 0..n-1.  This is really only
     *  needed for profiling and debugging and token stream rewriting.
     *  If you don't want to buffer up tokens, then this method makes no
     *  sense for you.  Naturally you can't use the rewrite stream feature.
     *  I believe DebugTokenStream can easily be altered to not use
     *  this method, removing the dependency.
     */
    pANTLR3_COMMON_TOKEN    (*get)		(struct ANTLR3_TOKEN_STREAM_struct * tokenStream, ANTLR3_UINT32 i);

    /** Where is this stream pulling tokens from?  This is not the name, but
     *  a pointer into an interface that contains a ANTLR3_TOKEN_SOURCE interface.
     *  The Token Source interface contains a pointer to the input stream and a pointer
     *  to a function that returns the next token.
     */
    pANTLR3_TOKEN_SOURCE    (*getTokenSource)	(struct ANTLR3_TOKEN_STREAM_struct * tokenStream);

    /** Function that installs a token source for teh stream
     */
    void		    (*setTokenSource)	(struct ANTLR3_TOKEN_STREAM_struct * tokenStream,
						 pANTLR3_TOKEN_SOURCE		   tokenSource);

    /** Return the text of all the tokens in the stream, as the old tramp in 
     *  Leeds market used to say; "Get the lot!"
     */
    pANTLR3_STRING	    (*toString)		(struct ANTLR3_TOKEN_STREAM_struct * tokenStream);

    /** Return the text of all tokens from start to stop, inclusive.
     *  If the stream does not buffer all the tokens then it can just
     *  return an empty ANTLR3_STRING or NULL;  Grammars should not access $ruleLabel.text in
     *  an action in that case.
     */
    pANTLR3_STRING	    (*toStringSS)	(struct ANTLR3_TOKEN_STREAM_struct * tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);

    /** Because the user is not required to use a token with an index stored
     *  in it, we must provide a means for two token objects themselves to
     *  indicate the start/end location.  Most often this will just delegate
     *  to the other toString(int,int).  This is also parallel with
     *  the pTREENODE_STREAM->toString(Object,Object).
     */
    pANTLR3_STRING	    (*toStringTT)	(struct ANTLR3_TOKEN_STREAM_struct * tokenStream, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);


    /** Function that sets the token stream into debugging mode
     */
    void		    (*setDebugListener)	    (struct ANTLR3_TOKEN_STREAM_struct * tokenStream, pANTLR3_DEBUG_EVENT_LISTENER debugger);



    /** Function that knows how to free the memory for an ANTLR3_TOKEN_STREAM
     */
    void		    (*free)		(struct ANTLR3_TOKEN_STREAM_struct * tokenStream);
}
    ANTLR3_TOKEN_STREAM;

/** Common token stream is an implementation of ANTLR_TOKEN_STREAM for the default
 *  parsers and recognizers. You may of course build your own implementation if
 *  you are so inclined.
 */
typedef	struct	ANTLR3_COMMON_TOKEN_STREAM_struct
{
    /** The ANTLR3_TOKEN_STREAM interface implementation, which also includes
     *  the intstream implementation. We could duplicate the pANTLR_INT_STREAM
     *  in this interface and initialize it to a copy, but this could be confusing
     *  it just results in one more level of indirection and I think that with
     *  judicial use of 'const' later, the optimizer will do decent job.
     */
    pANTLR3_TOKEN_STREAM    tstream;

    /** Whatever is supplying the COMMON_TOKEN_STREAM needs a pointer to itself
     *  so that this can be accessed by any of the API functions which it implements.
     */
    void		    * super;

    /** Records every single token pulled from the source indexed by the token index.
     *  There might be more efficient ways to do this, such as referencing directly in to
     *  the token factory pools, but for now this is convenient and the ANTLR3_LIST is not
     *  a huge overhead as it only stores pointers anyway, but allows for iterations and 
     *  so on.
     */
    pANTLR3_VECTOR	    tokens;

    /** Override map of tokens. If a token type has an entry in here, then
     *  the pointer in the table points to an int, being the override channel number
     *  that should always be used for this token type.
     */
    pANTLR3_LIST	    channelOverrides;

    /** Discared set. If a token has an entry in this table, then it is thrown
     *  away (data pointer is always NULL).
     */
    pANTLR3_LIST	    discardSet;

    /* The channel number that this token stream is tuned to. For instance, whitespace
     * is usually tuned to channel 99, which no token stream would normally tune to and
     * so it is thrown away.
     */
    ANTLR3_UINT32	    channel;

    /** If this flag is set to ANTLR3_TRUE, then tokens that the stream sees that are not
     *  in the channel that this stream is tuned to, are not tracked in the
     *  tokens table. When set to false, ALL tokens are added to the tracking.
     */
    ANTLR3_BOOLEAN	    discardOffChannel;

    /** The index into the tokens list of the current token (the next one that will be
     *  consumed. p = -1 indicates that the token list is empty.
     */
    ANTLR3_INT32	    p;

    /** A simple filter mechanism whereby you can tell this token stream
     *  to force all tokens of type ttype to be on channel.  For example,
     *  when interpreting, we cannot exec actions so we need to tell
     *  the stream to force all WS and NEWLINE to be a different, ignored
     *  channel.
     */
    void		    (*setTokenTypeChannel)  (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, 
							ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);

    /** Add a particular token type to the discard set. If a token is found to belong 
     *  to this set, then it is skipped/thrown away
     */
    void		    (*discardTokenType)	    (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, ANTLR3_INT32 ttype);

    /** Signal to discard off channel tokens from here on in.
     */
    void		    (*discardOffChannelToks)(struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, ANTLR3_BOOLEAN discard);

    /** Function that returns a pointer to the ANTLR3_LIST of all tokens
     *  in the stream (this causes the buffer to fill if we have not get any yet)
     */
    pANTLR3_VECTOR	    (*getTokens)	    (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream);

    /** Function that returns all the tokens between a start and a stop index.
     *  TODO: This is a new list (Ack! Maybe this is a reason to have factories for LISTS and HASHTABLES etc :-( come back to this)
     */
    pANTLR3_LIST	    (*getTokenRange)	    (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);

    /** Function that returns all the tokens indicated by the specified bitset, within a range of tokens
     */
    pANTLR3_LIST	    (*getTokensSet)	    (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, 
							ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);
    
    /** Function that returns all the tokens indicated by being a member of the supplied List
     */
    pANTLR3_LIST	    (*getTokensList)	    (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, 
							ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);

    /** Function that returns all tokens of a certain type within a range.
     */
    pANTLR3_LIST	    (*getTokensType)	    (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream, 
							ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);

    /** Function that resets the token stream so that it can be reused, but
     *  but that does not free up any resources, such as the token factory
     *  the factory pool and so on. This prevents the need to keep freeing
     *  and reallocating the token pools if the thing you are building is
     *  a multi-shot dameon or somethign like that. It is much faster to
     *  just reuse all the vectors.
     */
    void                    (*reset)            (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream);

    /** Function that knows how to free an ANTLR3_COMMON_TOKEN_STREAM
     */
    void		    (*free)		    (struct ANTLR3_COMMON_TOKEN_STREAM_struct * tokenStream);
}
    ANTLR3_COMMON_TOKEN_STREAM;

#ifdef __cplusplus
}
#endif

#endif