summaryrefslogtreecommitdiff
path: root/antlr/libantlr3c-3.4/src/antlr3commontoken.c
diff options
context:
space:
mode:
Diffstat (limited to 'antlr/libantlr3c-3.4/src/antlr3commontoken.c')
-rw-r--r--antlr/libantlr3c-3.4/src/antlr3commontoken.c586
1 files changed, 586 insertions, 0 deletions
diff --git a/antlr/libantlr3c-3.4/src/antlr3commontoken.c b/antlr/libantlr3c-3.4/src/antlr3commontoken.c
new file mode 100644
index 0000000..2627431
--- /dev/null
+++ b/antlr/libantlr3c-3.4/src/antlr3commontoken.c
@@ -0,0 +1,586 @@
+/**
+ * Contains the default implementation of the common token used within
+ * java. Custom tokens should create this structure and then append to it using the
+ * custom pointer to install their own structure and API.
+ */
+
+// [The "BSD licence"]
+// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
+// http://www.temporal-wave.com
+// http://www.linkedin.com/in/jimidle
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. The name of the author may not be used to endorse or promote products
+// derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <antlr3.h>
+
+/* Token API
+ */
+static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token);
+static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text);
+static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text);
+static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token);
+static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type);
+static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token);
+static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line);
+static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token);
+static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos);
+static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token);
+static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel);
+static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token);
+static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER);
+static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token);
+static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
+static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token);
+static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
+static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token);
+
+/* Factory API
+ */
+static void factoryClose (pANTLR3_TOKEN_FACTORY factory);
+static pANTLR3_COMMON_TOKEN newToken (void);
+static void setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input);
+static void factoryReset (pANTLR3_TOKEN_FACTORY factory);
+
+/* Internal management functions
+ */
+static void newPool (pANTLR3_TOKEN_FACTORY factory);
+static pANTLR3_COMMON_TOKEN newPoolToken (pANTLR3_TOKEN_FACTORY factory);
+
+
+ANTLR3_API pANTLR3_COMMON_TOKEN
+antlr3CommonTokenNew(ANTLR3_UINT32 ttype)
+{
+ pANTLR3_COMMON_TOKEN token;
+
+ // Create a raw token with the interface installed
+ //
+ token = newToken();
+
+ if (token != NULL)
+ {
+ token->setType(token, ttype);
+ }
+
+ // All good
+ //
+ return token;
+}
+
+ANTLR3_API pANTLR3_TOKEN_FACTORY
+antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input)
+{
+ pANTLR3_TOKEN_FACTORY factory;
+
+ /* allocate memory
+ */
+ factory = (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY));
+
+ if (factory == NULL)
+ {
+ return NULL;
+ }
+
+ /* Install factory API
+ */
+ factory->newToken = newPoolToken;
+ factory->close = factoryClose;
+ factory->setInputStream = setInputStream;
+ factory->reset = factoryReset;
+
+ /* Allocate the initial pool
+ */
+ factory->thisPool = -1;
+ factory->pools = NULL;
+ factory->maxPool = -1;
+ newPool(factory);
+
+ /* Factory space is good, we now want to initialize our cheating token
+ * which one it is initialized is the model for all tokens we manufacture
+ */
+ antlr3SetTokenAPI(&factory->unTruc);
+
+ /* Set some initial variables for future copying
+ */
+ factory->unTruc.factoryMade = ANTLR3_TRUE;
+
+ // Input stream
+ //
+ setInputStream(factory, input);
+
+ return factory;
+
+}
+
+static void
+setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input)
+{
+ factory->input = input;
+ factory->unTruc.input = input;
+ if (input != NULL)
+ {
+ factory->unTruc.strFactory = input->strFactory;
+ }
+ else
+ {
+ factory->unTruc.strFactory = NULL;
+ }
+}
+
+static void
+newPool(pANTLR3_TOKEN_FACTORY factory)
+{
+ /* Increment factory count
+ */
+ factory->thisPool++;
+
+ // If we were reusing this token factory then we may already have a pool
+ // allocated. If we exceeded the max avaible then we must allocate a new
+ // one.
+ if (factory->thisPool > factory->maxPool)
+ {
+ /* Ensure we have enough pointers allocated
+ */
+ factory->pools = (pANTLR3_COMMON_TOKEN *)
+ ANTLR3_REALLOC( (void *)factory->pools, /* Current pools pointer (starts at NULL) */
+ (ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN *)) /* Memory for new pool pointers */
+ );
+
+ /* Allocate a new pool for the factory
+ */
+ factory->pools[factory->thisPool] =
+ (pANTLR3_COMMON_TOKEN)
+ ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE));
+
+ // We now have a new pool and can track it as the maximum we have created so far
+ //
+ factory->maxPool = factory->thisPool;
+ }
+
+ /* Reset the counters
+ */
+ factory->nextToken = 0;
+
+ /* Done
+ */
+ return;
+}
+
+static pANTLR3_COMMON_TOKEN
+newPoolToken(pANTLR3_TOKEN_FACTORY factory)
+{
+ pANTLR3_COMMON_TOKEN token;
+
+ /* See if we need a new token pool before allocating a new
+ * one
+ */
+ if (factory->nextToken >= ANTLR3_FACTORY_POOL_SIZE)
+ {
+ /* We ran out of tokens in the current pool, so we need a new pool
+ */
+ newPool(factory);
+ }
+
+ /* Assuming everything went well (we are trying for performance here so doing minimal
+ * error checking. Then we can work out what the pointer is to the next token.
+ */
+ token = factory->pools[factory->thisPool] + factory->nextToken;
+ factory->nextToken++;
+
+ /* We have our token pointer now, so we can initialize it to the predefined model.
+ * We only need do this though if the token is not already initialized, we just check
+ * an api function pointer for this as they are allocated via calloc.
+ */
+ if (token->setStartIndex == NULL)
+ {
+ antlr3SetTokenAPI(token);
+
+ // It is factory made, and we need to copy the string factory pointer
+ //
+ token->factoryMade = ANTLR3_TRUE;
+ token->strFactory = factory->input == NULL ? NULL : factory->input->strFactory;
+ token->input = factory->input;
+ }
+
+ /* And we are done
+ */
+ return token;
+}
+
+static void
+factoryReset (pANTLR3_TOKEN_FACTORY factory)
+{
+ // Just start again with pool #0 when we are
+ // called.
+ //
+ factory->thisPool = -1;
+ newPool(factory);
+}
+
+static void
+factoryClose (pANTLR3_TOKEN_FACTORY factory)
+{
+ pANTLR3_COMMON_TOKEN pool;
+ ANTLR3_INT32 poolCount;
+ ANTLR3_UINT32 limit;
+ ANTLR3_UINT32 token;
+ pANTLR3_COMMON_TOKEN check;
+
+ /* We iterate the token pools one at a time
+ */
+ for (poolCount = 0; poolCount <= factory->thisPool; poolCount++)
+ {
+ /* Pointer to current pool
+ */
+ pool = factory->pools[poolCount];
+
+ /* Work out how many tokens we need to check in this pool.
+ */
+ limit = (poolCount == factory->thisPool ? factory->nextToken : ANTLR3_FACTORY_POOL_SIZE);
+
+ /* Marginal condition, we might be at the start of a brand new pool
+ * where the nextToken is 0 and nothing has been allocated.
+ */
+ if (limit > 0)
+ {
+ /* We have some tokens allocated from this pool
+ */
+ for (token = 0; token < limit; token++)
+ {
+ /* Next one in the chain
+ */
+ check = pool + token;
+
+ /* If the programmer made this a custom token, then
+ * see if we need to call their free routine.
+ */
+ if (check->custom != NULL && check->freeCustom != NULL)
+ {
+ check->freeCustom(check->custom);
+ check->custom = NULL;
+ }
+ }
+ }
+
+ /* We can now free this pool allocation
+ */
+ ANTLR3_FREE(factory->pools[poolCount]);
+ factory->pools[poolCount] = NULL;
+ }
+
+ /* All the pools are deallocated we can free the pointers to the pools
+ * now.
+ */
+ ANTLR3_FREE(factory->pools);
+
+ /* Finally, we can free the space for the factory itself
+ */
+ ANTLR3_FREE(factory);
+}
+
+
+static pANTLR3_COMMON_TOKEN
+newToken(void)
+{
+ pANTLR3_COMMON_TOKEN token;
+
+ /* Allocate memory for this
+ */
+ token = (pANTLR3_COMMON_TOKEN) ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN)));
+
+ if (token == NULL)
+ {
+ return NULL;
+ }
+
+ // Install the API
+ //
+ antlr3SetTokenAPI(token);
+ token->factoryMade = ANTLR3_FALSE;
+
+ return token;
+}
+
+ANTLR3_API void
+antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)
+{
+ token->getText = getText;
+ token->setText = setText;
+ token->setText8 = setText8;
+ token->getType = getType;
+ token->setType = setType;
+ token->getLine = getLine;
+ token->setLine = setLine;
+ token->setLine = setLine;
+ token->getCharPositionInLine = getCharPositionInLine;
+ token->setCharPositionInLine = setCharPositionInLine;
+ token->getChannel = getChannel;
+ token->setChannel = setChannel;
+ token->getTokenIndex = getTokenIndex;
+ token->setTokenIndex = setTokenIndex;
+ token->getStartIndex = getStartIndex;
+ token->setStartIndex = setStartIndex;
+ token->getStopIndex = getStopIndex;
+ token->setStopIndex = setStopIndex;
+ token->toString = toString;
+
+ return;
+}
+
+static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token)
+{
+ switch (token->textState)
+ {
+ case ANTLR3_TEXT_STRING:
+
+ // Someone already created a string for this token, so we just
+ // use it.
+ //
+ return token->tokText.text;
+ break;
+
+ case ANTLR3_TEXT_CHARP:
+
+ // We had a straight text pointer installed, now we
+ // must convert it to a string. Note we have to do this here
+ // or otherwise setText8() will just install the same char*
+ //
+ if (token->strFactory != NULL)
+ {
+ token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);
+ token->textState = ANTLR3_TEXT_STRING;
+ return token->tokText.text;
+ }
+ else
+ {
+ // We cannot do anything here
+ //
+ return NULL;
+ }
+ break;
+
+ default:
+
+ // EOF is a special case
+ //
+ if (token->type == ANTLR3_TOKEN_EOF)
+ {
+ token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");
+ token->textState = ANTLR3_TEXT_STRING;
+ token->tokText.text->factory = token->strFactory;
+ return token->tokText.text;
+ }
+
+
+ // We had nothing installed in the token, create a new string
+ // from the input stream
+ //
+
+ if (token->input != NULL)
+ {
+
+ return token->input->substr( token->input,
+ token->getStartIndex(token),
+ token->getStopIndex(token)
+ );
+ }
+
+ // Nothing to return, there is no input stream
+ //
+ return NULL;
+ break;
+ }
+}
+static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)
+{
+ // No text to set, so ignore
+ //
+ if (text == NULL) return;
+
+ switch (token->textState)
+ {
+ case ANTLR3_TEXT_NONE:
+ case ANTLR3_TEXT_CHARP: // Caller must free before setting again, if it needs to be freed
+
+ // Nothing in there yet, or just a char *, so just set the
+ // text as a pointer
+ //
+ token->textState = ANTLR3_TEXT_CHARP;
+ token->tokText.chars = (pANTLR3_UCHAR)text;
+ break;
+
+ default:
+
+ // It was already a pANTLR3_STRING, so just override it
+ //
+ token->tokText.text->set8(token->tokText.text, (const char *)text);
+ break;
+ }
+
+ // We are done
+ //
+ return;
+}
+
+/** \brief Install the supplied text string as teh text for the token.
+ * The method assumes that the existing text (if any) was created by a factory
+ * and so does not attempt to release any memory it is using.Text not created
+ * by a string fctory (not advised) should be released prior to this call.
+ */
+static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text)
+{
+ // Merely replaces and existing pre-defined text with the supplied
+ // string
+ //
+ token->textState = ANTLR3_TEXT_STRING;
+ token->tokText.text = text;
+
+ /* We are done
+ */
+ return;
+}
+
+static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token)
+{
+ return token->type;
+}
+
+static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type)
+{
+ token->type = type;
+}
+
+static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token)
+{
+ return token->line;
+}
+
+static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line)
+{
+ token->line = line;
+}
+
+static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token)
+{
+ return token->charPosition;
+}
+
+static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos)
+{
+ token->charPosition = pos;
+}
+
+static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token)
+{
+ return token->channel;
+}
+
+static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel)
+{
+ token->channel = channel;
+}
+
+static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token)
+{
+ return token->index;
+}
+
+static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index)
+{
+ token->index = index;
+}
+
+static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token)
+{
+ return token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start;
+}
+
+static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start)
+{
+ token->start = start;
+}
+
+static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token)
+{
+ return token->stop;
+}
+
+static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop)
+{
+ token->stop = stop;
+}
+
+static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token)
+{
+ pANTLR3_STRING text;
+ pANTLR3_STRING outtext;
+
+ text = token->getText(token);
+
+ if (text == NULL)
+ {
+ return NULL;
+ }
+
+ if (text->factory == NULL)
+ {
+ return text; // This usally means it is the EOF token
+ }
+
+ /* A new empty string to assemble all the stuff in
+ */
+ outtext = text->factory->newRaw(text->factory);
+
+ /* Now we use our handy dandy string utility to assemble the
+ * the reporting string
+ * return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
+ */
+ outtext->append8(outtext, "[Index: ");
+ outtext->addi (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
+ outtext->append8(outtext, " (Start: ");
+ outtext->addi (outtext, (ANTLR3_INT32)token->getStartIndex(token));
+ outtext->append8(outtext, "-Stop: ");
+ outtext->addi (outtext, (ANTLR3_INT32)token->getStopIndex(token));
+ outtext->append8(outtext, ") ='");
+ outtext->appendS(outtext, text);
+ outtext->append8(outtext, "', type<");
+ outtext->addi (outtext, token->type);
+ outtext->append8(outtext, "> ");
+
+ if (token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
+ {
+ outtext->append8(outtext, "(channel = ");
+ outtext->addi (outtext, (ANTLR3_INT32)token->getChannel(token));
+ outtext->append8(outtext, ") ");
+ }
+
+ outtext->append8(outtext, "Line: ");
+ outtext->addi (outtext, (ANTLR3_INT32)token->getLine(token));
+ outtext->append8(outtext, " LinePos:");
+ outtext->addi (outtext, token->getCharPositionInLine(token));
+ outtext->addc (outtext, ']');
+
+ return outtext;
+}
+
generated by cgit v1.2.3 (git 2.46.2) at 2024-11-10 07:14:40 +0000