Move antlr. Add `make test` to Makefile.

1 files changed, 586 insertions, 0 deletions

diff --git a/antlr/libantlr3c-3.4/src/antlr3commontoken.c b/antlr/libantlr3c-3.4/src/antlr3commontoken.c
new file mode 100644
index 0000000..2627431
--- /dev/null
+++ b/antlr/libantlr3c-3.4/src/antlr3commontoken.c
@@ -0,0 +1,586 @@
+/**
+ * Contains the default implementation of the common token used within
+ * java. Custom tokens should create this structure and then append to it using the 
+ * custom pointer to install their own structure and API.
+ */
+
+// [The "BSD licence"]
+// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
+// http://www.temporal-wave.com
+// http://www.linkedin.com/in/jimidle
+//
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+//    notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+//    notice, this list of conditions and the following disclaimer in the
+//    documentation and/or other materials provided with the distribution.
+// 3. The name of the author may not be used to endorse or promote products
+//    derived from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include    <antlr3.h>
+
+/* Token API
+ */
+static  pANTLR3_STRING	getText					(pANTLR3_COMMON_TOKEN token);
+static  void			setText					(pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text);
+static  void			setText8				(pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text);
+static	ANTLR3_UINT32   getType					(pANTLR3_COMMON_TOKEN token);
+static  void			setType					(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type);
+static  ANTLR3_UINT32   getLine					(pANTLR3_COMMON_TOKEN token);
+static  void			setLine					(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line);
+static  ANTLR3_INT32    getCharPositionInLine	(pANTLR3_COMMON_TOKEN token);
+static  void			setCharPositionInLine	(pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos);
+static  ANTLR3_UINT32   getChannel				(pANTLR3_COMMON_TOKEN token);
+static  void			setChannel				(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel);
+static  ANTLR3_MARKER   getTokenIndex			(pANTLR3_COMMON_TOKEN token);
+static  void			setTokenIndex			(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER);
+static  ANTLR3_MARKER   getStartIndex			(pANTLR3_COMMON_TOKEN token);
+static  void			setStartIndex			(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
+static  ANTLR3_MARKER   getStopIndex			(pANTLR3_COMMON_TOKEN token);
+static  void			setStopIndex			(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
+static  pANTLR3_STRING  toString				(pANTLR3_COMMON_TOKEN token);
+
+/* Factory API
+ */
+static	void			factoryClose	(pANTLR3_TOKEN_FACTORY factory);
+static	pANTLR3_COMMON_TOKEN	newToken	(void);
+static  void			setInputStream	(pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input);
+static	void                    factoryReset    (pANTLR3_TOKEN_FACTORY factory);
+
+/* Internal management functions
+ */
+static	void			newPool		(pANTLR3_TOKEN_FACTORY factory);
+static	pANTLR3_COMMON_TOKEN    newPoolToken	(pANTLR3_TOKEN_FACTORY factory);
+
+
+ANTLR3_API pANTLR3_COMMON_TOKEN
+antlr3CommonTokenNew(ANTLR3_UINT32 ttype)
+{
+	pANTLR3_COMMON_TOKEN    token;
+
+	// Create a raw token with the interface installed
+	//
+	token   = newToken();
+
+	if	(token != NULL)
+	{
+		token->setType(token, ttype);
+	}
+
+	// All good
+	//
+	return  token;
+}
+
+ANTLR3_API pANTLR3_TOKEN_FACTORY
+antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input)
+{
+    pANTLR3_TOKEN_FACTORY   factory;
+
+    /* allocate memory
+     */
+    factory	= (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY));
+
+    if	(factory == NULL)
+    {
+	return	NULL;
+    }
+
+    /* Install factory API
+     */
+    factory->newToken	    = newPoolToken;
+    factory->close	    = factoryClose;
+    factory->setInputStream = setInputStream;
+    factory->reset          = factoryReset;
+    
+    /* Allocate the initial pool
+     */
+    factory->thisPool	= -1;
+    factory->pools      = NULL;
+    factory->maxPool    = -1;
+    newPool(factory);
+
+    /* Factory space is good, we now want to initialize our cheating token
+     * which one it is initialized is the model for all tokens we manufacture
+     */
+    antlr3SetTokenAPI(&factory->unTruc);
+
+    /* Set some initial variables for future copying
+     */
+    factory->unTruc.factoryMade	= ANTLR3_TRUE;
+
+    // Input stream
+    //
+    setInputStream(factory, input);
+    
+    return  factory;
+
+}
+
+static void
+setInputStream	(pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input)
+{
+    factory->input          =  input;
+    factory->unTruc.input   =  input;
+	if	(input != NULL)
+	{
+		factory->unTruc.strFactory	= input->strFactory;
+	}
+	else
+	{
+		factory->unTruc.strFactory = NULL;
+    }
+}
+
+static void
+newPool(pANTLR3_TOKEN_FACTORY factory)
+{
+    /* Increment factory count
+     */
+    factory->thisPool++;
+
+    // If we were reusing this token factory then we may already have a pool
+    // allocated. If we exceeded the max avaible then we must allocate a new
+    // one.
+    if  (factory->thisPool > factory->maxPool)
+    {
+        /* Ensure we have enough pointers allocated
+         */
+        factory->pools = (pANTLR3_COMMON_TOKEN *)
+		         ANTLR3_REALLOC(	(void *)factory->pools,	    /* Current pools pointer (starts at NULL)	*/
+					    (ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN *))	/* Memory for new pool pointers */
+					    );
+
+        /* Allocate a new pool for the factory
+         */
+        factory->pools[factory->thisPool]	=
+			        (pANTLR3_COMMON_TOKEN) 
+				    ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE));
+
+        // We now have a new pool and can track it as the maximum we have created so far
+        //
+        factory->maxPool = factory->thisPool;
+    }
+
+    /* Reset the counters
+     */
+    factory->nextToken	= 0;
+  
+    /* Done
+     */
+    return;
+}
+
+static pANTLR3_COMMON_TOKEN
+newPoolToken(pANTLR3_TOKEN_FACTORY factory)
+{
+    pANTLR3_COMMON_TOKEN token;
+
+    /* See if we need a new token pool before allocating a new
+     * one
+     */
+    if (factory->nextToken >= ANTLR3_FACTORY_POOL_SIZE)
+    {
+        /* We ran out of tokens in the current pool, so we need a new pool
+         */
+        newPool(factory);
+    }
+
+    /* Assuming everything went well (we are trying for performance here so doing minimal
+     * error checking. Then we can work out what the pointer is to the next token.
+     */
+    token = factory->pools[factory->thisPool] + factory->nextToken;
+    factory->nextToken++;
+
+    /* We have our token pointer now, so we can initialize it to the predefined model.
+     * We only need do this though if the token is not already initialized, we just check
+     * an api function pointer for this as they are allocated via calloc.
+     */
+    if  (token->setStartIndex == NULL)
+    {
+        antlr3SetTokenAPI(token);
+
+        // It is factory made, and we need to copy the string factory pointer
+        //
+        token->factoryMade  = ANTLR3_TRUE;
+        token->strFactory   = factory->input == NULL ? NULL : factory->input->strFactory;
+        token->input        = factory->input;
+    }
+
+    /* And we are done
+     */
+    return token;
+}
+
+static	void
+factoryReset	    (pANTLR3_TOKEN_FACTORY factory)
+{
+    // Just start again with pool #0 when we are
+    // called.
+    //
+    factory->thisPool   = -1;
+    newPool(factory);
+}
+
+static	void
+factoryClose	    (pANTLR3_TOKEN_FACTORY factory)
+{
+    pANTLR3_COMMON_TOKEN    pool;
+    ANTLR3_INT32	    poolCount;
+    ANTLR3_UINT32	    limit;
+    ANTLR3_UINT32	    token;
+    pANTLR3_COMMON_TOKEN    check;
+
+    /* We iterate the token pools one at a time
+     */
+    for	(poolCount = 0; poolCount <= factory->thisPool; poolCount++)
+    {
+	/* Pointer to current pool
+	 */
+	pool	= factory->pools[poolCount];
+
+	/* Work out how many tokens we need to check in this pool.
+	 */
+	limit	= (poolCount == factory->thisPool ? factory->nextToken : ANTLR3_FACTORY_POOL_SIZE);
+	
+	/* Marginal condition, we might be at the start of a brand new pool
+	 * where the nextToken is 0 and nothing has been allocated.
+	 */
+	if  (limit > 0)
+	{
+	    /* We have some tokens allocated from this pool
+	     */
+	    for (token = 0; token < limit; token++)
+	    {
+		/* Next one in the chain
+		 */
+		check	= pool + token;
+
+		/* If the programmer made this a custom token, then
+		 * see if we need to call their free routine.
+		 */
+		if  (check->custom != NULL && check->freeCustom != NULL)
+		{
+		    check->freeCustom(check->custom);
+		    check->custom = NULL;
+		}
+	    }
+	}
+
+	/* We can now free this pool allocation
+	 */
+	ANTLR3_FREE(factory->pools[poolCount]);
+	factory->pools[poolCount] = NULL;
+    }
+
+    /* All the pools are deallocated we can free the pointers to the pools
+     * now.
+     */
+    ANTLR3_FREE(factory->pools);
+
+    /* Finally, we can free the space for the factory itself
+     */
+    ANTLR3_FREE(factory);
+}
+
+
+static	pANTLR3_COMMON_TOKEN	
+newToken(void)
+{
+    pANTLR3_COMMON_TOKEN    token;
+
+    /* Allocate memory for this
+     */
+    token   = (pANTLR3_COMMON_TOKEN) ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN)));
+
+    if	(token == NULL)
+    {
+	return	NULL;
+    }
+
+    // Install the API
+    //
+    antlr3SetTokenAPI(token);
+    token->factoryMade = ANTLR3_FALSE;
+
+    return  token;
+}
+
+ANTLR3_API void
+antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)
+{
+    token->getText		    = getText;
+    token->setText		    = setText;
+    token->setText8		    = setText8;
+    token->getType		    = getType;
+    token->setType		    = setType;
+    token->getLine		    = getLine;
+    token->setLine		    = setLine;
+    token->setLine		    = setLine;
+    token->getCharPositionInLine    = getCharPositionInLine;
+    token->setCharPositionInLine    = setCharPositionInLine;
+    token->getChannel		    = getChannel;
+    token->setChannel		    = setChannel;
+    token->getTokenIndex	    = getTokenIndex;
+    token->setTokenIndex	    = setTokenIndex;
+    token->getStartIndex	    = getStartIndex;
+    token->setStartIndex	    = setStartIndex;
+    token->getStopIndex		    = getStopIndex;
+    token->setStopIndex		    = setStopIndex;
+    token->toString		    = toString;
+
+    return;
+}
+
+static  pANTLR3_STRING  getText			(pANTLR3_COMMON_TOKEN token)
+{
+	switch (token->textState)
+	{
+		case ANTLR3_TEXT_STRING:
+
+			// Someone already created a string for this token, so we just
+			// use it.
+			//
+			return	token->tokText.text;
+			break;
+    
+		case ANTLR3_TEXT_CHARP:
+
+			// We had a straight text pointer installed, now we
+			// must convert it to a string. Note we have to do this here
+			// or otherwise setText8() will just install the same char*
+			//
+			if	(token->strFactory != NULL)
+			{
+				token->tokText.text	= token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);
+				token->textState	= ANTLR3_TEXT_STRING;
+				return token->tokText.text;
+			}
+			else
+			{
+				// We cannot do anything here
+				//
+				return NULL;
+			}
+			break;
+
+		default:
+
+			// EOF is a special case
+			//
+			if (token->type == ANTLR3_TOKEN_EOF)
+			{
+				token->tokText.text				= token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");
+				token->textState				= ANTLR3_TEXT_STRING;
+				token->tokText.text->factory	= token->strFactory;
+				return token->tokText.text;
+			}
+
+
+			// We had nothing installed in the token, create a new string
+			// from the input stream
+			//
+
+			if	(token->input != NULL)
+			{
+			
+				return	token->input->substr(	token->input, 
+												token->getStartIndex(token), 
+ 												token->getStopIndex(token)
+											);
+			}
+
+			// Nothing to return, there is no input stream
+			//
+			return NULL;
+			break;
+	}
+}
+static  void		setText8		(pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)
+{
+	// No text to set, so ignore
+	//
+	if	(text == NULL) return;
+
+	switch	(token->textState)
+	{
+		case	ANTLR3_TEXT_NONE:
+		case	ANTLR3_TEXT_CHARP:	// Caller must free before setting again, if it needs to be freed
+
+			// Nothing in there yet, or just a char *, so just set the
+			// text as a pointer
+			//
+			token->textState		= ANTLR3_TEXT_CHARP;
+			token->tokText.chars	= (pANTLR3_UCHAR)text;
+			break;
+
+		default:
+
+			// It was already a pANTLR3_STRING, so just override it
+			//
+			token->tokText.text->set8(token->tokText.text, (const char *)text);
+			break;
+	}
+
+	// We are done 
+	//
+	return;
+}
+
+/** \brief Install the supplied text string as teh text for the token.
+ * The method assumes that the existing text (if any) was created by a factory
+ * and so does not attempt to release any memory it is using.Text not created
+ * by a string fctory (not advised) should be released prior to this call.
+ */
+static  void		setText			(pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text)
+{
+	// Merely replaces and existing pre-defined text with the supplied
+	// string
+	//
+	token->textState	= ANTLR3_TEXT_STRING;
+	token->tokText.text	= text;
+
+	/* We are done 
+	*/
+	return;
+}
+
+static	ANTLR3_UINT32   getType			(pANTLR3_COMMON_TOKEN token)
+{
+    return  token->type;
+}
+
+static  void		setType			(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type)
+{
+    token->type = type;
+}
+
+static  ANTLR3_UINT32   getLine			(pANTLR3_COMMON_TOKEN token)
+{
+    return  token->line;
+}
+
+static  void		setLine			(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line)
+{
+    token->line = line;
+}
+
+static  ANTLR3_INT32    getCharPositionInLine	(pANTLR3_COMMON_TOKEN token)
+{
+    return  token->charPosition;
+}
+
+static  void		setCharPositionInLine	(pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos)
+{
+    token->charPosition = pos;
+}
+
+static  ANTLR3_UINT32   getChannel		(pANTLR3_COMMON_TOKEN token)
+{
+    return  token->channel;
+}
+
+static  void		setChannel		(pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel)
+{
+    token->channel  = channel;
+}
+
+static  ANTLR3_MARKER   getTokenIndex		(pANTLR3_COMMON_TOKEN token)
+{
+    return  token->index;
+}
+
+static  void		setTokenIndex		(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index)
+{
+    token->index    = index;
+}
+
+static  ANTLR3_MARKER   getStartIndex		(pANTLR3_COMMON_TOKEN token)
+{
+	return  token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start;
+}
+
+static  void		setStartIndex		(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start)
+{
+    token->start    = start;
+}
+
+static  ANTLR3_MARKER   getStopIndex		(pANTLR3_COMMON_TOKEN token)
+{
+    return  token->stop;
+}
+
+static  void		setStopIndex		(pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop)
+{
+    token->stop	= stop;
+}
+
+static  pANTLR3_STRING    toString		(pANTLR3_COMMON_TOKEN token)
+{
+    pANTLR3_STRING  text;
+    pANTLR3_STRING  outtext;
+
+    text    =	token->getText(token);
+    
+    if	(text == NULL)
+    {
+		return NULL;
+    }
+
+	if	(text->factory == NULL)
+	{
+		return text;		// This usally means it is the EOF token
+	}
+
+    /* A new empty string to assemble all the stuff in
+     */
+    outtext = text->factory->newRaw(text->factory);
+
+    /* Now we use our handy dandy string utility to assemble the
+     * the reporting string
+     * return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
+     */
+    outtext->append8(outtext, "[Index: ");
+    outtext->addi   (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
+    outtext->append8(outtext, " (Start: ");
+    outtext->addi   (outtext, (ANTLR3_INT32)token->getStartIndex(token));
+    outtext->append8(outtext, "-Stop: ");
+    outtext->addi   (outtext, (ANTLR3_INT32)token->getStopIndex(token));
+    outtext->append8(outtext, ") ='");
+    outtext->appendS(outtext, text);
+    outtext->append8(outtext, "', type<");
+    outtext->addi   (outtext, token->type);
+    outtext->append8(outtext, "> ");
+
+    if	(token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
+    {
+		outtext->append8(outtext, "(channel = ");
+		outtext->addi	(outtext, (ANTLR3_INT32)token->getChannel(token));
+		outtext->append8(outtext, ") ");
+    }
+
+    outtext->append8(outtext, "Line: ");
+    outtext->addi   (outtext, (ANTLR3_INT32)token->getLine(token));
+    outtext->append8(outtext, " LinePos:");
+    outtext->addi   (outtext, token->getCharPositionInLine(token));
+    outtext->addc   (outtext, ']');
+
+    return  outtext;
+}
+