/// \file
|
/// Default implementation of CommonTokenStream
|
///
|
|
// [The "BSD licence"]
|
// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
|
// http://www.temporal-wave.com
|
// http://www.linkedin.com/in/jimidle
|
//
|
// All rights reserved.
|
//
|
// Redistribution and use in source and binary forms, with or without
|
// modification, are permitted provided that the following conditions
|
// are met:
|
// 1. Redistributions of source code must retain the above copyright
|
// notice, this list of conditions and the following disclaimer.
|
// 2. Redistributions in binary form must reproduce the above copyright
|
// notice, this list of conditions and the following disclaimer in the
|
// documentation and/or other materials provided with the distribution.
|
// 3. The name of the author may not be used to endorse or promote products
|
// derived from this software without specific prior written permission.
|
//
|
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
#include <antlr3tokenstream.h>
|
|
#ifdef ANTLR3_WINDOWS
|
#pragma warning( disable : 4100 )
|
#endif
|
|
// COMMON_TOKEN_STREAM API
|
//
|
static void setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel);
|
static void discardTokenType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 ttype);
|
static void discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_BOOLEAN discard);
|
static pANTLR3_VECTOR getTokens (pANTLR3_COMMON_TOKEN_STREAM cts);
|
static pANTLR3_LIST getTokenRange (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
|
static pANTLR3_LIST getTokensSet (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types);
|
static pANTLR3_LIST getTokensList (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list);
|
static pANTLR3_LIST getTokensType (pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type);
|
static void reset (pANTLR3_COMMON_TOKEN_STREAM cts);
|
|
// TOKEN_STREAM API
|
//
|
static pANTLR3_COMMON_TOKEN tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
|
static pANTLR3_COMMON_TOKEN dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k);
|
static pANTLR3_COMMON_TOKEN get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i);
|
static pANTLR3_TOKEN_SOURCE getTokenSource (pANTLR3_TOKEN_STREAM ts);
|
static void setTokenSource (pANTLR3_TOKEN_STREAM ts, pANTLR3_TOKEN_SOURCE tokenSource);
|
static pANTLR3_STRING toString (pANTLR3_TOKEN_STREAM ts);
|
static pANTLR3_STRING toStringSS (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop);
|
static pANTLR3_STRING toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop);
|
static void setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger);
|
|
// INT STREAM API
|
//
|
static void consume (pANTLR3_INT_STREAM is);
|
static void dbgConsume (pANTLR3_INT_STREAM is);
|
static ANTLR3_UINT32 _LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
|
static ANTLR3_UINT32 dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i);
|
static ANTLR3_MARKER mark (pANTLR3_INT_STREAM is);
|
static ANTLR3_MARKER dbgMark (pANTLR3_INT_STREAM is);
|
static void release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark);
|
static ANTLR3_UINT32 size (pANTLR3_INT_STREAM is);
|
static ANTLR3_MARKER tindex (pANTLR3_INT_STREAM is);
|
static void rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
|
static void dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker);
|
static void rewindLast (pANTLR3_INT_STREAM is);
|
static void dbgRewindLast (pANTLR3_INT_STREAM is);
|
static void seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
|
static void dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index);
|
static pANTLR3_STRING getSourceName (pANTLR3_INT_STREAM is);
|
static void antlr3TokenStreamFree (pANTLR3_TOKEN_STREAM stream);
|
static void antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream);
|
|
// Helpers
|
//
|
static void fillBuffer (pANTLR3_COMMON_TOKEN_STREAM tokenStream);
|
static ANTLR3_UINT32 skipOffTokenChannels (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
|
static ANTLR3_UINT32 skipOffTokenChannelsReverse (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
|
static pANTLR3_COMMON_TOKEN LB (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i);
|
|
ANTLR3_API pANTLR3_TOKEN_STREAM
|
antlr3TokenStreamNew()
|
{
|
pANTLR3_TOKEN_STREAM stream;
|
|
// Memory for the interface structure
|
//
|
stream = (pANTLR3_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_TOKEN_STREAM));
|
|
if (stream == NULL)
|
{
|
return NULL;
|
}
|
|
// Install basic API
|
//
|
stream->free = antlr3TokenStreamFree;
|
|
|
return stream;
|
}
|
|
static void
|
antlr3TokenStreamFree(pANTLR3_TOKEN_STREAM stream)
|
{
|
ANTLR3_FREE(stream);
|
}
|
|
static void
|
antlr3CTSFree (pANTLR3_COMMON_TOKEN_STREAM stream)
|
{
|
// We only free up our subordinate interfaces if they belong
|
// to us, otherwise we let whoever owns them deal with them.
|
//
|
if (stream->tstream->super == stream)
|
{
|
if (stream->tstream->istream->super == stream->tstream)
|
{
|
stream->tstream->istream->free(stream->tstream->istream);
|
stream->tstream->istream = NULL;
|
}
|
stream->tstream->free(stream->tstream);
|
}
|
|
// Now we free our own resources
|
//
|
if (stream->tokens != NULL)
|
{
|
stream->tokens->free(stream->tokens);
|
stream->tokens = NULL;
|
}
|
if (stream->discardSet != NULL)
|
{
|
stream->discardSet->free(stream->discardSet);
|
stream->discardSet = NULL;
|
}
|
if (stream->channelOverrides != NULL)
|
{
|
stream->channelOverrides->free(stream->channelOverrides);
|
stream->channelOverrides = NULL;
|
}
|
|
// Free our memory now
|
//
|
ANTLR3_FREE(stream);
|
}
|
|
// Reset a token stream so it can be used again and can reuse it's
|
// resources.
|
//
|
static void
|
reset (pANTLR3_COMMON_TOKEN_STREAM cts)
|
{
|
|
// Free any resources that ar most like specifc to the
|
// run we just did.
|
//
|
if (cts->discardSet != NULL)
|
{
|
cts->discardSet->free(cts->discardSet);
|
cts->discardSet = NULL;
|
}
|
if (cts->channelOverrides != NULL)
|
{
|
cts->channelOverrides->free(cts->channelOverrides);
|
cts->channelOverrides = NULL;
|
}
|
|
// Now, if there were any existing tokens in the stream,
|
// then we just reset the vector count so that it starts
|
// again. We must traverse the entries unfortunately as
|
// there may be free pointers for custom token types and
|
// so on. However that is just a quick NULL check on the
|
// vector entries.
|
//
|
if (cts->tokens != NULL)
|
{
|
cts->tokens->clear(cts->tokens);
|
}
|
else
|
{
|
/* Install the token tracking tables
|
*/
|
cts->tokens = antlr3VectorNew(0);
|
}
|
|
// Reset to defaults
|
//
|
cts->discardOffChannel = ANTLR3_FALSE;
|
cts->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
|
cts->p = -1;
|
}
|
|
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
|
antlr3CommonTokenDebugStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source, pANTLR3_DEBUG_EVENT_LISTENER debugger)
|
{
|
pANTLR3_COMMON_TOKEN_STREAM stream;
|
|
// Create a standard token stream
|
//
|
stream = antlr3CommonTokenStreamSourceNew(hint, source);
|
|
// Install the debugger object
|
//
|
stream->tstream->debugger = debugger;
|
|
// Override standard token stream methods with debugging versions
|
//
|
stream->tstream->initialStreamState = ANTLR3_FALSE;
|
|
stream->tstream->_LT = dbgTokLT;
|
|
stream->tstream->istream->consume = dbgConsume;
|
stream->tstream->istream->_LA = dbgLA;
|
stream->tstream->istream->mark = dbgMark;
|
stream->tstream->istream->rewind = dbgRewindStream;
|
stream->tstream->istream->rewindLast = dbgRewindLast;
|
stream->tstream->istream->seek = dbgSeek;
|
|
return stream;
|
}
|
|
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
|
antlr3CommonTokenStreamSourceNew(ANTLR3_UINT32 hint, pANTLR3_TOKEN_SOURCE source)
|
{
|
pANTLR3_COMMON_TOKEN_STREAM stream;
|
|
stream = antlr3CommonTokenStreamNew(hint);
|
|
stream->channel = ANTLR3_TOKEN_DEFAULT_CHANNEL;
|
|
stream->channelOverrides = NULL;
|
stream->discardSet = NULL;
|
stream->discardOffChannel = ANTLR3_FALSE;
|
|
stream->tstream->setTokenSource(stream->tstream, source);
|
|
stream->free = antlr3CTSFree;
|
return stream;
|
}
|
|
ANTLR3_API pANTLR3_COMMON_TOKEN_STREAM
|
antlr3CommonTokenStreamNew(ANTLR3_UINT32 hint)
|
{
|
pANTLR3_COMMON_TOKEN_STREAM stream;
|
|
/* Memory for the interface structure
|
*/
|
stream = (pANTLR3_COMMON_TOKEN_STREAM) ANTLR3_MALLOC(sizeof(ANTLR3_COMMON_TOKEN_STREAM));
|
|
if (stream == NULL)
|
{
|
return NULL;
|
}
|
|
/* Create space for the token stream interface
|
*/
|
stream->tstream = antlr3TokenStreamNew();
|
stream->tstream->super = stream;
|
|
/* Create space for the INT_STREAM interfacce
|
*/
|
stream->tstream->istream = antlr3IntStreamNew();
|
stream->tstream->istream->super = (stream->tstream);
|
stream->tstream->istream->type = ANTLR3_TOKENSTREAM;
|
|
/* Install the token tracking tables
|
*/
|
stream->tokens = antlr3VectorNew(0);
|
|
/* Defaults
|
*/
|
stream->p = -1;
|
|
/* Install the common token stream API
|
*/
|
stream->setTokenTypeChannel = setTokenTypeChannel;
|
stream->discardTokenType = discardTokenType;
|
stream->discardOffChannelToks = discardOffChannel;
|
stream->getTokens = getTokens;
|
stream->getTokenRange = getTokenRange;
|
stream->getTokensSet = getTokensSet;
|
stream->getTokensList = getTokensList;
|
stream->getTokensType = getTokensType;
|
stream->reset = reset;
|
|
/* Install the token stream API
|
*/
|
stream->tstream->_LT = tokLT;
|
stream->tstream->get = get;
|
stream->tstream->getTokenSource = getTokenSource;
|
stream->tstream->setTokenSource = setTokenSource;
|
stream->tstream->toString = toString;
|
stream->tstream->toStringSS = toStringSS;
|
stream->tstream->toStringTT = toStringTT;
|
stream->tstream->setDebugListener = setDebugListener;
|
|
/* Install INT_STREAM interface
|
*/
|
stream->tstream->istream->_LA = _LA;
|
stream->tstream->istream->mark = mark;
|
stream->tstream->istream->release = release;
|
stream->tstream->istream->size = size;
|
stream->tstream->istream->index = tindex;
|
stream->tstream->istream->rewind = rewindStream;
|
stream->tstream->istream->rewindLast= rewindLast;
|
stream->tstream->istream->seek = seek;
|
stream->tstream->istream->consume = consume;
|
stream->tstream->istream->getSourceName = getSourceName;
|
|
return stream;
|
}
|
|
// Install a debug listener adn switch to debug mode methods
|
//
|
static void
|
setDebugListener (pANTLR3_TOKEN_STREAM ts, pANTLR3_DEBUG_EVENT_LISTENER debugger)
|
{
|
// Install the debugger object
|
//
|
ts->debugger = debugger;
|
|
// Override standard token stream methods with debugging versions
|
//
|
ts->initialStreamState = ANTLR3_FALSE;
|
|
ts->_LT = dbgTokLT;
|
|
ts->istream->consume = dbgConsume;
|
ts->istream->_LA = dbgLA;
|
ts->istream->mark = dbgMark;
|
ts->istream->rewind = dbgRewindStream;
|
ts->istream->rewindLast = dbgRewindLast;
|
ts->istream->seek = dbgSeek;
|
}
|
|
/** Get the ith token from the current position 1..n where k=1 is the
|
* first symbol of lookahead.
|
*/
|
static pANTLR3_COMMON_TOKEN
|
tokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
|
{
|
ANTLR3_INT32 i;
|
ANTLR3_INT32 n;
|
pANTLR3_COMMON_TOKEN_STREAM cts;
|
|
cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
|
|
if (k < 0)
|
{
|
return LB(cts, -k);
|
}
|
|
if (cts->p == -1)
|
{
|
fillBuffer(cts);
|
}
|
|
// Here we used to check for k == 0 and return 0, but this seems
|
// a superfluous check to me. LT(k=0) is therefore just undefined
|
// and we won't waste the clock cycles on the check
|
//
|
|
if ((cts->p + k - 1) >= (ANTLR3_INT32)ts->istream->cachedSize)
|
{
|
pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
|
|
teof->setStartIndex (teof, ts->istream->index (ts->istream));
|
teof->setStopIndex (teof, ts->istream->index (ts->istream));
|
return teof;
|
}
|
|
i = cts->p;
|
n = 1;
|
|
/* Need to find k good tokens, skipping ones that are off channel
|
*/
|
while ( n < k)
|
{
|
/* Skip off-channel tokens */
|
i = skipOffTokenChannels(cts, i+1); /* leave p on valid token */
|
n++;
|
}
|
if ( (ANTLR3_UINT32) i >= ts->istream->cachedSize)
|
{
|
pANTLR3_COMMON_TOKEN teof = &(ts->tokenSource->eofToken);
|
|
teof->setStartIndex (teof, ts->istream->index(ts->istream));
|
teof->setStopIndex (teof, ts->istream->index(ts->istream));
|
return teof;
|
}
|
|
// Here the token must be in the input vector. Rather then incur
|
// function call penalty, we just return the pointer directly
|
// from the vector
|
//
|
return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
|
//return (pANTLR3_COMMON_TOKEN)cts->tokens->get(cts->tokens, i);
|
}
|
|
/// Debug only method to flag consumption of initial off-channel
|
/// tokens in the input stream
|
///
|
static void
|
consumeInitialHiddenTokens(pANTLR3_INT_STREAM is)
|
{
|
ANTLR3_MARKER first;
|
ANTLR3_INT32 i;
|
pANTLR3_TOKEN_STREAM ts;
|
|
ts = (pANTLR3_TOKEN_STREAM) is->super;
|
first = is->index(is);
|
|
for (i=0; i<first; i++)
|
{
|
ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, i));
|
}
|
|
ts->initialStreamState = ANTLR3_FALSE;
|
|
}
|
|
/// As per the normal tokLT but sends information to the debugger
|
///
|
static pANTLR3_COMMON_TOKEN
|
dbgTokLT (pANTLR3_TOKEN_STREAM ts, ANTLR3_INT32 k)
|
{
|
if (ts->initialStreamState == ANTLR3_TRUE)
|
{
|
consumeInitialHiddenTokens(ts->istream);
|
}
|
return tokLT(ts, k);
|
}
|
|
#ifdef ANTLR3_WINDOWS
|
/* When fully optimized VC7 complains about non reachable code.
|
* Not yet sure if this is an optimizer bug, or a bug in the flow analysis
|
*/
|
#pragma warning( disable : 4702 )
|
#endif
|
|
static pANTLR3_COMMON_TOKEN
|
LB(pANTLR3_COMMON_TOKEN_STREAM cts, ANTLR3_INT32 k)
|
{
|
ANTLR3_INT32 i;
|
ANTLR3_INT32 n;
|
|
if (cts->p == -1)
|
{
|
fillBuffer(cts);
|
}
|
if (k == 0)
|
{
|
return NULL;
|
}
|
if ((cts->p - k) < 0)
|
{
|
return NULL;
|
}
|
|
i = cts->p;
|
n = 1;
|
|
/* Need to find k good tokens, going backwards, skipping ones that are off channel
|
*/
|
while (n <= (ANTLR3_INT32) k)
|
{
|
/* Skip off-channel tokens
|
*/
|
|
i = skipOffTokenChannelsReverse(cts, i - 1); /* leave p on valid token */
|
n++;
|
}
|
if (i < 0)
|
{
|
return NULL;
|
}
|
// Here the token must be in the input vector. Rather then incut
|
// function call penalty, we jsut return the pointer directly
|
// from the vector
|
//
|
return (pANTLR3_COMMON_TOKEN)cts->tokens->elements[i].element;
|
}
|
|
static pANTLR3_COMMON_TOKEN
|
get (pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 i)
|
{
|
pANTLR3_COMMON_TOKEN_STREAM cts;
|
|
cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
|
|
return (pANTLR3_COMMON_TOKEN)(cts->tokens->get(cts->tokens, i)); /* Token index is zero based but vectors are 1 based */
|
}
|
|
static pANTLR3_TOKEN_SOURCE
|
getTokenSource (pANTLR3_TOKEN_STREAM ts)
|
{
|
return ts->tokenSource;
|
}
|
|
static void
|
setTokenSource ( pANTLR3_TOKEN_STREAM ts,
|
pANTLR3_TOKEN_SOURCE tokenSource)
|
{
|
ts->tokenSource = tokenSource;
|
}
|
|
static pANTLR3_STRING
|
toString (pANTLR3_TOKEN_STREAM ts)
|
{
|
pANTLR3_COMMON_TOKEN_STREAM cts;
|
|
cts = (pANTLR3_COMMON_TOKEN_STREAM)ts->super;
|
|
if (cts->p == -1)
|
{
|
fillBuffer(cts);
|
}
|
|
return ts->toStringSS(ts, 0, ts->istream->size(ts->istream));
|
}
|
|
static pANTLR3_STRING
|
toStringSS(pANTLR3_TOKEN_STREAM ts, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
|
{
|
pANTLR3_STRING string;
|
pANTLR3_TOKEN_SOURCE tsource;
|
pANTLR3_COMMON_TOKEN tok;
|
ANTLR3_UINT32 i;
|
pANTLR3_COMMON_TOKEN_STREAM cts;
|
|
cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
|
|
if (cts->p == -1)
|
{
|
fillBuffer(cts);
|
}
|
if (stop >= ts->istream->size(ts->istream))
|
{
|
stop = ts->istream->size(ts->istream) - 1;
|
}
|
|
/* Who is giving us these tokens?
|
*/
|
tsource = ts->getTokenSource(ts);
|
|
if (tsource != NULL && cts->tokens != NULL)
|
{
|
/* Finally, let's get a string
|
*/
|
string = tsource->strFactory->newRaw(tsource->strFactory);
|
|
for (i = start; i <= stop; i++)
|
{
|
tok = ts->get(ts, i);
|
if (tok != NULL)
|
{
|
string->appendS(string, tok->getText(tok));
|
}
|
}
|
|
return string;
|
}
|
return NULL;
|
|
}
|
|
static pANTLR3_STRING
|
toStringTT (pANTLR3_TOKEN_STREAM ts, pANTLR3_COMMON_TOKEN start, pANTLR3_COMMON_TOKEN stop)
|
{
|
if (start != NULL && stop != NULL)
|
{
|
return ts->toStringSS(ts, (ANTLR3_UINT32)start->getTokenIndex(start), (ANTLR3_UINT32)stop->getTokenIndex(stop));
|
}
|
else
|
{
|
return NULL;
|
}
|
}
|
|
/** Move the input pointer to the next incoming token. The stream
|
* must become active with LT(1) available. consume() simply
|
* moves the input pointer so that LT(1) points at the next
|
* input symbol. Consume at least one token.
|
*
|
* Walk past any token not on the channel the parser is listening to.
|
*/
|
static void
|
consume (pANTLR3_INT_STREAM is)
|
{
|
pANTLR3_COMMON_TOKEN_STREAM cts;
|
pANTLR3_TOKEN_STREAM ts;
|
|
ts = (pANTLR3_TOKEN_STREAM) is->super;
|
cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
|
|
if ((ANTLR3_UINT32)cts->p < cts->tokens->count)
|
{
|
cts->p++;
|
cts->p = skipOffTokenChannels(cts, cts->p);
|
}
|
}
|
|
|
/// As per ordinary consume but notifies the debugger about hidden
|
/// tokens and so on.
|
///
|
static void
|
dbgConsume (pANTLR3_INT_STREAM is)
|
{
|
pANTLR3_TOKEN_STREAM ts;
|
ANTLR3_MARKER a;
|
ANTLR3_MARKER b;
|
pANTLR3_COMMON_TOKEN t;
|
|
ts = (pANTLR3_TOKEN_STREAM) is->super;
|
|
if (ts->initialStreamState == ANTLR3_TRUE)
|
{
|
consumeInitialHiddenTokens(is);
|
}
|
|
a = is->index(is); // Where are we right now?
|
t = ts->_LT(ts, 1); // Current token from stream
|
|
consume(is); // Standard consumer
|
|
b = is->index(is); // Where are we after consuming 1 on channel token?
|
|
ts->debugger->consumeToken(ts->debugger, t); // Tell the debugger that we consumed the first token
|
|
if (b>a+1)
|
{
|
// The standard consume caused the index to advance by more than 1,
|
// which can only happen if it skipped some off-channel tokens.
|
// we need to tell the debugger about those tokens.
|
//
|
ANTLR3_MARKER i;
|
|
for (i = a+1; i<b; i++)
|
{
|
ts->debugger->consumeHiddenToken(ts->debugger, ts->get(ts, (ANTLR3_UINT32)i));
|
}
|
|
}
|
}
|
|
/** A simple filter mechanism whereby you can tell this token stream
|
* to force all tokens of type ttype to be on channel. For example,
|
* when interpreting, we cannot execute actions so we need to tell
|
* the stream to force all WS and NEWLINE to be a different, ignored,
|
* channel.
|
*/
|
static void
|
setTokenTypeChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 ttype, ANTLR3_UINT32 channel)
|
{
|
if (tokenStream->channelOverrides == NULL)
|
{
|
tokenStream->channelOverrides = antlr3ListNew(10);
|
}
|
|
/* We add one to the channel so we can distinguish NULL as being no entry in the
|
* table for a particular token type.
|
*/
|
tokenStream->channelOverrides->put(tokenStream->channelOverrides, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)channel + 1), NULL);
|
}
|
|
static void
|
discardTokenType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 ttype)
|
{
|
if (tokenStream->discardSet == NULL)
|
{
|
tokenStream->discardSet = antlr3ListNew(31);
|
}
|
|
/* We add one to the channel so we can distinguish NULL as being no entry in the
|
* table for a particular token type. We could use bitsets for this I suppose too.
|
*/
|
tokenStream->discardSet->put(tokenStream->discardSet, ttype, ANTLR3_FUNC_PTR((ANTLR3_UINT32)ttype + 1), NULL);
|
}
|
|
static void
|
discardOffChannel (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_BOOLEAN discard)
|
{
|
tokenStream->discardOffChannel = discard;
|
}
|
|
static pANTLR3_VECTOR
|
getTokens (pANTLR3_COMMON_TOKEN_STREAM tokenStream)
|
{
|
if (tokenStream->p == -1)
|
{
|
fillBuffer(tokenStream);
|
}
|
|
return tokenStream->tokens;
|
}
|
|
static pANTLR3_LIST
|
getTokenRange (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop)
|
{
|
return tokenStream->getTokensSet(tokenStream, start, stop, NULL);
|
}
|
/** Given a start and stop index, return a List of all tokens in
|
* the token type BitSet. Return null if no tokens were found. This
|
* method looks at both on and off channel tokens.
|
*/
|
static pANTLR3_LIST
|
getTokensSet (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_BITSET types)
|
{
|
pANTLR3_LIST filteredList;
|
ANTLR3_UINT32 i;
|
ANTLR3_UINT32 n;
|
pANTLR3_COMMON_TOKEN tok;
|
|
if (tokenStream->p == -1)
|
{
|
fillBuffer(tokenStream);
|
}
|
if (stop > tokenStream->tstream->istream->size(tokenStream->tstream->istream))
|
{
|
stop = tokenStream->tstream->istream->size(tokenStream->tstream->istream);
|
}
|
if (start > stop)
|
{
|
return NULL;
|
}
|
|
/* We have the range set, now we need to iterate through the
|
* installed tokens and create a new list with just the ones we want
|
* in it. We are just moving pointers about really.
|
*/
|
filteredList = antlr3ListNew((ANTLR3_UINT32)tokenStream->tstream->istream->size(tokenStream->tstream->istream));
|
|
for (i = start, n = 0; i<= stop; i++)
|
{
|
tok = tokenStream->tstream->get(tokenStream->tstream, i);
|
|
if ( types == NULL
|
|| types->isMember(types, tok->getType(tok) == ANTLR3_TRUE)
|
)
|
{
|
filteredList->put(filteredList, n++, (void *)tok, NULL);
|
}
|
}
|
|
/* Did we get any then?
|
*/
|
if (filteredList->size(filteredList) == 0)
|
{
|
filteredList->free(filteredList);
|
filteredList = NULL;
|
}
|
|
return filteredList;
|
}
|
|
static pANTLR3_LIST
|
getTokensList (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, pANTLR3_LIST list)
|
{
|
pANTLR3_BITSET bitSet;
|
pANTLR3_LIST newlist;
|
|
bitSet = antlr3BitsetList(list->table);
|
|
newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
|
|
bitSet->free(bitSet);
|
|
return newlist;
|
|
}
|
|
static pANTLR3_LIST
|
getTokensType (pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_UINT32 start, ANTLR3_UINT32 stop, ANTLR3_UINT32 type)
|
{
|
pANTLR3_BITSET bitSet;
|
pANTLR3_LIST newlist;
|
|
bitSet = antlr3BitsetOf(type, -1);
|
newlist = tokenStream->getTokensSet(tokenStream, start, stop, bitSet);
|
|
bitSet->free(bitSet);
|
|
return newlist;
|
}
|
|
static ANTLR3_UINT32
|
_LA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
|
{
|
pANTLR3_TOKEN_STREAM ts;
|
pANTLR3_COMMON_TOKEN tok;
|
|
ts = (pANTLR3_TOKEN_STREAM) is->super;
|
|
tok = ts->_LT(ts, i);
|
|
if (tok != NULL)
|
{
|
return tok->getType(tok);
|
}
|
else
|
{
|
return ANTLR3_TOKEN_INVALID;
|
}
|
}
|
|
/// As per _LA() but for debug mode.
|
///
|
static ANTLR3_UINT32
|
dbgLA (pANTLR3_INT_STREAM is, ANTLR3_INT32 i)
|
{
|
pANTLR3_TOKEN_STREAM ts;
|
|
ts = (pANTLR3_TOKEN_STREAM) is->super;
|
|
if (ts->initialStreamState == ANTLR3_TRUE)
|
{
|
consumeInitialHiddenTokens(is);
|
}
|
ts->debugger->LT(ts->debugger, i, tokLT(ts, i));
|
return _LA(is, i);
|
}
|
|
static ANTLR3_MARKER
|
mark (pANTLR3_INT_STREAM is)
|
{
|
is->lastMarker = is->index(is);
|
return is->lastMarker;
|
}
|
|
/// As per mark() but with a call to tell the debugger we are doing this
|
///
|
static ANTLR3_MARKER
|
dbgMark (pANTLR3_INT_STREAM is)
|
{
|
pANTLR3_TOKEN_STREAM ts;
|
|
ts = (pANTLR3_TOKEN_STREAM) is->super;
|
|
is->lastMarker = is->index(is);
|
ts->debugger->mark(ts->debugger, is->lastMarker);
|
|
return is->lastMarker;
|
}
|
|
static void
|
release (pANTLR3_INT_STREAM is, ANTLR3_MARKER mark)
|
{
|
return;
|
}
|
|
static ANTLR3_UINT32
|
size (pANTLR3_INT_STREAM is)
|
{
|
pANTLR3_COMMON_TOKEN_STREAM cts;
|
pANTLR3_TOKEN_STREAM ts;
|
|
if (is->cachedSize > 0)
|
{
|
return is->cachedSize;
|
}
|
ts = (pANTLR3_TOKEN_STREAM) is->super;
|
cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
|
|
is->cachedSize = cts->tokens->count;
|
return is->cachedSize;
|
}
|
|
static ANTLR3_MARKER
|
tindex (pANTLR3_INT_STREAM is)
|
{
|
pANTLR3_COMMON_TOKEN_STREAM cts;
|
pANTLR3_TOKEN_STREAM ts;
|
|
ts = (pANTLR3_TOKEN_STREAM) is->super;
|
cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
|
|
return cts->p;
|
}
|
|
static void
|
dbgRewindLast (pANTLR3_INT_STREAM is)
|
{
|
pANTLR3_TOKEN_STREAM ts;
|
|
ts = (pANTLR3_TOKEN_STREAM) is->super;
|
|
ts->debugger->rewindLast(ts->debugger);
|
|
is->rewind(is, is->lastMarker);
|
}
|
static void
|
rewindLast (pANTLR3_INT_STREAM is)
|
{
|
is->rewind(is, is->lastMarker);
|
}
|
static void
|
rewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
|
{
|
is->seek(is, (ANTLR3_UINT32)(marker));
|
}
|
static void
|
dbgRewindStream (pANTLR3_INT_STREAM is, ANTLR3_MARKER marker)
|
{
|
pANTLR3_TOKEN_STREAM ts;
|
|
ts = (pANTLR3_TOKEN_STREAM) is->super;
|
|
ts->debugger->rewind(ts->debugger, marker);
|
|
is->seek(is, (ANTLR3_UINT32)(marker));
|
}
|
|
static void
|
seek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
|
{
|
pANTLR3_COMMON_TOKEN_STREAM cts;
|
pANTLR3_TOKEN_STREAM ts;
|
|
ts = (pANTLR3_TOKEN_STREAM) is->super;
|
cts = (pANTLR3_COMMON_TOKEN_STREAM) ts->super;
|
|
cts->p = (ANTLR3_UINT32)index;
|
}
|
static void
|
dbgSeek (pANTLR3_INT_STREAM is, ANTLR3_MARKER index)
|
{
|
// TODO: Implement seek in debugger when Ter adds it to Java
|
//
|
seek(is, index);
|
}
|
ANTLR3_API void
|
fillBufferExt(pANTLR3_COMMON_TOKEN_STREAM tokenStream)
|
{
|
fillBuffer(tokenStream);
|
}
|
static void
|
fillBuffer(pANTLR3_COMMON_TOKEN_STREAM tokenStream) {
|
ANTLR3_UINT32 index;
|
pANTLR3_COMMON_TOKEN tok;
|
ANTLR3_BOOLEAN discard;
|
void * channelI;
|
|
/* Start at index 0 of course
|
*/
|
index = 0;
|
|
/* Pick out the next token from the token source
|
* Remember we just get a pointer (reference if you like) here
|
* and so if we store it anywhere, we don't set any pointers to auto free it.
|
*/
|
tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
|
|
while (tok != NULL && tok->type != ANTLR3_TOKEN_EOF)
|
{
|
discard = ANTLR3_FALSE; /* Assume we are not discarding */
|
|
/* I employ a bit of a trick, or perhaps hack here. Rather than
|
* store a pointer to a structure in the override map and discard set
|
* we store the value + 1 cast to a void *. Hence on systems where NULL = (void *)0
|
* we can distinguish "not being there" from "being channel or type 0"
|
*/
|
|
if (tokenStream->discardSet != NULL
|
&& tokenStream->discardSet->get(tokenStream->discardSet, tok->getType(tok)) != NULL)
|
{
|
discard = ANTLR3_TRUE;
|
}
|
else if ( tokenStream->discardOffChannel == ANTLR3_TRUE
|
&& tok->getChannel(tok) != tokenStream->channel
|
)
|
{
|
discard = ANTLR3_TRUE;
|
}
|
else if (tokenStream->channelOverrides != NULL)
|
{
|
/* See if this type is in the override map
|
*/
|
channelI = tokenStream->channelOverrides->get(tokenStream->channelOverrides, tok->getType(tok) + 1);
|
|
if (channelI != NULL)
|
{
|
/* Override found
|
*/
|
tok->setChannel(tok, ANTLR3_UINT32_CAST(channelI) - 1);
|
}
|
}
|
|
/* If not discarding it, add it to the list at the current index
|
*/
|
if (discard == ANTLR3_FALSE)
|
{
|
/* Add it, indicating that we will delete it and the table should not
|
*/
|
tok->setTokenIndex(tok, index);
|
tokenStream->p++;
|
tokenStream->tokens->add(tokenStream->tokens, (void *) tok, NULL);
|
index++;
|
}
|
|
tok = tokenStream->tstream->tokenSource->nextToken(tokenStream->tstream->tokenSource);
|
}
|
|
/* Cache the size so we don't keep doing indirect method calls. We do this as
|
* early as possible so that anything after this may utilize the cached value.
|
*/
|
tokenStream->tstream->istream->cachedSize = tokenStream->tokens->count;
|
|
/* Set the consume pointer to the first token that is on our channel
|
*/
|
tokenStream->p = 0;
|
tokenStream->p = skipOffTokenChannels(tokenStream, tokenStream->p);
|
|
}
|
|
/// Given a starting index, return the index of the first on-channel
|
/// token.
|
///
|
static ANTLR3_UINT32
|
skipOffTokenChannels(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 i) {
|
ANTLR3_INT32 n;
|
pANTLR3_COMMON_TOKEN tok;
|
|
n = tokenStream->tstream->istream->cachedSize;
|
|
while (i < n)
|
{
|
tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[i].element;
|
|
if (tok->channel!= tokenStream->channel)
|
{
|
i++;
|
}
|
else
|
{
|
return i;
|
}
|
}
|
return i;
|
}
|
|
static ANTLR3_UINT32
|
skipOffTokenChannelsReverse(pANTLR3_COMMON_TOKEN_STREAM tokenStream, ANTLR3_INT32 x)
|
{
|
pANTLR3_COMMON_TOKEN tok;
|
|
while (x >= 0)
|
{
|
tok = (pANTLR3_COMMON_TOKEN)tokenStream->tokens->elements[x].element;
|
|
if ((tok->channel != tokenStream->channel))
|
{
|
x--;
|
}
|
else
|
{
|
return x;
|
}
|
}
|
return x;
|
}
|
|
/// Return a string that represents the name assoicated with the input source
|
///
|
/// /param[in] is The ANTLR3_INT_STREAM interface that is representing this token stream.
|
///
|
/// /returns
|
/// /implements ANTLR3_INT_STREAM_struct::getSourceName()
|
///
|
static pANTLR3_STRING
|
getSourceName (pANTLR3_INT_STREAM is)
|
{
|
// Slightly convoluted as we must trace back to the lexer's input source
|
// via the token source. The streamName that is here is not initialized
|
// because this is a token stream, not a file or string stream, which are the
|
// only things that have a context for a source name.
|
//
|
return ((pANTLR3_TOKEN_STREAM)(is->super))->tokenSource->fileName;
|
}
|
