Package gw.internal.gosu.parser
Class SourceCodeTokenizer
- java.lang.Object
-
- gw.internal.gosu.parser.SourceCodeTokenizer
-
- All Implemented Interfaces:
ISourceCodeTokenizer
public final class SourceCodeTokenizer extends Object implements ISourceCodeTokenizer
-
-
Field Summary
-
Fields inherited from interface gw.lang.parser.ISourceCodeTokenizer
TT_COMMENT, TT_EOF, TT_EOL, TT_INTEGER, TT_KEYWORD, TT_NOTHING, TT_NUMBER, TT_OPERATOR, TT_WHITESPACE, TT_WORD
-
-
Constructor Summary
Constructors Constructor Description SourceCodeTokenizer(SourceCodeReader reader)
SourceCodeTokenizer(SourceCodeReader reader, ITokenizerInstructor instructor)
SourceCodeTokenizer(Reader reader)
SourceCodeTokenizer(CharSequence sourceCode)
-
Method Summary
-
-
-
Constructor Detail
-
SourceCodeTokenizer
public SourceCodeTokenizer(CharSequence sourceCode)
-
SourceCodeTokenizer
public SourceCodeTokenizer(Reader reader)
-
SourceCodeTokenizer
public SourceCodeTokenizer(SourceCodeReader reader)
-
SourceCodeTokenizer
public SourceCodeTokenizer(SourceCodeReader reader, ITokenizerInstructor instructor)
-
-
Method Detail
-
copy
public IToken copy()
- Specified by:
copy
in interfaceISourceCodeTokenizer
-
copyInto
public IToken copyInto(IToken t)
- Specified by:
copyInto
in interfaceISourceCodeTokenizer
-
getInternal
public SourceCodeTokenizerInternal getInternal()
-
lightweightRestore
public SourceCodeTokenizer lightweightRestore()
- Specified by:
lightweightRestore
in interfaceISourceCodeTokenizer
- Returns:
- a very lightweight restored copy of this tokenizer, without deep cloning anything. This method should be used with care, as it does not clone internal state and cannot actually tokenize. It should be used only for offset information.
-
isPositioned
public boolean isPositioned()
- Specified by:
isPositioned
in interfaceISourceCodeTokenizer
-
reset
public void reset()
- Specified by:
reset
in interfaceISourceCodeTokenizer
-
reset
public void reset(Reader reader)
- Specified by:
reset
in interfaceISourceCodeTokenizer
-
reset
public void reset(SourceCodeReader reader)
- Specified by:
reset
in interfaceISourceCodeTokenizer
-
getReader
public SourceCodeReader getReader()
- Specified by:
getReader
in interfaceISourceCodeTokenizer
-
getSource
public String getSource()
- Specified by:
getSource
in interfaceISourceCodeTokenizer
-
getInstructor
public ITokenizerInstructor getInstructor()
- Specified by:
getInstructor
in interfaceISourceCodeTokenizer
-
setInstructor
public void setInstructor(ITokenizerInstructor instructor)
- Specified by:
setInstructor
in interfaceISourceCodeTokenizer
-
isWhitespaceSignificant
public boolean isWhitespaceSignificant()
- Specified by:
isWhitespaceSignificant
in interfaceISourceCodeTokenizer
-
setWhitespaceSignificant
public void setWhitespaceSignificant(boolean bWhitespaceSignificant)
- Specified by:
setWhitespaceSignificant
in interfaceISourceCodeTokenizer
-
isCommentsSignificant
public boolean isCommentsSignificant()
- Specified by:
isCommentsSignificant
in interfaceISourceCodeTokenizer
-
setCommentsSignificant
public void setCommentsSignificant(boolean bCommentsSignificant)
- Specified by:
setCommentsSignificant
in interfaceISourceCodeTokenizer
-
getCurrentToken
public final Token getCurrentToken()
- Specified by:
getCurrentToken
in interfaceISourceCodeTokenizer
-
getTokenAt
public Token getTokenAt(int iTokenIndex)
- Specified by:
getTokenAt
in interfaceISourceCodeTokenizer
-
getTokenAtPosition
public IToken getTokenAtPosition(int docPosition)
- Specified by:
getTokenAtPosition
in interfaceISourceCodeTokenizer
-
getLineNumber
public int getLineNumber()
- Specified by:
getLineNumber
in interfaceISourceCodeTokenizer
-
getLineOffset
public int getLineOffset()
- Specified by:
getLineOffset
in interfaceISourceCodeTokenizer
-
getTokenColumn
public int getTokenColumn()
- Specified by:
getTokenColumn
in interfaceISourceCodeTokenizer
-
wordChars
public void wordChars(int iLow, int iHigh)
- Specified by:
wordChars
in interfaceISourceCodeTokenizer
-
whitespaceChars
public void whitespaceChars(int iLow, int iHigh)
- Specified by:
whitespaceChars
in interfaceISourceCodeTokenizer
-
ordinaryChars
public void ordinaryChars(int iLow, int iHigh)
- Specified by:
ordinaryChars
in interfaceISourceCodeTokenizer
-
ordinaryChar
public void ordinaryChar(int ch)
- Specified by:
ordinaryChar
in interfaceISourceCodeTokenizer
-
operators
public void operators(String[] astrOperators)
- Specified by:
operators
in interfaceISourceCodeTokenizer
-
operatorChars
public void operatorChars(int iLow, int iHigh)
- Specified by:
operatorChars
in interfaceISourceCodeTokenizer
-
isOperator
public boolean isOperator(String strOperator)
- Specified by:
isOperator
in interfaceISourceCodeTokenizer
-
commentChar
public void commentChar(int ch)
- Specified by:
commentChar
in interfaceISourceCodeTokenizer
-
quoteChar
public void quoteChar(int ch)
- Specified by:
quoteChar
in interfaceISourceCodeTokenizer
-
parseNumbers
public void parseNumbers()
- Specified by:
parseNumbers
in interfaceISourceCodeTokenizer
-
eolIsSignificant
public void eolIsSignificant(boolean bFlag)
- Specified by:
eolIsSignificant
in interfaceISourceCodeTokenizer
-
lowerCaseMode
public void lowerCaseMode(boolean bLowerCaseMode)
- Specified by:
lowerCaseMode
in interfaceISourceCodeTokenizer
-
isUnterminatedString
public boolean isUnterminatedString()
- Specified by:
isUnterminatedString
in interfaceISourceCodeTokenizer
-
isUnterminatedComment
public boolean isUnterminatedComment()
- Specified by:
isUnterminatedComment
in interfaceISourceCodeTokenizer
-
setParseDotsAsOperators
public void setParseDotsAsOperators(boolean parseDotsAsOperators)
- Specified by:
setParseDotsAsOperators
in interfaceISourceCodeTokenizer
-
isParseDotsAsOperators
public boolean isParseDotsAsOperators()
- Specified by:
isParseDotsAsOperators
in interfaceISourceCodeTokenizer
-
popLastComment
public DocCommentBlock popLastComment()
-
getTokenStart
public int getTokenStart()
- Specified by:
getTokenStart
in interfaceISourceCodeTokenizer
-
getTokenEnd
public int getTokenEnd()
- Specified by:
getTokenEnd
in interfaceISourceCodeTokenizer
-
getTokenAsString
public String getTokenAsString()
- Specified by:
getTokenAsString
in interfaceISourceCodeTokenizer
-
mark
public int mark()
- Specified by:
mark
in interfaceISourceCodeTokenizer
-
restoreToMark
public void restoreToMark(int iMarkedOffset)
- Specified by:
restoreToMark
in interfaceISourceCodeTokenizer
-
getRestoreState
public int getRestoreState()
- Specified by:
getRestoreState
in interfaceISourceCodeTokenizer
-
getTokens
public final Stack<Token> getTokens()
- Specified by:
getTokens
in interfaceISourceCodeTokenizer
-
nextToken
public void nextToken()
- Specified by:
nextToken
in interfaceISourceCodeTokenizer
-
getStringValue
public String getStringValue()
- Specified by:
getStringValue
in interfaceISourceCodeTokenizer
-
getOffsetMarker
public ITokenizerOffsetMarker getOffsetMarker()
-
pushOffsetMarker
public void pushOffsetMarker(ITokenizerOffsetMarker offsetMarker)
- Specified by:
pushOffsetMarker
in interfaceISourceCodeTokenizer
-
popOffsetMarker
public void popOffsetMarker(ITokenizerOffsetMarker offsetMarker)
- Specified by:
popOffsetMarker
in interfaceISourceCodeTokenizer
-
getState
public int getState()
- Specified by:
getState
in interfaceISourceCodeTokenizer
-
countMatches
public int countMatches(String s)
- Specified by:
countMatches
in interfaceISourceCodeTokenizer
-
countMatches
public int countMatches(String s, int tokenType)
- Specified by:
countMatches
in interfaceISourceCodeTokenizer
-
getType
public int getType()
- Specified by:
getType
in interfaceISourceCodeTokenizer
-
isEOF
public boolean isEOF()
- Specified by:
isEOF
in interfaceISourceCodeTokenizer
-
isNOTHING
public boolean isNOTHING()
-
resetButKeepTokens
public void resetButKeepTokens()
-
isAnalyzingSeparately
public boolean isAnalyzingSeparately()
- Specified by:
isAnalyzingSeparately
in interfaceISourceCodeTokenizer
-
isAnalyzingDirective
public boolean isAnalyzingDirective()
- Specified by:
isAnalyzingDirective
in interfaceISourceCodeTokenizer
-
isSupportsKeywords
public boolean isSupportsKeywords()
- Specified by:
isSupportsKeywords
in interfaceISourceCodeTokenizer
-
setSupportsKeywords
public void setSupportsKeywords(boolean supportsKeywords)
- Specified by:
setSupportsKeywords
in interfaceISourceCodeTokenizer
-
getPriorToken
public Token getPriorToken()
-
getPriorToken
public Token getPriorToken(boolean bSkipWhitespace)
-
getPriorToken
public Token getPriorToken(boolean bSkipWhitespace, boolean bSkipComments)
-
lookaheadType
public int lookaheadType(int iTokens, boolean bSkipSpaces)
-
-