Package gw.internal.gosu.parser
Class SourceCodeTokenizer
java.lang.Object
gw.internal.gosu.parser.SourceCodeTokenizer
- All Implemented Interfaces:
ISourceCodeTokenizer
-
Field Summary
FieldsModifier and TypeFieldDescriptionprivate SourceCodeTokenizerInternal
private Stack<ITokenizerOffsetMarker>
private int
Fields inherited from interface gw.lang.parser.ISourceCodeTokenizer
TT_COMMENT, TT_EOF, TT_EOL, TT_INTEGER, TT_KEYWORD, TT_NOTHING, TT_NUMBER, TT_OPERATOR, TT_WHITESPACE, TT_WORD
-
Constructor Summary
ConstructorsModifierConstructorDescriptionprivate
private
SourceCodeTokenizer
(boolean initForCopy) SourceCodeTokenizer
(SourceCodeReader reader) SourceCodeTokenizer
(SourceCodeReader reader, ITokenizerInstructor instructor) SourceCodeTokenizer
(Reader reader) SourceCodeTokenizer
(CharSequence sourceCode) -
Method Summary
Modifier and TypeMethodDescriptionvoid
commentChar
(int ch) copy()
int
int
countMatches
(String s, int tokenType) void
eolIsSignificant
(boolean bFlag) final Token
int
int
getPriorToken
(boolean bSkipWhitespace) getPriorToken
(boolean bSkipWhitespace, boolean bSkipComments) int
int
getState()
getTokenAt
(int iTokenIndex) getTokenAtPosition
(int docPosition) int
int
int
int
getType()
(package private) void
goToPosition
(int iOffset) boolean
boolean
boolean
boolean
isEOF()
boolean
boolean
isOperator
(String strOperator) boolean
boolean
boolean
boolean
boolean
boolean
int
lookaheadType
(int iTokens, boolean bSkipSpaces) void
lowerCaseMode
(boolean bLowerCaseMode) int
mark()
private int
maybeSkipWhitespaceAndComments
(int state, Stack<Token> tokens, int count) void
void
operatorChars
(int iLow, int iHigh) void
void
ordinaryChar
(int ch) void
ordinaryChars
(int iLow, int iHigh) void
void
popOffsetMarker
(ITokenizerOffsetMarker offsetMarker) void
pushOffsetMarker
(ITokenizerOffsetMarker offsetMarker) void
quoteChar
(int ch) void
reset()
void
reset
(SourceCodeReader reader) private void
reset
(SourceCodeReader reader, boolean bResetReader) void
void
void
restoreToMark
(int iMarkedOffset) void
setCommentsSignificant
(boolean bCommentsSignificant) void
setInstructor
(ITokenizerInstructor instructor) void
setParseDotsAsOperators
(boolean parseDotsAsOperators) void
setSupportsKeywords
(boolean supportsKeywords) void
setWhitespaceSignificant
(boolean bWhitespaceSignificant) private int
skipSpaces
(int iPos) toString()
void
whitespaceChars
(int iLow, int iHigh) void
wordChars
(int iLow, int iHigh)
-
Field Details
-
_state
private int _state -
_internal
-
_offsetMarkers
-
-
Constructor Details
-
SourceCodeTokenizer
private SourceCodeTokenizer() -
SourceCodeTokenizer
private SourceCodeTokenizer(boolean initForCopy) -
SourceCodeTokenizer
-
SourceCodeTokenizer
-
SourceCodeTokenizer
-
SourceCodeTokenizer
-
-
Method Details
-
copy
- Specified by:
copy
in interfaceISourceCodeTokenizer
-
copyInto
- Specified by:
copyInto
in interfaceISourceCodeTokenizer
-
getInternal
-
lightweightRestore
- Specified by:
lightweightRestore
in interfaceISourceCodeTokenizer
- Returns:
- a very lightweight restored copy of this tokenizer, without deep cloning anything. This method should be used with care, as it does not clone internal state and cannot actually tokenize. It should be used only for offset information.
-
isPositioned
public boolean isPositioned()- Specified by:
isPositioned
in interfaceISourceCodeTokenizer
-
reset
public void reset()- Specified by:
reset
in interfaceISourceCodeTokenizer
-
reset
- Specified by:
reset
in interfaceISourceCodeTokenizer
-
reset
- Specified by:
reset
in interfaceISourceCodeTokenizer
-
reset
-
getReader
- Specified by:
getReader
in interfaceISourceCodeTokenizer
-
getSource
- Specified by:
getSource
in interfaceISourceCodeTokenizer
-
getInstructor
- Specified by:
getInstructor
in interfaceISourceCodeTokenizer
-
setInstructor
- Specified by:
setInstructor
in interfaceISourceCodeTokenizer
-
isWhitespaceSignificant
public boolean isWhitespaceSignificant()- Specified by:
isWhitespaceSignificant
in interfaceISourceCodeTokenizer
-
setWhitespaceSignificant
public void setWhitespaceSignificant(boolean bWhitespaceSignificant) - Specified by:
setWhitespaceSignificant
in interfaceISourceCodeTokenizer
-
isCommentsSignificant
public boolean isCommentsSignificant()- Specified by:
isCommentsSignificant
in interfaceISourceCodeTokenizer
-
setCommentsSignificant
public void setCommentsSignificant(boolean bCommentsSignificant) - Specified by:
setCommentsSignificant
in interfaceISourceCodeTokenizer
-
getCurrentToken
- Specified by:
getCurrentToken
in interfaceISourceCodeTokenizer
-
getTokenAt
- Specified by:
getTokenAt
in interfaceISourceCodeTokenizer
-
getTokenAtPosition
- Specified by:
getTokenAtPosition
in interfaceISourceCodeTokenizer
-
getLineNumber
public int getLineNumber()- Specified by:
getLineNumber
in interfaceISourceCodeTokenizer
-
getLineOffset
public int getLineOffset()- Specified by:
getLineOffset
in interfaceISourceCodeTokenizer
-
getTokenColumn
public int getTokenColumn()- Specified by:
getTokenColumn
in interfaceISourceCodeTokenizer
-
wordChars
public void wordChars(int iLow, int iHigh) - Specified by:
wordChars
in interfaceISourceCodeTokenizer
-
whitespaceChars
public void whitespaceChars(int iLow, int iHigh) - Specified by:
whitespaceChars
in interfaceISourceCodeTokenizer
-
ordinaryChars
public void ordinaryChars(int iLow, int iHigh) - Specified by:
ordinaryChars
in interfaceISourceCodeTokenizer
-
ordinaryChar
public void ordinaryChar(int ch) - Specified by:
ordinaryChar
in interfaceISourceCodeTokenizer
-
operators
- Specified by:
operators
in interfaceISourceCodeTokenizer
-
operatorChars
public void operatorChars(int iLow, int iHigh) - Specified by:
operatorChars
in interfaceISourceCodeTokenizer
-
isOperator
- Specified by:
isOperator
in interfaceISourceCodeTokenizer
-
commentChar
public void commentChar(int ch) - Specified by:
commentChar
in interfaceISourceCodeTokenizer
-
quoteChar
public void quoteChar(int ch) - Specified by:
quoteChar
in interfaceISourceCodeTokenizer
-
parseNumbers
public void parseNumbers()- Specified by:
parseNumbers
in interfaceISourceCodeTokenizer
-
eolIsSignificant
public void eolIsSignificant(boolean bFlag) - Specified by:
eolIsSignificant
in interfaceISourceCodeTokenizer
-
lowerCaseMode
public void lowerCaseMode(boolean bLowerCaseMode) - Specified by:
lowerCaseMode
in interfaceISourceCodeTokenizer
-
isUnterminatedString
public boolean isUnterminatedString()- Specified by:
isUnterminatedString
in interfaceISourceCodeTokenizer
-
isUnterminatedComment
public boolean isUnterminatedComment()- Specified by:
isUnterminatedComment
in interfaceISourceCodeTokenizer
-
setParseDotsAsOperators
public void setParseDotsAsOperators(boolean parseDotsAsOperators) - Specified by:
setParseDotsAsOperators
in interfaceISourceCodeTokenizer
-
isParseDotsAsOperators
public boolean isParseDotsAsOperators()- Specified by:
isParseDotsAsOperators
in interfaceISourceCodeTokenizer
-
popLastComment
-
getTokenStart
public int getTokenStart()- Specified by:
getTokenStart
in interfaceISourceCodeTokenizer
-
getTokenEnd
public int getTokenEnd()- Specified by:
getTokenEnd
in interfaceISourceCodeTokenizer
-
getTokenAsString
- Specified by:
getTokenAsString
in interfaceISourceCodeTokenizer
-
mark
public int mark()- Specified by:
mark
in interfaceISourceCodeTokenizer
-
restoreToMark
public void restoreToMark(int iMarkedOffset) - Specified by:
restoreToMark
in interfaceISourceCodeTokenizer
-
getRestoreState
public int getRestoreState()- Specified by:
getRestoreState
in interfaceISourceCodeTokenizer
-
getTokens
- Specified by:
getTokens
in interfaceISourceCodeTokenizer
-
nextToken
public void nextToken()- Specified by:
nextToken
in interfaceISourceCodeTokenizer
-
maybeSkipWhitespaceAndComments
-
getStringValue
- Specified by:
getStringValue
in interfaceISourceCodeTokenizer
-
toString
-
getOffsetMarker
-
pushOffsetMarker
- Specified by:
pushOffsetMarker
in interfaceISourceCodeTokenizer
-
popOffsetMarker
- Specified by:
popOffsetMarker
in interfaceISourceCodeTokenizer
-
getState
public int getState()- Specified by:
getState
in interfaceISourceCodeTokenizer
-
countMatches
- Specified by:
countMatches
in interfaceISourceCodeTokenizer
-
countMatches
- Specified by:
countMatches
in interfaceISourceCodeTokenizer
-
getType
public int getType()- Specified by:
getType
in interfaceISourceCodeTokenizer
-
isEOF
public boolean isEOF()- Specified by:
isEOF
in interfaceISourceCodeTokenizer
-
isNOTHING
public boolean isNOTHING() -
goToPosition
- Throws:
IOException
-
resetButKeepTokens
public void resetButKeepTokens() -
isAnalyzingSeparately
public boolean isAnalyzingSeparately()- Specified by:
isAnalyzingSeparately
in interfaceISourceCodeTokenizer
-
isAnalyzingDirective
public boolean isAnalyzingDirective()- Specified by:
isAnalyzingDirective
in interfaceISourceCodeTokenizer
-
isSupportsKeywords
public boolean isSupportsKeywords()- Specified by:
isSupportsKeywords
in interfaceISourceCodeTokenizer
-
setSupportsKeywords
public void setSupportsKeywords(boolean supportsKeywords) - Specified by:
setSupportsKeywords
in interfaceISourceCodeTokenizer
-
getPriorToken
-
getPriorToken
-
getPriorToken
-
lookaheadType
public int lookaheadType(int iTokens, boolean bSkipSpaces) -
skipSpaces
private int skipSpaces(int iPos)
-