Package gw.internal.gosu.parser
Class SourceCodeTokenizerInternal
java.lang.Object
gw.internal.gosu.parser.SourceCodeTokenizerInternal
This class is adapted from java.io.SourceCodeTokenizer. It adds the notion
of operator to the mix. You can define your own operators or use the
default set of operators, which are taken from the Java Language Spec.
It also captures state information for use by our parser e.g., current token
location, line number and column.
-
Field Summary
FieldsModifier and TypeFieldDescriptionprivate boolean
private boolean
private boolean
private boolean
boolean
boolean
private boolean
private int[]
private Token
private int
int
private int
private int
private ITokenizerInstructor
private int
private int
private int
int
private DocCommentBlock
(package private) int
private SourceCodeReader
private boolean
protected static final int
protected static final int
protected static final int
protected static final int
protected static final int
protected static final int
protected static final int
protected static final int
-
Constructor Summary
ConstructorsConstructorDescriptionSourceCodeTokenizerInternal
(boolean initForCopy) SourceCodeTokenizerInternal
(SourceCodeReader reader, ITokenizerInstructor instructor) SourceCodeTokenizerInternal
(Reader reader) SourceCodeTokenizerInternal
(CharSequence sourceCode) -
Method Summary
Modifier and TypeMethodDescriptionprivate int
void
bitshiftOperatorChars
(int iLow, int iHigh) void
charQuoteChar
(int ch) void
commentChar
(int ch) private void
copy()
void
eolIsSignificant
(boolean bFlag) static String[]
static String[]
int
int
int
int
int
int
getType()
(package private) void
goToPosition
(int iOffset) protected void
private void
(package private) Token
boolean
boolean
private boolean
boolean
boolean
isEOF()
private boolean
isHexDigit
(int c) private boolean
isLeftOpenIntervalOp
(String strOpTest) boolean
boolean
isOperator
(String strOperator) boolean
private boolean
boolean
boolean
boolean
boolean
void
lowerCaseMode
(boolean bLowerCaseMode) int
void
operatorChars
(int iLow, int iHigh) void
void
ordinaryChar
(int ch) void
ordinaryChars
(int iLow, int iHigh) void
private void
private void
private void
pushWhitespaceToken
(StringBuilder sbWhitespace) void
quoteChar
(int ch) private int
read()
private int
readOne()
protected int
readWithInstructions
(int c) void
reset()
void
reset
(SourceCodeReader reader) (package private) void
reset
(SourceCodeReader reader, boolean bResetReader) void
(package private) void
rip()
private void
setBitshiftOperatorChars
(Set<String> operators) void
setCommentsSignificant
(boolean bCommentsSignificant) void
setInstructor
(ITokenizerInstructor instructor) private void
setOperatorChars
(Set<String> operators) void
setParseDotsAsOperators
(boolean parseDotsAsOperators) void
setSupportsKeywords
(boolean supportsKeywords) (package private) void
void
setWhitespaceSignificant
(boolean bWhitespaceSignificant) private boolean
stopOnDot
(int c) toString()
void
whitespaceChars
(int iLow, int iHigh) void
wordChars
(int iLow, int iHigh)
-
Field Details
-
CT_WHITESPACE
protected static final int CT_WHITESPACE- See Also:
-
CT_DIGIT
protected static final int CT_DIGIT- See Also:
-
CT_ALPHA
protected static final int CT_ALPHA- See Also:
-
CT_QUOTE
protected static final int CT_QUOTE- See Also:
-
CT_COMMENT
protected static final int CT_COMMENT- See Also:
-
CT_OPERATOR
protected static final int CT_OPERATOR- See Also:
-
CT_BITSHIFT_OPERATOR
protected static final int CT_BITSHIFT_OPERATOR- See Also:
-
CT_CHARQUOTE
protected static final int CT_CHARQUOTE- See Also:
-
DEFAULT_OPERATORS
-
BITSHIFT_OPERATORS
-
_reader
-
_peekc
int _peekc -
_bForceLower
private boolean _bForceLower -
_bEOLIsSignificantP
private boolean _bEOLIsSignificantP -
_ctype
private int[] _ctype -
_operators
-
_iPos
private int _iPos -
_iLineNum
private int _iLineNum -
_iColumn
private int _iColumn -
_iTokenStart
private int _iTokenStart -
_iTokenColumn
private int _iTokenColumn -
_bWhitespaceSignificant
private boolean _bWhitespaceSignificant -
_bCommentsSignificant
private boolean _bCommentsSignificant -
_iType
public int _iType -
_keyword
-
_strValue
-
_iInvalidCharPos
public int _iInvalidCharPos -
_bUnterminatedString
public boolean _bUnterminatedString -
_bUnterminatedComment
public boolean _bUnterminatedComment -
_instructor
-
_lastComment
-
_bParseDotsAsOperators
private boolean _bParseDotsAsOperators -
_iLineOffset
private int _iLineOffset -
_tokens
-
_eof
-
_supportsKeywords
private boolean _supportsKeywords
-
-
Constructor Details
-
SourceCodeTokenizerInternal
SourceCodeTokenizerInternal(boolean initForCopy) -
SourceCodeTokenizerInternal
-
SourceCodeTokenizerInternal
-
SourceCodeTokenizerInternal
-
SourceCodeTokenizerInternal
-
-
Method Details
-
copy
-
reset
public void reset() -
reset
-
reset
-
reset
-
getReader
-
getSource
-
getInstructor
-
setInstructor
-
isWhitespaceSignificant
public boolean isWhitespaceSignificant() -
setWhitespaceSignificant
public void setWhitespaceSignificant(boolean bWhitespaceSignificant) -
isCommentsSignificant
public boolean isCommentsSignificant() -
setCommentsSignificant
public void setCommentsSignificant(boolean bCommentsSignificant) -
getLineNumber
public int getLineNumber() -
getLineOffset
public int getLineOffset() -
incrementLineNumber
protected void incrementLineNumber() -
getTokenColumn
public int getTokenColumn() -
wordChars
public void wordChars(int iLow, int iHigh) -
whitespaceChars
public void whitespaceChars(int iLow, int iHigh) -
ordinaryChars
public void ordinaryChars(int iLow, int iHigh) -
ordinaryChar
public void ordinaryChar(int ch) -
getDefaultOperators
-
getDefaultBindingOperators
-
getBitshiftOperators
-
operators
-
setOperatorChars
-
operatorChars
public void operatorChars(int iLow, int iHigh) -
setBitshiftOperatorChars
-
bitshiftOperatorChars
public void bitshiftOperatorChars(int iLow, int iHigh) -
isOperator
-
commentChar
public void commentChar(int ch) -
quoteChar
public void quoteChar(int ch) -
charQuoteChar
public void charQuoteChar(int ch) -
parseNumbers
public void parseNumbers() -
eolIsSignificant
public void eolIsSignificant(boolean bFlag) -
lowerCaseMode
public void lowerCaseMode(boolean bLowerCaseMode) -
isUnterminatedString
public boolean isUnterminatedString() -
isUnterminatedComment
public boolean isUnterminatedComment() -
setParseDotsAsOperators
public void setParseDotsAsOperators(boolean parseDotsAsOperators) -
isParseDotsAsOperators
public boolean isParseDotsAsOperators() -
popLastComment
-
getTokenStart
public int getTokenStart() -
getTokenEnd
public int getTokenEnd() -
read
- Throws:
IOException
-
readOne
- Throws:
IOException
-
readWithInstructions
- Throws:
IOException
-
pushWhitespaceToken
-
pushToken
private void pushToken() -
initEofToken
private void initEofToken() -
initToken
-
pushToken
-
isAnalyzingSeparately
public boolean isAnalyzingSeparately() -
isAnalyzingDirective
public boolean isAnalyzingDirective() -
getTokens
-
rip
void rip() -
nextToken
- Throws:
IOException
-
_nextTokenImpl
- Throws:
IOException
-
consumeBlockComment
- Throws:
IOException
-
isReserved
private boolean isReserved() -
isAtIgnorePos
private boolean isAtIgnorePos() -
isHexDigit
private boolean isHexDigit(int c) -
isLeftOpenIntervalOp
-
stopOnDot
private boolean stopOnDot(int c) -
toString
-
getType
public int getType() -
isEOF
public boolean isEOF() -
isNOTHING
public boolean isNOTHING() -
goToPosition
- Throws:
IOException
-
setTokens
-
getEofToken
-
isSupportsKeywords
public boolean isSupportsKeywords() -
setSupportsKeywords
public void setSupportsKeywords(boolean supportsKeywords)
-