Class TokenStream


  • class TokenStream
    extends java.lang.Object
    This class implements the JavaScript scanner.

    It is based on the C source files jsscan.c and jsscan.h in the jsref package.

    See Also:
    Parser
    • Field Detail

      • REPORT_NUMBER_FORMAT_ERROR

        private static final int REPORT_NUMBER_FORMAT_ERROR
        See Also:
        Constant Field Values
      • rawString

        private java.lang.StringBuilder rawString
      • dirtyLine

        private boolean dirtyLine
      • regExpFlags

        java.lang.String regExpFlags
      • string

        private java.lang.String string
      • number

        private double number
      • bigInt

        private java.math.BigInteger bigInt
      • isBinary

        private boolean isBinary
      • isOldOctal

        private boolean isOldOctal
      • isOctal

        private boolean isOctal
      • isHex

        private boolean isHex
      • quoteChar

        private int quoteChar
      • stringBuffer

        private char[] stringBuffer
      • stringBufferTop

        private int stringBufferTop
      • ungetBuffer

        private final int[] ungetBuffer
      • ungetCursor

        private int ungetCursor
      • hitEOF

        private boolean hitEOF
      • lineStart

        private int lineStart
      • lineEndChar

        private int lineEndChar
      • lineno

        int lineno
      • sourceString

        private java.lang.String sourceString
      • sourceReader

        private java.io.Reader sourceReader
      • sourceBuffer

        private char[] sourceBuffer
      • sourceEnd

        private int sourceEnd
      • sourceCursor

        int sourceCursor
      • cursor

        int cursor
      • tokenBeg

        int tokenBeg
      • tokenEnd

        int tokenEnd
      • xmlIsAttribute

        private boolean xmlIsAttribute
      • xmlIsTagContent

        private boolean xmlIsTagContent
      • xmlOpenTagsCount

        private int xmlOpenTagsCount
      • parser

        private Parser parser
      • commentPrefix

        private java.lang.String commentPrefix
      • commentCursor

        private int commentCursor
    • Constructor Detail

      • TokenStream

        TokenStream​(Parser parser,
                    java.io.Reader sourceReader,
                    java.lang.String sourceString,
                    int lineno)
    • Method Detail

      • tokenToString

        java.lang.String tokenToString​(int token)
      • isKeyword

        static boolean isKeyword​(java.lang.String s,
                                 int version,
                                 boolean isStrict)
      • stringToKeyword

        private static int stringToKeyword​(java.lang.String name,
                                           int version,
                                           boolean isStrict)
      • stringToKeywordForJS

        private static int stringToKeywordForJS​(java.lang.String name)
        JavaScript 1.8 and earlier
      • stringToKeywordForES

        private static int stringToKeywordForES​(java.lang.String name,
                                                boolean isStrict)
        ECMAScript 6.
      • isValidIdentifierName

        private static boolean isValidIdentifierName​(java.lang.String str)
      • getSourceString

        final java.lang.String getSourceString()
      • getLineno

        final int getLineno()
      • getString

        final java.lang.String getString()
      • getQuoteChar

        final char getQuoteChar()
      • getNumber

        final double getNumber()
      • getBigInt

        final java.math.BigInteger getBigInt()
      • isNumericBinary

        final boolean isNumericBinary()
      • isNumericOldOctal

        final boolean isNumericOldOctal()
      • isNumericOctal

        final boolean isNumericOctal()
      • isNumericHex

        final boolean isNumericHex()
      • eof

        final boolean eof()
      • getToken

        final int getToken()
                    throws java.io.IOException
        Throws:
        java.io.IOException
      • readDigits

        private int readDigits​(int base,
                               int c)
                        throws java.io.IOException
        Throws:
        java.io.IOException
      • isAlpha

        private static boolean isAlpha​(int c)
      • isDigit

        private static boolean isDigit​(int base,
                                       int c)
      • isDualDigit

        private static boolean isDualDigit​(int c)
      • isOctalDigit

        private static boolean isOctalDigit​(int c)
      • isDigit

        private static boolean isDigit​(int c)
      • isHexDigit

        private static boolean isHexDigit​(int c)
      • isJSSpace

        private static boolean isJSSpace​(int c)
      • isJSFormatChar

        private static boolean isJSFormatChar​(int c)
      • readRegExp

        void readRegExp​(int startToken)
                 throws java.io.IOException
        Parser calls the method when it gets / or /= in literal context.
        Throws:
        java.io.IOException
      • readAndClearRegExpFlags

        java.lang.String readAndClearRegExpFlags()
      • getRawString

        java.lang.String getRawString()
      • getTemplateLiteralChar

        private int getTemplateLiteralChar()
                                    throws java.io.IOException
        Throws:
        java.io.IOException
      • ungetTemplateLiteralChar

        private void ungetTemplateLiteralChar​(int c)
      • matchTemplateLiteralChar

        private boolean matchTemplateLiteralChar​(int test)
                                          throws java.io.IOException
        Throws:
        java.io.IOException
      • peekTemplateLiteralChar

        private int peekTemplateLiteralChar()
                                     throws java.io.IOException
        Throws:
        java.io.IOException
      • readTemplateLiteral

        int readTemplateLiteral​(boolean isTaggedLiteral)
                         throws java.io.IOException
        Throws:
        java.io.IOException
      • isXMLAttribute

        boolean isXMLAttribute()
      • getFirstXMLToken

        int getFirstXMLToken()
                      throws java.io.IOException
        Throws:
        java.io.IOException
      • getNextXMLToken

        int getNextXMLToken()
                     throws java.io.IOException
        Throws:
        java.io.IOException
      • readQuotedString

        private boolean readQuotedString​(int quote)
                                  throws java.io.IOException
        Throws:
        java.io.IOException
      • readXmlComment

        private boolean readXmlComment()
                                throws java.io.IOException
        Throws:
        java.io.IOException
      • readCDATA

        private boolean readCDATA()
                           throws java.io.IOException
        Throws:
        java.io.IOException
      • readEntity

        private boolean readEntity()
                            throws java.io.IOException
        Throws:
        java.io.IOException
      • readPI

        private boolean readPI()
                        throws java.io.IOException
        Throws:
        java.io.IOException
      • getStringFromBuffer

        private java.lang.String getStringFromBuffer()
      • addToString

        private void addToString​(int c)
      • canUngetChar

        private boolean canUngetChar()
      • ungetChar

        private void ungetChar​(int c)
      • matchChar

        private boolean matchChar​(int test)
                           throws java.io.IOException
        Throws:
        java.io.IOException
      • peekChar

        private int peekChar()
                      throws java.io.IOException
        Throws:
        java.io.IOException
      • getChar

        private int getChar()
                     throws java.io.IOException
        Throws:
        java.io.IOException
      • getChar

        private int getChar​(boolean skipFormattingChars)
                     throws java.io.IOException
        Throws:
        java.io.IOException
      • getChar

        private int getChar​(boolean skipFormattingChars,
                            boolean ignoreLineEnd)
                     throws java.io.IOException
        Throws:
        java.io.IOException
      • getCharIgnoreLineEnd

        private int getCharIgnoreLineEnd()
                                  throws java.io.IOException
        Throws:
        java.io.IOException
      • getCharIgnoreLineEnd

        private int getCharIgnoreLineEnd​(boolean skipFormattingChars)
                                  throws java.io.IOException
        Throws:
        java.io.IOException
      • ungetCharIgnoreLineEnd

        private void ungetCharIgnoreLineEnd​(int c)
      • skipLine

        private void skipLine()
                       throws java.io.IOException
        Throws:
        java.io.IOException
      • getOffset

        final int getOffset()
        Returns the offset into the current line.
      • charAt

        private final int charAt​(int index)
      • substring

        private final java.lang.String substring​(int beginIndex,
                                                 int endIndex)
      • getLine

        final java.lang.String getLine()
      • getLine

        final java.lang.String getLine​(int position,
                                       int[] linep)
      • fillSourceBuffer

        private boolean fillSourceBuffer()
                                  throws java.io.IOException
        Throws:
        java.io.IOException
      • getCursor

        public int getCursor()
        Return the current position of the scanner cursor.
      • getTokenBeg

        public int getTokenBeg()
        Return the absolute source offset of the last scanned token.
      • getTokenEnd

        public int getTokenEnd()
        Return the absolute source end-offset of the last scanned token.
      • getTokenLength

        public int getTokenLength()
        Return tokenEnd - tokenBeg
      • getCommentType

        public Token.CommentType getCommentType()
        Return the type of the last scanned comment.
        Returns:
        type of last scanned comment, or 0 if none have been scanned.
      • markCommentStart

        private void markCommentStart()
      • markCommentStart

        private void markCommentStart​(java.lang.String prefix)
      • isMarkingComment

        private boolean isMarkingComment()
      • getAndResetCurrentComment

        final java.lang.String getAndResetCurrentComment()
      • convertLastCharToHex

        private static java.lang.String convertLastCharToHex​(java.lang.String str)