Class Tokenizer
- java.lang.Object
-
- org.sonatype.maven.polyglot.atom.parsing.Tokenizer
-
public class Tokenizer extends java.lang.Object
Taken from the Loop programming language compiler pipeline.
-
-
Field Summary
Fields Modifier and Type Field Description private static int[]
DELIMITERS
private java.lang.String
input
private static int
NON
private static int
SEQUENCE_TOKEN
private static int
SINGLE_TOKEN
private static boolean[]
STRING_TERMINATORS
-
Constructor Summary
Constructors Constructor Description Tokenizer(java.lang.String input)
-
Method Summary
All Methods Static Methods Instance Methods Concrete Methods Modifier and Type Method Description private static void
bakeToken(java.util.List<Token> tokens, char[] input, int i, int start)
static java.lang.String
detokenize(java.util.List<Token> tokens)
private static boolean
isDelimiter(char c)
(package private) static boolean
isSingleTokenDelimiter(char c)
private static boolean
isWhitespace(char c)
java.util.List<Token>
tokenize()
-
-
-
Field Detail
-
input
private final java.lang.String input
-
NON
private static final int NON
- See Also:
- Constant Field Values
-
SINGLE_TOKEN
private static final int SINGLE_TOKEN
- See Also:
- Constant Field Values
-
SEQUENCE_TOKEN
private static final int SEQUENCE_TOKEN
- See Also:
- Constant Field Values
-
DELIMITERS
private static final int[] DELIMITERS
-
STRING_TERMINATORS
private static final boolean[] STRING_TERMINATORS
-
-
Method Detail
-
tokenize
public java.util.List<Token> tokenize()
-
isWhitespace
private static boolean isWhitespace(char c)
-
isSingleTokenDelimiter
static boolean isSingleTokenDelimiter(char c)
-
detokenize
public static java.lang.String detokenize(java.util.List<Token> tokens)
-
isDelimiter
private static boolean isDelimiter(char c)
-
bakeToken
private static void bakeToken(java.util.List<Token> tokens, char[] input, int i, int start)
-
-