Package org.mozilla.javascript
Class TokenStream
java.lang.Object
org.mozilla.javascript.TokenStream
This class implements the JavaScript scanner.
It is based on the C source files jsscan.c and jsscan.h in the jsref package.
- See Also:
-
Field Summary
FieldsModifier and TypeFieldDescriptionprivate ObjToIntMap
private BigInteger
private static final char
private int
private String
(package private) Token.CommentType
(package private) int
private boolean
private static final int
private boolean
private boolean
private boolean
private boolean
private boolean
private int
(package private) int
private int
private double
private static final char
private Parser
private int
private StringBuilder
(package private) String
private static final int
private char[]
(package private) int
private int
private Reader
private String
private String
private char[]
private int
(package private) int
(package private) int
private final int[]
private int
private boolean
private boolean
private int
-
Constructor Summary
ConstructorsConstructorDescriptionTokenStream
(Parser parser, Reader sourceReader, String sourceString, int lineno) -
Method Summary
Modifier and TypeMethodDescriptionprivate void
addToString
(int c) private boolean
private final int
charAt
(int index) private static String
(package private) final boolean
eof()
private boolean
(package private) final String
(package private) final BigInteger
private int
getChar()
private int
getChar
(boolean skipFormattingChars) private int
getChar
(boolean skipFormattingChars, boolean ignoreLineEnd) private int
private int
getCharIgnoreLineEnd
(boolean skipFormattingChars) Return the type of the last scanned comment.int
Return the current position of the scanner cursor.(package private) int
(package private) final String
getLine()
(package private) final String
getLine
(int position, int[] linep) (package private) final int
(package private) int
(package private) final double
(package private) final int
Returns the offset into the current line.(package private) final char
(package private) String
(package private) final String
(package private) final String
private String
private int
(package private) final int
getToken()
int
Return the absolute source offset of the last scanned token.int
Return the absolute source end-offset of the last scanned token.int
Return tokenEnd - tokenBegprivate static boolean
isAlpha
(int c) private static boolean
isDigit
(int c) private static boolean
isDigit
(int base, int c) private static boolean
isDualDigit
(int c) private static boolean
isHexDigit
(int c) private static boolean
isJSFormatChar
(int c) private static boolean
isJSSpace
(int c) (package private) static boolean
private boolean
(package private) final boolean
(package private) final boolean
(package private) final boolean
(package private) final boolean
private static boolean
isOctalDigit
(int c) (package private) boolean
private void
private void
markCommentStart
(String prefix) private boolean
matchChar
(int test) private boolean
matchTemplateLiteralChar
(int test) private int
peekChar()
private int
(package private) String
private boolean
private int
readDigits
(int base, int c) private boolean
private boolean
readPI()
private boolean
readQuotedString
(int quote) (package private) void
readRegExp
(int startToken) Parser calls the method when it gets / or /= in literal context.(package private) int
readTemplateLiteral
(boolean isTaggedLiteral) private boolean
private void
skipLine()
private static int
stringToKeyword
(String name, int version, boolean isStrict) private static int
stringToKeywordForES
(String name, boolean isStrict) ECMAScript 6.private static int
stringToKeywordForJS
(String name) JavaScript 1.8 and earlierprivate final String
substring
(int beginIndex, int endIndex) (package private) String
tokenToString
(int token) private void
ungetChar
(int c) private void
ungetCharIgnoreLineEnd
(int c) private void
ungetTemplateLiteralChar
(int c)
-
Field Details
-
EOF_CHAR
private static final int EOF_CHAR- See Also:
-
REPORT_NUMBER_FORMAT_ERROR
private static final int REPORT_NUMBER_FORMAT_ERROR- See Also:
-
BYTE_ORDER_MARK
private static final char BYTE_ORDER_MARK- See Also:
-
NUMERIC_SEPARATOR
private static final char NUMERIC_SEPARATOR- See Also:
-
rawString
-
dirtyLine
private boolean dirtyLine -
regExpFlags
String regExpFlags -
string
-
number
private double number -
bigInt
-
isBinary
private boolean isBinary -
isOldOctal
private boolean isOldOctal -
isOctal
private boolean isOctal -
isHex
private boolean isHex -
quoteChar
private int quoteChar -
stringBuffer
private char[] stringBuffer -
stringBufferTop
private int stringBufferTop -
allStrings
-
ungetBuffer
private final int[] ungetBuffer -
ungetCursor
private int ungetCursor -
hitEOF
private boolean hitEOF -
lineStart
private int lineStart -
lineEndChar
private int lineEndChar -
lineno
int lineno -
sourceString
-
sourceReader
-
sourceBuffer
private char[] sourceBuffer -
sourceEnd
private int sourceEnd -
sourceCursor
int sourceCursor -
cursor
int cursor -
tokenBeg
int tokenBeg -
tokenEnd
int tokenEnd -
commentType
Token.CommentType commentType -
xmlIsAttribute
private boolean xmlIsAttribute -
xmlIsTagContent
private boolean xmlIsTagContent -
xmlOpenTagsCount
private int xmlOpenTagsCount -
parser
-
commentPrefix
-
commentCursor
private int commentCursor
-
-
Constructor Details
-
TokenStream
-
-
Method Details
-
tokenToString
-
isKeyword
-
stringToKeyword
-
stringToKeywordForJS
JavaScript 1.8 and earlier -
stringToKeywordForES
ECMAScript 6. -
getSourceString
-
getLineno
final int getLineno() -
getString
-
getQuoteChar
final char getQuoteChar() -
getNumber
final double getNumber() -
getBigInt
-
isNumericBinary
final boolean isNumericBinary() -
isNumericOldOctal
final boolean isNumericOldOctal() -
isNumericOctal
final boolean isNumericOctal() -
isNumericHex
final boolean isNumericHex() -
eof
final boolean eof() -
getToken
- Throws:
IOException
-
readDigits
- Throws:
IOException
-
isAlpha
private static boolean isAlpha(int c) -
isDigit
private static boolean isDigit(int base, int c) -
isDualDigit
private static boolean isDualDigit(int c) -
isOctalDigit
private static boolean isOctalDigit(int c) -
isDigit
private static boolean isDigit(int c) -
isHexDigit
private static boolean isHexDigit(int c) -
isJSSpace
private static boolean isJSSpace(int c) -
isJSFormatChar
private static boolean isJSFormatChar(int c) -
readRegExp
Parser calls the method when it gets / or /= in literal context.- Throws:
IOException
-
readAndClearRegExpFlags
String readAndClearRegExpFlags() -
getRawString
String getRawString() -
getTemplateLiteralChar
- Throws:
IOException
-
ungetTemplateLiteralChar
private void ungetTemplateLiteralChar(int c) -
matchTemplateLiteralChar
- Throws:
IOException
-
peekTemplateLiteralChar
- Throws:
IOException
-
readTemplateLiteral
- Throws:
IOException
-
isXMLAttribute
boolean isXMLAttribute() -
getFirstXMLToken
- Throws:
IOException
-
getNextXMLToken
- Throws:
IOException
-
readQuotedString
- Throws:
IOException
-
readXmlComment
- Throws:
IOException
-
readCDATA
- Throws:
IOException
-
readEntity
- Throws:
IOException
-
readPI
- Throws:
IOException
-
getStringFromBuffer
-
addToString
private void addToString(int c) -
canUngetChar
private boolean canUngetChar() -
ungetChar
private void ungetChar(int c) -
matchChar
- Throws:
IOException
-
peekChar
- Throws:
IOException
-
getChar
- Throws:
IOException
-
getChar
- Throws:
IOException
-
getChar
- Throws:
IOException
-
getCharIgnoreLineEnd
- Throws:
IOException
-
getCharIgnoreLineEnd
- Throws:
IOException
-
ungetCharIgnoreLineEnd
private void ungetCharIgnoreLineEnd(int c) -
skipLine
- Throws:
IOException
-
getOffset
final int getOffset()Returns the offset into the current line. -
charAt
private final int charAt(int index) -
substring
-
getLine
-
getLine
-
fillSourceBuffer
- Throws:
IOException
-
getCursor
public int getCursor()Return the current position of the scanner cursor. -
getTokenBeg
public int getTokenBeg()Return the absolute source offset of the last scanned token. -
getTokenEnd
public int getTokenEnd()Return the absolute source end-offset of the last scanned token. -
getTokenLength
public int getTokenLength()Return tokenEnd - tokenBeg -
getCommentType
Return the type of the last scanned comment.- Returns:
- type of last scanned comment, or 0 if none have been scanned.
-
markCommentStart
private void markCommentStart() -
markCommentStart
-
isMarkingComment
private boolean isMarkingComment() -
getAndResetCurrentComment
-
convertLastCharToHex
-