digit = true;
continue;
}
}
if (sb.length() > 0) {
return new SimpleToken(new SimpleTokenType(TokenType.numericValue, sb.toString()), index);
}
}
boolean escapeAllowed = allowEscape && acceptType(TokenType.escape, filters);
if (escapeAllowed) {
StringBuilder sb = new StringBuilder();
char ch = expression.charAt(index);
boolean escaped = '\\' == ch;
if (escaped && index < expression.length() - 1) {
// grab next character to escape
char next = expression.charAt(++index);
// special for new line, tabs and carriage return
if ('n' == next) {
sb.append("\n");
} else if ('t' == next) {
sb.append("\t");
} else if ('r' == next) {
sb.append("\r");
} else {
// append the next
sb.append(next);
}
// force 2 as length
return new SimpleToken(new SimpleTokenType(TokenType.character, sb.toString()), index, 2);
}
}
// it could be any of the known tokens
String text = expression.substring(index);
for (SimpleTokenType token : KNOWN_TOKENS) {
if (acceptType(token.getType(), filters)) {
if (text.startsWith(token.getValue())) {
return new SimpleToken(token, index);
}
}
}
// fallback and create a character token
char ch = expression.charAt(index);
SimpleToken token = new SimpleToken(new SimpleTokenType(TokenType.character, "" + ch), index);
return token;
}