* Collects the next set of emitting tokens from a token and accumulates them in the active or result lists
*
* @param token the token to collect successors from
*/
protected void collectSuccessorTokens(Token token) {
SearchState state = token.getSearchState();
// If this is a final state, add it to the final list
if (token.isFinal()) {
resultList.add(token);
}
if (token.getScore() < threshold) {
return;
}
if (state instanceof WordSearchState
&& token.getScore() < wordThreshold) {
return;
}
SearchStateArc[] arcs = state.getSuccessors();
// For each successor
// calculate the entry score for the token based upon the
// predecessor token score and the transition probabilities
// if the score is better than the best score encountered for
// the SearchState and frame then create a new token, add
// it to the lattice and the SearchState.
// If the token is an emitting token add it to the list,
// otherwise recursively collect the new tokens successors.
for (SearchStateArc arc : arcs) {
SearchState nextState = arc.getState();
// We're actually multiplying the variables, but since
// these come in log(), multiply gets converted to add
float logEntryScore = token.getScore() + arc.getProbability();
if (wantEntryPruning) { // false by default
if (logEntryScore < threshold) {