tokenizer.addAttribute(PositionLengthAttribute.class);
CharTermAttribute term =
(CharTermAttribute)tokenizer.addAttribute(CharTermAttribute.class);
TypeAttribute type =
(TypeAttribute)tokenizer.addAttribute(TypeAttribute.class);
SemanticClassAttribute semanticClass =
(SemanticClassAttribute)tokenizer.addAttribute(SemanticClassAttribute.class);
PartOfSpeechAttribute pos =
(PartOfSpeechAttribute)tokenizer.addAttribute(PartOfSpeechAttribute.class);
StringBuilder result = new StringBuilder();
while (tokenizer.incrementToken() == true) {
result.append(new String(term.buffer(), 0, term.length())).append(":");
result.append(type.type()).append(":");
result.append(pos.partOfSpeech()).append(":");
result.append(semanticClass.semanticClass()).append(":");
result.append(String.valueOf(posIncrAtt.getPositionIncrement())).append(":");
result.append(String.valueOf(posLengthAtt.getPositionLength())).append(":");
result.append(String.valueOf(extOffset.startOffset())).append(":");
result.append(String.valueOf(extOffset.endOffset()));
result.append(",");