Package edu.isi.karma.cleaning

Examples of edu.isi.karma.cleaning.Tokenizer


    fnames = new Vector<String>();
  }

  public Vector<TNode> tokenizer(String Org) {
    CharStream cs = new ANTLRStringStream(Org);
    Tokenizer tk = new Tokenizer(cs);
    Token t;
    t = tk.nextToken();
    Vector<TNode> x = new Vector<TNode>();
    while (t.getType() != -1) {
      int mytype = -1;
      if (t.getType() == 15) {
        mytype = TNode.UWRDTYP;
      } else if (t.getType() == 4) {
        mytype = TNode.BNKTYP;
      } else if (t.getType() == 10) {
        mytype = TNode.NUMTYP;
      } else if (t.getType() == 12) {
        mytype = TNode.SYBSTYP;
      } else if (t.getType() == 9) {
        mytype = TNode.LWRDTYP;
      }
      TNode tx = new TNode(mytype, t.getText());
      x.add(tx);
      t = tk.nextToken();
    }
    return x;
  }
View Full Code Here

TOP

Related Classes of edu.isi.karma.cleaning.Tokenizer

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.