Package edu.harvard.wcfia.yoshikoder.document.tokenizer

Examples of edu.harvard.wcfia.yoshikoder.document.tokenizer.TokenListImpl

272
273
274
275
276
277
278
279
280
281
282
283
284
285
                super(name, type);
            }
            public void setPatternEngine(PatternEngine eng){}
            public PatternEngine getPatternEngine(){return null;}
            public long[] test(){
                TokenList tl = new TokenListImpl();
                for (int ii = 0; ii < 1000; ii++) {
                    tl.add(new TokenImpl("china", 0, 5));
                    tl.add(new TokenImpl("sausage", 0, 5));
                }
                PatternNode p = new PatternNodeImpl("chin*", null, Pattern.compile("chin*"));
                System.out.println(p.getPattern());
                long start1 = new Date().getTime();
                Set<Token> l = getMatchingTokens(tl, p);
View Full Code Here
272
273
274
275
276
277
278
279
280
281
282
283
284
285
                super(name, type);
            }
            public void setPatternEngine(PatternEngine eng){}
            public PatternEngine getPatternEngine(){return null;}
            public long[] test(){
                TokenList tl = new TokenListImpl();
                for (int ii = 0; ii < 1000; ii++) {
                    tl.add(new TokenImpl("china", 0, 5));
                    tl.add(new TokenImpl("sausage", 0, 5));
                }
                PatternNode p = new PatternNodeImpl("chin*", null, Pattern.compile("chin*"));
                System.out.println(p.getPattern());
                long start1 = new Date().getTime();
                Set<Token> l = getMatchingTokens(tl, p);
View Full Code Here
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
          Map<YKDocument,EntryFrequencyMap> efmMap;
           
          protected void doWork() throws Exception {
            EntryFrequencyMap efm = null;
            for (YKDocument doc : concmap.keySet()) {
              TokenList tlist = new TokenListImpl();
              Concordance conc = concmap.get(doc);
              for (Iterator iter = conc.iterator(); iter.hasNext();) {
                ConcordanceLine line = (ConcordanceLine) iter.next();
                for (Iterator iterator = line.getLeftHandSide().iterator(); iterator.hasNext();) {
                  Token token = (Token) iterator.next();
                  tlist.add(token);
                }
                for (Iterator iterator = line.getRightHandSide().iterator(); iterator.hasNext();) {
                  Token token = (Token) iterator.next();
                  tlist.add(token);
                }
              }
              efm = new EntryFrequencyMap(yoshikoder.getDictionary(), tlist);
              efmMap.put(doc, efm);
            }
View Full Code Here
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
        }
    }

    public void endElement(String uri, String localName, String qName){                                                 
        if (qName.equals("line")) { //$NON-NLS-1$
            TokenList tl = new TokenListImpl();
            for (Iterator iter = lhs.iterator(); iter.hasNext();) {
                String lhs = (String) iter.next();
                Token t = new TokenImpl(lhs, 0, 0);
                tl.add(t);
            }
            TokenList rl = new TokenListImpl();
            for (Iterator iter = rhs.iterator(); iter.hasNext();) {
                String rhs = (String) iter.next();
                Token t = new TokenImpl(rhs, 0, 0);
                rl.add(t);
            }
            Token targ = new TokenImpl(target, 0, 0);
            ConcordanceLine line = new ConcordanceLineImpl(tl, targ, rl);
            clist.add(line);
        }
View Full Code Here
TOP

Related Classes of edu.harvard.wcfia.yoshikoder.document.tokenizer.TokenListImpl

  • edu.harvard.wcfia.yoshikoder.concordance.Concordance
  • edu.harvard.wcfia.yoshikoder.concordance.ConcordanceImpl
  • edu.harvard.wcfia.yoshikoder.concordance.ConcordanceLine
  • edu.harvard.wcfia.yoshikoder.concordance.ConcordanceLineImpl
  • edu.harvard.wcfia.yoshikoder.ConcordanceFrequencyReportAction
  • edu.harvard.wcfia.yoshikoder.dictionary.AbstractYKDictionary
  • edu.harvard.wcfia.yoshikoder.dictionary.Dict
  • edu.harvard.wcfia.yoshikoder.reporting.WordFrequencyMap
  • edu.harvard.wcfia.yoshikoder.util.ConcordanceHandler
  • java.util.regex.Matcher
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.