Package opennlp.tools.dictionary

Examples of opennlp.tools.dictionary.Dictionary

57
58
59
60
61
62
63
64
65
66
67
68
    OutputStream out = null;
    try {
      in = new InputStreamReader(new FileInputStream(dictInFile), encoding);
      out = new FileOutputStream(dictOutFile);

      Dictionary dict = Dictionary.parseOneEntryPerLine(in);
      dict.serialize(out);

    } catch (IOException e) {
      throw new TerminateToolException(-1, "IO error while reading training data or indexing data: " + e.getMessage(), e);
    } finally {
      try {
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
   *    built from the input file.
   * @throws IOException IOException
   */
  public static Dictionary createDictionary(ObjectStream<StringList> sampleStream) throws IOException {

    Dictionary mNameDictionary = new Dictionary(true);
    StringList entry;

    entry = sampleStream.read();
    while (entry != null) {
      if (!mNameDictionary.contains(entry)) {
        mNameDictionary.put(entry);
      }
      entry = sampleStream.read();
    }

    return mNameDictionary;
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
    FileInputStream sampleDataIn = CmdLineUtil.openInFile(testData);
    ObjectStream<StringList> sampleStream = new NameFinderCensus90NameStream(sampleDataIn,
        Charset.forName(params.getEncoding()));
   
    Dictionary mDictionary;
    try {
      System.out.println("Creating Dictionary...");
      mDictionary = createDictionary(sampleStream);
    } catch (IOException e) {
      throw new TerminateToolException(-1, "IO error while reading training data or indexing data: "
          + e.getMessage(), e);
    } finally {
      try {
        sampleStream.close();
      } catch(IOException e) {
        // sorry this can fail..
      }
    }

    System.out.println("Saving Dictionary...");
   
    OutputStream out = null;
   
    try {
      out = new FileOutputStream(dictOutFile);
      mDictionary.serialize(out);
    } catch (IOException e) {
      throw new TerminateToolException(-1, "IO error while writing dictionary file: "
          + e.getMessage(), e);
    }
    finally {
View Full Code Here
211
212
213
214
215
216
217
218
219
220
221
222
223
   * Gets the context generator
   */
  public TokenContextGenerator getContextGenerator() {
    Factory f = new Factory();
    Set<String> abbs = null;
    Dictionary abbDict = getAbbreviationDictionary();
    if (abbDict != null) {
      abbs = abbDict.asStringSet();
    } else {
      abbs = Collections.emptySet();
    }
    return f.createTokenContextGenerator(getLanguageCode(), abbs);
  }
View Full Code Here
61
62
63
64
65
66
67
  static Dictionary loadAbbDictionary() throws IOException {
    InputStream in = SentenceDetectorFactoryTest.class.getClassLoader()
        .getResourceAsStream("opennlp/tools/sentdetect/abb.xml");

    return new Dictionary(in);
  }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
  }

  @Test
  public void testDefault() throws IOException {

    Dictionary dic = loadAbbDictionary();

    char[] eos = { '.', '?' };
    SentenceModel sdModel = train(new SentenceDetectorFactory("en", true, dic,
        eos));
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
    assertTrue(Arrays.equals(eos, factory.getEOSCharacters()));
  }

  @Test
  public void testNullDict() throws IOException {
    Dictionary dic = null;

    char[] eos = { '.', '?' };
    SentenceModel sdModel = train(new SentenceDetectorFactory("en", true, dic,
        eos));
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
    assertTrue(Arrays.equals(eos, factory.getEOSCharacters()));
  }

  @Test
  public void testDefaultEOS() throws IOException {
    Dictionary dic = null;

    char[] eos = null;
    SentenceModel sdModel = train(new SentenceDetectorFactory("en", true, dic,
        eos));
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
  }

  @Test
  public void testDummyFactory() throws IOException {

    Dictionary dic = loadAbbDictionary();

    char[] eos = { '.', '?' };
    SentenceModel sdModel = train(new DummySentenceDetectorFactory("en", true,
        dic, eos));
View Full Code Here
183
184
185
186
187
188
189
190
191
192
193
        sdModel.getEosCharacters()));
  }
 
  @Test
  public void testCreateDummyFactory() throws IOException {
    Dictionary dic = loadAbbDictionary();
    char[] eos = { '.', '?' };
   
    SentenceDetectorFactory factory = SentenceDetectorFactory.create(
        DummySentenceDetectorFactory.class.getCanonicalName(), "es", false,
        dic, eos);
View Full Code Here
TOP

Related Classes of opennlp.tools.dictionary.Dictionary

  • io.lumify.opennlpDictionary.OpenNLPDictionaryExtractorGraphPropertyWorker
  • io.lumify.opennlpDictionary.OpenNLPDictionaryExtractorGraphPropertyWorkerTest
  • opennlp.tools.cmdline.dictionary.DictionaryBuilderTool
  • opennlp.tools.cmdline.namefind.CensusDictionaryCreatorTool
  • opennlp.tools.cmdline.parser.BuildModelUpdaterTool
  • opennlp.tools.cmdline.parser.CheckModelUpdaterTool
  • opennlp.tools.cmdline.parser.ParserTrainerTool
  • opennlp.tools.cmdline.postag.POSTaggerTrainerTool
  • opennlp.tools.cmdline.sentdetect.SentenceDetectorCrossValidatorTool
  • opennlp.tools.cmdline.sentdetect.SentenceDetectorTrainerTool
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.