Package opennlp.model

Examples of opennlp.model.AbstractModel

457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
    // build
    System.err.println("Training builder");
    opennlp.model.EventStream bes = new ParserEventStream(parseSamples, rules,
        ParserEventTypeEnum.BUILD, mdict);
    AbstractModel buildModel = train(bes, iterations, cut);

    parseSamples.reset();

    // check
    System.err.println("Training checker");
    opennlp.model.EventStream kes = new ParserEventStream(parseSamples, rules,
        ParserEventTypeEnum.CHECK);
    AbstractModel checkModel = train(kes, iterations, cut);

    parseSamples.reset();
   
    // attach
    System.err.println("Training attacher");
    opennlp.model.EventStream attachEvents = new ParserEventStream(parseSamples, rules,
        ParserEventTypeEnum.ATTACH);
    AbstractModel attachModel = train(attachEvents, iterations, cut);
   
    // TODO: Remove cast for HeadRules
    return new ParserModel(languageCode, buildModel, checkModel,
        attachModel, posModel, chunkModel,
        (opennlp.tools.parser.lang.en.HeadRules) rules, ParserType.TREEINSERT);
View Full Code Here
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
      Parse.useFunctionTags(true);
    }
    if (tag || all) {
      System.err.println("Training tagger");
      opennlp.model.EventStream tes = new ParserEventStream(new ParseSampleStream(new PlainTextByLineStream(new java.io.FileReader(inFile))), rules, ParserEventTypeEnum.TAG);
      AbstractModel tagModel = train(tes, iterations, cutoff);
      System.out.println("Saving the tagger model as: " + tagFile);
      new opennlp.maxent.io.SuffixSensitiveGISModelWriter(tagModel, tagFile).persist();
    }

    if (chunk || all) {
      System.err.println("Training chunker");
      opennlp.model.EventStream ces = new ParserEventStream(new ParseSampleStream(new PlainTextByLineStream(new java.io.FileReader(inFile))), rules, ParserEventTypeEnum.CHUNK);
      AbstractModel chunkModel = train(ces, iterations, cutoff);
      System.out.println("Saving the chunker model as: " + chunkFile);
      new opennlp.maxent.io.SuffixSensitiveGISModelWriter(chunkModel, chunkFile).persist();
    }

    if (build || all) {
      System.err.println("Training builder");
      opennlp.model.EventStream bes = new ParserEventStream(new ParseSampleStream(new PlainTextByLineStream(new java.io.FileReader(inFile))), rules, ParserEventTypeEnum.BUILD,null);
      AbstractModel buildModel = train(bes, iterations, cutoff);
      System.out.println("Saving the build model as: " + buildFile);
      new opennlp.maxent.io.SuffixSensitiveGISModelWriter(buildModel, buildFile).persist();
    }

    if (attach || all) {
      System.err.println("Training attacher");
      opennlp.model.EventStream kes = new ParserEventStream(new ParseSampleStream(new PlainTextByLineStream(new java.io.FileReader(inFile))), rules, ParserEventTypeEnum.ATTACH);
      AbstractModel attachModel = train(kes, iterations, cutoff);
      System.out.println("Saving the attach model as: " + attachFile);
      new opennlp.maxent.io.SuffixSensitiveGISModelWriter(attachModel, attachFile).persist();
    }

    if (check || all) {
      System.err.println("Training checker");
      opennlp.model.EventStream ces = new ParserEventStream(new ParseSampleStream(new PlainTextByLineStream(new java.io.FileReader(inFile))), rules, ParserEventTypeEnum.CHECK);
      AbstractModel checkModel = train(ces, iterations, cutoff);
      System.out.println("Saving the check model as: " + checkFile);
      new opennlp.maxent.io.SuffixSensitiveGISModelWriter(checkModel, checkFile).persist();
    }
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
      // TODO: Maybe that should be part of the ChunkingParser ...
      // Training build
      System.out.println("Training check model");
      opennlp.model.EventStream bes = new ParserEventStream(parseSamples,
          originalModel.getHeadRules(), ParserEventTypeEnum.CHECK, mdict);
      AbstractModel checkModel = Parser.train(bes,
          parameters.getNumberOfIterations(), parameters.getCutoff());
     
      parseSamples.close();
     
      return originalModel.updateCheckModel(checkModel);
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
    String lang = args[1];
    String packageName = args[2];
    String modelName = args[3];

    AbstractModel chunkerModel = new GenericModelReader(
        new BinaryFileDataReader(new FileInputStream(modelName))).getModel();

    ChunkerModel packageModel = new ChunkerModel(lang, chunkerModel);
    packageModel.serialize(new FileOutputStream(packageName));
  }
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
    ModelUtil.addCutoffAndIterations(manifestInfoEntries, cutoff, iterations);
   
    EventStream es = new ChunkerEventStream(in, contextGenerator);
    HashSumEventStream hses = new HashSumEventStream(es);
   
    AbstractModel maxentModel = opennlp.maxent.GIS.trainModel(iterations,
        new TwoPassDataIndexer(hses, cutoff));
   
    manifestInfoEntries.put(BaseModel.TRAINING_EVENTHASH_PROPERTY,
        hses.calculateHashSum().toString(16));
   
View Full Code Here
340
341
342
343
344
345
346
347
348
349
350
       featureGenerator = createFeatureGenerator();
    
     EventStream eventStream = new NameFinderEventStream(samples, type,
         new DefaultNameContextGenerator(featureGenerator));
     HashSumEventStream hses = new HashSumEventStream(eventStream);
     AbstractModel nameFinderModel = GIS.trainModel(iterations, new TwoPassDataIndexer(hses, cutoff));
    
     manifestInfoEntries.put(BaseModel.TRAINING_EVENTHASH_PROPERTY,
         hses.calculateHashSum().toString(16));
    
     return new TokenNameFinderModel(languageCode, nameFinderModel,
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
        es = new BasicEventStream(new PlainTextByLineDataStream(datafr), ",");
      } else {
        es = new RealBasicEventStream(new PlainTextByLineDataStream(datafr));
      }
      GIS.SMOOTHING_OBSERVATION = SMOOTHING_OBSERVATION;
      AbstractModel model;
      if (type.equals("maxent")) {

        if (!real) {
          model = GIS.trainModel(es, maxit, cutoff, sigma);
        } else {
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
      // TODO: training individual models should be in the chunking parser, not here
      // Training build
      System.out.println("Training builder");
      opennlp.model.EventStream bes = new ParserEventStream(parseSamples,
          originalModel.getHeadRules(), ParserEventTypeEnum.BUILD, mdict);
      AbstractModel buildModel = Parser.train(bes,
          parameters.getNumberOfIterations(), parameters.getCutoff());
     
      parseSamples.close();
     
      return originalModel.updateBuildModel(buildModel);
View Full Code Here
157
158
159
160
161
162
163
164
165
166
      throws IOException {
   
    Map<String, String> manifestInfoEntries = new HashMap<String, String>();
    ModelUtil.addCutoffAndIterations(manifestInfoEntries, cutoff, iterations);
   
    AbstractModel model = GIS.trainModel(iterations, new TwoPassDataIndexer(
        new DocumentCategorizerEventStream(samples, featureGenerators), cutoff));
   
    return new DoccatModel(languageCode, model, manifestInfoEntries);
  }
View Full Code Here
TOP

Related Classes of opennlp.model.AbstractModel

  • CreateModel
  • opennlp.maxent.GISModel
  • opennlp.maxent.io.SuffixSensitiveGISModelReader
  • opennlp.maxent.MaxentPrepAttachTest
  • opennlp.maxent.ModelTrainer
  • opennlp.maxent.quasinewton.QNTrainerTest
  • opennlp.perceptron.PerceptronModel
  • opennlp.perceptron.PerceptronPrepAttachTest
  • opennlp.tools.chunker.ChunkerME
  • opennlp.tools.chunker.ChunkerModel
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.