Package edu.stanford.nlp.parser.lexparser

Examples of edu.stanford.nlp.parser.lexparser.LexicalizedParser

72
73
74
75
76
77
78
79
80
81
  }
  */

  // todo: rewrite to test all Chinese models, as for English
  public void testChineseTagSet() {
    LexicalizedParser lp = LexicalizedParser.loadModel("edu/stanford/nlp/models/lexparser/chineseFactored.ser.gz");
    MaxentTagger tagger = new MaxentTagger("edu/stanford/nlp/models/pos-tagger/chinese-distsim/chinese-distsim.tagger");
    assertEquals("Chinese (Fact/distsim) parser/tagger tag set mismatch",
            lp.getLexicon().tagSet(lp.treebankLanguagePack().getBasicCategoryFunction()), tagger.tagSet());
  }
View Full Code Here
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
      tb.add(t);
    } else if (treeFileName != null) {
      tb.loadPath(treeFileName);
    } else {
      String[] options = {"-retainNPTmpSubcategories"};
      LexicalizedParser lp = LexicalizedParser.loadModel("/u/nlp/data/lexparser/englishPCFG.ser.gz", options);
      BufferedReader reader = null;
      try {
        reader = IOUtils.readerFromString(sentFileName);
      } catch (IOException e) {
        throw new RuntimeIOException("Cannot find or open " + sentFileName, e);
      }
      try {
        System.out.println("Processing sentence file " + sentFileName);
        for  (String line; (line = reader.readLine()) != null; ) {
          System.out.println("Processing sentence: " + line);
          PTBTokenizer<Word> ptb = PTBTokenizer.newPTBTokenizer(new StringReader(line));
          List<Word> words = ptb.tokenize();
          Tree parseTree = lp.parseTree(words);
          tb.add(parseTree);
        }
        reader.close();
      } catch (Exception e) {
        throw new RuntimeException("Exception reading key file " + sentFileName, e);
View Full Code Here
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
      }
    }

    System.err.println("Loading lexparser from: " + lexparserFile);
    String[] newArgs = unusedArgs.toArray(new String[unusedArgs.size()]);
    LexicalizedParser lexparser = LexicalizedParser.loadModel(lexparserFile, newArgs);
    System.err.println("... done");

    Treebank testTreebank = null;
    if (testTreebankPath != null) {
      System.err.println("Reading in trees from " + testTreebankPath);
      if (testTreebankFilter != null) {
        System.err.println("Filtering on " + testTreebankFilter);
      }
      testTreebank = lexparser.getOp().tlpParams.memoryTreebank();;
      testTreebank.loadPath(testTreebankPath, testTreebankFilter);
      System.err.println("Read in " + testTreebank.size() + " trees for testing");
    }

    double[] labelResults = new double[weights.length];
    double[] tagResults = new double[weights.length];

    for (int i = 0; i < weights.length; ++i) {
      lexparser.getOp().baseParserWeight = weights[i];
      EvaluateTreebank evaluator = new EvaluateTreebank(lexparser);
      evaluator.testOnTreebank(testTreebank);
      labelResults[i] = evaluator.getLBScore();
      tagResults[i] = evaluator.getTagScore();
    }
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
      throw new IllegalArgumentException("Need to specify -output");
    }

    String[] newArgs = unusedArgs.toArray(new String[unusedArgs.size()]);

    LexicalizedParser lexparser = LexicalizedParser.loadModel(modelPath, newArgs);

    Treebank testTreebank = null;
    if (testTreebankPath != null) {
      System.err.println("Reading in trees from " + testTreebankPath);
      if (testTreebankFilter != null) {
        System.err.println("Filtering on " + testTreebankFilter);
      }
      testTreebank = lexparser.getOp().tlpParams.memoryTreebank();;
      testTreebank.loadPath(testTreebankPath, testTreebankFilter);
      System.err.println("Read in " + testTreebank.size() + " trees for testing");
    }

    FileWriter out = new FileWriter(outputPath);
    BufferedWriter bout = new BufferedWriter(out);

    System.err.println("Parsing " + testTreebank.size() + " trees");
    int count = 0;
    List<ParseRecord> records = Generics.newArrayList();
    for (Tree goldTree : testTreebank) {
      List<Word> tokens = goldTree.yieldWords();
      ParserQuery parserQuery = lexparser.parserQuery();
      if (!parserQuery.parse(tokens)) {
        throw new AssertionError("Could not parse: " + tokens);
      }
      if (!(parserQuery instanceof RerankingParserQuery)) {
        throw new IllegalArgumentException("Expected a LexicalizedParser with a Reranker attached");
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
    File outputFile = new File(outputDir);
    FileSystem.checkNotExistsOrFail(outputFile);

    FileSystem.mkdirOrFail(outputFile);

    LexicalizedParser parser = LexicalizedParser.loadModel(modelPath);
    DVModel model = DVParser.getModelFromLexicalizedParser(parser);

    String binaryWDir = outputDir + File.separator + "binaryW";
    FileSystem.mkdirOrFail(binaryWDir);
    for (TwoDimensionalMap.Entry<String, String, SimpleMatrix> entry : model.binaryTransform) {
View Full Code Here
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
    if (parserFile == null) {
      System.err.println("Must specify a model file with -model");
      System.exit(2);
    }

    LexicalizedParser parser = LexicalizedParser.loadModel(parserFile);

    Set<String> tags = Generics.newTreeSet();
    for (String tag : parser.tagIndex) {
      tags.add(parser.treebankLanguagePack().basicCategory(tag));
    }
    System.out.println("Basic tags: " + tags.size());
    for (String tag : tags) {
      System.out.print("  " + tag);
    }
    System.out.println();
    System.out.println("All tags size: " + parser.tagIndex.size());

    Set<String> states = Generics.newTreeSet();
    for (String state : parser.stateIndex) {
      states.add(parser.treebankLanguagePack().basicCategory(state));
    }
    System.out.println("Basic states: " + states.size());
    for (String tag : states) {
      System.out.print("  " + tag);
    }
View Full Code Here
48
49
50
51
52
53
54
55
56
57
58
59
      //System.out.println(tree.pennString());
      checkTree(tree);

      // System.err.println("Four.");
      // check a tree generated by the parser
      LexicalizedParser parser = LexicalizedParser.loadModel();
      tree = parser.parse("Mary had a little lamb .");
      // System.out.println(tree.pennString());
      tree.indexLeaves();
      checkTree(tree);

    } catch (IOException e) {
View Full Code Here
33
34
35
36
37
38
39
40
      } else {
        extraArgs.add(args[argIndex++]);
      }
    }

    LexicalizedParser parser = LexicalizedParser.loadModel(input, extraArgs);
    parser.saveParserToSerialized(output);
  }
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
136
137
138
    }

    System.err.println("Averaging " + inputModelFilenames);
    System.err.println("Outputting result to " + outputModelFilename);

    LexicalizedParser lexparser = null;
    List<DVModel> models = Generics.newArrayList();
    for (String filename : inputModelFilenames) {
      LexicalizedParser parser = LexicalizedParser.loadModel(filename);
      if (lexparser == null) {
        lexparser = parser;
      }
      models.add(DVParser.getModelFromLexicalizedParser(parser));
    }
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
    // parser object (unless it's a text file parser?)
    Options op = new Options(ctpp);
    op.doDep = false;
    op.testOptions.maxLength = 90;

    LexicalizedParser lp;
    try {
      FileFilter trainFilt = new NumberRangesFileFilter(args[1], false);

      lp = LexicalizedParser.trainFromTreebank(args[0], trainFilt, op);
      try {
        String filename = "chineseCharTagPCFG.ser.gz";
        System.err.println("Writing parser in serialized format to file " + filename + " ");
        System.err.flush();
        ObjectOutputStream out = IOUtils.writeStreamFromString(filename);

        out.writeObject(lp);
        out.close();
        System.err.println("done.");
      } catch (IOException ioe) {
        ioe.printStackTrace();
      }
    } catch (IllegalArgumentException e) {
      lp = LexicalizedParser.loadModel(args[1], op);
    }

    FileFilter testFilt = new NumberRangesFileFilter(args[2], false);
    MemoryTreebank testTreebank = ctpp.memoryTreebank();
    testTreebank.loadPath(new File(args[0]), testFilt);
    PrintWriter pw = new PrintWriter(new OutputStreamWriter(new FileOutputStream("out.chi"), "GB18030"), true);
    WordCatEquivalenceClasser eqclass = new WordCatEquivalenceClasser();
    WordCatEqualityChecker eqcheck = new WordCatEqualityChecker();
    EquivalenceClassEval eval = new EquivalenceClassEval(eqclass, eqcheck);
    //    System.out.println("Preterminals:" + preterminals);
    System.out.println("Testing...");
    for (Tree gold : testTreebank) {
      Tree tree;
      try {
        tree = lp.parseTree(gold.yieldHasWord());
        if (tree == null) {
          System.out.println("Failed to parse " + gold.yieldHasWord());
          continue;
        }
      } catch (Exception e) {
View Full Code Here
TOP

Related Classes of edu.stanford.nlp.parser.lexparser.LexicalizedParser

  • edu.stanford.nlp.ling.Word
  • edu.stanford.nlp.parser.common.ParserQuery
  • edu.stanford.nlp.parser.DependencyIndexITest
  • edu.stanford.nlp.parser.dvparser.AverageDVModels
  • edu.stanford.nlp.parser.dvparser.CacheParseHypotheses
  • edu.stanford.nlp.parser.dvparser.CombineDVModels
  • edu.stanford.nlp.parser.dvparser.CrossValidateTestOptions
  • edu.stanford.nlp.parser.dvparser.DumpMatrices
  • edu.stanford.nlp.parser.dvparser.DVParser
  • edu.stanford.nlp.parser.dvparser.FindNearestNeighbors
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.