Package org.apache.nutch.parse

Examples of org.apache.nutch.parse.ParseUtil

100
101
102
103
104
105
106
107
108
109
110
    Configuration conf = NutchConfiguration.create();
    urlString = "file:" + sampleDir + fileSeparator + id3v1;
    protocol = new ProtocolFactory(conf).getProtocol(urlString);
    content = protocol.getProtocolOutput(new UTF8(urlString), new CrawlDatum())
                      .getContent();
    parse = new ParseUtil(conf).parseByExtensionId("parse-mp3", content);

    Metadata metadata = parse.getData().getParseMeta();
    assertEquals("postgresql comment id3v1", metadata.get("COMM-Text"));
    assertEquals("postgresql artist id3v1", metadata.get("TPE1-Text"));
    assertEquals("(28)", metadata.get("TCON-Text"));
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
    Configuration conf = NutchConfiguration.create();
    urlString = "file:" + sampleDir + fileSeparator + none;
    protocol = new ProtocolFactory(conf).getProtocol(urlString);
    content = protocol.getProtocolOutput(new UTF8(urlString), new CrawlDatum())
                      .getContent();
    parse = new ParseUtil(conf).parseByExtensionId("parse-mp3", content);
//    Metadata metadata = parse.getData().getParseMeta();
    if (parse.getData().getStatus().isSuccess()) {
      fail("Expected ParseException");
    }
  }
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
          try {
            Metadata metadata = new Metadata();
            metadata.set(Response.CONTENT_LENGTH, Long.toString(entry.getSize()));
            metadata.set(Response.CONTENT_TYPE, contentType);
            Content content = new Content(newurl, base, b, contentType, metadata, this.conf);
            Parse parse = new ParseUtil(this.conf).parse(content);
            ParseData theParseData = parse.getData();
            Outlink[] theOutlinks = theParseData.getOutlinks();
           
            for(int count = 0; count < theOutlinks.length; count++) {
              outLinksList.add(new Outlink(theOutlinks[count].getToUrl(), theOutlinks[count].getAnchor(), this.conf));
View Full Code Here
85
86
87
88
89
90
91
92
93
94
95
        for (int i = 0; i < sampleFiles.length; i++) {
            urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

            protocol = new ProtocolFactory(conf).getProtocol(urlString);
            content = protocol.getProtocolOutput(new UTF8(urlString), new CrawlDatum()).getContent();
            parse = new ParseUtil(conf).parseByExtensionId("parse-rss", content);

            //check that there are 3 outlinks:
            //http://test.channel.com
            //http://www-scf.usc.edu/~mattmann/
            //http://www.nutch.org
View Full Code Here
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
    Protocol protocol;
    Content content;
    Parse parse;

    Configuration conf = NutchConfiguration.create();
    ParseUtil parser = new ParseUtil(conf);
    ProtocolFactory factory = new ProtocolFactory(conf);
    for (int i = 0; i < sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      protocol = factory.getProtocol(urlString);
      content = protocol.getProtocolOutput(new UTF8(urlString),
                                           new CrawlDatum()).getContent();
      parse = parser.parseByExtensionId("parse-msexcel", content);

      assertTrue(parse.getText().equals(expectedText));
    }
  }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
   * Test parsing of language identifiers from html
   **/
  public void testMetaHTMLParsing() {

    try {
      ParseUtil parser = new ParseUtil(NutchConfiguration.create());
      /* loop through the test documents and validate result */
      for (int t = 0; t < docs.length; t++) {
        Content content = getContent(docs[t]);
        Parse parse = parser.parse(content);
        assertEquals(metalanguages[t], (String) parse.getData().getParseMeta().get(Metadata.LANGUAGE));
      }
    } catch (Exception e) {
      e.printStackTrace(System.out);
      fail(e.toString());
View Full Code Here
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
    // loop alternately, total 10*2 times of invoking external command
    for (int i=0; i<10; i++) {
      // check external parser that does 'cat'
      contentType = "application/vnd.nutch.example.cat";
      content.setContentType(contentType);
      parse = new ParseUtil(conf).parseByExtensionId("parse-ext", content);
      assertEquals(expectedText,parse.getText());

      // check external parser that does 'md5sum'
      contentType = "application/vnd.nutch.example.md5sum";
      content.setContentType(contentType);
      parse = new ParseUtil(conf).parseByExtensionId("parse-ext", content);
      assertTrue(parse.getText().startsWith(expectedMD5sum));
    }
  }
View Full Code Here
91
92
93
94
95
96
97
98
99
100
101
    if (LOG.isInfoEnabled()) {
      LOG.info("parsing: "+url);
      LOG.info("contentType: "+contentType);
    }

    Parse parse = new ParseUtil(conf).parse(content);

    System.out.print("---------\nParseData\n---------\n");
    System.out.print(parse.getData().toString());
    if (dumpText) {
      System.out.print("---------\nParseText\n---------\n");
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
          try {
            Metadata metadata = new Metadata();
            metadata.set(Response.CONTENT_LENGTH, Long.toString(entry.getSize()));
            metadata.set(Response.CONTENT_TYPE, contentType);
            Content content = new Content(newurl, base, b, contentType, metadata, this.conf);
            Parse parse = new ParseUtil(this.conf).parse(content).get(content.getUrl());
            ParseData theParseData = parse.getData();
            Outlink[] theOutlinks = theParseData.getOutlinks();
           
            for(int count = 0; count < theOutlinks.length; count++) {
              outLinksList.add(new Outlink(theOutlinks[count].getToUrl(), theOutlinks[count].getAnchor()));
View Full Code Here
69
70
71
72
73
74
75
76
77
    for (int i = 0; i < sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();
      parse = new ParseUtil(conf).parseByExtensionId("parse-zip",content).get(content.getUrl());
      assertTrue(parse.getText().equals(expectedText));
    }
  }
View Full Code Here
TOP

Related Classes of org.apache.nutch.parse.ParseUtil

  • com.google.common.util.concurrent.ThreadFactoryBuilder
  • org.apache.avro.util.Utf8
  • org.apache.nutch.analysis.lang.LanguageIdentifier
  • org.apache.nutch.analysis.lang.TestHTMLLanguageParser
  • org.apache.nutch.crawl.URLWebPage
  • org.apache.nutch.fetcher.FetcherReducer
  • org.apache.nutch.indexer.IndexingFiltersChecker
  • org.apache.nutch.microformats.reltag.TestRelTagParser
  • org.apache.nutch.net.URLFilters
  • org.apache.nutch.net.URLNormalizers
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.