Examples of ParseUtil

Examples of org.apache.nutch.parse.ParseUtil

92
93
94
95
96
97
98
99
100
101
102
    if (LOG.isInfoEnabled()) {
      LOG.info("parsing: "+url);
      LOG.info("contentType: "+contentType);
    }

    Parse parse = new ParseUtil(conf).parse(content);

    System.out.print("---------\nParseData\n---------\n");
    System.out.print(parse.getData().toString());
    if (dumpText) {
      System.out.print("---------\nParseText\n---------\n");
View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

68
69
70
71
72
73
74
75
76
77
78
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      Configuration conf = NutchConfiguration.create();
      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();
      parse = new ParseUtil(conf).parseByExtensionId("parse-pdf", content);

      int index = parse.getText().indexOf(expectedText);
      assertTrue(index > 0);
    }
  }
View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

95
96
97
98
99
100
101
102
103
104
105
          try {
            Metadata metadata = new Metadata();
            metadata.set(Response.CONTENT_LENGTH, Long.toString(entry.getSize()));
            metadata.set(Response.CONTENT_TYPE, contentType);
            Content content = new Content(newurl, base, b, contentType, metadata, this.conf);
            Parse parse = new ParseUtil(this.conf).parse(content);
            ParseData theParseData = parse.getData();
            Outlink[] theOutlinks = theParseData.getOutlinks();
           
            for(int count = 0; count < theOutlinks.length; count++) {
              outLinksList.add(new Outlink(theOutlinks[count].getToUrl(), theOutlinks[count].getAnchor(), this.conf));
View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

68
69
70
71
72
73
74
75
76
    for (int i = 0; i < sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();
      parse = new ParseUtil(conf).parseByExtensionId("parse-zip",content);
      assertTrue(parse.getText().equals(expectedText));
    }
  }
View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
    // loop alternately, total 10*2 times of invoking external command
    for (int i=0; i<10; i++) {
      // check external parser that does 'cat'
      contentType = "application/vnd.nutch.example.cat";
      content.setContentType(contentType);
      parse = new ParseUtil(conf).parseByExtensionId("parse-ext", content);
      assertEquals(expectedText,parse.getText());

      // check external parser that does 'md5sum'
      contentType = "application/vnd.nutch.example.md5sum";
      content.setContentType(contentType);
      parse = new ParseUtil(conf).parseByExtensionId("parse-ext", content);
      assertTrue(parse.getText().startsWith(expectedMD5sum));
    }
  }
View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
    Protocol protocol;
    Content content;
    Parse parse;

    Configuration conf = NutchConfiguration.create();
    ParseUtil parser = new ParseUtil(conf);
    ProtocolFactory factory = new ProtocolFactory(conf);
    for (int i = 0; i < sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      protocol = factory.getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString),
                                           new CrawlDatum()).getContent();
      parse = parser.parseByExtensionId("parse-msexcel", content);

      assertTrue(parse.getText().equals(expectedText));
    }
  }
View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

86
87
88
89
90
91
92
93
94
95
96
        for (int i = 0; i < sampleFiles.length; i++) {
            urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

            protocol = new ProtocolFactory(conf).getProtocol(urlString);
            content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();
            parse = new ParseUtil(conf).parseByExtensionId("parse-rss", content);

            //check that there are 3 outlinks:
            //http://test.channel.com
            //http://www-scf.usc.edu/~mattmann/
            //http://www.nutch.org
View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

73
74
75
76
77
78
79
80
81
82
83
    Configuration conf = NutchConfiguration.create();
    urlString = "file:" + sampleDir + fileSeparator + rtfFile;
    protocol = new ProtocolFactory(conf).getProtocol(urlString);
    content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum())
                      .getContent();
    parse = new ParseUtil(conf).parseByExtensionId("parse-rtf", content);
    String text = parse.getText();
    assertEquals("The quick brown fox jumps over the lazy dog", text.trim());

    String title = parse.getData().getTitle();
    Metadata meta = parse.getData().getParseMeta();
View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

86
87
88
89
90
91
92
93
94
95
96
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();

      parse = new ParseUtil(conf).parse(content);

      String text = parse.getText().replaceAll("[ \t\r\n]+", " ").trim();
      assertTrue(sampleTexts[i].equals(text));
    }
  }
View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

68
69
70
71
72
73
74
75
76
77
    for (int i=0; i<sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];

      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();
      parse = new ParseUtil(conf).parseByExtensionId("parse-msword", content);

      assertTrue(parse.getText().startsWith(expectedText));
    }
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.