Examples of ParseUtil

org.apache.nutch.parse.ParseUtil
A Utility class containing methods to simply perform parsing utilities such as iterating through a preferred list of {@link Parser}s to obtain {@link Parse} objects. @author mattmann @author Jérôme Charron @author Sébastien Le Callonnec

Examples of org.apache.nutch.parse.ParseUtil

    Protocol protocol;
    Content content;
    Parse parse;


    Configuration conf = NutchConfiguration.create();
    ParseUtil parser = new ParseUtil(conf);
    ProtocolFactory factory = new ProtocolFactory(conf);
    for (int i = 0; i < sampleFiles.length; i++) {
      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];


      protocol = factory.getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString),
                                           new CrawlDatum()).getContent();
      parse = parser.parseByExtensionId("parse-msexcel", content).get(content.getUrl());


      assertTrue(parse.getText().equals(expectedText));
    }
  }

View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];


      Configuration conf = NutchConfiguration.create();
      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();
      parse = new ParseUtil(conf).parseByExtensionId("parse-pdf", content).get(content.getUrl());


      int index = parse.getText().indexOf(expectedText);
      assertTrue(index > 0);
    }
  }

View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

   * Test parsing of language identifiers from html 
   **/
  public void testMetaHTMLParsing() {


    try {
      ParseUtil parser = new ParseUtil(NutchConfiguration.create());
      /* loop through the test documents and validate result */
      for (int t = 0; t < docs.length; t++) {
        Content content = getContent(docs[t]);
        Parse parse = parser.parse(content).get(content.getUrl());
        assertEquals(metalanguages[t], (String) parse.getData().getParseMeta().get(Metadata.LANGUAGE));
      }
    } catch (Exception e) {
      e.printStackTrace(System.out);
      fail(e.toString());

View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

  public void map(final WritableComparable key, final Writable value,
      final OutputCollector output, final Reporter reporter) {


    InputStream inputFile = null;
    TRECParser trec = null;
    ParseUtil pu = null;
    
    LOG.info("Getting: " + value.toString());
    try {
        inputFile = getTrec(value.toString());    
    } catch(MalformedURLException e) {
      LOG.error(e.getMessage());
    } catch(IOException e) {
      LOG.error(e.getMessage());
    }


    //Get parser for nutch
    pu = new ParseUtil(this.conf);
    //Get a javacc parser
    trec = new TRECParser(inputFile);


    // Run the parser
    try {

View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

    this.sha1 = job.getBoolean("wax.digest.sha1", false);


    this.urlNormalizers = new URLNormalizers(job, URLNormalizers.SCOPE_FETCHER);
    this.filters = new URLFilters(job);


    this.parseUtil = new ParseUtil(job);


    this.collectionName = job.get(ImportArcs.WAX_SUFFIX + ImportArcs.ARCCOLLECTION_KEY);


    // Get ARCName by reading first record in ARC?  Otherwise, we parse
    // the name of the file we've been passed to find an ARC name.

View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

    byte[] bytes = out.toByteArray();
    Configuration conf = NutchConfiguration.create();


    Content content =
      new Content(url, url, bytes, contentType, new Metadata(), conf);
    Parse parse =  new ParseUtil(conf).parse(content).get(content.getUrl());
    
    Metadata metadata = parse.getData().getParseMeta();
    Assert.assertEquals(license, metadata.get("License-Url"));
    Assert.assertEquals(location, metadata.get("License-Location"));
    Assert.assertEquals(type, metadata.get("Work-Type"));

View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil


      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString),
          new CrawlDatum()).getContent();


      parseResult = new ParseUtil(conf).parseByExtensionId("feed", content);


      Assert.assertEquals(3, parseResult.size());


      boolean hasLink1 = false, hasLink2 = false, hasLink3=false;

View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

   **/
  @Test
  public void testMetaHTMLParsing() {


    try {
      ParseUtil parser = new ParseUtil(NutchConfiguration.create());
      /* loop through the test documents and validate result */
      for (int t = 0; t < docs.length; t++) {
        Content content = getContent(docs[t]);
        Parse parse = parser.parse(content).get(content.getUrl());
        Assert.assertEquals(metalanguages[t], (String) parse.getData().getParseMeta().get(Metadata.LANGUAGE));
      }
    } catch (Exception e) {
      e.printStackTrace(System.out);
      Assert.fail(e.toString());

View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

          try {
            Metadata metadata = new Metadata();
            metadata.set(Response.CONTENT_LENGTH, Long.toString(entry.getSize()));
            metadata.set(Response.CONTENT_TYPE, contentType);
            Content content = new Content(newurl, base, b, contentType, metadata, this.conf);
            Parse parse = new ParseUtil(this.conf).parse(content).get(content.getUrl());
            ParseData theParseData = parse.getData();
            Outlink[] theOutlinks = theParseData.getOutlinks();
            
            for(int count = 0; count < theOutlinks.length; count++) {
              outLinksList.add(new Outlink(theOutlinks[count].getToUrl(), theOutlinks[count].getAnchor()));

View Full Code Here

Examples of org.apache.nutch.parse.ParseUtil

      urlString = "file:" + sampleDir + fileSeparator + sampleFiles[i];


      protocol = new ProtocolFactory(conf).getProtocol(urlString);
      content = protocol.getProtocolOutput(new Text(urlString), new CrawlDatum()).getContent();


      parse = new ParseUtil(conf).parse(content).get(content.getUrl());


      String text = parse.getText().replaceAll("[ \t\r\n]+", " ").trim();
      Assert.assertTrue(sampleTexts[i].equals(text));
    }
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.