Package org.apache.nutch.metadata

Examples of org.apache.nutch.metadata.Metadata

60
61
62
63
64
65
66
67
68
69
    conf.addResource("nutch-default.xml");
    conf.addResource("crawl-tests.xml");

    IndexingFilters filters = new IndexingFilters(conf);
    NutchDocument doc = filters.filter(null, new ParseImpl("text", new ParseData(
      new ParseStatus(), "title", new Outlink[0], new Metadata())), new Text(
      "http://www.example.com/"), new CrawlDatum(), new Inlinks());
    
    Assert.assertNull(doc);
  }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
    String class1 = "org.apache.nutch.indexer.basic.BasicIndexingFilter";
    conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1);

    IndexingFilters filters1 = new IndexingFilters(conf);
    NutchDocument fdoc1 = filters1.filter(new NutchDocument(),new ParseImpl("text",new ParseData(
      new ParseStatus(),"title",new Outlink[0],new Metadata())),new Text("http://www.example.com/"),
      new CrawlDatum(),new Inlinks());

    // add another index filter
    String class2 = "org.apache.nutch.indexer.metadata.MetadataIndexer";
    // set content metadata
    Metadata md = new Metadata();
    md.add("example","data");
    // set content metadata property defined in MetadataIndexer
    conf.set("index.content.md","example");
    // add MetadataIndxer filter
    conf.set(IndexingFilters.INDEXINGFILTER_ORDER, class1 + " " + class2);
    IndexingFilters filters2 = new IndexingFilters(conf);
View Full Code Here
155
156
157
158
159
160
161
162
163
164
165
166
167
      Text key = new Text();
      ParseData value = new ParseData();
      if(!reader.next(key, value)) break READ_PARSE_DATA;
      // make sure they all contain "nutch.segment.name" and "nutch.content.digest"
      // keys in parse metadata
      Metadata contentMeta = value.getContentMeta();
      if (contentMeta.get(Nutch.SEGMENT_NAME_KEY) != null
            && contentMeta.get(Nutch.SIGNATURE_KEY) != null) {
        handledurls.add(key.toString());
      }
    } while(true);
   
    Collections.sort(handledurls);
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
    in.close();
    byte[] bytes = out.toByteArray();
    Configuration conf = NutchConfiguration.create();

    Content content =
      new Content(url, url, bytes, contentType, new Metadata(), conf);
    Parse parse =  new ParseUtil(conf).parse(content).get(content.getUrl());
   
    Metadata metadata = parse.getData().getParseMeta();
    Assert.assertEquals(license, metadata.get("License-Url"));
    Assert.assertEquals(location, metadata.get("License-Location"));
    Assert.assertEquals(type, metadata.get("Work-Type"));
  }
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
        if (i != -1) {
          // Trying to resolve the Mime-Type
          Tika tika = new Tika();
          String contentType = tika.detect(fname);
          try {
            Metadata metadata = new Metadata();
            metadata.set(Response.CONTENT_LENGTH, Long.toString(entry.getSize()));
            metadata.set(Response.CONTENT_TYPE, contentType);
            Content content = new Content(newurl, base, b, contentType, metadata, this.conf);
            Parse parse = new ParseUtil(this.conf).parse(content).get(content.getUrl());
            ParseData theParseData = parse.getData();
            Outlink[] theOutlinks = theParseData.getOutlinks();
           
View Full Code Here
122
123
124
125
126
127
128
129
130
    }
  }
 
 
  private Content getContent(String text) {
    Metadata meta = new Metadata();
    meta.add("Content-Type", "text/html");
    return new Content(URL, BASE, text.getBytes(), "text/html", meta, NutchConfiguration.create());
  }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
70
    NutchDocument doc = new NutchDocument();

    String title = "The Foo Page";
    Outlink[] outlinks = new Outlink[] { new Outlink("http://foo.com/", "Foo") };
    Metadata metaData = new Metadata();
    metaData.add("Language", "en/us");
    ParseData parseData = new ParseData(ParseStatus.STATUS_SUCCESS, title, outlinks, metaData);
    ParseImpl parse = new ParseImpl("this is a sample foo bar page. hope you enjoy it.", parseData);

    CrawlDatum crawlDatum = new CrawlDatum();
    crawlDatum.setFetchTime(100L);
View Full Code Here
54
55
56
57
58
59
60
61
62
63
64
  /** Unit tests for getContentType(String, String, byte[]) method. */
  @Test
  public void testGetContentType() throws Exception {
    Content c = null;
    Metadata p = new Metadata();

    c = new Content("http://www.foo.com/",
        "http://www.foo.com/",
        "".getBytes("UTF8"),
        "text/html; charset=UTF-8", p, conf);
View Full Code Here
102
103
104
105
106
107
108
109
  }

  protected Parse parse(byte[] contentBytes) {
    String dummyUrl = "http://dummy.url/";
    return parser.getParse(
        new Content(dummyUrl, dummyUrl, contentBytes, "text/html", new Metadata(),
            conf)).get(dummyUrl);
  }
View Full Code Here
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
    }

    String text = "";
    String title = "";
    Outlink[] outlinks = new Outlink[0];
    Metadata metadata = new Metadata();

    // parse the content
    DocumentFragment root;
    try {
      byte[] contentInOctets = content.getContent();
      InputSource input = new InputSource(new ByteArrayInputStream(contentInOctets));

      EncodingDetector detector = new EncodingDetector(conf);
      detector.autoDetectClues(content, true);
      detector.addClue(sniffCharacterEncoding(contentInOctets), "sniffed");
      String encoding = detector.guessEncoding(content, defaultCharEncoding);

      metadata.set(Metadata.ORIGINAL_CHAR_ENCODING, encoding);
      metadata.set(Metadata.CHAR_ENCODING_FOR_CONVERSION, encoding);

      input.setEncoding(encoding);
      if (LOG.isTraceEnabled()) { LOG.trace("Parsing..."); }
      root = parse(input);
    } catch (IOException e) {
View Full Code Here
TOP

Related Classes of org.apache.nutch.metadata.Metadata

  • com.atlantbh.nutch.filter.index.omit.OmitIndexingFilter
  • com.atlantbh.nutch.filter.xpath.XPathHtmlParserFilter
  • com.atlantbh.nutch.filter.xpath.XPathHtmlParserFilterTest
  • com.atlantbh.nutch.filter.xpath.XPathIndexingFilter
  • com.atlantbh.nutch.filter.xpath.XPathIndexingFilterTest
  • com.atlantbh.nutch.index.alternativedataflow.flow.CsvDataFlow
  • com.flaptor.hounder.crawler.Nutch9Fetcher$NutchSegment$SegmentIterator
  • org.apache.nutch.analysis.lang.TestHTMLLanguageParser
  • org.apache.nutch.fetcher.Fetcher$FetcherThread
  • org.apache.nutch.fetcher.Fetcher2$FetcherThread
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.