Examples of org.apache.nutch.crawl.Generator

org.apache.nutch.crawl.Generator
Generates a subset of a crawl db to fetch. This version allows to generate fetchlists for several segments in one go. Unlike in the initial version (OldGenerator), the IP resolution is done ONLY on the entries which have been selected for fetching. The URLs are partitioned by IP, domain or host within a segment. We can chose separately how to count the URLS i.e. by domain or host to limit the entries.

    //inject
    Injector injector=new Injector(conf);
    injector.inject(crawldbPath, urlPath);


    //generate
    Generator g=new Generator(conf);
    Path[] generatedSegment = g.generate(crawldbPath, segmentsPath, 1,
        Long.MAX_VALUE, Long.MAX_VALUE, false, false);


    long time=System.currentTimeMillis();
    //fetch
    Fetcher fetcher=new Fetcher(conf);

View Full Code Here

    //inject
    Injector injector=new Injector(conf);
    injector.inject(crawldbPath, urlPath);


    //generate
    Generator g=new Generator(conf);
    Path generatedSegment = g.generate(crawldbPath, segmentsPath, 1,
        Long.MAX_VALUE, Long.MAX_VALUE, false, false);


    long time=System.currentTimeMillis();
    //fetch
    conf.setBoolean("fetcher.parse", true);

View Full Code Here

            // inject
            Injector injector = new Injector(conf);
            injector.inject(crawldbPath, urlPath);


            // generate
            Generator g = new Generator(conf);
            // fetch
            conf.setBoolean("fetcher.parse", true);
            Fetcher fetcher = new Fetcher(conf);
            CrawlDb crawlDbTool = new CrawlDb(conf);


            int depth = 5;
            int threads = 4;
            for (int i = 0; i < depth; i++) { // generate new segment
                Path generatedSegment = g.generate(crawldbPath, segmentsPath, 1, Long.MAX_VALUE, Long.MAX_VALUE, false,
                        false);


                if (generatedSegment == null) {
                    logger.info("Stopping at depth=" + i + " - no more URLs to fetch.");
                    break;

View Full Code Here

    //inject
    Injector injector=new Injector(conf);
    injector.inject(crawldbPath, urlPath);


    //generate
    Generator g=new Generator(conf);
    Path generatedSegment = g.generate(crawldbPath, segmentsPath, 1,
        Long.MAX_VALUE, Long.MAX_VALUE, false, false);


    long time=System.currentTimeMillis();
    //fetch
    Fetcher fetcher=new Fetcher(conf);

View Full Code Here

    //inject
    Injector injector=new Injector(conf);
    injector.inject(crawldbPath, urlPath);


    //generate
    Generator g=new Generator(conf);
    Path[] generatedSegment = g.generate(crawldbPath, segmentsPath, 1,
        Long.MAX_VALUE, Long.MAX_VALUE, false, false);


    long time=System.currentTimeMillis();
    //fetch
    Fetcher fetcher=new Fetcher(conf);

View Full Code Here

    Path crawlDb = new Path(dir + "/crawldb");
    Path linkDb = new Path(dir + "/linkdb");
    Path segments = new Path(dir + "/segments");
    res.elapsed = System.currentTimeMillis();
    Injector injector = new Injector(getConf());
    Generator generator = new Generator(getConf());
    Fetcher fetcher = new Fetcher(getConf());
    ParseSegment parseSegment = new ParseSegment(getConf());
    CrawlDb crawlDbTool = new CrawlDb(getConf());
    LinkDb linkDbTool = new LinkDb(getConf());
      
    // initialize crawlDb
    long start = System.currentTimeMillis();
    injector.inject(crawlDb, rootUrlDir);
    long delta = System.currentTimeMillis() - start;
    res.addTiming("inject", "0", delta);
    int i;
    for (i = 0; i < depth; i++) {             // generate new segment
      start = System.currentTimeMillis();
      Path[] segs = generator.generate(crawlDb, segments, -1, topN, System
          .currentTimeMillis());
      delta = System.currentTimeMillis() - start;
      res.addTiming("generate", i + "", delta);
      if (segs == null) {
        LOG.info("Stopping at depth=" + i + " - no more URLs to fetch.");

View Full Code Here

    //inject
    Injector injector=new Injector(conf);
    injector.inject(crawldbPath, urlPath);


    //generate
    Generator g=new Generator(conf);
    Path[] generatedSegment = g.generate(crawldbPath, segmentsPath, 1,
        Long.MAX_VALUE, Long.MAX_VALUE, false, false);


    long time=System.currentTimeMillis();
    //fetch
    Fetcher fetcher=new Fetcher(conf);

View Full Code Here

    Path crawlDb = new Path(dir + "/crawldb");
    Path linkDb = new Path(dir + "/linkdb");
    Path segments = new Path(dir + "/segments");
    res.elapsed = System.currentTimeMillis();
    Injector injector = new Injector(getConf());
    Generator generator = new Generator(getConf());
    Fetcher fetcher = new Fetcher(getConf());
    ParseSegment parseSegment = new ParseSegment(getConf());
    CrawlDb crawlDbTool = new CrawlDb(getConf());
    LinkDb linkDbTool = new LinkDb(getConf());
      
    // initialize crawlDb
    long start = System.currentTimeMillis();
    injector.inject(crawlDb, rootUrlDir);
    long delta = System.currentTimeMillis() - start;
    res.addTiming("inject", "0", delta);
    int i;
    for (i = 0; i < depth; i++) {             // generate new segment
      start = System.currentTimeMillis();
      Path[] segs = generator.generate(crawlDb, segments, -1, topN, System
          .currentTimeMillis());
      delta = System.currentTimeMillis() - start;
      res.addTiming("generate", i + "", delta);
      if (segs == null) {
        LOG.info("Stopping at depth=" + i + " - no more URLs to fetch.");

View Full Code Here

TOP

Related Classes of org.apache.nutch.crawl.Generator

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.Path

org.apache.nutch.fetcher.TestFetcher

org.apache.nutch.net.URLFilters

org.apache.nutch.net.URLNormalizers

org.apache.nutch.scoring.ScoringFilters

org.apache.nutch.tools.Benchmark

org.apache.nutch.util.NutchJob

org.jahia.services.content.CrawlingPageVisitorTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.