Package org.apache.droids.robot.crawler

Examples of org.apache.droids.robot.crawler.CrawlingDroid


    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
   
    CrawlingDroid crawler = new SaveCrawlingDroid( simpleQueue, taskMaster, new SaveHandler() );
    crawler.setFiltersFactory(filtersFactory);
    crawler.setParserFactory(parserFactory);
    crawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURI );
    crawler.setInitialLocations(initialLocations);
    return crawler;
  }
View Full Code Here


    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
   
    Queue<Link> queue = new LinkedList<Link>();
   
    CrawlingDroid crawler = new ReportCrawlingDroid( queue, taskMaster );
    crawler.setFiltersFactory(filtersFactory);
    crawler.setParserFactory(parserFactory);
    crawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURI );
    crawler.setInitialLocations(initialLocations);
    return crawler;
  }
View Full Code Here

    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
//    MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
   
    CrawlingDroid crawler = new ExceptionCrawlingDroid( queue, taskMaster );
    crawler.setFiltersFactory(filtersFactory);
    crawler.setParserFactory(parserFactory);
    crawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURI );
    crawler.setInitialLocations(initialLocations);
    return crawler;
  }
View Full Code Here

    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler( new DefaultTaskExceptionHandler() );
   
    CrawlingDroid helloCrawler = new SysoutCrawlingDroid( simpleQueue, taskMaster );
    helloCrawler.setFiltersFactory(filtersFactory);
    helloCrawler.setParserFactory(parserFactory);
    helloCrawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURL );
    helloCrawler.setInitialLocations(initialLocations);
   
    // Initialize and start the crawler
    helloCrawler.init();
    helloCrawler.start();
   
    // Await termination
    helloCrawler.getTaskMaster().awaitTermination(0, TimeUnit.MILLISECONDS);
    // Shut down the HTTP connection manager
    httpclient.getConnectionManager().shutdown();
  }
View Full Code Here

    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
   
    CrawlingDroid crawler = new SaveCrawlingDroid( simpleQueue, taskMaster, new SaveHandler() );
    crawler.setFiltersFactory(filtersFactory);
    crawler.setParserFactory(parserFactory);
    crawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURI );
    crawler.setInitialLocations(initialLocations);
    return crawler;
  }
View Full Code Here

    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
   
    Queue<Link> queue = new LinkedList<Link>();
   
    CrawlingDroid crawler = new ReportCrawlingDroid( queue, taskMaster );
    crawler.setFiltersFactory(filtersFactory);
    crawler.setParserFactory(parserFactory);
    crawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURI );
    crawler.setInitialLocations(initialLocations);
    return crawler;
  }
View Full Code Here

    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
//    MultiThreadedTaskMaster<Link> taskMaster = new MultiThreadedTaskMaster<Link>();
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler(new DefaultTaskExceptionHandler());
   
    CrawlingDroid crawler = new ExceptionCrawlingDroid( queue, taskMaster );
    crawler.setFiltersFactory(filtersFactory);
    crawler.setParserFactory(parserFactory);
    crawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURI );
    crawler.setInitialLocations(initialLocations);
    return crawler;
  }
View Full Code Here

    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
    taskMaster.setDelayTimer( simpleDelayTimer );
    taskMaster.setExceptionHandler( new DefaultTaskExceptionHandler() );
   
    CrawlingDroid helloCrawler = new SysoutCrawlingDroid( simpleQueue, taskMaster );
    helloCrawler.setFiltersFactory(filtersFactory);
    helloCrawler.setParserFactory(parserFactory);
    helloCrawler.setProtocolFactory(protocolFactory);
   
    Collection<String> initialLocations = new ArrayList<String>();
    initialLocations.add( targetURL );
    helloCrawler.setInitialLocations(initialLocations);
   
    // Initialize and start the crawler
    helloCrawler.init();
    helloCrawler.start();
   
    // Await termination
    helloCrawler.getTaskMaster().awaitTermination(0, TimeUnit.MILLISECONDS);
    // Shut down the HTTP connection manager
    httpclient.getConnectionManager().shutdown();
  }
View Full Code Here

TOP

Related Classes of org.apache.droids.robot.crawler.CrawlingDroid

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.