Package org.apache.droids.helper.factories

Examples of org.apache.droids.helper.factories.URLFiltersFactory

70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
    protocolFactory.getMap().put("http", httpProtocol);
    return protocolFactory;
  }
 
  public static URLFiltersFactory createDefaultURLFiltersFactory() {
    URLFiltersFactory filtersFactory = new URLFiltersFactory();
    URLFilter defaultURLFilter = new URLFilter() {

      public String filter(String urlString) {
        return urlString;
      }
     
    };
    filtersFactory.setMap(new HashMap<String, Object>());
    filtersFactory.getMap().put("default", defaultURLFilter);
    return filtersFactory;
  }
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
 
  public static Droid<Link> createSimpleSaveCrawlingDroid(
      String targetURI) {
    ParserFactory parserFactory = createDefaultParserFactory();
    ProtocolFactory protocolFactory = createDefaultProtocolFactory();
    URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();

    SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
    simpleDelayTimer.setDelayMillis(100);
   
    SimpleTaskQueueWithHistory<Link> simpleQueue = new SimpleTaskQueueWithHistory<Link>();
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
 
  public static Droid<Link> createSimpleReportCrawlingDroid(
      String targetURI) {
    ParserFactory parserFactory = createDefaultParserFactory();
    ProtocolFactory protocolFactory = createDefaultProtocolFactory();
    URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();

    SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
    simpleDelayTimer.setDelayMillis(100);

    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
View Full Code Here
147
148
149
150
151
152
153
154
155
156
157
 
  public static Droid<Link> createSimpleExceptionCrawlingDroid(
      String targetURI) {
    ParserFactory parserFactory = createDefaultParserFactory();
    ProtocolFactory protocolFactory = createDefaultProtocolFactory();
    URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();

    SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
    simpleDelayTimer.setDelayMillis(100);
   
    Queue<Link> queue = new LinkedList<Link>();
View Full Code Here
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
    protocolFactory.setMap(new HashMap<String, Object>());
    protocolFactory.getMap().put("http", httpProtocol);
    protocolFactory.getMap().put("https", httpProtocol);
   
    // Create URL filter factory.
    URLFiltersFactory filtersFactory = new URLFiltersFactory();
    RegexURLFilter defaultURLFilter = new RegexURLFilter();
    defaultURLFilter.setFile("classpath:/regex-urlfilter.txt");
    filtersFactory.setMap(new HashMap<String, Object>());
    filtersFactory.getMap().put("default", defaultURLFilter);
   
    // Create handler factory. Provide sysout handler only.
    HandlerFactory handlerFactory = new HandlerFactory();
    SysoutHandler defaultHandler = new SysoutHandler();
    handlerFactory.setMap(new HashMap<String, Object>());
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
    getHandlerFactory().handle(link.getURI(), entity);
  }
 
  protected Collection<Link> getFilteredOutlinks( Parse parse )
  {
    URLFiltersFactory filters = droid.getFiltersFactory();
    TaskValidator< Link > linkValidator = droid.getLinkValidator();
  
    // TODO -- make the hashvalue for Outlink...
    Map<String,Link> filtered = new LinkedHashMap<String,Link>();
    for( Link outlink : parse.getOutlinks() ) {
      String id = outlink.getId();
      if (filters.accept(id) && !filtered.containsKey(id)) {
      if( linkValidator == null ){
        filtered.put(id,outlink);
      }
      else if( linkValidator.validate( outlink ) ){
        filtered.put(id,outlink);
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    protocolFactory.getMap().put("http", httpProtocol);
    return protocolFactory;
  }
 
  public static URLFiltersFactory createDefaultURLFiltersFactory() {
    URLFiltersFactory filtersFactory = new URLFiltersFactory();
    URLFilter defaultURLFilter = new URLFilter() {

      public String filter(String urlString) {
        return urlString;
      }
     
    };
    filtersFactory.getMap().put("default", defaultURLFilter);
    return filtersFactory;
  }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
 
  public static Droid<Link> createSimpleSaveCrawlingDroid(
      String targetURI) {
    ParserFactory parserFactory = createDefaultParserFactory();
    ProtocolFactory protocolFactory = createDefaultProtocolFactory();
    URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();

    SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
    simpleDelayTimer.setDelayMillis(100);
   
    SimpleTaskQueueWithHistory<Link> simpleQueue = new SimpleTaskQueueWithHistory<Link>();
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
 
  public static Droid<Link> createSimpleReportCrawlingDroid(
      String targetURI) {
    ParserFactory parserFactory = createDefaultParserFactory();
    ProtocolFactory protocolFactory = createDefaultProtocolFactory();
    URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();

    SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
    simpleDelayTimer.setDelayMillis(100);

    SequentialTaskMaster<Link> taskMaster = new SequentialTaskMaster<Link>();
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
 
  public static Droid<Link> createSimpleExceptionCrawlingDroid(
      String targetURI) {
    ParserFactory parserFactory = createDefaultParserFactory();
    ProtocolFactory protocolFactory = createDefaultProtocolFactory();
    URLFiltersFactory filtersFactory = createDefaultURLFiltersFactory();

    SimpleDelayTimer simpleDelayTimer = new SimpleDelayTimer();
    simpleDelayTimer.setDelayMillis(100);
   
    Queue<Link> queue = new LinkedList<Link>();
View Full Code Here
TOP

Related Classes of org.apache.droids.helper.factories.URLFiltersFactory

  • org.apache.droids.api.URLFilter
  • org.apache.droids.DroidsFactory
  • org.apache.droids.examples.SimpleRuntime
  • org.apache.droids.robot.crawler.CrawlingWorker
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.