Examples of UrlFilter

Examples of bixo.operations.UrlFilter

235
236
237
238
239
240
241
242
243
244
245
            }
       
        Pipe urlFromOutlinksPipe = new Pipe("url from outlinks", parsePipe.getTailPipe());
        urlFromOutlinksPipe = new Each(urlFromOutlinksPipe, new CreateUrlDatumFromOutlinksFunction(new SimpleUrlNormalizer(), new SimpleUrlValidator()));
        if (urlFilter != null) {
            urlFromOutlinksPipe = new Each(urlFromOutlinksPipe, new UrlFilter(urlFilter));
        }
       
        urlFromOutlinksPipe = TupleLogger.makePipe(urlFromOutlinksPipe, true);

        // Take status and output urls from it 
View Full Code Here

Examples of com.flaptor.hounder.util.UrlFilter

102
103
104
105
106
107
108
109
110
111
112
        fetchlistQueue = new CloseableQueue<FetchList>(1); // max one fetchlists in the queue
        injectedFetchlistQueue = new CloseableQueue<FetchList>(); //TODO: put a limit, a large injectdb causes an OutOfMemoryError.
        fetchdataQueue = new CloseableQueue<FetchData>(1); //TODO: analyze if the fetchdata should be written to disk.
        cycleFinishedMonitor = new Object();
        stopMonitor = new StopMonitor("stop");
        urlFilter = new UrlFilter();

      if (config.getBoolean("clustering.enable")) {
          int port = PortUtil.getPort("clustering.rpc.crawler");
        nodeListener = new NodeListener(port, config);
        MonitorModule.addModuleListener(nodeListener, new CrawlerMonitoredNode(this));
View Full Code Here

Examples of com.github.zhangkaitao.shiro.chapter19.entity.UrlFilter

28
29
30
31
32
33
34
35
36
        return "urlFilter/list";
    }

    @RequestMapping(value = "/create", method = RequestMethod.GET)
    public String showCreateForm(Model model) {
        model.addAttribute("urlFilter", new UrlFilter());
        model.addAttribute("op", "新增");
        return "urlFilter/edit";
    }
View Full Code Here

Examples of org.apache.droids.api.URLFilter

71
72
73
74
75
76
77
78
79
80
81
    return protocolFactory;
  }
 
  public static URLFiltersFactory createDefaultURLFiltersFactory() {
    URLFiltersFactory filtersFactory = new URLFiltersFactory();
    URLFilter defaultURLFilter = new URLFilter() {

      public String filter(String urlString) {
        return urlString;
      }
     
View Full Code Here

Examples of org.apache.droids.api.URLFilter

36
37
38
39
40
41
42
43
44
45
46
47
  public boolean accept(String urlString) {
    for (String key : getMap().keySet()) {
      if (urlString == null) {
        return false;
      }
      URLFilter filter = getMap().get(key);
      urlString = filter.filter(urlString);
      if (urlString == null) {
        return false;
      }
    }
    return true;
View Full Code Here

Examples of org.apache.droids.api.URLFilter

58
59
60
61
62
63
64
65
66
67
68
69
   */
  public boolean accept(String urlString, String filterName) {
    if (urlString == null) {
      return false;
    }
    URLFilter filter = getMap().get(filterName);
    urlString = filter.filter(urlString);
    if (urlString == null) {
      return false;
    }
    return true;
  }
View Full Code Here

Examples of org.apache.droids.api.URLFilter

68
69
70
71
72
73
74
75
76
77
78
    return protocolFactory;
  }
 
  public static URLFiltersFactory createDefaultURLFiltersFactory() {
    URLFiltersFactory filtersFactory = new URLFiltersFactory();
    URLFilter defaultURLFilter = new URLFilter() {

      public String filter(String urlString) {
        return urlString;
      }
     
View Full Code Here

Examples of org.apache.nutch.net.URLFilter

58
59
60
61
62
63
64
65
66
67
68
  }

  protected void bench(int loops, Reader rules, Reader urls) {
    long start = System.currentTimeMillis();
    try {
      URLFilter filter = getURLFilter(rules);
      FilteredURL[] expected = readURLFile(urls);
      for (int i=0; i<loops; i++) {
        test(filter, expected);
      }
    } catch (Exception e) {
View Full Code Here

Examples of org.apache.nutch.net.URLFilter

65
66
67
68
69
70
71
72
73
74
75
  }
 
  protected void bench(int loops, Reader rules, Reader urls) {
    long start = System.currentTimeMillis();
    try {
      URLFilter filter = getURLFilter(rules);
      FilteredURL[] expected = readURLFile(urls);
      for (int i=0; i<loops; i++) {
        test(filter, expected);
      }
    } catch (Exception e) {
View Full Code Here

Examples of org.apache.nutch.net.URLFilter

65
66
67
68
69
70
71
72
73
74
75
  }
 
  protected void bench(int loops, Reader rules, Reader urls) {
    long start = System.currentTimeMillis();
    try {
      URLFilter filter = getURLFilter(rules);
      FilteredURL[] expected = readURLFile(urls);
      for (int i=0; i<loops; i++) {
        test(filter, expected);
      }
    } catch (Exception e) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.