Package bixo.config

Examples of bixo.config.FetcherPolicy

308
309
310
311
312
313
314
    public SimpleHttpFetcher(UserAgent userAgent) {
        this(DEFAULT_MAX_THREADS, userAgent);
    }
   
    public SimpleHttpFetcher(int maxThreads, UserAgent userAgent) {
        this(maxThreads, new FetcherPolicy(), userAgent);
    }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
    }

    public static BaseFetcher createFetcher(UserAgent userAgent, int maxThreads) {
        // TODO KKr - add static createRobotsFetcher method somewhere that
        // I can use here, and also in SimpleGroupingKeyGenerator
        FetcherPolicy policy = new FetcherPolicy();
        policy.setMaxContentSize(MAX_ROBOTS_SIZE);
        policy.setMaxConnectionsPerHost(MAX_CONNECTIONS_PER_HOST);
        SimpleHttpFetcher fetcher = new SimpleHttpFetcher(maxThreads, policy, userAgent);
        fetcher.setMaxRetryCount(ROBOTS_RETRY_COUNT);
        fetcher.setConnectionTimeout(ROBOTS_CONNECTION_TIMEOUT);
        fetcher.setSocketTimeout(ROBOTS_SOCKET_TIMEOUT);
       
View Full Code Here
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
    @Override
    public void operate(FlowProcess process, BufferCall<NullContext> buffCall) {
        QueuedValues values = new QueuedValues(buffCall.getArgumentsIterator());

        _collector = buffCall.getOutputCollector();
        FetcherPolicy fetcherPolicy = _fetcher.getFetcherPolicy();
       
        // Each value is a PreFetchedDatum that contains a set of URLs to fetch in one request from
        // a single server, plus other values needed to set state properly.
        while (!Thread.interrupted() && !fetcherPolicy.isTerminateFetch() && !values.isEmpty()) {
            FetchSetDatum datum = values.nextOrNull(_fetcherMode);
           
            try {
                if (datum == null) {
                    trace("Nothing ready to fetch, sleeping...");
View Full Code Here
49
50
51
52
53
54
55
        "<html><head><meta http-equiv=\"content-type\" content=\"text/html; charset=utf-8\">\n" +
        "<title>LoggingFetcher</title>\n" +
        "</head><body>URL = %s</body></html>\n";
   
    public LoggingFetcher(int maxThreads) {
        super(maxThreads, new FetcherPolicy(), new UserAgent("agentName", "agentName@domain.com", "http://agentName.domain.com"));
    }
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
     * @param maxThreads - number of requests to make in parallel. Should be 1 to 100?
     * @param userAgent - what to use when making requests.
     * @return BaseFetcher that can be passed to the UrlLengthener constructor.
     */
    public static BaseFetcher makeFetcher(int maxThreads, UserAgent userAgent) {
        FetcherPolicy policy = new FetcherPolicy();
        policy.setRedirectMode(RedirectMode.FOLLOW_NONE);
        policy.setMaxRedirects(MAX_REDIRECTS);
        policy.setMaxConnectionsPerHost(maxThreads);

        SimpleHttpFetcher result = new SimpleHttpFetcher(maxThreads, policy, userAgent);
        result.setDefaultMaxContentSize(MAX_CONTENT_SIZE);
       
        // We don't want any encoding (compression) of the data.
View Full Code Here
TOP

Related Classes of bixo.config.FetcherPolicy

  • bixo.examples.crawl.DemoCrawlTool
  • bixo.examples.crawl.DemoCrawlWorkflowLRTest
  • bixo.examples.webmining.DemoWebMiningTool
  • bixo.examples.webmining.DemoWebMiningWorkflowTest
  • bixo.fetcher.FetcherTest
  • bixo.fetcher.LoggingFetcher
  • bixo.fetcher.SimpleHttpFetcher
  • bixo.fetcher.SimpleHttpFetcherTest
  • bixo.fetcher.simulation.FakeHttpFetcher
  • bixo.fetcher.simulation.NullHttpFetcher
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.