Examples of bixo.config.FetcherPolicy

bixo.config.FetcherPolicy
Definition of policy for fetches.

    public final void testInterruptedFetch() throws Exception {
        // Need to read in lots of data that we get very slowly
        Server server = startServer(new RandomResponseHandler(20000, 2 * 1000L), 8089);


        // Set no response rate, so that doesn't trigger an exception
        FetcherPolicy policy = new FetcherPolicy();
        policy.setMinResponseRate(FetcherPolicy.NO_MIN_RESPONSE_RATE);


        final BaseFetcher fetcher = new SimpleHttpFetcher(1, policy, ConfigUtils.BIXO_TEST_AGENT);
        final String[] failMsg = new String[1];
        
        Thread t = new Thread(new Runnable() {

View Full Code Here

        // Return 1K bytes at 2K bytes/second - would normally trigger an
        // error.
        Server server = startServer(new RandomResponseHandler(1000, 500), 8089);


        // Set up for no minimum response rate.
        FetcherPolicy policy = new FetcherPolicy();
        policy.setMinResponseRate(FetcherPolicy.NO_MIN_RESPONSE_RATE);


        BaseFetcher fetcher = new SimpleHttpFetcher(1, policy, ConfigUtils.BIXO_TEST_AGENT);


        String url = "http://localhost:8089/test.html";
        fetcher.get(new ScoredUrlDatum(url));

View Full Code Here

        server.stop();
    }
    
    @Test
    public final void testLargeContent() throws Exception {
        FetcherPolicy policy = new FetcherPolicy();
        Server server = startServer(new RandomResponseHandler(policy.getMaxContentSize() * 2), 8089);
        BaseFetcher fetcher = new SimpleHttpFetcher(1, policy, ConfigUtils.BIXO_TEST_AGENT);
        String url = "http://localhost:8089/test.html";
        FetchedDatum result = fetcher.get(new ScoredUrlDatum(url));
        server.stop();


        assertTrue("Content size should be truncated", result.getContentLength() <= policy.getMaxContentSize());
    }

View Full Code Here

    
    @Test
    public final void testTruncationWithKeepAlive() throws Exception {
        Server server = startServer(new ResourcesResponseHandler(), 8089);


        FetcherPolicy policy = new FetcherPolicy();
        BaseFetcher fetcher = new SimpleHttpFetcher(1, policy, ConfigUtils.BIXO_TEST_AGENT);
        fetcher.setDefaultMaxContentSize(1000);
        fetcher.setMaxContentSize("image/png", 5000);
        ScoredUrlDatum datumToFetch = new ScoredUrlDatum("http://localhost:8089/karlie.html");

View Full Code Here

        server.stop();
    }
    
    @Test
    public final void testLargeHtml() throws Exception {
        FetcherPolicy policy = new FetcherPolicy();
        Server server = startServer(new ResourcesResponseHandler(), 8089);
        BaseFetcher fetcher = new SimpleHttpFetcher(1, policy, ConfigUtils.BIXO_TEST_AGENT);
        String url = "http://localhost:8089/karlie.html";
        FetchedDatum result = fetcher.get(new ScoredUrlDatum(url));
        server.stop();


        assertTrue("Content size should be truncated", result.getContentLength() <= policy.getMaxContentSize());


    }

View Full Code Here


    }
    
    @Test
    public final void testContentTypeHeader() throws Exception {
        FetcherPolicy policy = new FetcherPolicy();
        Server server = startServer(new ResourcesResponseHandler(), 8089);
        BaseFetcher fetcher = new SimpleHttpFetcher(1, policy, ConfigUtils.BIXO_TEST_AGENT);
        String url = "http://localhost:8089/simple-page.html";
        FetchedDatum result = fetcher.get(new ScoredUrlDatum(url));
        server.stop();

View Full Code Here

        assertEquals("text/html", contentType);
    }
    
    @Test
    public final void testTempRedirectHandling() throws Exception {
        FetcherPolicy policy = new FetcherPolicy();
        Server server = startServer(new RedirectResponseHandler(), 8089);
        BaseFetcher fetcher = new SimpleHttpFetcher(1, policy, ConfigUtils.BIXO_TEST_AGENT);
        String url = "http://localhost:8089/base";
        FetchedDatum result = fetcher.get(new ScoredUrlDatum(url));
        server.stop();

View Full Code Here

        assertEquals(1, result.getNumRedirects());
    }
    
    @Test
    public final void testPermRedirectHandling() throws Exception {
        FetcherPolicy policy = new FetcherPolicy();
        Server server = startServer(new RedirectResponseHandler(true), 8089);
        BaseFetcher fetcher = new SimpleHttpFetcher(1, policy, ConfigUtils.BIXO_TEST_AGENT);
        String url = "http://localhost:8089/base";
        ScoredUrlDatum scoredUrl = new ScoredUrlDatum(url);
        scoredUrl.setPayloadValue("payload-field-1", 1);

View Full Code Here

        assertEquals(1, result.getPayloadValue("payload-field-1"));
    }
    
    @Test
    public final void testRedirectPolicy() throws Exception {
        FetcherPolicy policy = new FetcherPolicy();
        policy.setRedirectMode(RedirectMode.FOLLOW_TEMP);
        Server server = startServer(new RedirectResponseHandler(true), 8089);
        BaseFetcher fetcher = new SimpleHttpFetcher(1, policy, ConfigUtils.BIXO_TEST_AGENT);
        String url = "http://localhost:8089/base";
        
        try {
            fetcher.get(new ScoredUrlDatum(url));
            fail("Exception should have been thrown");
        } catch (RedirectFetchException e) {
            assertEquals("Redirected URL", "http://localhost:8089/redirect", e.getRedirectedUrl());
            assertEquals(RedirectExceptionReason.PERM_REDIRECT_DISALLOWED, e.getReason());
        } finally {
            server.stop();
        }
        
        // Now try setting the mode to follow none
        policy.setRedirectMode(RedirectMode.FOLLOW_NONE);
        server = startServer(new RedirectResponseHandler(false), 8089);
        fetcher = new SimpleHttpFetcher(1, policy, ConfigUtils.BIXO_TEST_AGENT);
        
        try {
            fetcher.get(new ScoredUrlDatum(url));

View Full Code Here

    @Test
    public final void testAcceptLanguage() throws Exception {
        final String englishContent = "English";
        final String foreignContent = "Foreign";
        
        FetcherPolicy policy = new FetcherPolicy();
        Server server = startServer(new LanguageResponseHandler(englishContent, foreignContent), 8089);
        BaseFetcher fetcher = new SimpleHttpFetcher(1, policy, ConfigUtils.BIXO_TEST_AGENT);
        String url = "http://localhost:8089/";
        FetchedDatum result = fetcher.get(new ScoredUrlDatum(url));
        server.stop();

View Full Code Here

0 1 2 3

TOP

Related Classes of bixo.config.FetcherPolicy

bixo.examples.crawl.DemoCrawlTool

bixo.examples.crawl.DemoCrawlWorkflowLRTest

bixo.examples.webmining.DemoWebMiningTool

bixo.examples.webmining.DemoWebMiningWorkflowTest

bixo.fetcher.FetcherTest

bixo.fetcher.LoggingFetcher

bixo.fetcher.SimpleHttpFetcher

bixo.fetcher.SimpleHttpFetcherTest

bixo.fetcher.simulation.FakeHttpFetcher

bixo.fetcher.simulation.NullHttpFetcher

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.