Examples of FetchRequest

Examples of bixo.fetcher.FetchRequest

49
50
51
52
53
54
55
56
    }
   
    public FetchRequest getFetchRequest(long now, long crawlDelay, int maxUrls) {
        // Ignore crawlDelay, and always use our delay.
        long nextRequestTime = now + getCrawlDelay();
        FetchRequest result = new FetchRequest(Math.min(maxUrls, 1), nextRequestTime);
        return result;
    }
View Full Code Here

Examples of bixo.fetcher.FetchRequest

47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
   
    public FetchRequest getFetchRequest(long now, long crawlDelay, int maxUrls) {
        // we want to fetch maxUrls in the remaining time, but the min delay might constrain us.
       
        if ((getCrawlDelay() == 0) || (maxUrls == 0)) {
            return new FetchRequest(Math.min(maxUrls, getMaxRequestsPerConnection()), now);
        }

        // Even if we're at the end of the crawl, we still want to do our calculation using our
        // default fetch interval, to avoid not crawling anything if we run over. We rely on
        // an external mechanism to do any pruning of remaining URLs, so that they get properly
        // aborted.
        long fetchInterval = Math.max(DEFAULT_FETCH_INTERVAL, getCrawlEndTime() - now);
       
        // Crawl delay must be between the min crawl delay and the default crawl delay.
        long customCrawlDelay = Math.max(getCrawlDelay(), Math.min(DEFAULT_CRAWL_DELAY, fetchInterval / maxUrls));
       
        // Figure out how many URLs we can get in 5 minutes,or the remaining time (whatever is less).
        int numUrls = Math.min((int)(Math.min(DEFAULT_FETCH_INTERVAL, fetchInterval) / customCrawlDelay), maxUrls);
        long nextFetchTime = now + (numUrls * customCrawlDelay);
        return new FetchRequest(numUrls, nextFetchTime);
    }
View Full Code Here

Examples of bixo.fetcher.FetchRequest

43
44
45
46
47
48
49
50
    }
   
    @Test
    public void testEmptyFetchList() {
        FakeUserFetcherPolicy policy = new FakeUserFetcherPolicy();
        FetchRequest request = policy.getFetchRequest(System.currentTimeMillis(), 30 * 1000L, 0);
        Assert.assertEquals(0, request.getNumUrls());
    }
View Full Code Here

Examples of bixo.fetcher.FetchRequest

60
61
62
63
64
65
66
67
68
69
70
71
72
       
        for (int i = 0; i < 100; i++) {
            long curTime = System.currentTimeMillis();
           
            // Pass in longer crawl delay, that we'll ignore
            FetchRequest request = policy.getFetchRequest(curTime, 100 * 1000L, 100);
            Assert.assertEquals(1, request.getNumUrls());
            int delayInSeconds = (int)(request.getNextRequestTime() - curTime)/1000;
           
            Assert.assertTrue("Delay #" + i + " must be at least 5 seconds: " + delayInSeconds, delayInSeconds >= 30);
            Assert.assertTrue(delayInSeconds <= 91);
            delayCounts[delayInSeconds - 30] += 1;
        }
View Full Code Here

Examples of bixo.fetcher.FetchRequest

32
33
34
35
36
37
38
39
40
41
        AdaptiveFetcherPolicy policy = new AdaptiveFetcherPolicy(now + (10 * 1000), 1000L);
       
        // Ask for more than we can get in the remaining time, so it has to use a 5 minutes
        // window.
        final long crawlDelay = 30 * 1000L;
        FetchRequest request = policy.getFetchRequest(now, crawlDelay, 1000);
       
        // No way we can get more than 300.
        Assert.assertEquals(300, request.getNumUrls());
    }
View Full Code Here

Examples of bixo.fetcher.FetchRequest

47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
        AdaptiveFetcherPolicy policy = new AdaptiveFetcherPolicy(now + (600 * 1000), 1000L);

        // Ask for more than will fit in the duration assuming a 30 second delay, but less than what
        // would be constrained by the 1 second min delay.
        final long crawlDelay = 30 * 1000L;
        FetchRequest request = policy.getFetchRequest(now, crawlDelay, 100);
       
        // Our per-request rate is about 600/100 = 6 seconds. So in 5 minutes (the default
        // window) we should get back 50
        Assert.assertEquals(50, request.getNumUrls());
       
        // Our next fetch time should be around 5 minutes.
        long deltaFromTarget = Math.abs(request.getNextRequestTime() - (now + (5 * 60 * 1000L)));
        Assert.assertTrue(deltaFromTarget < 100);
    }
View Full Code Here

Examples of bixo.fetcher.FetchRequest

68
69
70
71
72
73
74
75
76
77
78
79
80
        AdaptiveFetcherPolicy policy = new AdaptiveFetcherPolicy(now - 1000, 10 * 1000L);
       
        // Ask for more than we can get in the remaining time, so it has to use a 5 minutes
        // window.
        final long crawlDelay = 30 * 1000L;
        FetchRequest request = policy.getFetchRequest(now, crawlDelay, 1000);
       
        Assert.assertEquals(30, request.getNumUrls());

        // Our next fetch time should be around 5 minutes.
        long deltaFromTarget = Math.abs(request.getNextRequestTime() - (now + (5 * 60 * 1000L)));
        Assert.assertTrue(deltaFromTarget < 100);
    }
View Full Code Here

Examples of com.google.dataconnector.protocol.proto.SdcFrame.FetchRequest

45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
  @Override
  protected void setUp() throws Exception {
  }
 
  public void testGetFromFrameInfo() throws Exception {
    FetchRequest request = FetchRequest.newBuilder()
      .setId("requestId").setStrategy("HttpClient")
      .setResource("http://www.google.com").build();
   
      String sessionId = UUID.randomUUID().toString();

      SdcKeysManager sm = new SdcKeysManager();
      sm.storeSessionKey(sessionId,
          SessionEncryption.JCE_ALGO, SessionEncryption.newKeyBytes());
   
    FrameInfo frameInfo = FrameInfo.newBuilder()
      .setSessionId(sessionId)
      .setPayload(sm.getSessionEncryption().encrypt(request.toByteString())).build();

    FetchRequestHandler handler = new FetchRequestHandler(
        sm,
        EasyMock.createMock(ThreadPoolExecutor.class),
        EasyMock.createMock(Injector.class),
        EasyMock.createMock(ClockUtil.class));

    FetchRequest parsed = sm.getSessionEncryption().getFrom(frameInfo,
        new SessionEncryption.Parse<FetchRequest>() {
      public FetchRequest parse(ByteString s) throws InvalidProtocolBufferException {
        return FetchRequest.parseFrom(s);
      }
    });
    assertEquals(request, parsed);

      FrameInfo frameInfo2 = FrameInfo.newBuilder()
        .setSessionId(UUID.randomUUID().toString())
        .setPayload(sm.getSessionEncryption().encrypt(request.toByteString())).build();

      FetchRequest parsed2 = sm.getSessionEncryption().getFrom(frameInfo2,
          new SessionEncryption.Parse<FetchRequest>() {
        public FetchRequest parse(ByteString s) throws InvalidProtocolBufferException {
          return FetchRequest.parseFrom(s);
        }
      });
View Full Code Here

Examples of com.google.step2.http.FetchRequest

425
426
427
428
429
430
431
432
433
434
435
   *
   * @throws FetchException
   */
  private XrdRepresentations fetchXrd(URI uri) throws FetchException {

    FetchRequest request = FetchRequest.createGetRequest(uri);

    XRDS xrds;
    byte[] documentBytes;
    String signature;

View Full Code Here

Examples of com.google.step2.http.FetchRequest

51
52
53
54
55
56
57
58
59
60
61
    InputStream responseStream = null;

    try {
      URI uri = getHostMetaUriForHost(host);
      FetchRequest request = FetchRequest.createGetRequest(uri);

      FetchResponse response = fetcher.fetch(request);

      int status = response.getStatusCode();
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.