Package org.commoncrawl.io.internal

Examples of org.commoncrawl.io.internal.NIOHttpConnection


        _writeLock.unlock();
      }
      super.ensureBuffer();
    }
    if (pausedConnection.get() != null && _bufferQueue.available() < MAX_BUFFER_SIZE) {
      final NIOHttpConnection connection = pausedConnection.get();
      pausedConnection.set(null);
      downloader.getEventLoop().queueAsyncCallback(new Callbacks.Callback() {

        @Override
        public void execute() {
          System.out.println("*** RESUMING DOWNLOADS ***");
          try {
            connection.enableReads();
          } catch (IOException e) {
            LOG.error(CCStringUtils.stringifyException(e));
          }
        }
       
View Full Code Here


    }
  }
 
  private void downloadItem(S3DownloadItem item) {

    NIOHttpConnection connection = null;
    try {
      // construct the url for the item
      URL theURL = _callingFormat.getURL(false, S3Utils.DEFAULT_HOST, S3Utils.INSECURE_PORT, _s3BucketName, item.getKey(), null);
      connection = new NIOHttpConnection(theURL,_eventLoop.getSelector(),_eventLoop.getResolver(),null);
      connection.getContentBuffer().setMinBufferSize(DEFAULT_MIN_HTTP_BUFFER_SIZE);
      connection.getContentBuffer().setMaxBufferSize(DEFAULT_MAX_HTTP_BUFFER_SIZE);
      // set up the item as the context for the connection ...
      connection.setContext(item);
      // we don't want to populate default http headers ...
      connection.setPopulateDefaultHeaderItems(false);
      // get at headers object
      NIOHttpHeaders headers = connection.getRequestHeaders();
      // populate http request string
      headers.prepend("GET" + " " + theURL.getFile() +" "  + "HTTP/1.1", null);
      // populate host entry ...
      if (theURL.getPort() != -1 && theURL.getPort() != 80) {
        headers.set("Host",theURL.getHost() +":"+String.valueOf(theURL.getPort()));
      }
      else {
        headers.set("Host",theURL.getHost());
      }
      // create a tree map in parallel (to pass to canonicalization routine for s3 auth)
      Map amazonHeaders = new TreeMap();
     
      // add date ...
      String theDate = httpDate();
     
      headers.set("Date", theDate);
      // and set it in amazon headers ...
      addToAmazonHeader("Date", theDate, amazonHeaders);
      // add requester pays if specified ...
      if (_isRequesterPays) {
        headers.set("x-amz-request-payer", "requester");
        addToAmazonHeader("x-amz-request-payer", "requester",amazonHeaders);
      }

      String canonicalString =  S3Utils.makeCanonicalString("GET", _s3BucketName, item.getKey(), null,amazonHeaders );
      String encodedCanonical = S3Utils.encode(_s3SecretKey, canonicalString, false);
     
      // add auth string to headers ...
      headers.set("Authorization","AWS " + _s3AccessId + ":" + encodedCanonical);
     
      // figure out of this is a continuation ...
      if (item.isContinuation()) {
        // figure out where to start ...
        String rangeString = "bytes=" + item.getLastReadPos() + "-" + item.getContentLength();
        // set the range header ...
        headers.set("Range",rangeString);
        // and if etag is valid ...
        if (item.getLastKnownETag() != null) {
          headers.set("If-match",item.getLastKnownETag());
        }
      }
      // add cache control pragmas ...
      headers.set ("Connection", "close");
      headers.set("Cache-Control", "no-cache");
      headers.set("Pragma", "no-cache");
      headers.remove("Accept-Encoding");
      headers.set("Accept-Encoding","identity");
     
      // set up the listener relationship
      connection.setListener(this);
      // and open the  connection
      connection.open();
     
      // add it to list for tracking purposes
      _activeConnections.add(connection);
     
    }
    catch (IOException e) {
      LOG.error(StringUtils.stringifyException(e));
      LOG.error("Failure Opening Connection For Key:" + item.getKey());

      synchronized (_queuedItems) {
        _queuedItems.addLast(item);
      }
      if (connection != null) {
        // null out listener
        connection.setListener(null);
        connection.setContext(null);
        // close connection
        connection.close();
      }
    }
  }
View Full Code Here

TOP

Related Classes of org.commoncrawl.io.internal.NIOHttpConnection

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.