Package org.commoncrawl.io.internal

Examples of org.commoncrawl.io.internal.NIOBufferList


  }

  @Test
  public void testEncoderDecoder() throws Exception {

    NIOBufferList output = new NIOBufferList();
    NIOBufferList input = new NIOBufferList();

    NIOBufferListOutputStream outputStream = new NIOBufferListOutputStream(
        output);
    NIOBufferListInputStream inputStream = new NIOBufferListInputStream(input);

    RPCFrame.Encoder encoder = new RPCFrame.Encoder(outputStream);
    RPCFrame.Decoder decoder = new RPCFrame.Decoder(inputStream);

    /**
     * UnitTestStruct1 based on rpc defiinition: module org.commoncrawl.rpc {
     *
     * class UnitTestStruct1 {
     *
     * enum EnumeratedValue { ONE = 1; TWO = 2; }
     *
     * int intType = 1; long longType = 2; ustring stringType = 3;
     * vector<ustring> vectorOfStrings = 4; }
     *
     * }
     * */
    UnitTestStruct1 inputStruct = new UnitTestStruct1();
    UnitTestStruct1 outputStruct = new UnitTestStruct1();

    inputStruct.setIntType(10);
    inputStruct.setLongType(20);
    inputStruct.setStringType("one");
    inputStruct.setFieldDirty(UnitTestStruct1.Field_VECTOROFSTRINGS);
    inputStruct.getVectorOfStrings().add(new TextBytes("one"));
    inputStruct.getVectorOfStrings().add(new TextBytes("two"));
    inputStruct.getVectorOfStrings().add(new TextBytes("three"));

    OutgoingMessageContext<UnitTestStruct1, UnitTestStruct1> request = new OutgoingMessageContext<UnitTestStruct1, UnitTestStruct1>(
        "testService", "testMethod", inputStruct, outputStruct, null);

    request.setRequestId(10);
   
    encoder.encodeRequest(request);

    // stream the data in one byte at a time
    while (output.available() != 0) {

      ByteBuffer buffer = output.read();

      input.getWriteBuf().put(buffer.get());
      input.flush();
      if (buffer.remaining() != 0) {
        output.putBack(buffer);
      }
      // validate that the decoder doesn't return the frame until the
      // appropriate time
View Full Code Here


   * @param s3Secret
   * @param bufferSize set this to be at least 1MB or higher to ensure decent performance
   * @throws IOException
   */
  public S3InputStream(URI uri,String s3AccessKey,String s3Secret,int bufferSize) throws IOException {
    super(new NIOBufferList());
   
    this.uri = uri;
   
    downloader = new S3Downloader(uri.getHost(), s3AccessKey,s3Secret, false);
    // we are download a single stream ...
View Full Code Here

TOP

Related Classes of org.commoncrawl.io.internal.NIOBufferList

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.