Examples of TCTLSeparatedProtocol


Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

      schema.setProperty(Constants.MAPKEY_DELIM, "4");

      DynamicSerDe serde = new DynamicSerDe();
      serde.initialize(new Configuration(), schema);

      TCTLSeparatedProtocol prot = (TCTLSeparatedProtocol)serde.oprot_;
      assertTrue(prot.getPrimarySeparator().equals("\u0009"));
     
      ObjectInspector oi = serde.getObjectInspector();

      // Try to serialize
      BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi);
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

      schema.setProperty(Constants.MAPKEY_DELIM, "4");

      DynamicSerDe serde = new DynamicSerDe();
      serde.initialize(new Configuration(), schema);

      TCTLSeparatedProtocol prot = (TCTLSeparatedProtocol)serde.oprot_;
      assertTrue(prot.getPrimarySeparator().equals("\u0009"));
     
      ObjectInspector oi = serde.getObjectInspector();

      // Try to serialize
      BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi);
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

      schema.setProperty(Constants.MAPKEY_DELIM, "4");

      DynamicSerDe serde = new DynamicSerDe();
      serde.initialize(new Configuration(), schema);

      TCTLSeparatedProtocol prot = (TCTLSeparatedProtocol)serde.oprot_;
      assertTrue(prot.getPrimarySeparator().equals("\u0009"));
     
      ObjectInspector oi = serde.getObjectInspector();

      // Try to serialize
      BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi);
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

      schema.setProperty(Constants.MAPKEY_DELIM, "4");

      DynamicSerDe serde = new DynamicSerDe();
      serde.initialize(new Configuration(), schema);

      TCTLSeparatedProtocol prot = (TCTLSeparatedProtocol)serde.oprot_;
      assertTrue(prot.getPrimarySeparator().equals("\u0009"));
     
      ObjectInspector oi = serde.getObjectInspector();

      // Try to serialize
      BytesWritable bytes = (BytesWritable) serde.serialize(struct, oi);
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

    trans.flush();


    // use 3 as the row buffer size to force lots of re-buffering.
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 1024);
    prot.initialize(new Configuration(), new Properties());

    prot.readStructBegin();

    prot.readFieldBegin();
    String hello = prot.readString();
    prot.readFieldEnd();

    assertTrue(hello.equals(foo));

    prot.readFieldBegin();
    assertTrue(prot.readString().equals(""));
    prot.readFieldEnd();

    prot.readFieldBegin();
    assertTrue(prot.readString().equals(bar));
    prot.readFieldEnd();

    prot.readFieldBegin();
    TMap mapHeader = prot.readMapBegin();
    assertTrue(mapHeader.size == 2);

    assertTrue(prot.readI32() == 22);
    assertTrue(prot.readString().equals(value));
    assertTrue(prot.readI32() == 24);
    assertTrue(prot.readString().equals(value2));
    prot.readMapEnd();
    prot.readFieldEnd();

    prot.readFieldBegin();
    hello = prot.readString();
    prot.readFieldEnd();
    assertNull(hello);

    prot.readStructEnd();
  }
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

  }


  public void testWrites() throws Exception {
    TMemoryBuffer trans = new TMemoryBuffer(1024);
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 1024);

    prot.writeStructBegin(new TStruct());
    prot.writeFieldBegin(new TField());
    prot.writeI32(100);
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeListBegin(new TList());
    prot.writeDouble(348.55);
    prot.writeDouble(234.22);
    prot.writeListEnd();
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeString("hello world!");
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeMapBegin(new TMap());
    prot.writeString("key1");
    prot.writeString("val1");
    prot.writeString("key2");
    prot.writeString("val2");
    prot.writeString("key3");
    prot.writeString("val3");
    prot.writeMapEnd();
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeListBegin(new TList());
    prot.writeString("elem1");
    prot.writeString("elem2");
    prot.writeListEnd();
    prot.writeFieldEnd();

   
    prot.writeFieldBegin(new TField());
    prot.writeString("bye!");
    prot.writeFieldEnd();

    prot.writeStructEnd();
    trans.flush();
    byte b[] = new byte[3*1024];
    int len = trans.read(b,0,b.length);
    String test = new String(b, 0, len);

    String testRef = "100348.55234.22hello world!key1val1key2val2key3val3elem1elem2bye!";
    assertTrue(test.equals(testRef));

    trans = new TMemoryBuffer(1023);
    trans.write(b, 0, len);

    //
    // read back!
    //

    prot = new TCTLSeparatedProtocol(trans, 10);
    prot.initialize(new Configuration(), new Properties());
   
    // 100 is the start
    prot.readStructBegin();
    prot.readFieldBegin();
    assertTrue(prot.readI32() == 100);
    prot.readFieldEnd();

    // let's see if doubles work ok
    prot.readFieldBegin();
    TList l = prot.readListBegin();
    assertTrue(l.size == 2);
    assertTrue(prot.readDouble() == 348.55);
    assertTrue(prot.readDouble() == 234.22);
    prot.readListEnd();
    prot.readFieldEnd();

    // nice message
    prot.readFieldBegin();
    assertTrue(prot.readString().equals("hello world!"));
    prot.readFieldEnd();

    // 3 element map
    prot.readFieldBegin();
    TMap m = prot.readMapBegin();
    assertTrue(m.size == 3);
    assertTrue(prot.readString().equals("key1"));
    assertTrue(prot.readString().equals("val1"));
    assertTrue(prot.readString().equals("key2"));
    assertTrue(prot.readString().equals("val2"));
    assertTrue(prot.readString().equals("key3"));
    assertTrue(prot.readString().equals("val3"));
    prot.readMapEnd();
    prot.readFieldEnd();

    // the 2 element list
    prot.readFieldBegin();
    l = prot.readListBegin();
    assertTrue(l.size == 2);
    assertTrue(prot.readString().equals("elem1"));
    assertTrue(prot.readString().equals("elem2"));
    prot.readListEnd();
    prot.readFieldEnd();

    // final string
    prot.readFieldBegin();
    assertTrue(prot.readString().equals("bye!"));
    prot.readFieldEnd();

    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();

    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();

    prot.readStructEnd();
  }
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

    prot.readStructEnd();
  }

  public void testQuotedWrites() throws Exception {
    TMemoryBuffer trans = new TMemoryBuffer(4096);
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 4096);
    Properties schema = new Properties();
    schema.setProperty(Constants.QUOTE_CHAR, "\"");
    schema.setProperty(Constants.FIELD_DELIM, ",");
    prot.initialize(new Configuration(), schema);

    String testStr = "\"hello, world!\"";

    prot.writeStructBegin(new TStruct());

    prot.writeFieldBegin(new TField());
    prot.writeString(testStr);
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeListBegin(new TList());
    prot.writeString("elem1");
    prot.writeString("elem2");
    prot.writeListEnd();
    prot.writeFieldEnd();

    prot.writeStructEnd();
    prot.writeString("\n");

    trans.flush();

    byte b[] = new byte[4096];
    int len = trans.read(b,0,b.length);


    trans = new TMemoryBuffer(4096);
    trans.write(b,0,len);
    prot = new TCTLSeparatedProtocol(trans, 1024);
    prot.initialize(new Configuration(), schema);

    prot.readStructBegin();
    prot.readFieldBegin();
    final String firstRead = prot.readString();
    prot.readFieldEnd();

    testStr = testStr.replace("\"","");

    assertEquals(testStr, firstRead);


    // the 2 element list
    prot.readFieldBegin();
    TList l = prot.readListBegin();
    assertTrue(l.size == 2);
    assertTrue(prot.readString().equals("elem1"));
    assertTrue(prot.readString().equals("elem2"));
    prot.readListEnd();
    prot.readFieldEnd();

    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();

    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();

    prot.readStructEnd();
  }
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

    TMemoryBuffer trans = new TMemoryBuffer(4096);
    trans.write(sample.getBytes(), 0, sample.getBytes().length);
    trans.flush();

    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 4096);
    Properties schema = new Properties();

    // this is a hacky way of doing the quotes since it will match any 2 of these, so
    // "[ hello this is something to split [" would be considered to be quoted.
    schema.setProperty(Constants.QUOTE_CHAR, "(\"|\\[|\\])");

    schema.setProperty(Constants.FIELD_DELIM, " ");
    schema.setProperty(Constants.SERIALIZATION_NULL_FORMAT, "-");
    prot.initialize(new Configuration(), schema);

    prot.readStructBegin();

    // ip address
    prot.readFieldBegin();
    final String ip = prot.readString();
    prot.readFieldEnd();

    assertEquals("127.0.0.1", ip);

    //  identd
    prot.readFieldBegin();
    final String identd = prot.readString();
    prot.readFieldEnd();

    assertNull(identd);

    //  user
    prot.readFieldBegin();
    final String user = prot.readString();
    prot.readFieldEnd();

    assertEquals("frank",user);

    //  finishTime
    prot.readFieldBegin();
    final String finishTime = prot.readString();
    prot.readFieldEnd();

    assertEquals("10/Oct/2000:13:55:36 -0700",finishTime);

    //  requestLine
    prot.readFieldBegin();
    final String requestLine = prot.readString();
    prot.readFieldEnd();

    assertEquals("GET /apache_pb.gif HTTP/1.0",requestLine);

    //  returncode
    prot.readFieldBegin();
    final int returnCode = prot.readI32();
    prot.readFieldEnd();

    assertEquals(200, returnCode);

    //  return size
    prot.readFieldBegin();
    final int returnSize = prot.readI32();
    prot.readFieldEnd();

    assertEquals(2326, returnSize);

    prot.readStructEnd();
  }
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol



  public void testNulls() throws Exception {
    TMemoryBuffer trans = new TMemoryBuffer(1024);
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 10);
    prot.initialize(new Configuration(), new Properties());

    prot.writeStructBegin(new TStruct());

    prot.writeFieldBegin(new TField());
    prot.writeString(null);
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeString(null);
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeI32(100);
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeString(null);
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeMapBegin(new TMap());
    prot.writeString(null);
    prot.writeString(null);
    prot.writeString("key2");
    prot.writeString(null);
    prot.writeString(null);
    prot.writeString("val3");
    prot.writeMapEnd();
    prot.writeFieldEnd();

    prot.writeStructEnd();

    byte b[] = new byte[3*1024];
    int len = trans.read(b,0,b.length);
    String written = new String(b,0,len);

    String testRef = "\\N\\N100\\N\\N\\Nkey2\\N\\Nval3";

    assertTrue(testRef.equals(written));

    trans = new TMemoryBuffer(1023);
    trans.write(b, 0, len);

    prot = new TCTLSeparatedProtocol(trans, 3);
    prot.initialize(new Configuration(), new Properties());
   
    prot.readStructBegin();

    prot.readFieldBegin();
    String ret = prot.readString();
    prot.readFieldEnd();

    assertNull(ret);

    prot.readFieldBegin();
    ret = prot.readString();
    prot.readFieldEnd();
   
    assertNull(ret);

    prot.readFieldBegin();
    int ret1 = prot.readI32();
    prot.readFieldEnd();

    assertTrue(ret1 == 100);


    prot.readFieldBegin();
    ret1 = prot.readI32();
    prot.readFieldEnd();

    prot.readFieldBegin();
    TMap map =  prot.readMapBegin();

    assertTrue(map.size == 3);

    assertNull(prot.readString());
    assertNull(prot.readString());

    assertTrue(prot.readString().equals("key2"));
    assertNull(prot.readString());

    assertNull(prot.readString());
    assertTrue(prot.readString().equals("val3"));
   
    prot.readMapEnd();
    prot.readFieldEnd();

    assertTrue(ret1 == 0);
  }
View Full Code Here

Examples of org.apache.hadoop.hive.serde2.thrift.TCTLSeparatedProtocol

    prot.readStructEnd();
  }

  public void testQuotedWrites() throws Exception {
    TMemoryBuffer trans = new TMemoryBuffer(4096);
    TCTLSeparatedProtocol prot = new TCTLSeparatedProtocol(trans, 4096);
    Properties schema = new Properties();
    schema.setProperty(Constants.QUOTE_CHAR, "\"");
    schema.setProperty(Constants.FIELD_DELIM, ",");
    prot.initialize(new Configuration(), schema);

    String testStr = "\"hello, world!\"";

    prot.writeStructBegin(new TStruct());

    prot.writeFieldBegin(new TField());
    prot.writeString(testStr);
    prot.writeFieldEnd();

    prot.writeFieldBegin(new TField());
    prot.writeListBegin(new TList());
    prot.writeString("elem1");
    prot.writeString("elem2");
    prot.writeListEnd();
    prot.writeFieldEnd();

    prot.writeStructEnd();
    prot.writeString("\n");

    trans.flush();

    byte b[] = new byte[4096];
    int len = trans.read(b, 0, b.length);

    trans = new TMemoryBuffer(4096);
    trans.write(b, 0, len);
    prot = new TCTLSeparatedProtocol(trans, 1024);
    prot.initialize(new Configuration(), schema);

    prot.readStructBegin();
    prot.readFieldBegin();
    final String firstRead = prot.readString();
    prot.readFieldEnd();

    testStr = testStr.replace("\"", "");

    assertEquals(testStr, firstRead);

    // the 2 element list
    prot.readFieldBegin();
    TList l = prot.readListBegin();
    assertTrue(l.size == 2);
    assertTrue(prot.readString().equals("elem1"));
    assertTrue(prot.readString().equals("elem2"));
    prot.readListEnd();
    prot.readFieldEnd();

    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();

    // shouldl return nulls at end
    prot.readFieldBegin();
    assertNull(prot.readString());
    prot.readFieldEnd();

    prot.readStructEnd();
  }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.