Examples of org.apache.hadoop.zebra.io.TableInserter

org.apache.hadoop.zebra.io.TableInserter
Inserter interface allows application to to insert a number of rows into table.

    ParseException {
    fs.delete(path, true);
    System.out.println("testFailureInsertAfterClose");
    writer = new ColumnGroup.Writer(path, "abc:string, def:map(string)", false, path.getName(), "pig", "gz",
        null, null, (short) -1, true, conf);
    TableInserter ins = writer.getInserter("part1", true);


    Tuple row = TypesUtils.createTuple(writer.getSchema());
    row.set(0, new String("val1"));


    SortedMap<String, String> map = new TreeMap<String, String>();
    map.put("john", "boy");
    row.set(1, map);


    ins.insert(new BytesWritable("key".getBytes()), row);


    ins.close();
    writer.close();


    try {
      TableInserter ins2 = writer.getInserter("part2", true);
      Assert.fail("Failed to catch insertion after closure.");
    } catch (IOException e) {
      // noop, expecting exceptions
    } finally {
      close();

View Full Code Here

    ParseException {
    fs.delete(path, true);
    System.out.println("testFailureInsertXtraColumn");
    writer = new ColumnGroup.Writer(path, "abc ", false, path.getName(), "pig", "gz", null, null, (short) -1, true,
        conf);
    TableInserter ins = writer.getInserter("part1", true);


    try {
      Tuple row = TypesUtils.createTuple(writer.getSchema());
      row.set(0, new String("val1"));


      SortedMap<String, String> map = new TreeMap<String, String>();
      map.put("john", "boy");
      row.set(1, map);
      Assert
          .fail("Failed to catch insertion an extra column not defined in schema.");
    } catch (IndexOutOfBoundsException e) {
      // noop, expecting exceptions
    } finally {
      ins.close();
      close();
    }
  }

View Full Code Here

    ParseException {
    fs.delete(path, true);
    System.out.println("testInsertOneRow");
    writer = new ColumnGroup.Writer(path, "abc:string, def:map(string)", false, path.getName(), "pig", "gz",
        null, null, (short) -1, true, conf);
    TableInserter ins = writer.getInserter("part1", true);


    Tuple row = TypesUtils.createTuple(writer.getSchema());
    row.set(0, new String("val1"));


    SortedMap<String, String> map = new TreeMap<String, String>();
    map.put("john", "boy");
    row.set(1, map);


    ins.insert(new BytesWritable("key".getBytes()), row);


    ins.close();
    close();
  }

View Full Code Here

    ParseException {
    fs.delete(path, true);
    System.out.println("testInsert2Rows");
    writer = new ColumnGroup.Writer(path, "abc:string, def:map(string)", false, path.getName(), "pig", "gz",
        null, null, (short) -1, true, conf);
    TableInserter ins = writer.getInserter("part1", true);


    // row 1
    Tuple row = TypesUtils.createTuple(writer.getSchema());
    row.set(0, new String("val1"));


    SortedMap<String, String> map = new TreeMap<String, String>();
    map.put("john", "boy");
    row.set(1, map);


    ins.insert(new BytesWritable("key".getBytes()), row);


    // row 2
    TypesUtils.resetTuple(row);
    row.set(0, new String("val2"));
    map.put("joe", "boy");
    map.put("jane", "girl");
    // map should contain 3 k->v pairs
    row.set(1, map);


    ins.insert(new BytesWritable("key".getBytes()), row);
    ins.close();


    ins.close();
    close();
  }

View Full Code Here

    ParseException {
    fs.delete(path, true);
    System.out.println("testInsert2Inserters");
    writer = new ColumnGroup.Writer(path, "abc:string, def:map(string)", false, path.getName(), "pig", "gz",
        null, null, (short) -1, true, conf);
    TableInserter ins1 = writer.getInserter("part1", true);
    TableInserter ins2 = writer.getInserter("part2", true);


    // row 1
    Tuple row = TypesUtils.createTuple(writer.getSchema());
    row.set(0, new String("val1"));


    SortedMap<String, String> map = new TreeMap<String, String>();
    map.put("john", "boy");


    ins1.insert(new BytesWritable("key11".getBytes()), row);
    ins2.insert(new BytesWritable("key21".getBytes()), row);


    // row 2
    TypesUtils.resetTuple(row);
    row.set(0, new String("val2"));
    map.put("joe", "boy");
    map.put("jane", "girl");
    // map should contain 3 k->v pairs
    row.set(1, map);


    ins2.insert(new BytesWritable("key22".getBytes()), row);
    // ins2.close();
    ins1.insert(new BytesWritable("key12".getBytes()), row);


    ins1.close();
    ins2.close();
    close();
  }

View Full Code Here

    ParseException {
    fs.delete(path, true);
    System.out.println("testFailureOverlappingKeys");
    writer = new ColumnGroup.Writer(path, "abc:string, def:map(string)", true, path.getName(), "pig", "gz",
        null, null, (short) -1, true, conf);
    TableInserter ins1 = writer.getInserter("part1", false);
    TableInserter ins2 = writer.getInserter("part2", false);


    // row 1


    Tuple row = TypesUtils.createTuple(writer.getSchema());
    row.set(0, new String("val1"));


    SortedMap<String, String> map = new TreeMap<String, String>();
    map.put("john", "boy");
    row.set(1, map);


    ins1.insert(new BytesWritable("key1".getBytes()), row);
    ins2.insert(new BytesWritable("key2".getBytes()), row);


    // row 2
    TypesUtils.resetTuple(row);
    row.set(0, new String("val2"));
    map.put("joe", "boy");
    map.put("jane", "girl");
    // map should contain 3 k->v pairs
    row.set(1, map);


    ins2.insert(new BytesWritable("key3".getBytes()), row);
    // ins2.close();
    ins1.insert(new BytesWritable("key4".getBytes()), row);
    try {
      ins1.close();
      ins2.close();
      close();
      Assert.fail("Failed to detect overlapping keys.");
    } catch (IOException e) {
      // noop, exceptions expected
    } finally {

View Full Code Here

        STR_STORAGE, conf);
    Schema schema = writer.getSchema();


    BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
    int part = 0;
    TableInserter inserter = writer1.getInserter("part" + part, true);


    Tuple tuple = TypesUtils.createTuple(schema);
    TypesUtils.resetTuple(tuple);


    // insert data in row 1
    int row = 0;
    tuple.set(0, true); // bool
    tuple.set(1, 1); // int
    tuple.set(2, 1001L); // long
    tuple.set(3, 1.1); // float
    tuple.set(4, "hello world 1"); // string
    tuple.set(5, new DataByteArray("hello byte 1")); // byte
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);


    row++;
    TypesUtils.resetTuple(tuple);


    tuple.set(0, false);
    tuple.set(1, 2); // int
    tuple.set(2, 1002L); // long
    tuple.set(3, 3.1); // float
    tuple.set(4, "hello world 2"); // string
    tuple.set(5, new DataByteArray("hello byte 2")); // byte
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);


    inserter.close();
    writer1.finish();
    writer.close();
  }

View Full Code Here


    schema = new Schema(STR_SCHEMA);


    ColumnGroup.Writer writer = new ColumnGroup.Writer(path, schema, false, path.getName(),
        "pig", "gz", null, null, (short) -1, true, conf);
    TableInserter ins = writer.getInserter("part0", true);


    Tuple row = TypesUtils.createTuple(schema);
    // schema for "r:record..."
    Schema schRecord = writer.getSchema().getColumn(1).getSchema();
    Tuple tupRecord = TypesUtils.createTuple(schRecord);
    Schema schColl = schema.getColumn(3).getSchema().getColumn(0).getSchema();
    DataBag bagColl = TypesUtils.createBag();
    Tuple tupColl1 = TypesUtils.createTuple(schColl);
    Tuple tupColl2 = TypesUtils.createTuple(schColl);
    Map<String, String> map = new HashMap<String, String>();
    byte[] abs1 = new byte[3];
    byte[] abs2 = new byte[4];


    // row 1
    tupRecord.set(0, 1);
    tupRecord.set(1, 1001L);
    map.put("a", "x");
    map.put("b", "y");
    map.put("c", "z");
    tupColl1.set(0, 3.1415926);
    tupColl1.set(1, 1.6);
    abs1[0] = 11;
    abs1[1] = 12;
    abs1[2] = 13;
    tupColl1.set(2, new DataByteArray(abs1));
    bagColl.add(tupColl1);
    tupColl2.set(0, 123.456789);
    tupColl2.set(1, 100);
    abs2[0] = 21;
    abs2[1] = 22;
    abs2[2] = 23;
    abs2[3] = 24;
    tupColl2.set(2, new DataByteArray(abs2));
    bagColl.add(tupColl2);
    row.set(0, true);
    row.set(1, tupRecord);
    row.set(2, map);
    row.set(3, bagColl);
    ins.insert(new BytesWritable("k1".getBytes()), row);


    // row 2
    TypesUtils.resetTuple(row);
    TypesUtils.resetTuple(tupRecord);
    map.clear();
    tupRecord.set(0, 2);
    tupRecord.set(1, 1002L);
    map.put("boy", "girl");
    map.put("adam", "amy");
    map.put("bob", "becky");
    map.put("carl", "cathy");
    bagColl.clear();
    TypesUtils.resetTuple(tupColl1);
    TypesUtils.resetTuple(tupColl2);
    tupColl1.set(0, 7654.321);
    tupColl1.set(1, 0.0001);
    abs1[0] = 31;
    abs1[1] = 32;
    abs1[2] = 33;
    tupColl1.set(2, new DataByteArray(abs1));
    bagColl.add(tupColl1);
    tupColl2.set(0, 0.123456789);
    tupColl2.set(1, 0.3333);
    abs2[0] = 41;
    abs2[1] = 42;
    abs2[2] = 43;
    abs2[3] = 44;
    tupColl2.set(2, new DataByteArray(abs2));
    bagColl.add(tupColl2);
    row.set(0, false);
    row.set(1, tupRecord);
    row.set(2, map);
    row.set(3, bagColl);
    ins.insert(new BytesWritable("k2".getBytes()), row);
    ins.close();


    writer.close();
  }

View Full Code Here

    //
    BasicTable.Writer writer = new BasicTable.Writer(path, schemaString, storageString, conf);
    
    Schema schema = writer.getSchema();
    Tuple tuple = TypesUtils.createTuple(schema);
    TableInserter inserter = writer.getInserter("ins", false);
    
    for (int i = 0; i < tableData.length; ++i) {
      TypesUtils.resetTuple(tuple);
      for (int k = 0; k < tableData[i].length; ++k) {
        tuple.set(k, tableData[i][k]);
        System.out.println("DEBUG: setting tuple k=" + k + "value= " + tableData[i][k]);
      }
      inserter.insert(new BytesWritable(("key" + i).getBytes()), tuple);
    }
    inserter.close();
    writer.close();
  }

View Full Code Here


    Schema schema = writer.getSchema();
    Tuple tuple = TypesUtils.createTuple(schema);
    BasicTable.Writer writer1 = new BasicTable.Writer(path, conf);
    int part = 0;
    TableInserter inserter = null;
    try {
      inserter = writer1.getInserter("part" + part, true);
    } catch (Exception e) {
      e.printStackTrace();
    }
    TypesUtils.resetTuple(tuple);
    Tuple tupRecord1 = null;
    try {
      tupRecord1 = TypesUtils.createTuple(schema.getColumnSchema("r1")
          .getSchema());
    } catch (ParseException e) {
      e.printStackTrace();
    }


    Tuple tupRecord2 = null;
    try {
      tupRecord2 = TypesUtils.createTuple(schema.getColumnSchema("r2")
          .getSchema());
    } catch (ParseException e) {
      e.printStackTrace();
    }


    Tuple tupRecord3 = null;
    try {
      tupRecord3 = TypesUtils.createTuple(new Schema("f3:float, f4"));
    } catch (ParseException e) {
      e.printStackTrace();
    }


    // row 1
    tuple.set(0, true); // bool
    tuple.set(1, 1); // int
    tuple.set(2, 1001L); // long
    tuple.set(3, 1.1); // float
    tuple.set(4, "hello world 1"); // string
    tuple.set(5, new DataByteArray("hello byte 1")); // byte


    // r1:record(f1:int, f2:long
    tupRecord1.set(0, 1);
    tupRecord1.set(1, 1001L);
    tuple.set(6, tupRecord1);


    // r2:record(r3:record(f3:float, f4))
    tupRecord2.set(0, tupRecord3);
    tupRecord3.set(0, 1.3);
    tupRecord3.set(1, new DataByteArray("r3 row 1 byte array "));
    tuple.set(7, tupRecord2);


    // m1:map(string)
    Map<String, String> m1 = new HashMap<String, String>();
    m1.put("a", "A");
    m1.put("b", "B");
    m1.put("c", "C");
    tuple.set(8, m1);


    // m2:map(map(int))
    HashMap<String, Map> m2 = new HashMap<String, Map>();
    Map<String, Integer> m3 = new HashMap<String, Integer>();
    m3.put("m311", 311);
    m3.put("m321", 321);
    m3.put("m331", 331);
    Map<String, Integer> m4 = new HashMap<String, Integer>();
    m4.put("m411", 411);
    m4.put("m421", 421);
    m4.put("m431", 431);
    m2.put("x", m3);
    m2.put("y", m4);
    tuple.set(9, m2);


    // c:collection(f13:double, f14:float, f15:bytes)
    DataBag bagColl = TypesUtils.createBag();
    Schema schColl = schema.getColumn(10).getSchema().getColumn(0).getSchema();
    Tuple tupColl1 = TypesUtils.createTuple(schColl);
    Tuple tupColl2 = TypesUtils.createTuple(schColl);
    byte[] abs1 = new byte[3];
    byte[] abs2 = new byte[4];
    tupColl1.set(0, 3.1415926);
    tupColl1.set(1, 1.6);
    abs1[0] = 11;
    abs1[1] = 12;
    abs1[2] = 13;
    tupColl1.set(2, new DataByteArray(abs1));
    bagColl.add(tupColl1);
    tupColl2.set(0, 123.456789);
    tupColl2.set(1, 100);
    abs2[0] = 21;
    abs2[1] = 22;
    abs2[2] = 23;
    abs2[3] = 24;
    tupColl2.set(2, new DataByteArray(abs2));
    bagColl.add(tupColl2);
    tuple.set(10, bagColl);


    // set s7 to s23
    for (int i = 7; i <= 23; i++) {
      tuple.set(i + 4, "s" + "i" + ", line1");
    }


    int row = 0;
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);


    // row 2
    row++;
    TypesUtils.resetTuple(tuple);
    TypesUtils.resetTuple(tupRecord1);
    TypesUtils.resetTuple(tupRecord2);
    TypesUtils.resetTuple(tupRecord3);
    m1.clear();
    m2.clear();
    m3.clear();
    m4.clear();
    tuple.set(0, false);
    tuple.set(1, 2); // int
    tuple.set(2, 1002L); // long
    tuple.set(3, 3.1); // float
    tuple.set(4, "hello world 2"); // string
    tuple.set(5, new DataByteArray("hello byte 2")); // byte


    // r1:record(f1:int, f2:long
    tupRecord1.set(0, 2);
    tupRecord1.set(1, 1002L);
    tuple.set(6, tupRecord1);


    // r2:record(r3:record(f3:float, f4))
    tupRecord2.set(0, tupRecord3);
    tupRecord3.set(0, 2.3);
    tupRecord3.set(1, new DataByteArray("r3 row2  byte array"));
    tuple.set(7, tupRecord2);


    // m1:map(string)
    m1.put("a2", "A2");
    m1.put("b2", "B2");
    m1.put("c2", "C2");
    tuple.set(8, m1);


    // m2:map(map(int))
    m3.put("m321", 321);
    m3.put("m322", 322);
    m3.put("m323", 323);
    m2.put("z", m3);
    tuple.set(9, m2);


    // c:collection(f13:double, f14:float, f15:bytes)
    bagColl.clear();
    TypesUtils.resetTuple(tupColl1);
    TypesUtils.resetTuple(tupColl2);
    tupColl1.set(0, 7654.321);
    tupColl1.set(1, 0.0001);
    abs1[0] = 31;
    abs1[1] = 32;
    abs1[2] = 33;
    tupColl1.set(2, new DataByteArray(abs1));
    bagColl.add(tupColl1);
    tupColl2.set(0, 0.123456789);
    tupColl2.set(1, 0.3333);
    abs2[0] = 41;
    abs2[1] = 42;
    abs2[2] = 43;
    abs2[3] = 44;
    tupColl2.set(2, new DataByteArray(abs2));
    bagColl.add(tupColl2);
    tuple.set(10, bagColl);
    // set s7 to s23
    for (int i = 7; i <= 23; i++) {
      tuple.set(i + 4, "s" + "i" + ", line2");
    }
    inserter.insert(new BytesWritable(String.format("k%d%d", part + 1, row + 1)
        .getBytes()), tuple);


    inserter.close();
    writer1.finish();


    writer.close();
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.hadoop.zebra.io.TableInserter

org.apache.hadoop.zebra.io.TestBasicTable

org.apache.hadoop.zebra.io.TestBasicTableMapSplits

org.apache.hadoop.zebra.io.TestBasicTableProjections

org.apache.hadoop.zebra.io.TestBasicTableSplits

org.apache.hadoop.zebra.io.TestCollection

org.apache.hadoop.zebra.io.TestColumnGroup

org.apache.hadoop.zebra.io.TestColumnGroupInserters

org.apache.hadoop.zebra.io.TestColumnGroupName1

org.apache.hadoop.zebra.io.TestColumnGroupName2

org.apache.hadoop.zebra.io.TestColumnGroupName3

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.