Package org.apache.hadoop.zebra.types

Examples of org.apache.hadoop.zebra.types.Partition$PartitionedColumn


        // set default projection that contains everything
        schema = schemaFile.getLogical();
        projection = new Projection(schema);
        String storage = schemaFile.getStorageString();
        String comparator = schemaFile.getComparator();
        partition = new Partition(schema, projection, storage, comparator);
        for (int nx = 0; nx < numCGs; nx++) {
          if (!schemaFile.isCGDeleted(nx)) {
            colGroups[nx] =
              new ColumnGroup.Reader(new Path(path, partition.getCGSchema(nx).getName()),
                                     conf, mapper);
View Full Code Here


    public synchronized void setProjection(String projection)
        throws ParseException, IOException {
      if (projection == null) {
        this.projection = new Projection(schemaFile.getLogical());
        partition =
            new Partition(schemaFile.getLogical(), this.projection, schemaFile
                .getStorageString(), schemaFile.getComparator());
      }
      else {
        /**
         * the typed schema from projection which is untyped or actually typed
         * as "bytes"
         */
        this.projection =
            new Projection(schemaFile.getLogical(), projection);
        partition =
            new Partition(schemaFile.getLogical(), this.projection, schemaFile
                .getStorageString(), schemaFile.getComparator());
      }
      inferredMapping = false;
    }
View Full Code Here

    public SchemaFile(Path path, String btSchemaStr, String btStorageStr, String sortColumns,
        String btComparator, Configuration conf)
        throws IOException {
      storage = btStorageStr;
      try {
        partition = new Partition(btSchemaStr, btStorageStr, btComparator, sortColumns);
      }
      catch (Exception e) {
        throw new IOException("Partition constructor failed :" + e.getMessage());
      }
      this.sortInfo = partition.getSortInfo();
View Full Code Here

        ;
        throw new IOException("Schema build failed :" + e.getMessage());
      }
      storage = WritableUtils.readString(in);
      try {
        partition = new Partition(logicalStr, storage, comparator);
      }
      catch (Exception e) {
        throw new IOException("Partition constructor failed :" + e.getMessage());
      }
      cgschemas = partition.getCGSchemas();
View Full Code Here

  @Test
  public void testStorageValid1() {
    try {
      String strStorage = "[c1]; [c2]";
      Partition p = new Partition(schema.toString(), strStorage, null);
      CGSchema[] cgschemas = p.getCGSchemas();

      // 2 column group;
      int size = cgschemas.length;
      Assert.assertEquals(size, 2);
      System.out.println("********** Column Groups **********");
      for (int i = 0; i < cgschemas.length; i++) {
        System.out.println(cgschemas[i]);
        System.out.println("--------------------------------");
      }
      CGSchema cgs1 = cgschemas[0];
      CGSchema cgs2 = cgschemas[1];

      ColumnSchema f11 = cgs1.getSchema().getColumn(0);
      Assert.assertEquals("c1", f11.getName());
      Assert.assertEquals(ColumnType.COLLECTION, f11.getType());
      ColumnSchema f21 = cgs2.getSchema().getColumn(0);
      Assert.assertEquals("c2", f21.getName());
      Assert.assertEquals(ColumnType.COLLECTION, f21.getType());

      System.out.println("*********** Column Map **********");
      Map<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> colmap = p
          .getPartitionInfo().getColMap();
      Assert.assertEquals(colmap.size(), 2);
      Iterator<Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>> it = colmap
          .entrySet().iterator();
      for (int i = 0; i < colmap.size(); i++) {
View Full Code Here

  @Test
  public void testStorageValid2() {
    try {
      String strStorage = "[c1.f1]";
      Partition p = new Partition(schema.toString(), strStorage, null);
      Assert.assertTrue(false);
      CGSchema[] cgschemas = p.getCGSchemas();

      // 2 column group;
      int size = cgschemas.length;
      Assert.assertEquals(size, 2);
      System.out.println("********** Column Groups **********");
      for (int i = 0; i < cgschemas.length; i++) {
        System.out.println(cgschemas[i]);
        System.out.println("--------------------------------");
      }
      CGSchema cgs1 = cgschemas[0];
      CGSchema cgs2 = cgschemas[1];

      ColumnSchema f11 = cgs1.getSchema().getColumn(0);
      Assert.assertEquals("c1.f1", f11.getName());
      Assert.assertEquals(ColumnType.INT, f11.getType());
      ColumnSchema f21 = cgs2.getSchema().getColumn(0);
      Assert.assertEquals("c1.f2", f21.getName());
      Assert.assertEquals(ColumnType.INT, f21.getType());
      ColumnSchema f22 = cgs2.getSchema().getColumn(1);
      Assert.assertEquals("c2", f22.getName());
      Assert.assertEquals(ColumnType.COLLECTION, f22.getType());

      System.out.println("*********** Column Map **********");
      Map<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> colmap = p
          .getPartitionInfo().getColMap();
      Assert.assertEquals(colmap.size(), 3);
      Iterator<Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>> it = colmap
          .entrySet().iterator();
      for (int i = 0; i < colmap.size(); i++) {
View Full Code Here

    if (schema == null)
      throw new ParseException("Schema has not been set");

    // for sanity check purpose only
    new Partition(schema, storehint, null);

    conf.set(OUTPUT_STORAGEHINT, storehint);
  }
View Full Code Here

    } else {
      storageHintStr = zStorageHint.toString();
    }
   
    try {
      new Partition(schemaStr, storageHintStr, null);
    } catch (ParseException e) {
      throw new ParseException("[" + zStorageHint + "] " + " is not a valid storage hint string: " + e.getMessage()  );
    } catch (IOException e) {
      throw new ParseException("[" + zStorageHint + "] " + " is not a valid storage hint string: " + e.getMessage()  );
    }
View Full Code Here

  @Test
  public void testStorageValid1() {
    try {
      String strStorage = "[c] compress by gz; [m1] serialize by avro";
      Partition p = new Partition(schema.toString(), strStorage, null);
      CGSchema[] cgschemas = p.getCGSchemas();

      // 2 column group;
      int size = cgschemas.length;
      Assert.assertEquals(size, 2);
      System.out.println("********** Column Groups **********");
      for (int i = 0; i < cgschemas.length; i++) {
        System.out.println(cgschemas[i]);
        System.out.println("--------------------------------");
      }
      CGSchema cgs1 = cgschemas[0];
      CGSchema cgs2 = cgschemas[1];

      ColumnSchema f11 = cgs1.getSchema().getColumn(0);
      Assert.assertEquals("c", f11.getName());
      Assert.assertEquals(ColumnType.COLLECTION, f11.getType());

      ColumnSchema f21 = cgs2.getSchema().getColumn(0);
      Assert.assertEquals("m1", f21.getName());
      Assert.assertEquals(ColumnType.MAP, f21.getType());

      Assert.assertEquals(cgs1.getCompressor(), "gz");
      Assert.assertEquals(cgs2.getSerializer(), "avro");

      System.out.println("*********** Column Map **********");
      Map<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> colmap = p
          .getPartitionInfo().getColMap();
      Assert.assertEquals(colmap.size(), 2);
      Iterator<Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>> it = colmap
          .entrySet().iterator();
      for (int i = 0; i < colmap.size(); i++) {
View Full Code Here

  @Test
  public void testStorageValid2() {
    try {
      String strStorage = "[c.r] compress by gz; [m1] serialize by avro";
      Partition p = new Partition(schema.toString(), strStorage, null);
      Assert.assertTrue(false);
      CGSchema[] cgschemas = p.getCGSchemas();

      // 2 column group;
      int size = cgschemas.length;
      Assert.assertEquals(size, 2);
      System.out.println("********** Column Groups **********");
      for (int i = 0; i < cgschemas.length; i++) {
        System.out.println(cgschemas[i]);
        System.out.println("--------------------------------");
      }
      CGSchema cgs1 = cgschemas[0];
      CGSchema cgs2 = cgschemas[1];

      ColumnSchema f11 = cgs1.getSchema().getColumn(0);
      Assert.assertEquals("c.r", f11.getName());
      Assert.assertEquals(ColumnType.RECORD, f11.getType());

      ColumnSchema f21 = cgs2.getSchema().getColumn(0);
      Assert.assertEquals("m1", f21.getName());
      Assert.assertEquals(ColumnType.MAP, f21.getType());

      Assert.assertEquals(cgs1.getCompressor(), "gz");
      Assert.assertEquals(cgs2.getSerializer(), "avro");

      System.out.println("*********** Column Map **********");
      Map<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>> colmap = p
          .getPartitionInfo().getColMap();
      Assert.assertEquals(colmap.size(), 2);
      Iterator<Map.Entry<String, HashSet<Partition.PartitionInfo.ColumnMappingEntry>>> it = colmap
          .entrySet().iterator();
      for (int i = 0; i < colmap.size(); i++) {
View Full Code Here

TOP

Related Classes of org.apache.hadoop.zebra.types.Partition$PartitionedColumn

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.