Package org.kitesdk.data.hbase.avro

Examples of org.kitesdk.data.hbase.avro.AvroEntitySchema


      throw new DatasetIOException("Cannot open schema table", e);
    }

    Schema schema = getEmbeddedSchema(descriptor);
    String entitySchemaString = schema.toString(true);
    AvroEntitySchema entitySchema = new AvroEntitySchema(
        schema, entitySchemaString, descriptor.getColumnMapping());

    String tableName = getTableName(name);
    String entityName = getEntityName(name);

    schemaManager.refreshManagedSchemaCache(tableName, entityName);
    schemaManager.createSchema(tableName, entityName, entitySchemaString,
        "org.kitesdk.data.hbase.avro.AvroKeyEntitySchemaParser",
        "org.kitesdk.data.hbase.avro.AvroKeySerDe",
        "org.kitesdk.data.hbase.avro.AvroEntitySerDe");

    try {
      if (!hbaseAdmin.tableExists(tableName)) {
        HTableDescriptor desc = new HTableDescriptor(tableName);
        desc.addFamily(columnFamily(Constants.SYS_COL_FAMILY, descriptor));
        desc.addFamily(columnFamily(Constants.OBSERVABLE_COL_FAMILY, descriptor));
        for (String columnFamily : entitySchema.getColumnMappingDescriptor()
            .getRequiredColumnFamilies()) {
          desc.addFamily(columnFamily(columnFamily, descriptor));
        }
        hbaseAdmin.createTable(desc);
      } else {
        Set<String> familiesToAdd = entitySchema.getColumnMappingDescriptor()
            .getRequiredColumnFamilies();
        familiesToAdd.add(new String(Constants.SYS_COL_FAMILY));
        familiesToAdd.add(new String(Constants.OBSERVABLE_COL_FAMILY));
        HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName
            .getBytes());
View Full Code Here


    schemaManager.refreshManagedSchemaCache(tableName, entityName);

    Schema newSchema = getEmbeddedSchema(descriptor);

    String schemaString = newSchema.toString(true);
    EntitySchema entitySchema = new AvroEntitySchema(
        newSchema, schemaString, descriptor.getColumnMapping());

    if (!schemaManager.hasSchemaVersion(tableName, entityName, entitySchema)) {
      schemaManager.migrateSchema(tableName, entityName, schemaString);
    } else {
View Full Code Here

   *          exist, and will create families if they don't exist.
   */
  public void createOrMigrateSchema(String tableName,
      String entitySchemaString, boolean createTableAndFamilies) {
    String entityName = getEntityNameFromSchemaString(entitySchemaString);
    AvroEntitySchema entitySchema = parser
        .parseEntitySchema(entitySchemaString);
    AvroKeySchema keySchema = parser.parseKeySchema(entitySchemaString);
    if (schemaManager.hasManagedSchema(tableName, entityName)) {
      KeySchema currentKeySchema = schemaManager.getKeySchema(tableName,
          entityName);
      if (!keySchema.equals(currentKeySchema)) {
        String msg = "Migrating schema with different keys. Current: "
            + currentKeySchema.getRawSchema() + " New: "
            + keySchema.getRawSchema();
        LOG.error(msg);
        throw new ValidationException(msg);
      }
      if (!schemaManager.hasSchemaVersion(tableName, entityName, entitySchema)) {
        LOG.info("Migrating Schema: (" + tableName + ", " + entityName + ")");
        schemaManager.migrateSchema(tableName, entityName, entitySchemaString);
      } else {
        // don't set createTableAndFamilies to false, becasue we may still need
        // to update the table to support what exists in the meta store.
        LOG.info("Schema hasn't changed, not migrating: (" + tableName + ", "
            + entityName + ")");
      }
    } else {
      LOG.info("Creating Schema: (" + tableName + ", " + entityName + ")");
      parser.parseEntitySchema(entitySchemaString).getColumnMappingDescriptor()
          .getRequiredColumnFamilies();
      schemaManager.createSchema(tableName, entityName, entitySchemaString,
          "org.kitesdk.data.hbase.avro.AvroKeyEntitySchemaParser",
          "org.kitesdk.data.hbase.avro.AvroKeySerDe",
          "org.kitesdk.data.hbase.avro.AvroEntitySerDe");
    }

    if (createTableAndFamilies) {
      try {
        if (!hbaseAdmin.tableExists(tableName)) {
          HTableDescriptor desc = new HTableDescriptor(tableName);
          desc.addFamily(new HColumnDescriptor(Constants.SYS_COL_FAMILY));
          desc.addFamily(new HColumnDescriptor(Constants.OBSERVABLE_COL_FAMILY));
          for (String columnFamily : entitySchema.getColumnMappingDescriptor()
              .getRequiredColumnFamilies()) {
            desc.addFamily(new HColumnDescriptor(columnFamily));
          }
          hbaseAdmin.createTable(desc);
        } else {
          Set<String> familiesToAdd = entitySchema.getColumnMappingDescriptor()
              .getRequiredColumnFamilies();
          familiesToAdd.add(new String(Constants.SYS_COL_FAMILY));
          familiesToAdd.add(new String(Constants.OBSERVABLE_COL_FAMILY));
          HTableDescriptor desc = hbaseAdmin.getTableDescriptor(tableName
              .getBytes());
View Full Code Here

TOP

Related Classes of org.kitesdk.data.hbase.avro.AvroEntitySchema

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.