Examples of com.cloudera.cdk.data.DatasetDescriptor

com.cloudera.cdk.data.DatasetDescriptor

The structural definition of a {@link Dataset}.

Each {@code Dataset} has an associated {@link Schema} and optional{@link PartitionStrategy} defined at the time of creation. Instances of thisclass are used to hold this information. Users are strongly encouraged to use the inner {@link Builder} to create new instances.

    }


    logger.info("Creating an external Hive table named: " + name);


    // create a new descriptor with the dataset's location
    final DatasetDescriptor newDescriptor =
        new DatasetDescriptor.Builder(descriptor)
        .location(pathForDataset(name))
        .build();


    // create the data directory first so it is owned by the current user, not Hive

View Full Code Here

        .getMetadataProvider();
    Assert.assertTrue("Repo is using a HCatalogExternalMetadataProvider",
        provider instanceof HCatalogExternalMetadataProvider);


    // verify location
    DatasetDescriptor created = provider.create("test",
        new DatasetDescriptor.Builder()
        .schemaLiteral("\"string\"")
        .build());
    Assert.assertEquals("Location should be in HDFS",
        "hdfs", created.getLocation().getScheme());
    Assert.assertEquals("Location should have the correct HDFS host",
        hdfsUri.getHost(), created.getLocation().getHost());
    Assert.assertEquals("Location should have the correct HDFS port",
        hdfsUri.getPort(), created.getLocation().getPort());
    Assert.assertTrue("Location should be in the repo path",
        created.getLocation().getPath().startsWith("/tmp/hive-repo"));
  }

View Full Code Here

    return getDatasetDescriptor(schemaManager.getEntitySchema(tableName, entityName).getRawSchema());
  }


  @Override
  public boolean delete(String name) {
    DatasetDescriptor descriptor = load(name);
    String tableName = getTableName(name);
    String entityName = getEntityName(name);


    schemaManager.deleteSchema(tableName, entityName);


    String entitySchemaString = descriptor.getSchema().toString(true);


    AvroKeyEntitySchemaParser parser = new AvroKeyEntitySchemaParser();
    AvroEntitySchema entitySchema = parser.parseEntitySchema(entitySchemaString);


    // TODO: this may delete columns for other entities if they share column families

View Full Code Here


  @Test
  public void testCreateAssignsCorrectLocation() {
    ensureCreated();


    DatasetDescriptor loaded = provider.load(NAME);
    Path assignedPath = new Path(loaded.getLocation().getPath());
    Assert.assertEquals("Path should be in the test directory",
        new Path(testDirectory, NAME), assignedPath);
  }

View Full Code Here

    reader.hasNext();
  }


  @Test(expected = UnknownFormatException.class)
  public void testUnknownFormat() throws IOException {
    final DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schema(STRING_SCHEMA)
        .format(Accessor.getDefault().newFormat("explode!"))
        .build();


    MultiFileDatasetReader<Record> reader = new MultiFileDatasetReader<Record>(

View Full Code Here

  @Before
  @SuppressWarnings("unchecked")
  public void setup() throws Exception {
    repo = new HBaseDatasetRepository.Builder().configuration(
        HBaseTestUtils.getConf()).build();
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schemaLiteral(testEntity).build();
    ds = (DaoDataset) repo.create(tableName, descriptor);
  }

View Full Code Here

  @SuppressWarnings("unchecked")
  public void testGeneric() throws Exception {
    HBaseDatasetRepository repo = new HBaseDatasetRepository.Builder()
        .configuration(HBaseTestUtils.getConf()).build();
    
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schemaLiteral(testGenericEntity)
        .build();
    DaoDataset<GenericRecord> ds = (DaoDataset) repo.create(tableName, descriptor);


    // Create the new entities

View Full Code Here

  @Test
  public void testSpecific() throws Exception {
    HBaseDatasetRepository repo = new HBaseDatasetRepository.Builder()
        .configuration(HBaseTestUtils.getConf()).build();


    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schemaLiteral(testEntity)
        .build();
    RandomAccessDataset<TestEntity> ds = repo.create(tableName, descriptor);


    // Create the new entities

View Full Code Here

  public void testDeleteDataset() throws Exception {


    HBaseDatasetRepository repo = new HBaseDatasetRepository.Builder()
        .configuration(HBaseTestUtils.getConf()).build();


    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schemaLiteral(testGenericEntity)
        .build();
    RandomAccessDataset<GenericRecord> ds = repo.create(tableName, descriptor);


    // Create a new entity

View Full Code Here

          "cdk.avroSchemaReflectClass must be specified");
    }


    DatasetRepository repo = getDatasetRepository();


    DatasetDescriptor descriptor = repo.load(datasetName).getDescriptor();
    DatasetDescriptor.Builder descriptorBuilder =
        new DatasetDescriptor.Builder(descriptor);
    configureSchema(descriptorBuilder, avroSchemaFile, avroSchemaReflectClass);


    repo.update(datasetName, descriptorBuilder.build());

View Full Code Here

0 1 2 3 4

TOP

Related Classes of com.cloudera.cdk.data.DatasetDescriptor

com.cloudera.cdk.data.filesystem.FileSystemDatasetRepository

com.cloudera.cdk.data.filesystem.FileSystemMetadataProvider

com.cloudera.cdk.data.filesystem.PartitionedDatasetWriter

com.cloudera.cdk.data.filesystem.TestCSVFileReader

com.cloudera.cdk.data.filesystem.TestFileSystemDatasetRepository

com.cloudera.cdk.data.filesystem.TestFileSystemMetadataProvider

com.cloudera.cdk.data.filesystem.TestMultiFileDatasetReader

com.cloudera.cdk.data.hbase.avro.example.UserProfileDatasetExample

com.cloudera.cdk.data.hbase.avro.HBaseDatasetReaderTest

com.cloudera.cdk.data.hbase.DaoViewTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.