Package com.cloudera.cdk.data

Examples of com.cloudera.cdk.data.DatasetDescriptor

80
81
82
83
84
85
86
87
88
89
90
    Dataset<Record> dataset = repo.create(NAME,
        new DatasetDescriptor.Builder(testDescriptor)
            .format(Formats.AVRO)
            .build());

    DatasetDescriptor changed =
        new DatasetDescriptor.Builder(dataset.getDescriptor())
        .format(Formats.PARQUET)
        .build();

    try {
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
    Dataset<Record> dataset = repo.create(NAME,
        new DatasetDescriptor.Builder(testDescriptor)
            .partitionStrategy(ps1)
            .build());

    DatasetDescriptor changed =
        new DatasetDescriptor.Builder(dataset.getDescriptor())
            .partitionStrategy(ps2)
            .build();

    try {
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
  public void testUpdateFailsWithLocationChange() {
    ensureCreated();
    Dataset<Record> dataset = repo.load(NAME);
    URI location = dataset.getDescriptor().getLocation();

    DatasetDescriptor changed =
        new DatasetDescriptor.Builder(dataset.getDescriptor())
            .location(new Path(testDirectory, "newDataLocation").toUri())
            .build();

    try {
View Full Code Here
49
50
51
52
53
54
55
56
    this.metadataProvider = new HBaseMetadataProvider(hBaseAdmin, schemaManager);
  }

  @Override
  public <E> RandomAccessDataset<E> create(String name, DatasetDescriptor descriptor) {
    DatasetDescriptor newDescriptor = metadataProvider.create(name, descriptor);
    return newDataset(name, newDescriptor);
  }
View Full Code Here
55
56
57
58
59
60
61
62
    return newDataset(name, newDescriptor);
  }

  @Override
  public <E> RandomAccessDataset<E> update(String name, DatasetDescriptor descriptor) {
    DatasetDescriptor newDescriptor = metadataProvider.update(name, descriptor);
    return newDataset(name, newDescriptor);
  }
View Full Code Here
70
71
72
73
74
75
76
77
78
      for (String subEntityName : entityName.split("\\.")) {
        descriptors.add(metadataProvider.load(tableName + "." + subEntityName));
      }
      return newCompositeDataset(name, tableName, descriptors);
    } else {
      DatasetDescriptor descriptor = metadataProvider.load(name);
      return newDataset(name, descriptor);
    }
  }
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
    // Construct an HCatalog dataset repository using managed Hive tables
    DatasetRepository repo = DatasetRepositories.open("repo:hive");

    // Create a dataset of users with the Avro schema in the repository
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schemaUri("resource:user.avsc")
        .build();
    Dataset<GenericRecord> users = repo.create("users", descriptor);

    // Get a writer for the dataset and write some users to it
    DatasetWriter<GenericRecord> writer = users.newWriter();
    try {
      writer.open();
      String[] colors = { "green", "blue", "pink", "brown", "yellow" };
      Random rand = new Random();
      GenericRecordBuilder builder = new GenericRecordBuilder(descriptor.getSchema());
      for (int i = 0; i < 100; i++) {
        GenericRecord record = builder.set("username", "user-" + i)
            .set("creationDate", System.currentTimeMillis())
            .set("favoriteColor", colors[rand.nextInt(colors.length)]).build();
        writer.write(record);
View Full Code Here
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
    // Construct a filesystem dataset repository rooted at /tmp/data
    DatasetRepository repo = DatasetRepositories.open("repo:hdfs:/tmp/data");

    // Create a dataset of users with the Avro schema in the repository
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schemaUri("resource:user.avsc")
        .build();
    Dataset<GenericRecord> users = repo.create("users", descriptor);

    // Get a writer for the dataset and write some users to it
    DatasetWriter<GenericRecord> writer = users.newWriter();
    try {
      writer.open();
      String[] colors = { "green", "blue", "pink", "brown", "yellow" };
      Random rand = new Random();
      GenericRecordBuilder builder = new GenericRecordBuilder(descriptor.getSchema());
      for (int i = 0; i < 100; i++) {
        GenericRecord record = builder.set("username", "user-" + i)
            .set("creationDate", System.currentTimeMillis())
            .set("favoriteColor", colors[rand.nextInt(colors.length)]).build();
        writer.write(record);
View Full Code Here
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
    // Create a partition strategy that hash partitions on username with 10 buckets
    PartitionStrategy partitionStrategy =
        new PartitionStrategy.Builder().hash("username", 10).build();

    // Create a dataset of users with the Avro schema in the repository
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schemaUri("resource:user.avsc")
        .partitionStrategy(partitionStrategy)
        .build();
    Dataset<GenericRecord> users = repo.create("users", descriptor);

    // Get a writer for the dataset and write some users to it
    DatasetWriter<GenericRecord> writer = users.newWriter();
    try {
      writer.open();
      String[] colors = { "green", "blue", "pink", "brown", "yellow" };
      Random rand = new Random();
      GenericRecordBuilder builder = new GenericRecordBuilder(descriptor.getSchema());
      for (int i = 0; i < 100; i++) {
        GenericRecord record = builder.set("username", "user-" + i)
            .set("creationDate", System.currentTimeMillis())
            .set("favoriteColor", colors[rand.nextInt(colors.length)]).build();
        writer.write(record);
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
    // Construct a filesystem dataset repository rooted at /tmp/data
    DatasetRepository repo = DatasetRepositories.open("repo:hdfs:/tmp/data");

    // Create a dataset of users with the Avro schema, and Parquet format in the
    // repository
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schemaUri("resource:user.avsc")
        .format(Formats.PARQUET)
        .build();
    Dataset<GenericRecord> users = repo.create("users", descriptor);

    // Get a writer for the dataset and write some users to it
    DatasetWriter<GenericRecord> writer = users.newWriter();
    try {
      writer.open();
      String[] colors = { "green", "blue", "pink", "brown", "yellow" };
      Random rand = new Random();
      GenericRecordBuilder builder = new GenericRecordBuilder(descriptor.getSchema());
      for (int i = 0; i < 100; i++) {
        GenericRecord record = builder.set("username", "user-" + i)
            .set("creationDate", System.currentTimeMillis())
            .set("favoriteColor", colors[rand.nextInt(colors.length)]).build();
        writer.write(record);
View Full Code Here
TOP

Related Classes of com.cloudera.cdk.data.DatasetDescriptor

  • com.cloudera.cdk.data.filesystem.FileSystemDatasetRepository
  • com.cloudera.cdk.data.filesystem.FileSystemMetadataProvider
  • com.cloudera.cdk.data.filesystem.PartitionedDatasetWriter
  • com.cloudera.cdk.data.filesystem.TestCSVFileReader
  • com.cloudera.cdk.data.filesystem.TestFileSystemDatasetRepository
  • com.cloudera.cdk.data.filesystem.TestFileSystemMetadataProvider
  • com.cloudera.cdk.data.filesystem.TestMultiFileDatasetReader
  • com.cloudera.cdk.data.hbase.avro.example.UserProfileDatasetExample
  • com.cloudera.cdk.data.hbase.avro.HBaseDatasetReaderTest
  • com.cloudera.cdk.data.hbase.DaoViewTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.