Package com.cloudera.cdk.data

Examples of com.cloudera.cdk.data.DatasetDescriptor

61
62
63
64
65
66
67
68
69
70
71
    Schema schema = new Schema.Parser().parse(
        Resources.getResource("combined_log_format.avsc").openStream());

    // Create the dataset
    DatasetRepository repo = DatasetRepositories.open(datasetUri);
    DatasetDescriptor datasetDescriptor = new DatasetDescriptor.Builder()
        .schema(schema).build();
    Dataset<Object> dataset = repo.create(datasetName, datasetDescriptor);

    // Run the job
    final String schemaString = schema.toString();
View Full Code Here
45
46
47
48
49
50
51
52
53
54
55
    HBaseTestUtils.util.deleteTable(Bytes.toBytes(managedTableName));
    HBaseDatasetRepository repo = new HBaseDatasetRepository.Builder()
        .configuration(HBaseTestUtils.getConf()).build();
    String testGenericEntity = AvroUtils.inputStreamToString(
        HBaseDatasetRepositoryTest.class.getResourceAsStream("/TestGenericEntity.avsc"));
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schemaLiteral(testGenericEntity)
        .build();
    dataset = repo.create("testtable", descriptor);
    for (int i = 0; i < 10; i++) {
      dataset.put(HBaseDatasetRepositoryTest.createGenericEntity(i));
View Full Code Here
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
  @Test
  public void testLoadSetsURIs() throws IOException {
    ensureCreated();

    DatasetDescriptor loaded = provider.load(NAME);
    Assert.assertNotNull("Loaded descriptor should have a location",
        loaded.getLocation());
    if (distributed) {
      // purposely call new Configuration() to test that the URI has HDFS info
      Assert.assertEquals(
          getDFS(),
          FileSystem.get(loaded.getLocation(), new Configuration()));
      Assert.assertEquals(
          "hdfs",
          loaded.getLocation().getScheme());
      Assert.assertEquals(
          getDFS().getUri().getAuthority(),
          loaded.getLocation().getAuthority());
    } else {
      // purposely call new Configuration() to test that the URI has FS info
      Assert.assertEquals(
          getFS(),
          FileSystem.get(loaded.getLocation(), new Configuration()));
      Assert.assertEquals(
          "file",
          loaded.getLocation().getScheme());
      Assert.assertEquals(
          getFS().getUri().getAuthority(),
          loaded.getLocation().getAuthority());
    }
  }
View Full Code Here
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
    }
  }

  @Test
  public void testCreateSetsURIs() throws IOException {
    DatasetDescriptor created = provider.create(NAME, testDescriptor);
    Assert.assertNotNull("Created descriptor should have a location",
        created.getLocation());
    if (distributed) {
      Assert.assertEquals(
          "hdfs",
          created.getLocation().getScheme());
      Assert.assertEquals(
          getDFS().getUri().getAuthority(),
          created.getLocation().getAuthority());
    } else {
      Assert.assertEquals(
          "file",
          created.getLocation().getScheme());
      Assert.assertEquals(
          getFS().getUri().getAuthority(),
          created.getLocation().getAuthority());
    }
  }
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
141
142
  @Test
  public void testDeleteRemovesMetadataFiles() throws IOException {
    testCreateMetadataFiles();

    DatasetDescriptor loaded = provider.load(NAME);

    Path namedDirectory = new Path(loaded.getLocation());
    Path metadataDirectory = new Path(namedDirectory, ".metadata");
    Path propertiesFile = new Path(metadataDirectory, "descriptor.properties");
    Path schemaFile = new Path(metadataDirectory, "schema.avsc");

    boolean result = provider.delete(NAME);
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
  private final StorageKey reusedKey;

  private ReaderWriterState state;

  public PartitionedDatasetWriter(FileSystemView<E> view) {
    final DatasetDescriptor descriptor = view.getDataset().getDescriptor();
    Preconditions.checkArgument(descriptor.isPartitioned(),
        "Dataset " + view.getDataset() + " is not partitioned");

    this.view = view;
    this.partitionStrategy = descriptor.getPartitionStrategy();
    this.maxWriters = Math.min(10, partitionStrategy.getCardinality());
    this.state = ReaderWriterState.NEW;
    this.reusedKey = new StorageKey(partitionStrategy);
  }
View Full Code Here
879
880
881
882
883
884
885
886
887
888
889
  }

  @Test
  @SuppressWarnings("unchecked")
  public void testNotPartitioned() throws Exception {
    final DatasetDescriptor flatDescriptor = new DatasetDescriptor
        .Builder(testDescriptor).partitionStrategy(null).build();
    final Dataset<StandardEvent> flatDataset =
        repo.create("flat", flatDescriptor);
    final RangeView<StandardEvent> notPartitioned = (RangeView<StandardEvent>)
        flatDataset;
View Full Code Here
113
114
115
116
117
118
119
120
121
122
123
    out.close();
  }

  @Override
  public DatasetReader<GenericData.Record> newReader() throws IOException {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .property("cdk.csv.lines-to-skip", "1")
        .schema(VALIDATOR_SCHEMA)
        .build();
    return new CSVFileReader<GenericData.Record>(localfs, validatorFile, desc);
  }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
    };
  }

  @Test(expected = IllegalArgumentException.class)
  public void testRejectsNonRecordSchemas() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .schema(SchemaBuilder.array().items().stringType())
        .build();
    new CSVFileReader(localfs, csvFile, desc);
  }
View Full Code Here
148
149
150
151
152
153
154
155
156
157
158
    new CSVFileReader(localfs, csvFile, desc);
  }

  @Test
  public void testStringSchema() {
    final DatasetDescriptor desc = new DatasetDescriptor.Builder()
        .schema(STRINGS)
        .build();
    final CSVFileReader<GenericData.Record> reader =
        new CSVFileReader<GenericData.Record>(localfs, csvFile, desc);
View Full Code Here
TOP

Related Classes of com.cloudera.cdk.data.DatasetDescriptor

  • com.cloudera.cdk.data.filesystem.FileSystemDatasetRepository
  • com.cloudera.cdk.data.filesystem.FileSystemMetadataProvider
  • com.cloudera.cdk.data.filesystem.PartitionedDatasetWriter
  • com.cloudera.cdk.data.filesystem.TestCSVFileReader
  • com.cloudera.cdk.data.filesystem.TestFileSystemDatasetRepository
  • com.cloudera.cdk.data.filesystem.TestFileSystemMetadataProvider
  • com.cloudera.cdk.data.filesystem.TestMultiFileDatasetReader
  • com.cloudera.cdk.data.hbase.avro.example.UserProfileDatasetExample
  • com.cloudera.cdk.data.hbase.avro.HBaseDatasetReaderTest
  • com.cloudera.cdk.data.hbase.DaoViewTest
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.