Examples of com.cloudera.cdk.data.PartitionKey

com.cloudera.cdk.data.PartitionKey

A key for retrieving partitions from a {@link Dataset}.

A {@code PartitionKey} is a ordered sequence of values corresponding to the{@link FieldPartitioner}s in a {@link PartitionStrategy}. A {@link PartitionKey} may be obtained using{@link PartitionStrategy#partitionKey(Object)} or{@link PartitionStrategy#partitionKeyForEntity(Object)}.

Implementations of {@link PartitionKey} are typically not thread-safe; that is,the behavior when accessing a single instance from multiple threads is undefined.
@see PartitionStrategy @see FieldPartitioner @see Dataset

      throw new DatasetException("Unable to list partition directory for directory " + directory, e);
    }


    for (FileStatus stat : fileStatuses) {
      Path p = fileSystem.makeQualified(stat.getPath());
      PartitionKey key = fromDirectoryName(p);
      PartitionStrategy subPartitionStrategy = Accessor.getDefault()
          .getSubpartitionStrategy(partitionStrategy, 1);
      Builder builder = new FileSystemDataset.Builder()
          .name(name)
          .fileSystem(fileSystem)

View Full Code Here

    partitionKeyForPath(dataset, new Path(testDirectory, "username_part").toUri());
  }


  @Test
  public void testValidPartition() throws Exception {
    PartitionKey key = partitionKeyForPath(dataset,
        new Path(testDirectory, "username_part=1").toUri());
    Assert.assertEquals(Accessor.getDefault().newPartitionKey(1), key);
  }

View Full Code Here

   *          The last name of users to scan.
   */
  public void printUserProfileActionsForLastName(String lastName) {
    // Create a partial key that will allow us to start the scanner from the
    // first user record that has last name equal to the one provided.
    PartitionKey startKey = userProfileActionsDao.getPartitionStrategy()
        .partitionKey("lastName");


    // Get the scanner with the start key. Null for stopKey in the getScanner
    // method indicates that the scanner will scan to the end of the table. Our
    // loop will break out when it encounters a record without the last name.

View Full Code Here

    // Get the timestamp we'll use to set the value of the profile_updated
    // action.
    long ts = System.currentTimeMillis();


    // Construct the key we'll use to fetch the user.
    PartitionKey key = userProfileActionsDao.getPartitionStrategy()
        .partitionKey(lastName, firstName);


    // Get the profile and actions entity from the composite dao.
    UserProfileActionsModel profileActionsModel = userProfileActionsDao
        .get(key);

View Full Code Here

  }


  @Override
  public E mapToEntity(Result result) {
    boolean allNull = true;
    PartitionKey partitionKey = keySerDe.deserialize(result.getRow());
    EntityComposer.Builder<E> builder = getEntityComposer().getBuilder();
    for (FieldMapping fieldMapping : entitySchema.getFieldMappings()) {
      Object fieldValue;
      if (fieldMapping.getMappingType() == MappingType.KEY) {
        fieldValue = partitionKey.get(Integer.parseInt(fieldMapping
            .getMappingValue()));
      } else {
        fieldValue = entitySerDe.deserialize(fieldMapping, result);
      }
      if (fieldValue != null) {

View Full Code Here

  @Override
  public PutAction mapFromEntity(E entity) {
    List<PutAction> putActionList = new ArrayList<PutAction>();
    List<Object> keyParts = entitySerDe.getEntityComposer()
        .getPartitionKeyParts(entity);
    PartitionKey partitionKey = keySchema.getPartitionStrategy().partitionKey(
        keyParts.toArray());
    byte[] keyBytes = keySerDe.serialize(partitionKey);
    for (FieldMapping fieldMapping : entitySchema.getFieldMappings()) {
      if (fieldMapping.getMappingType() == MappingType.KEY) {
        continue;

View Full Code Here


  @Override
  public boolean delete(E entity) {
    VersionCheckAction checkAction = entityMapper.mapFromEntity(entity)
        .getVersionCheckAction();
    PartitionKey key = getPartitionStrategy().partitionKeyForEntity(entity);
    return clientTemplate.delete(key, entityMapper.getRequiredColumns(),
        checkAction, entityMapper.getKeySerDe());
  }

View Full Code Here

    Assert.assertTrue("Partitioned directory 0 exists",
      fileSystem.exists(new Path(testDirectory, "username=0")));
    Assert.assertTrue("Partitioned directory 1 exists",
      fileSystem.exists(new Path(testDirectory, "username=1")));
    checkTestUsers(ds, 10);
    PartitionKey key0 = partitionStrategy.partitionKey(0);
    PartitionKey key1 = partitionStrategy.partitionKey(1);
    int total = readTestUsersInPartition(ds, key0, null)
      + readTestUsersInPartition(ds, key1, null);
    Assert.assertEquals(10, total);


    testPartitionKeysAreEqual(ds, key0, key1);

View Full Code Here

    Assert.assertEquals(partitionStrategy, ds.getDescriptor()
      .getPartitionStrategy());


    writeTestUsers(ds, 10);
    checkTestUsers(ds, 10);
    PartitionKey key0 = Accessor.getDefault().newPartitionKey(0);
    PartitionKey key1 = Accessor.getDefault().newPartitionKey(1);
    int total = readTestUsersInPartition(ds, key0, "email")
      + readTestUsersInPartition(ds, key0, "email");
    Assert.assertEquals(10, total);


    total = 0;

View Full Code Here

            .location(testDirectory)
            .partitionStrategy(partitionStrategy)
            .build())
        .build();


    PartitionKey key = Accessor.getDefault().newPartitionKey(1);
    FileSystemDataset<Record> userPartition = (FileSystemDataset<Record>) ds.getPartition(key, true);
    Assert.assertEquals(key, userPartition.getPartitionKey());


    writeTestUsers(userPartition, 1);
    Assert.assertTrue("Partitioned directory exists",

View Full Code Here

0 1 2

TOP

Related Classes of com.cloudera.cdk.data.PartitionKey

com.cloudera.cdk.data.crunch.TestCrunchDatasets

com.cloudera.cdk.data.filesystem.FileSystemDataset

com.cloudera.cdk.data.filesystem.TestFileSystemDataset

com.cloudera.cdk.data.filesystem.TestFileSystemDatasetPartitionKeyForPath

com.cloudera.cdk.data.hbase.avro.AvroDaoTest

com.cloudera.cdk.data.hbase.avro.CompositeDaoTest

com.cloudera.cdk.data.hbase.avro.example.UserProfileExample

com.cloudera.cdk.data.hbase.avro.ManagedDaoTest

com.cloudera.cdk.data.hbase.filters.ScannerFilterTest

com.cloudera.cdk.data.hbase.impl.BaseDao

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.