Examples of org.kitesdk.data.spi.PartitionKey

org.kitesdk.data.spi.PartitionKey

A key for retrieving partitions from a {@link org.kitesdk.data.Dataset}.

A {@code PartitionKey} is an ordered sequence of values corresponding to the{@link org.kitesdk.data.spi.FieldPartitioner}s in a {@link org.kitesdk.data.PartitionStrategy}. You can obtain a {@link PartitionKey} using{@link PartitionKey#partitionKeyForEntity(org.kitesdk.data.PartitionStrategy,Object,EntityAccessor,PartitionKey)}.

Implementations of {@link PartitionKey} are typically not thread-safe; that is, the behavior when accessing a single instance from multiple threads is undefined.
@see org.kitesdk.data.PartitionStrategy @see org.kitesdk.data.spi.FieldPartitioner @see org.kitesdk.data.Dataset

          "FileSystemDataset. Dataset: " + dataset);
    }
    FileSystemDataset<E> fsDataset = (FileSystemDataset<E>) dataset;
    LOG.debug("Getting delegate input format for dataset {} with partition directory {}",
        dataset, partitionDir);
    PartitionKey key = fsDataset.keyFromDirectory(new Path(partitionDir));
    LOG.debug("Partition key: {}", key);
    if (key != null) {
      PartitionedDataset<E> partition = fsDataset.getPartition(key, false);
      LOG.debug("Partition: {}", partition);
      return getDelegateInputFormat(partition, conf);

View Full Code Here

      if (!(target instanceof FileSystemDataset)) {
        throw new UnsupportedOperationException("Partitions only supported for " +
            "FileSystemDataset. Dataset: " + target);
      }
      FileSystemDataset fsDataset = (FileSystemDataset) target;
      PartitionKey key = fsDataset.keyFromDirectory(new Path(partitionDir));
      if (key != null) {
        working = fsDataset.getPartition(key, true);
      }
      return new DatasetRecordWriter<E>(working);
    } else {

View Full Code Here


    for (int i = 0; i < size; i += 1) {
      values[i] = key.get(i);
    }


    return new PartitionKey(values);
  }

View Full Code Here

    partitionKeyForPath(dataset, new Path(testDirectory, "username_part").toUri());
  }


  @Test
  public void testValidPartition() throws Exception {
    PartitionKey key = partitionKeyForPath(dataset,
        new Path(testDirectory, "username_part=1").toUri());
    Assert.assertEquals(new PartitionKey(1), key);
  }

View Full Code Here

    Assert.assertTrue("Partitioned directory 0 exists",
      fileSystem.exists(new Path(testDirectory, "username_hash=0")));
    Assert.assertTrue("Partitioned directory 1 exists",
      fileSystem.exists(new Path(testDirectory, "username_hash=1")));
    checkTestUsers(ds, 10);
    PartitionKey key0 = new PartitionKey(0);
    PartitionKey key1 = new PartitionKey(1);
    int total = readTestUsersInPartition(ds, key0, null)
      + readTestUsersInPartition(ds, key1, null);
    Assert.assertEquals(10, total);


    testPartitionKeysAreEqual(ds, key0, key1);

View Full Code Here

    Assert.assertEquals(partitionStrategy, ds.getDescriptor()
      .getPartitionStrategy());


    writeTestUsers(ds, 10);
    checkTestUsers(ds, 10);
    PartitionKey key0 = new PartitionKey(0);
    PartitionKey key1 = new PartitionKey(1);
    int total = readTestUsersInPartition(ds, key0, "email_hash")
      + readTestUsersInPartition(ds, key0, "email_hash");
    Assert.assertEquals(10, total);


    total = 0;
    for (int i1 = 0; i1 < 2; i1++) {
      for (int i2 = 0; i2 < 3; i2++) {
        String part = "username_hash=" + i1 + "/email_hash=" + i2;
        Assert.assertTrue("Partitioned directory " + part + " exists",
          fileSystem.exists(new Path(testDirectory, part)));
        total += readTestUsersInPartition(ds,
          new PartitionKey(i1, i2), null);
      }
    }
    Assert.assertEquals(10, total);


    testPartitionKeysAreEqual(ds, key0, key1);

View Full Code Here

            .build())
        .type(Record.class)
        .build();


    Assert
      .assertNull(ds.getPartition(new PartitionKey(1), false));
  }

View Full Code Here

            .partitionStrategy(partitionStrategy)
            .build())
        .type(Record.class)
        .build();


    PartitionKey key = new PartitionKey(1);
    FileSystemDataset<Record> userPartition = (FileSystemDataset<Record>) ds.getPartition(key, true);
    Assert.assertEquals(key, userPartition.getPartitionKey());


    writeTestUsers(userPartition, 1);
    Assert.assertTrue("Partitioned directory exists",

View Full Code Here

    Assert.assertTrue(
      fileSystem.isDirectory(new Path(testDirectory, "username_hash=0")));
    Assert.assertTrue(
      fileSystem.isDirectory(new Path(testDirectory, "username_hash=1")));


    ds.dropPartition(new PartitionKey(0));
    Assert.assertFalse(
      fileSystem.isDirectory(new Path(testDirectory, "username_hash=0")));


    ds.dropPartition(new PartitionKey(1));
    Assert.assertFalse(
      fileSystem.isDirectory(new Path(testDirectory, "username_hash=1")));


    DatasetException caught = null;


    try {
      ds.dropPartition(new PartitionKey(0));
    } catch (DatasetException e) {
      caught = e;
    }


    Assert.assertNotNull(caught);

View Full Code Here

    Assert.assertEquals(6, dirPaths.size());


    Assert.assertTrue("dirIterator should yield absolute paths.", dirPaths.get(0).isAbsolute());


    FileSystemDataset<Record> partition = (FileSystemDataset<Record>)
        ds.getPartition(new PartitionKey(1, 2), false);
    List<Path> leafPaths = Lists.newArrayList(partition.dirIterator());
    Assert.assertEquals(1, leafPaths.size());
    final Path leafPath = leafPaths.get(0);
    Assert.assertTrue("dirIterator should yield absolute paths.", leafPath.isAbsolute());


    Assert.assertEquals(new PartitionKey(1, 2), ds.keyFromDirectory(leafPath));
    Assert.assertEquals(new PartitionKey(1), ds.keyFromDirectory(leafPath.getParent()));
    Assert.assertEquals(new PartitionKey(), ds.keyFromDirectory(leafPath.getParent().getParent()));


    TestHelpers.assertThrows("Path with too many components",
        IllegalStateException.class, new Runnable() {
      @Override
      public void run() {

View Full Code Here

0 1 2 3 4

TOP

Related Classes of org.kitesdk.data.spi.PartitionKey

org.kitesdk.data.crunch.TestCrunchDatasets

org.kitesdk.data.hbase.avro.AvroDaoTest

org.kitesdk.data.hbase.avro.AvroKeySerDe

org.kitesdk.data.hbase.avro.CompositeDaoTest

org.kitesdk.data.hbase.avro.example.UserProfileExample

org.kitesdk.data.hbase.avro.ManagedDaoTest

org.kitesdk.data.hbase.DaoDataset

org.kitesdk.data.hbase.DaoView

org.kitesdk.data.hbase.filters.ScannerFilterTest

org.kitesdk.data.hbase.impl.BaseDao

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.