Package org.kitesdk.data

Examples of org.kitesdk.data.PartitionStrategy

23
24
25
26
27
28
29
30
31
32
33
public class DatasetTemplatePartitioningAvroTests extends AbstractDatasetTemplatePartitioningTests {

  @Autowired
  public void setDatasetOperations(DatasetOperations datasetOperations) {
    PartitionStrategy partitionStrategy =
        new PartitionStrategy.Builder().year("birthDate").month("birthDate").build();
    ((DatasetTemplate) datasetOperations).setDatasetDefinitions(Arrays.asList(
            new DatasetDefinition[] {new DatasetDefinition(SimplePojo.class, "avro", partitionStrategy)}));
    this.datasetOperations = datasetOperations;
  }
View Full Code Here
33
34
35
36
37
38
39
40
41
42
43
@MiniHadoopCluster
public class DatasetTemplatePartitioningParquetTests extends AbstractDatasetTemplatePartitioningTests {

  @Autowired
  public void setDatasetOperations(DatasetOperations datasetOperations) {
    PartitionStrategy partitionStrategy =
        new PartitionStrategy.Builder().year("birthDate").month("birthDate").build();
    ((DatasetTemplate) datasetOperations).setDatasetDefinitions(Arrays.asList(
        new DatasetDefinition[]{new DatasetDefinition(SimplePojo.class, "parquet", partitionStrategy)}));
    this.datasetOperations = datasetOperations;
  }
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
  }

  @Test
  public void runWithoutSettingWriterCacheSize() throws IOException {
    PartitionStrategy partitionStrategy =
        new PartitionStrategy.Builder().year("birthDate").month("birthDate").build();
    DatasetDefinition datasetDefinition = new DatasetDefinition(SimplePojo.class, "avro", partitionStrategy);
    final DatasetTemplate datasetTemplete = new DatasetTemplate(datasetRepositoryFactory, datasetDefinition);

    // clean up
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
    assertTrue(fsShell.ls(true, path + "/test/" + datasetTemplete.getDatasetName(SimplePojo.class) + "/year=1997/month=12/*.avro").size() >= 2);
  }

  @Test
  public void runSettingWriterCacheSize() throws IOException {
    PartitionStrategy partitionStrategy =
        new PartitionStrategy.Builder().year("birthDate").month("birthDate").build();
    DatasetDefinition datasetDefinition = new DatasetDefinition(SimplePojo.class, "avro", partitionStrategy);
    datasetDefinition.setWriterCacheSize(20);
    final DatasetTemplate datasetTemplete = new DatasetTemplate(datasetRepositoryFactory, datasetDefinition);

View Full Code Here
75
76
77
78
79
80
81
82
83
  @Test
  public void testBasic() {
    DatasetDescriptor desc = provider.create("default", tableName + ".TestEntity",
        new DatasetDescriptor.Builder().schemaLiteral(testEntity).build());
    ColumnMapping columnMapping = desc.getColumnMapping();
    PartitionStrategy partStrat = desc.getPartitionStrategy();
    assertEquals(9, columnMapping.getFieldMappings().size());
    assertEquals(2, partStrat.getFieldPartitioners().size());
  }
View Full Code Here
46
47
48
49
50
51
52
53
54
55
56
57
  public void testBasic() throws Exception {
    command.partitions = Lists.newArrayList(
        "username:hash[16]", "username:copy");
    command.run();

    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .hash("username", 16)
        .identity("username")
        .build();
    verify(console).info(strategy.toString(true));
    verifyNoMoreInteractions(console);
  }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
        "created_at:year", "created_at:month", "created_at:day",
        "created_at:hour", "created_at:minute"
    );
    command.run();

    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .year("created_at")
        .month("created_at")
        .day("created_at")
        .hour("created_at")
        .minute("created_at")
        .build();
    verify(console).info(strategy.toString(true));
    verifyNoMoreInteractions(console);
  }
View Full Code Here
28
29
30
31
32
33
34
35
36
37
38
import org.kitesdk.data.ValidationException;

public class TestPartitionStrategyParser {

  public static void checkParser(PartitionStrategy expected, String json) {
    PartitionStrategy parsed = PartitionStrategyParser.parse(json);
    Assert.assertEquals(expected, parsed);

    parsed = PartitionStrategyParser.parse(expected.toString());
    Assert.assertEquals("Should reparse properly", expected, parsed);
  }
View Full Code Here
370
371
372
373
374
375
376
377
378
379
380
    );
  }

  @Test
  public void testAddEmbeddedPartitionStrategy() {
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .hash("username", 16)
        .identity("username", "u")
        .build();
    Schema original = new Schema.Parser().parse("{" +
        "  \"type\": \"record\"," +
View Full Code Here
391
392
393
394
395
396
397
398
399
400
401
    Assert.assertEquals(strategy, PartitionStrategyParser.parseFromSchema(embedded));
  }

  @Test
  public void testReplaceEmbeddedPartitionStrategy() {
    PartitionStrategy strategy = new PartitionStrategy.Builder()
        .hash("username", 16)
        .identity("username", "u")
        .build();
    Schema original = new Schema.Parser().parse("{" +
        "  \"type\": \"record\"," +
View Full Code Here
TOP

Related Classes of org.kitesdk.data.PartitionStrategy

  • org.kitesdk.cli.commands.TestCreatePartitionStrategyCommand
  • org.kitesdk.data.crunch.CrunchDatasets$GetStorageKey
  • org.kitesdk.data.crunch.TestCrunchDatasets
  • org.kitesdk.data.hbase.avro.AvroKeyEntitySchemaParserTest
  • org.kitesdk.data.hbase.avro.AvroUtils
  • org.kitesdk.data.hbase.HBaseMetadataProviderTest
  • org.kitesdk.data.spi.Constraints
  • org.kitesdk.data.spi.EntityAccessor
  • org.kitesdk.data.spi.filesystem.FileSystemDataset
  • org.kitesdk.data.spi.filesystem.FileSystemDatasetRepository
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.