Examples of org.kitesdk.data.DatasetDescriptor

org.kitesdk.data.DatasetDescriptor

The structural definition of a {@link Dataset}.

Each {@code Dataset} has an associated {@link Schema} and optional{@link PartitionStrategy} defined at the time of creation. You use instancesof this class to hold this information. You are strongly encouraged to use the inner {@link Builder} to create new instances.

        public Dataset<GenericRecord> run() {
          return Datasets.load(uri);
        }
      });


    DatasetDescriptor descriptor = view.getDataset().getDescriptor();
    String formatName = descriptor.getFormat().getName();
    Preconditions.checkArgument(allowedFormats().contains(formatName),
      "Unsupported format: " + formatName);


    Schema newSchema = descriptor.getSchema();
    if (targetSchema == null || !newSchema.equals(targetSchema)) {
      this.targetSchema = descriptor.getSchema();
      // target dataset schema has changed, invalidate all readers based on it
      readers.invalidateAll();
    }


    this.reuseDatum = !("parquet".equals(formatName));

View Full Code Here

    // run the sink
    sink.start();
    sink.process();


    // update the dataset's schema
    DatasetDescriptor updated = new DatasetDescriptor
      .Builder(Datasets.load(FILE_DATASET_URI).getDataset().getDescriptor())
      .schema(UPDATED_SCHEMA)
      .build();
    Datasets.update(FILE_DATASET_URI, updated);

View Full Code Here

    readWithCallback(targetClass, callback, null);
  }


  @Override
  public <T> Collection<T> read(Class<T> targetClass) {
    DatasetDescriptor descriptor = getDatasetDescriptor(targetClass);
    if (descriptor == null) {
      throw new StoreException("Unable to locate dataset for target class " + targetClass.getName());
    }
    if (Formats.PARQUET.equals(descriptor.getFormat())) {
      return readGenericRecords(targetClass, null);
    } else {
      return readPojo(targetClass, null);
    }
  }

View Full Code Here

    readWithCallback(targetClass, callback, viewCallback);
  }


  @Override
  public <T> Collection<T> read(Class<T> targetClass, ViewCallback viewCallback) {
    DatasetDescriptor descriptor = getDatasetDescriptor(targetClass);
    if (descriptor == null) {
      throw new StoreException("Unable to locate dataset for target class " + targetClass.getName());
    }
    if (Formats.PARQUET.equals(descriptor.getFormat())) {
      return readGenericRecords(targetClass, viewCallback);
    } else {
      return readPojo(targetClass, viewCallback);
    }
  }

View Full Code Here

          fields.add(new Schema.Field(f.name(), f.schema(), f.doc(), f.defaultValue()));
        }
        genericSchema.setFields(fields);
        schema = genericSchema;
      }
      DatasetDescriptor descriptor;
      if (datasetDefinition.getPartitionStrategy() == null) {
        descriptor = new DatasetDescriptor.Builder()
            .schema(schema)
            .format(datasetDefinition.getFormat())
            .build();

View Full Code Here

  }


  @Test
  public void testGetDatasetDescriptor() {
    datasetOperations.write(records);
    DatasetDescriptor desc1 = datasetOperations.getDatasetDescriptor(TestPojo.class);
    assertNotNull(desc1);
    DatasetDescriptor desc2 = datasetOperations.getDatasetDescriptor(RandomPojo.class);
    assertNull(desc2);
  }

View Full Code Here

    conf.set(CONF_AVRO_SCHEMA, schema.toString());
    DatasetKeyOutputFormat.configure(conf).writeTo(dataset);
  }


  private static Dataset createDataset(Schema schema, String uri) {
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schema(schema)
        .format(Formats.PARQUET)
        .build();
    return Datasets.create(uri, descriptor, GenericRecord.class);
  }

View Full Code Here

          public Dataset<GenericRecord> run() {
            return Datasets.load(uri);
          }
        });


    DatasetDescriptor descriptor = view.getDataset().getDescriptor();
    String formatName = descriptor.getFormat().getName();
    Preconditions.checkArgument(allowedFormats().contains(formatName),
        "Unsupported format: " + formatName);


    Schema newSchema = descriptor.getSchema();
    if (targetSchema == null || !newSchema.equals(targetSchema)) {
      this.targetSchema = descriptor.getSchema();
      // target dataset schema has changed, invalidate all readers based on it
      readers.invalidateAll();
    }


    this.reuseDatum = !("parquet".equals(formatName));

View Full Code Here


  @Before
  public void setup() throws Exception {
    repo = new HBaseDatasetRepository.Builder().configuration(
        HBaseTestUtils.getConf()).build();
    DatasetDescriptor descriptor = new DatasetDescriptor.Builder()
        .schemaLiteral(testEntity).build();
    ds = (DaoDataset<TestEntity>) repo.create(
        "default", tableName, descriptor, TestEntity.class);
  }

View Full Code Here

    Preconditions.checkArgument(DEFAULT_NAMESPACE.equals(namespace),
        "Non-default namespaces are not supported");
    Preconditions.checkNotNull(name, "Dataset name cannot be null");
    Preconditions.checkNotNull(descriptor, "Descriptor cannot be null");


    DatasetDescriptor newDescriptor = metadataProvider.create(namespace, name, descriptor);
    return newDataset(namespace, name, newDescriptor, type);
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.kitesdk.data.DatasetDescriptor

org.apache.avro.Schema

org.apache.flume.sink.kite.DatasetSink

org.apache.flume.sink.kite.TestDatasetSink

org.apache.hadoop.fs.FileSystem

org.apache.hadoop.fs.Path

org.apache.sqoop.mapreduce.ParquetJob

org.kitesdk.cli.commands.CreateColumnMappingCommand

org.kitesdk.cli.commands.CreateDatasetCommand

org.kitesdk.cli.commands.CreatePartitionStrategyCommand

org.kitesdk.cli.commands.CSVImportCommand

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.