Package org.kitesdk.data

Examples of org.kitesdk.data.Format

97
98
99
100
101
102
103
104
105
106
107
      .build();

  static DatasetDescriptor descriptorForTable(Configuration conf, Table table) {
    final DatasetDescriptor.Builder builder = new DatasetDescriptor.Builder();

    Format format;
    final String serializationLib = table.getSd().getSerdeInfo().getSerializationLib();
    if (SERDE_TO_FORMAT.containsKey(serializationLib)) {
      format = SERDE_TO_FORMAT.get(serializationLib);
      builder.format(format);
    } else {
View Full Code Here
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
    }

    addPropertiesForDescriptor(table, descriptor);

    // translate from Format to SerDe
    final Format format = descriptor.getFormat();
    if (FORMAT_TO_SERDE.containsKey(format)) {
      table.getSd().getSerdeInfo().setSerializationLib(FORMAT_TO_SERDE.get(format));
      table.getSd().setInputFormat(FORMAT_TO_INPUT_FORMAT.get(format));
      table.getSd().setOutputFormat(FORMAT_TO_OUTPUT_FORMAT.get(format));
    } else {
      throw new UnknownFormatException(
          "No known serde for format:" + format.getName());
    }

    // copy schema info
    boolean useLiteral;
    final URL schemaURL = descriptor.getSchemaUrl();
View Full Code Here
82
83
84
85
86
87
88
89
90
91
    this.formatBundle = inputBundle(temp);

    Dataset<E> dataset = view.getDataset();

    // Disable CombineFileInputFormat in Crunch unless we're dealing with Avro or Parquet files
    Format format = dataset.getDescriptor().getFormat();
    boolean isAvroOrParquetFile = (dataset instanceof FileSystemDataset)
        && (Formats.AVRO.equals(format) || Formats.PARQUET.equals(format));
    formatBundle.set(RuntimeParameters.DISABLE_COMBINE_FILE, Boolean.toString(!isAvroOrParquetFile));
  }
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
80
81
  @Override
  public void initialize() {
    Preconditions.checkState(state.equals(ReaderWriterState.NEW),
      "A reader may not be opened more than once - current state:%s", state);

    final Format format = descriptor.getFormat();
    if (!(Formats.AVRO.equals(format) || Formats.PARQUET.equals(format)
        || Formats.CSV.equals(format))) {
      throw new UnknownFormatException("Cannot open format:" + format.getName());
    }

    this.state = ReaderWriterState.OPEN;
  }
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
  public FileSystemViewKeyInputFormat(FileSystemDataset<E> dataset,
      Configuration conf) {
    this.dataset = dataset;
    LOG.debug("Dataset: {}", dataset);

    Format format = dataset.getDescriptor().getFormat();
    if (Formats.AVRO.equals(format)) {
      setModel.invoke(conf,
          DataModelUtil.getDataModelForType(dataset.getType()).getClass());
    }
  }
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
  @Override
  @SuppressWarnings({"unchecked", "deprecation"})
  public List<InputSplit> getSplits(JobContext jobContext) throws IOException {
    Configuration conf = Hadoop.JobContext.getConfiguration.invoke(jobContext);
    Job job = new Job(conf);
    Format format = dataset.getDescriptor().getFormat();

    if (setInputPaths(jobContext, job)) {
      if (Formats.AVRO.equals(format)) {
        AvroJob.setInputKeySchema(job, dataset.getDescriptor().getSchema());
        AvroKeyInputFormat<E> delegate = new AvroKeyInputFormat<E>();
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
  }

  @SuppressWarnings("unchecked")
  private RecordReader<E, Void> createUnfilteredRecordReader(InputSplit inputSplit,
      TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
    Format format = dataset.getDescriptor().getFormat();
    if (Formats.AVRO.equals(format)) {
      AvroKeyInputFormat<E> delegate = new AvroKeyInputFormat<E>();
      return new KeyReaderWrapper(
          delegate.createRecordReader(inputSplit, taskAttemptContext));
    } else if (Formats.PARQUET.equals(format)) {
View Full Code Here
216
217
218
219
220
221
222
223
224
225
226
  }

  @VisibleForTesting
  @SuppressWarnings("unchecked")
  <E> FileAppender<E> newAppender(Path temp) {
    Format format = descriptor.getFormat();
    if (Formats.PARQUET.equals(format)) {
      // by default, Parquet is not durable
      if (DescriptorUtil.isDisabled(
          FileSystemProperties.NON_DURABLE_PARQUET_PROP, descriptor)) {
        return (FileAppender<E>) new DurableParquetAppender(
View Full Code Here
TOP

Related Classes of org.kitesdk.data.Format

  • org.kitesdk.data.crunch.DatasetSourceTarget
  • org.kitesdk.data.spi.filesystem.FileSystemViewKeyInputFormat
  • org.kitesdk.data.spi.filesystem.FileSystemWriter
  • org.kitesdk.data.spi.filesystem.MultiFileDatasetReader
  • org.kitesdk.data.spi.hive.HiveUtils
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.