Package co.cask.cdap.internal.app.runtime.batch.dataset

Examples of co.cask.cdap.internal.app.runtime.batch.dataset.DataSetInputSplit

43
44
45
46
47
48
49
50
  @Override
  public void initialize(final InputSplit split, final TaskAttemptContext context) throws IOException,
    InterruptedException {
    LoggingContextAccessor.setLoggingContext(this.context.getLoggingContext());
    DataSetInputSplit inputSplit = (DataSetInputSplit) split;
    splitReader.initialize(inputSplit.getSplit());
  }
View Full Code Here
53
54
55
56
57
58
59
60
61
  public List<InputSplit> getSplits(final JobContext context) throws IOException, InterruptedException {
    SparkContextConfig sparkContextConfig = new SparkContextConfig(context.getConfiguration());
    List<Split> splits = sparkContextConfig.getInputSelection();
    List<InputSplit> list = new ArrayList<InputSplit>();
    for (Split split : splits) {
      list.add(new DataSetInputSplit(split));
    }
    return list;
  }
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
  @Override
  public RecordReader<KEY, VALUE> createRecordReader(final InputSplit split,
                                                     final TaskAttemptContext context)
    throws IOException, InterruptedException {

    DataSetInputSplit inputSplit = (DataSetInputSplit) split;

    Configuration conf = context.getConfiguration();
    SparkContextProvider contextProvider = new SparkContextProvider(context.getConfiguration());
    BasicSparkContext sparkContext = contextProvider.get();
    //TODO: Metrics should be started here when implemented
    String dataSetName = getInputName(conf);
    BatchReadable<KEY, VALUE> inputDataset = (BatchReadable<KEY, VALUE>) sparkContext.getDataSet(dataSetName);
    SplitReader<KEY, VALUE> splitReader = inputDataset.createSplitReader(inputSplit.getSplit());

    // the record reader now owns the context and will close it
    return new DatasetRecordReader<KEY, VALUE>(splitReader, sparkContext, dataSetName);
  }
View Full Code Here
TOP

Related Classes of co.cask.cdap.internal.app.runtime.batch.dataset.DataSetInputSplit

  • co.cask.cdap.internal.app.runtime.spark.dataset.DatasetRecordReader
  • co.cask.cdap.internal.app.runtime.spark.dataset.SparkDatasetInputFormat
  • com.google.gson.Gson
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.