@Override
public RecordWriter<KEY, VALUE> getRecordWriter(final TaskAttemptContext context)
throws IOException, InterruptedException {
Configuration conf = context.getConfiguration();
SparkContextProvider contextProvider = new SparkContextProvider(context.getConfiguration());
BasicSparkContext sparkContext = contextProvider.get();
//TODO: Metrics collection needs to be started here once implemented
BatchWritable<KEY, VALUE> dataset = (BatchWritable<KEY, VALUE>) sparkContext.getDataSet(getOutputDataSet(conf));
// the record writer now owns the context and will close it
return new DatasetRecordWriter<KEY, VALUE>(dataset, sparkContext);