Package org.apache.flink.core.fs

Examples of org.apache.flink.core.fs.FileInputSplit

71
72
73
74
75
76
77
78
79
80
81
82
83
    final long splitLength = inFile.length() / noSplits;
    long pos = 0;

    for (int i = 0; i < noSplits - 1; i++) {
      tmp[i] = new FileInputSplit(i, new Path(path), pos, splitLength, hosts);
      pos += splitLength;
    }

    tmp[noSplits - 1] = new FileInputSplit(noSplits - 1, new Path(path), pos, inFile.length() - pos, hosts);

    this.inputSplits = tmp;
  }
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
  @SuppressWarnings("unchecked")
  @Test
  public void readWithEmptyFieldInstanceParameters() {
    try {
      final String fileContent = "abc|def|ghijk\nabc||hhg\n|||";
      final FileInputSplit split = createTempFile(fileContent); 
   
      final Configuration parameters = new Configuration();

      format.setFieldDelimiter('|');
      format.setFieldTypes(StringValue.class, StringValue.class, StringValue.class);
View Full Code Here
129
130
131
132
133
134
135
136
137
138
139
 
  @Test
  public void readWithEmptyFieldConfigParameters() {
    try {
      final String fileContent = "abc|def|ghijk\nabc||hhg\n|||";
      final FileInputSplit split = createTempFile(fileContent); 
   
      final Configuration parameters = new Configuration();
      new CsvInputFormat.ConfigBuilder(null, parameters)
        .field(StringValue.class, 0).field(StringValue.class, 1).field(StringValue.class, 2);
     
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
 
  @Test
  public void testReadAll() throws IOException {
    try {
      final String fileContent = "111|222|333|444|555\n666|777|888|999|000|";
      final FileInputSplit split = createTempFile(fileContent); 
   
      final Configuration parameters = new Configuration();
     
      new CsvInputFormat.ConfigBuilder(null, parameters)
        .fieldDelimiter('|')
View Full Code Here
206
207
208
209
210
211
212
213
214
215
216
 
  @Test
  public void testReadFirstN() throws IOException {
    try {
      final String fileContent = "111|222|333|444|555|\n666|777|888|999|000|";
      final FileInputSplit split = createTempFile(fileContent); 
   
      final Configuration parameters = new Configuration();
     
      new CsvInputFormat.ConfigBuilder(null, parameters)
      .fieldDelimiter('|')
View Full Code Here
254
255
256
257
258
259
260
261
262
263
264
 
  @Test
  public void testReadSparse() throws IOException {
    try {
      final String fileContent = "111|222|333|444|555|666|777|888|999|000|\n000|999|888|777|666|555|444|333|222|111|";
      final FileInputSplit split = createTempFile(fileContent); 
   
      final Configuration parameters = new Configuration();
     
      new CsvInputFormat.ConfigBuilder(null, parameters)
        .fieldDelimiter('|')
View Full Code Here
289
290
291
292
293
294
295
296
297
298
299
 
  @Test
  public void testReadSparseShufflePosition() throws IOException {
    try {
      final String fileContent = "111|222|333|444|555|666|777|888|999|000|\n000|999|888|777|666|555|444|333|222|111|";
      final FileInputSplit split = createTempFile(fileContent); 
   
      final Configuration parameters = new Configuration();
     
      new CsvInputFormat.ConfigBuilder(null, parameters)
        .fieldDelimiter('|')
View Full Code Here
328
329
330
331
332
333
334
   
    DataOutputStream dos = new DataOutputStream(new FileOutputStream(tempFile));
    dos.writeBytes(content);
    dos.close();
     
    return new FileInputSplit(0, new Path(this.tempFile.toURI().toString()), 0, this.tempFile.length(), new String[] {"localhost"});
  }
View Full Code Here
64
65
66
67
68
69
70
71
72
73
74
  }

  @Test
  public void testOpen() throws IOException {
    final int[] fileContent = {1,2,3,4,5,6,7,8};
    final FileInputSplit split = createTempFile(fileContent); 
 
    final Configuration parameters = new Configuration();
    parameters.setInteger(FixedLengthInputFormat.RECORDLENGTH_PARAMETER_KEY, 8);
   
    format.configure(parameters);
View Full Code Here
94
95
96
97
98
99
100
101
102
103
104
  }
 
  @Test
  public void testRead() throws IOException {
    final int[] fileContent = {1,2,3,4,5,6,7,8};
    final FileInputSplit split = createTempFile(fileContent);
   
    final Configuration parameters = new Configuration();
   
    parameters.setInteger(FixedLengthInputFormat.RECORDLENGTH_PARAMETER_KEY, 8);
   
View Full Code Here
TOP

Related Classes of org.apache.flink.core.fs.FileInputSplit

  • org.apache.flink.api.common.io.BinaryInputFormat
  • org.apache.flink.api.common.io.DelimitedInputFormat
  • org.apache.flink.api.common.io.DelimitedInputFormatTest
  • org.apache.flink.api.common.io.FileInputFormat
  • org.apache.flink.api.common.io.FormatUtil
  • org.apache.flink.api.common.io.GenericCsvInputFormatTest
  • org.apache.flink.api.java.io.CsvInputFormatTest
  • org.apache.flink.api.java.record.io.CsvInputFormatTest
  • org.apache.flink.api.java.record.io.FixedLenghtInputFormatTest
  • org.apache.flink.runtime.operators.testutils.MockInputSplitProvider
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.