Package eu.stratosphere.core.fs

Examples of eu.stratosphere.core.fs.FileInputSplit

55
56
57
58
59
60
61
62
63
64
65
 
  @Test
  public void readStringFields() {
    try {
      final String fileContent = "abc|def|ghijk\nabc||hhg\n|||";
      final FileInputSplit split = createTempFile(fileContent);
     
      final CsvInputFormat<Tuple3<String, String, String>> format = new CsvInputFormat<Tuple3<String, String, String>>(PATH, "\n", '|', String.class, String.class, String.class);
   
      final Configuration parameters = new Configuration();
      format.configure(parameters);
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
 
  @Test
  public void readStringFieldsWithTrailingDelimiters() {
    try {
      final String fileContent = "abc|def|ghijk\nabc||hhg\n|||\n";
      final FileInputSplit split = createTempFile(fileContent);
   
      final CsvInputFormat<Tuple3<String, String, String>> format = new CsvInputFormat<Tuple3<String, String, String>>(PATH);
     
      format.setFieldDelimiter('|');
      format.setFieldTypes(String.class, String.class, String.class);
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
 
  @Test
  public void testIntegerFieldsl() throws IOException {
    try {
      final String fileContent = "111|222|333|444|555\n666|777|888|999|000|\n";
      final FileInputSplit split = createTempFile(fileContent); 
   
      final CsvInputFormat<Tuple5<Integer, Integer, Integer, Integer, Integer>> format = new CsvInputFormat<Tuple5<Integer, Integer, Integer, Integer, Integer>>(PATH);
     
      format.setFieldDelimiter('|');
      format.setFieldTypes(Integer.class, Integer.class, Integer.class, Integer.class, Integer.class);
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
 
  @Test
  public void testReadFirstN() throws IOException {
    try {
      final String fileContent = "111|222|333|444|555|\n666|777|888|999|000|\n";
      final FileInputSplit split = createTempFile(fileContent); 
   
      final CsvInputFormat<Tuple2<Integer, Integer>> format = new CsvInputFormat<Tuple2<Integer, Integer>>(PATH);
     
      format.setFieldDelimiter('|');
      format.setFieldTypes(Integer.class, Integer.class);
View Full Code Here
217
218
219
220
221
222
223
224
225
226
227
 
  @Test
  public void testReadSparseWithNullFieldsForTypes() throws IOException {
    try {
      final String fileContent = "111|222|333|444|555|666|777|888|999|000|\n000|999|888|777|666|555|444|333|222|111|";
      final FileInputSplit split = createTempFile(fileContent); 
     
      final CsvInputFormat<Tuple3<Integer, Integer, Integer>> format = new CsvInputFormat<Tuple3<Integer, Integer, Integer>>(PATH);
     
      format.setFieldDelimiter('|');
      format.setFieldTypes(Integer.class, null, null, Integer.class, null, null, null, Integer.class);
View Full Code Here
254
255
256
257
258
259
260
261
262
263
264
 
  @Test
  public void testReadSparseWithPositionSetter() throws IOException {
    try {
      final String fileContent = "111|222|333|444|555|666|777|888|999|000|\n000|999|888|777|666|555|444|333|222|111|";
      final FileInputSplit split = createTempFile(fileContent); 
     
      final CsvInputFormat<Tuple3<Integer, Integer, Integer>> format = new CsvInputFormat<Tuple3<Integer, Integer, Integer>>(PATH);
     
      format.setFieldDelimiter('|');
     
View Full Code Here
293
294
295
296
297
298
299
300
301
302
303
 
  @Test
  public void testReadSparseWithMask() throws IOException {
    try {
      final String fileContent = "111|222|333|444|555|666|777|888|999|000|\n000|999|888|777|666|555|444|333|222|111|";
      final FileInputSplit split = createTempFile(fileContent); 
     
      final CsvInputFormat<Tuple3<Integer, Integer, Integer>> format = new CsvInputFormat<Tuple3<Integer, Integer, Integer>>(PATH);
     
      format.setFieldDelimiter('|');
View Full Code Here
356
357
358
359
360
361
362
   
    FileWriter wrt = new FileWriter(tempFile);
    wrt.write(content);
    wrt.close();
     
    return new FileInputSplit(0, new Path(tempFile.toURI().toString()), 0, tempFile.length(), new String[] {"localhost"});
  }
View Full Code Here
96
97
98
99
100
101
102
103
104
105
106
  @SuppressWarnings("unchecked")
  @Test
  public void readWithEmptyFieldInstanceParameters() {
    try {
      final String fileContent = "abc|def|ghijk\nabc||hhg\n|||";
      final FileInputSplit split = createTempFile(fileContent); 
   
      final Configuration parameters = new Configuration();

      format.setFieldDelimiter('|');
      format.setFieldTypes(StringValue.class, StringValue.class, StringValue.class);
View Full Code Here
133
134
135
136
137
138
139
140
141
142
143
 
  @Test
  public void readWithEmptyFieldConfigParameters() {
    try {
      final String fileContent = "abc|def|ghijk\nabc||hhg\n|||";
      final FileInputSplit split = createTempFile(fileContent); 
   
      final Configuration parameters = new Configuration();
      new CsvInputFormat.ConfigBuilder(null, parameters)
        .field(StringValue.class, 0).field(StringValue.class, 1).field(StringValue.class, 2);
     
View Full Code Here
TOP

Related Classes of eu.stratosphere.core.fs.FileInputSplit

  • eu.stratosphere.api.common.io.BinaryInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormatTest
  • eu.stratosphere.api.common.io.FileInputFormat
  • eu.stratosphere.api.common.io.FormatUtil
  • eu.stratosphere.api.common.io.GenericCsvInputFormatTest
  • eu.stratosphere.api.java.io.CsvInputFormatTest
  • eu.stratosphere.api.java.record.io.CsvInputFormatTest
  • eu.stratosphere.api.java.record.io.FixedLenghtInputFormatTest
  • eu.stratosphere.nephele.jobmanager.DoubleSourceTask
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.