Package org.apache.flink.core.fs

Examples of org.apache.flink.core.fs.FileInputSplit


 
 
  @Test
  public void testReadFail() throws IOException {
    final int[] fileContent = {1,2,3,4,5,6,7,8,9};
    final FileInputSplit split = createTempFile(fileContent);
   
    final Configuration parameters = new Configuration();
    parameters.setInteger(FixedLengthInputFormat.RECORDLENGTH_PARAMETER_KEY, 8);
   
    format.configure(parameters);
View Full Code Here


      dos.writeInt(i);
    }
   
    dos.close();
     
    return new FileInputSplit(0, new Path(this.tempFile.toURI().toString()), 0, this.tempFile.length(), new String[] {"localhost"});
  }
View Full Code Here

      // take the samples
      while (samplesTaken < numSamples && fileNum < allFiles.size()) {
        // make a split for the sample and use it to read a record
        FileStatus file = allFiles.get(fileNum);
        FileInputSplit split = new FileInputSplit(0, file.getPath(), offset, file.getLen() - offset, null);

        // we open the split, read one line, and take its length
        try {
          open(split);
          if (readLine()) {
View Full Code Here

        // get the block locations and make sure they are in order with respect to their offset
        final BlockLocation[] blocks = fs.getFileBlockLocations(file, pos, remainingLength);
        Arrays.sort(blocks);

        inputSplits.add(new FileInputSplit(inputSplits.size(), file.getPath(), pos, remainingLength,
          blocks[0].getHosts()));
      }
    }

    if (inputSplits.size() < minNumSplits) {
      LOG.warn(String.format(
        "With the given block size %d, the file %s cannot be split into %d blocks. Filling up with empty splits...",
        blockSize, this.filePath, minNumSplits));
      FileStatus last = files.get(files.size() - 1);
      final BlockLocation[] blocks = fs.getFileBlockLocations(last, 0, last.getLen());
      for (int index = files.size(); index < minNumSplits; index++) {
        inputSplits.add(new FileInputSplit(index, last.getPath(), last.getLen(), 0, blocks[0].getHosts()));
      }
    }

    return inputSplits.toArray(new FileInputSplit[0]);
  }
View Full Code Here

  @Test
  public void readStringFields() {
    try {
      final String fileContent = "abc|def|ghijk\nabc||hhg\n|||";
      final FileInputSplit split = createTempFile(fileContent);
     
      final CsvInputFormat<Tuple3<String, String, String>> format = new CsvInputFormat<Tuple3<String, String, String>>(PATH, "\n", '|', String.class, String.class, String.class);
   
      final Configuration parameters = new Configuration();
      format.configure(parameters);
View Full Code Here

 
  @Test
  public void readStringFieldsWithTrailingDelimiters() {
    try {
      final String fileContent = "abc|def|ghijk\nabc||hhg\n|||\n";
      final FileInputSplit split = createTempFile(fileContent);
   
      final CsvInputFormat<Tuple3<String, String, String>> format = new CsvInputFormat<Tuple3<String, String, String>>(PATH);
     
      format.setFieldDelimiter('|');
      format.setFieldTypes(String.class, String.class, String.class);
View Full Code Here

 
  @Test
  public void testIntegerFieldsl() throws IOException {
    try {
      final String fileContent = "111|222|333|444|555\n666|777|888|999|000|\n";
      final FileInputSplit split = createTempFile(fileContent)
   
      final CsvInputFormat<Tuple5<Integer, Integer, Integer, Integer, Integer>> format = new CsvInputFormat<Tuple5<Integer, Integer, Integer, Integer, Integer>>(PATH);
     
      format.setFieldDelimiter('|');
      format.setFieldTypes(Integer.class, Integer.class, Integer.class, Integer.class, Integer.class);
View Full Code Here

 
  @Test
  public void testReadFirstN() throws IOException {
    try {
      final String fileContent = "111|222|333|444|555|\n666|777|888|999|000|\n";
      final FileInputSplit split = createTempFile(fileContent)
   
      final CsvInputFormat<Tuple2<Integer, Integer>> format = new CsvInputFormat<Tuple2<Integer, Integer>>(PATH);
     
      format.setFieldDelimiter('|');
      format.setFieldTypes(Integer.class, Integer.class);
View Full Code Here

 
  @Test
  public void testReadSparseWithNullFieldsForTypes() throws IOException {
    try {
      final String fileContent = "111|222|333|444|555|666|777|888|999|000|\n000|999|888|777|666|555|444|333|222|111|";
      final FileInputSplit split = createTempFile(fileContent)
     
      final CsvInputFormat<Tuple3<Integer, Integer, Integer>> format = new CsvInputFormat<Tuple3<Integer, Integer, Integer>>(PATH);
     
      format.setFieldDelimiter('|');
      format.setFieldTypes(Integer.class, null, null, Integer.class, null, null, null, Integer.class);
View Full Code Here

 
  @Test
  public void testReadSparseWithPositionSetter() throws IOException {
    try {
      final String fileContent = "111|222|333|444|555|666|777|888|999|000|\n000|999|888|777|666|555|444|333|222|111|";
      final FileInputSplit split = createTempFile(fileContent)
     
      final CsvInputFormat<Tuple3<Integer, Integer, Integer>> format = new CsvInputFormat<Tuple3<Integer, Integer, Integer>>(PATH);
     
      format.setFieldDelimiter('|');
     
View Full Code Here

TOP

Related Classes of org.apache.flink.core.fs.FileInputSplit

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.