Examples of org.apache.flink.core.fs.FSDataInputStream

org.apache.flink.core.fs.FSDataInputStream
Interface for a data input stream to a file on a {@link FileSystem}.

        final FileSystem fs = FileSystem.get(this.split.getPath().toUri());
        this.fdis = fs.open(this.split.getPath());
        
        // check for canceling and close the stream in that case, because no one will obtain it
        if (this.aborted) {
          final FSDataInputStream f = this.fdis;
          this.fdis = null;
          f.close();
        }
      }
      catch (Throwable t) {
        this.error = t;
      }

View Full Code Here

    /**
     * Double checked procedure setting the abort flag and closing the stream.
     */
    private void abortWait() {
      this.aborted = true;
      final FSDataInputStream inStream = this.fdis;
      this.fdis = null;
      if (inStream != null) {
        try {
          inStream.close();
        } catch (Throwable t) {}
      }
    }

View Full Code Here

    final FSDataOutputStream outputStream = fs.create(objectPath, false);
    generateTestData(outputStream, fileSize);
    outputStream.close();


    // Now read the same file back from S3
    final FSDataInputStream inputStream = fs.open(objectPath);
    testReceivedData(inputStream, fileSize);
    inputStream.close();


    // Delete test bucket
    fs.delete(bucketPath, true);
  }

View Full Code Here

          copy(content.getPath(), new Path(localPath), executable);
        }
      } else {
        try {
          FSDataOutputStream lfsOutput = tFS.create(targetPath, false);
          FSDataInputStream fsInput = sFS.open(sourcePath);
          IOUtils.copyBytes(fsInput, lfsOutput);
          new File(targetPath.toString()).setExecutable(executable);
        } catch (IOException ioe) {
          LOG.error("could not copy file to local file cache.", ioe);
        }

View Full Code Here

        pw.append("line\n");
      }
      pw.close();


      LocalFileSystem lfs = new LocalFileSystem();
      FSDataInputStream fis = lfs.open(pathtotestfile);


      // first, we test under "usual" conditions
      final LineReader lr = new LineReader(fis, 0, testfile.length(), 256);


      byte[] buffer;

View Full Code Here

      final FileOutputStream fosfile2 = new FileOutputStream(testfile2);
      fosfile2.write(testbytes);
      fosfile2.close();


      testbytestest = new byte[5];
      final FSDataInputStream lfsinput2 = lfs.open(pathtotestfile2);
      assertEquals(lfsinput2.read(testbytestest), 5);
      lfsinput2.close();
      assertTrue(Arrays.equals(testbytes, testbytestest));


      // does lfs see two files?
      assertEquals(lfs.listStatus(pathtotmpdir).length, 2);

View Full Code Here


      for (final Iterator<Path> it = this.userJars.iterator(); it.hasNext();) {


        final Path jar = it.next();
        final FileSystem fs = jar.getFileSystem();
        FSDataInputStream is = null;
        try {
          is = fs.open(jar);
          final BlobKey key = bc.put(is);
          this.userJarBlobKeys.add(key);
        } finally {
          if (is != null) {
            is.close();
          }
        }
      }


    } finally {

View Full Code Here

      // invalid file
      if (file.getLen() < blockInfo.getInfoSize()) {
        continue;
      }


      FSDataInputStream fdis = file.getPath().getFileSystem().open(file.getPath(), blockInfo.getInfoSize());
      fdis.seek(file.getLen() - blockInfo.getInfoSize());


      DataInputStream input = new DataInputStream(fdis);
      blockInfo.read(new InputViewDataInputStreamWrapper(input));
      totalCount += blockInfo.getAccumulatedRecordCount();
    }

View Full Code Here

TOP

Related Classes of org.apache.flink.core.fs.FSDataInputStream

org.apache.flink.api.common.io.BinaryInputFormat

org.apache.flink.api.common.io.FileInputFormat$InputSplitOpenThread

org.apache.flink.core.fs.local.LocalFileSystemTest

org.apache.flink.runtime.filecache.FileCache

org.apache.flink.runtime.fs.LineReaderTest

org.apache.flink.runtime.fs.s3.S3FileSystemTest

org.apache.flink.runtime.jobgraph.JobGraph

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.