Examples of FSDataOutputStream

eu.stratosphere.core.fs.FSDataOutputStream
Interface for a data output stream to a file on a {@link FileSystem}.
org.apache.flink.core.fs.FSDataOutputStream
Interface for a data output stream to a file on a {@link FileSystem}.
org.apache.hadoop.fs.FSDataOutputStream
Utility that wraps a {@link OutputStream} in a {@link DataOutputStream}, buffers output through a {@link BufferedOutputStream} and creates a checksumfile.

Examples of org.apache.hadoop.fs.FSDataOutputStream

    }


    public void createFile(String file, String content) throws IOException {
        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);
        byte[] buff = content.getBytes();
        FSDataOutputStream os = null;
        try {
            os = fs.create(new Path(file));
            os.write(buff, 0, buff.length);
            System.out.println("Create: " + file);
        } finally {
            if (os != null)
                os.close();
        }
        fs.close();
    }

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataOutputStream

    // is set to roughly same count as expected number of writers
    // and strategy init is accurate enough to find a good starting
    // position for naming, we should always get a next available
    // path and its stream.
    Path p = null;
    FSDataOutputStream wout = null;
    int openAttempt = 0;
    do {
      try {
        p = getResolvedPath();
        if (isAppendable() && p.getFileSystem(getConfiguration()).exists(p)) {

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataOutputStream

    assertThat(context.getLong(DataStoreItemReader.READ_POSITION), is(300l));
  }


  private void createTestData(Path path) throws IOException {
    FileSystem fs = FileSystem.get(getYarnCluster().getConfiguration());
    FSDataOutputStream out = fs.create(path);
    for (int i = 0; i < 300; i++) {
      out.writeBytes("line" + i + "\n");
    }
    out.close();
    assertTrue(fs.exists(path));
    assertThat(fs.getFileStatus(path).getLen(), greaterThan(0l));
  }

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataOutputStream

    assertThat(fs.exists(new Path(path)), is(false));
  }


  private void createFile(Configuration configuration, String path) throws Exception {
    FileSystem fs = FileSystem.get(configuration);
    FSDataOutputStream create = fs.create(new Path(path));
    create.close();
    assertThat(fs.exists(new Path(path)), is(true));
  }

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataOutputStream

  }


  public void createTextFileInHDFS(String inputData, String filePath,
      String nameOfFile) throws IOException {
    FileSystem fs = getFileSystem();
    FSDataOutputStream out = null;
    Path inputFile = new Path(filePath + "/" + nameOfFile);
    try {
      out = fs.create(inputFile, false);
      out.write(inputData.getBytes(), 0, inputData.getBytes().length);
      out.close();
      out = null;
      // Cheking input file exists or not.
      Path inputPath = new Path(fs.getHomeDirectory(), filePath + "/"
          + nameOfFile);
      assertTrue(fs.exists(inputPath));
    } finally {
      if (out != null) {
        out.close();
      }
    }
  }

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataOutputStream

                Path checkSumValueFile = new Path(nodeDir, fileNamePrefix + ".data.checksum");


                if(outputFs.exists(checkSumIndexFile)) {
                    outputFs.delete(checkSumIndexFile);
                }
                FSDataOutputStream output = outputFs.create(checkSumIndexFile);
                outputFs.setPermission(checkSumIndexFile,
                                       new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION));
                output.write(this.checkSumDigestIndex.getCheckSum());
                output.close();


                if(outputFs.exists(checkSumValueFile)) {
                    outputFs.delete(checkSumValueFile);
                }
                output = outputFs.create(checkSumValueFile);
                outputFs.setPermission(checkSumValueFile,
                                       new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION));
                output.write(this.checkSumDigestValue.getCheckSum());
                output.close();
            } else {
                throw new RuntimeException("Failed to open checksum digest for node " + nodeId
                                           + " ( partition - " + this.partitionId + ", chunk - "
                                           + chunkId + " )");
            }

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataOutputStream

                    Path checkSumValueFile = new Path(nodeDir, chunkFileName + ".data.checksum");


                    if(outputFs.exists(checkSumIndexFile)) {
                        outputFs.delete(checkSumIndexFile);
                    }
                    FSDataOutputStream output = outputFs.create(checkSumIndexFile);
                    outputFs.setPermission(checkSumIndexFile,
                                           new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION));
                    output.write(this.checkSumDigestIndex[chunkId].getCheckSum());
                    output.close();


                    if(outputFs.exists(checkSumValueFile)) {
                        outputFs.delete(checkSumValueFile);
                    }
                    output = outputFs.create(checkSumValueFile);
                    outputFs.setPermission(checkSumValueFile,
                                           new FsPermission(HadoopStoreBuilder.HADOOP_FILE_PERMISSION));
                    output.write(this.checkSumDigestValue[chunkId].getCheckSum());
                    output.close();
                } else {
                    throw new RuntimeException("Failed to open checksum digest for node " + nodeId
                                               + " ( partition - " + this.partitionId
                                               + ", chunk - " + chunkId + " )");
                }

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataOutputStream

                    }
                }


                // Write metadata
                Path metadataPath = new Path(nodePath, ".metadata");
                FSDataOutputStream metadataStream = outputFs.create(metadataPath);
                outputFs.setPermission(metadataPath, new FsPermission(HADOOP_FILE_PERMISSION));
                logger.info("Setting permission to 755 for " + metadataPath);
                metadataStream.write(metadata.toJsonString().getBytes());
                metadataStream.flush();
                metadataStream.close();


            }


        } catch(Exception e) {
            logger.error("Error in Store builder", e);

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataOutputStream

      ext = codec.getDefaultExtension();
    }


    Path file = getDefaultWorkFile(context, ext);
    FileSystem fs = file.getFileSystem(conf);
    FSDataOutputStream fileOut = fs.create(file, false);
    DataOutputStream ostream = fileOut;


    if (isCompressed) {
      ostream = new DataOutputStream(codec.createOutputStream(fileOut));
    }

View Full Code Here

Examples of org.apache.hadoop.fs.FSDataOutputStream


  private void writeFile(FileSystem fileSys, Path name, int repl,
                         int fileSize, int blockSize)
    throws IOException {
    // create and write a file that contains three blocks of data
    FSDataOutputStream stm = fileSys.create(name, true,
                                            fileSys.getConf().getInt("io.file.buffer.size", 4096),
                                            (short)repl, (long)blockSize);
    byte[] buffer = new byte[fileSize];
    Random rand = new Random(seed);
    rand.nextBytes(buffer);
    stm.write(buffer);
    stm.close();
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.