Examples of org.apache.hadoop.fs.FileSystem.create()

org.apache.hadoop.fs.FileSystem.create()
create a file with the provided permission The permission of the file is set to be the provided permission as in setPermission, not permission&~umask It is implemented using two RPCs. It is understood that it is inefficient, but the implementation is thread-safe. The other option is to change the value of umask in configuration to be 0, but it is not thread-safe. @param fs file system handle @param file the name of the file to be created @param permission the permission of the file @return an output stream @throws IOException

          this.region.getRegionInfo(), compactionDescriptor, new AtomicLong(1));


      Path recoveredEditsDir = HLogUtil.getRegionDirRecoveredEditsDir(regiondir);


      Path recoveredEdits = new Path(recoveredEditsDir, String.format("%019d", 1000));
      fs.create(recoveredEdits);
      HLog.Writer writer = HLogFactory.createRecoveredEditsWriter(fs, recoveredEdits, conf);


      long time = System.nanoTime();


      writer.append(new HLog.Entry(new HLogKey(regionName, tableName, 10, time,

View Full Code Here

    public RecordWriter<K, V> getRecordWriter(FileSystem ignored, JobConf job,
        String name, Progressable progress) throws IOException {


      Path dir = getWorkOutputPath(job);
      FileSystem fs = dir.getFileSystem(job);
      FSDataOutputStream fileOut = fs.create(new Path(dir, name), progress);
      return new LineRecordWriter<K, V>(fileOut);
    }
    
  }

View Full Code Here

        JobConf localJobConf = new JobConf(localJobFile);
        String jarFile = localJobConf.getJar();
        if (jarFile != null) {
          fs.copyToLocalFile(new Path(jarFile), localJarFile);
          localJobConf.setJar(localJarFile.toString());
          OutputStream out = localFs.create(localJobFile);
          try {
            localJobConf.write(out);
          } finally {
            out.close();
          }

View Full Code Here

                       fConf.get("mapred.local.dir"));
            
      localJobConf.set("mapred.task.id", task.getTaskId());
      keepFailedTaskFiles = localJobConf.getKeepFailedTaskFiles();
      task.localizeConfiguration(localJobConf);
      OutputStream out = localFs.create(localTaskFile);
      try {
        localJobConf.write(out);
      } finally {
        out.close();
      }

View Full Code Here

    public void createFile(String file, String content) throws IOException {
        FileSystem fs = FileSystem.get(URI.create(hdfsPath), conf);
        byte[] buff = content.getBytes();
        FSDataOutputStream os = null;
        try {
            os = fs.create(new Path(file));
            os.write(buff, 0, buff.length);
            System.out.println("Create: " + file);
        } finally {
            if (os != null)
                os.close();

View Full Code Here

      try {
        p = getResolvedPath();
        if (isAppendable() && p.getFileSystem(getConfiguration()).exists(p)) {
          wout = fs.append(p);
        } else {
          wout = fs.create(p, isOverwrite());
        }
        break;
      } catch (Exception e) {
        getOutputContext().rollStrategies();
      }

View Full Code Here

    assertThat(context.getLong(DataStoreItemReader.READ_POSITION), is(300l));
  }


  private void createTestData(Path path) throws IOException {
    FileSystem fs = FileSystem.get(getYarnCluster().getConfiguration());
    FSDataOutputStream out = fs.create(path);
    for (int i = 0; i < 300; i++) {
      out.writeBytes("line" + i + "\n");
    }
    out.close();
    assertTrue(fs.exists(path));

View Full Code Here

    assertThat(fs.exists(new Path(path)), is(false));
  }


  private void createFile(Configuration configuration, String path) throws Exception {
    FileSystem fs = FileSystem.get(configuration);
    FSDataOutputStream create = fs.create(new Path(path));
    create.close();
    assertThat(fs.exists(new Path(path)), is(true));
  }


  private static String getHdfsUrl(Configuration configuration) {

View Full Code Here

      String nameOfFile) throws IOException {
    FileSystem fs = getFileSystem();
    FSDataOutputStream out = null;
    Path inputFile = new Path(filePath + "/" + nameOfFile);
    try {
      out = fs.create(inputFile, false);
      out.write(inputData.getBytes(), 0, inputData.getBytes().length);
      out.close();
      out = null;
      // Cheking input file exists or not.
      Path inputPath = new Path(fs.getHomeDirectory(), filePath + "/"

View Full Code Here

    if (conf.getBoolean("hdfs.append.support", false) == true && hdfs.isFile
    (dstPath)) {
      fsOut = hdfs.append(dstPath);
      appending = true;
    } else {
      fsOut = hdfs.create(dstPath);
    }
    if(compressor == null) {
      compressor = CodecPool.getCompressor(codec, conf);
    }
    cmpOut = codec.createOutputStream(fsOut, compressor);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.