Examples of org.apache.hadoop.fs.FileSystem.create()

org.apache.hadoop.fs.FileSystem.create()
create a file with the provided permission The permission of the file is set to be the provided permission as in setPermission, not permission&~umask It is implemented using two RPCs. It is understood that it is inefficient, but the implementation is thread-safe. The other option is to change the value of umask in configuration to be 0, but it is not thread-safe. @param fs file system handle @param file the name of the file to be created @param permission the permission of the file @return an output stream @throws IOException

      fs.mkdirs(inDir);
    }
    String input = "The quick brown fox\n" + "has many silly\n"
        + "red fox sox\n";
    for (int i = 0; i < numMaps; ++i) {
      DataOutputStream file = fs.create(new Path(inDir, "part-" + i));
      file.writeBytes(input);
      file.close();
    }    


    conf.setInputFormat(TextInputFormat.class);

View Full Code Here

      jmapper.configure(
          SerializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, true);
      JsonFactory jfactory = jmapper.getJsonFactory();
      FileSystem jobFS = jobTraceFilename.getFileSystem(getConf());
      jobTraceGen = jfactory.createJsonGenerator(
          jobFS.create(jobTraceFilename), JsonEncoding.UTF8);
      if (prettyprintTrace) {
        jobTraceGen.useDefaultPrettyPrinter();
      }


      if (topologyFilename != null) {

View Full Code Here

        tmapper.configure(
            SerializationConfig.Feature.CAN_OVERRIDE_ACCESS_MODIFIERS, true);
        JsonFactory tfactory = tmapper.getJsonFactory();
        FileSystem topoFS = topologyFilename.getFileSystem(getConf());
        topologyGen = tfactory.createJsonGenerator(
            topoFS.create(topologyFilename), JsonEncoding.UTF8);
        topologyGen.useDefaultPrettyPrinter();
      }
    }


    return 0;

View Full Code Here

    MiniDFSCluster cluster = new MiniDFSCluster(conf, 2, true, null);
    FileSystem fileSys = cluster.getFileSystem();


    try {
      // create two files
      fileSys.create(new Path("/test/dfsclose/file-0"));
      fileSys.create(new Path("/test/dfsclose/file-1"));


      fileSys.close();
    }
    finally {

View Full Code Here

    FileSystem fileSys = cluster.getFileSystem();


    try {
      // create two files
      fileSys.create(new Path("/test/dfsclose/file-0"));
      fileSys.create(new Path("/test/dfsclose/file-1"));


      fileSys.close();
    }
    finally {
      if (cluster != null) {cluster.shutdown();}

View Full Code Here

      throw new IOException("Failed to create the input directory:" 
            + inDir.toString());
    }
    fs.setPermission(inDir, new FsPermission(FsAction.ALL, 
            FsAction.ALL, FsAction.ALL));
    DataOutputStream file = fs.create(new Path(inDir, "data.txt"));
    int i = 0;
    while(i < 1000 * 3000) {
      file.writeBytes(input);
      i++;
    }

View Full Code Here

      }
      if (!fs.exists(inDir)) {
        fs.mkdirs(inDir);
      }
      String input = "The quick brown fox jumped over the lazy dog";
      DataOutputStream file = fs.create(new Path(inDir, "part-0"));
      file.writeBytes(input);
      file.close();


      conf.setInputFormat(TextInputFormat.class);
      conf.setOutputKeyClass(LongWritable.class);

View Full Code Here

      }
      if (!fs.exists(inDir)) {
        fs.mkdirs(inDir);
      }
      String input = "The quick brown fox jumped over the lazy dog";
      DataOutputStream file = fs.create(new Path(inDir, "part-0"));
      file.writeBytes(input);
      file.close();


      conf.setInputFormat(TextInputFormat.class);
      conf.setOutputKeyClass(LongWritable.class);

View Full Code Here

      Path scriptPath = new Path(TEST_ROOT_DIR, "debug-script.txt");
      String debugScriptContent =
          "for ((i=0;i<1000;i++)); " + "do "
              + "echo \"Lots of logs! Lots of logs! "
              + "Waiting to be truncated! Lots of logs!\";" + "done";
      DataOutputStream scriptFile = fs.create(scriptPath);
      scriptFile.writeBytes(debugScriptContent);
      scriptFile.close();
      new File(scriptPath.toUri().getPath()).setExecutable(true);


      URI uri = scriptPath.toUri();

View Full Code Here

    if (fs.exists(inDir)) {
      fs.delete(inDir, true);
    }
    fs.mkdirs(inDir);
    for (int i = 0; i < numInputFiles; ++i) {
      DataOutputStream file = fs.create(new Path(inDir, "part-" + i));
      file.writeBytes(input);
      file.close();
    }    


    FileInputFormat.setInputPaths(job, inDir);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.