Examples of org.apache.flink.core.fs.Path

org.apache.flink.core.fs.Path
Names a file or directory in a {@link FileSystem}. Path strings use slash as the directory separator. A path string is absolute if it begins with a slash.

    if (path.length() == 0) {
      throw new InvalidProgramException("File path of FileDataSource is empty string.");
    }
    
    try {
      Path p = new Path(path);
      String scheme = p.toUri().getScheme();
      
      if (scheme == null) {
        throw new InvalidProgramException("File path \"" + path + "\" of FileDataSource has no file system scheme (like 'file:// or hdfs://').");
      }
    } catch (Exception e) {

View Full Code Here

  }
  
  public void registerFileOutputTask(AbstractInvokable outTask, FileOutputFormat outputFormat, String outPath) {
    TaskConfig dsConfig = new TaskConfig(this.mockEnv.getTaskConfiguration());
    
    outputFormat.setOutputFilePath(new Path(outPath));
    outputFormat.setWriteMode(WriteMode.OVERWRITE);


    dsConfig.setStubWrapper(new UserCodeObjectWrapper<FileOutputFormat>(outputFormat));


    outTask.setEnvironment(this.mockEnv);

View Full Code Here

    long latestModTime = 0L;
    
    // get the file info and check whether the cached statistics are still valid.
    for(org.apache.hadoop.fs.Path hadoopPath : hadoopFilePaths) {
      
      final Path filePath = new Path(hadoopPath.toUri());
      final FileSystem fs = FileSystem.get(filePath.toUri());
      
      final FileStatus file = fs.getFileStatus(filePath);
      latestModTime = Math.max(latestModTime, file.getModificationTime());
      
      // enumerate all files and check their modification time stamp.

View Full Code Here

    long latestModTime = 0L;
    
    // get the file info and check whether the cached statistics are still valid.
    for(org.apache.hadoop.fs.Path hadoopPath : hadoopFilePaths) {
      
      final Path filePath = new Path(hadoopPath.toUri());
      final FileSystem fs = FileSystem.get(filePath.toUri());
      
      final FileStatus file = fs.getFileStatus(filePath);
      latestModTime = Math.max(latestModTime, file.getModificationTime());
      
      // enumerate all files and check their modification time stamp.

View Full Code Here

    DeleteProcess dp = new DeleteProcess(name, entry, jobID, count.get(new ImmutablePair<JobID, String>(jobID,name)));
    executorService.schedule(dp, 5000L, TimeUnit.MILLISECONDS);
  }


  public Path getTempDir(JobID jobID, String childPath) {
    return new Path(GlobalConfiguration.getString(ConfigConstants.TASK_MANAGER_TMP_DIR_KEY,
      ConfigConstants.DEFAULT_TASK_MANAGER_TMP_PATH), DistributedCache.TMP_PREFIX + jobID.toString() + "/" + childPath);
  }

View Full Code Here

            if (distPath.endsWith("/")) {
              distPath = distPath.substring(0, distPath.length() - 1);
            }
          }
          String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/"));
          copy(content.getPath(), new Path(localPath), executable);
        }
      } else {
        try {
          FSDataOutputStream lfsOutput = tFS.create(targetPath, false);
          FSDataInputStream fsInput = sFS.open(sourcePath);

View Full Code Here

      this.executable = e.isExecutable;
      this.jobID = jobID;
    }
    @Override
    public Path call()  {
      Path tmp = getTempDir(jobID, filePath.substring(filePath.lastIndexOf("/") + 1));
      try {
        synchronized (lock) {
          copy(new Path(filePath), tmp, this.executable);
        }
      } catch (IOException e) {
        LOG.error("Could not copy file to local file cache.", e);
      }
      return tmp;

View Full Code Here

      synchronized (count) {
        if (count.get(new ImmutablePair<JobID, String>(jobID, name)) != oldCount) {
          return;
        }
      }
      Path tmp = getTempDir(jobID, "");
      try {
        if (lfs.exists(tmp)) {
          lfs.delete(tmp, true);
        }
      } catch (IOException e) {

View Full Code Here

   */
  @Test
  public void testLineReader() {
    final File testfile = new File(CommonTestUtils.getTempDir() + File.separator
      + CommonTestUtils.getRandomFilename());
    final Path pathtotestfile = new Path(testfile.toURI().getPath());


    try {
      PrintWriter pw = new PrintWriter(testfile, "UTF8");


      for (int i = 0; i < 100; i++) {

View Full Code Here

  public static void main(String[] args) throws Exception {
    String inputPath = args[0];
    
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    
    DataSet<MyUser> input = env.createInput(new AvroInputFormat<MyUser>(new Path(inputPath), MyUser.class));
  
    DataSet<Tuple2<String, MyUser>> result = input.map(new NameExtractor()).groupBy(0).reduce(new NameGrouper());
    
    result.output(new DiscardingOuputFormat<Tuple2<String,MyUser>>());
    env.execute();

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.flink.core.fs.Path

org.apache.flink.api.avro.testjar.AvroExternalJarProgram

org.apache.flink.api.common.cache.DistributedCache

org.apache.flink.api.common.io.BinaryInputFormat

org.apache.flink.api.common.io.DelimitedInputFormat

org.apache.flink.api.common.io.DelimitedInputFormatTest

org.apache.flink.api.common.io.FileInputFormat

org.apache.flink.api.common.io.FileOutputFormat

org.apache.flink.api.common.io.FileOutputFormatTest

org.apache.flink.api.common.io.FormatUtil

org.apache.flink.api.common.io.GenericCsvInputFormatTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.