Examples of eu.stratosphere.core.fs.Path

eu.stratosphere.core.fs.Path
Names a file or directory in a {@link FileSystem}. Path strings use slash as the directory separator. A path string is absolute if it begins with a slash.

    
    String tmpFilePath = tmpOutPath.toURI().toString();


    // check fail if file exists
    DummyFileOutputFormat dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.NO_OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());
    
    boolean exception = false;
    try {
      dfof.open(0, 1);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(exception);


    // check fail if directory exists
    tmpOutPath.delete();
    Assert.assertTrue("Directory could not be created.", tmpOutPath.mkdir());


    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.NO_OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 1);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(exception);
    
    // check success
    tmpOutPath.delete();
    
    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.NO_OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 1);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(!exception);
    Assert.assertTrue(tmpOutPath.exists() && tmpOutPath.isFile());
    
    // ----------- test again with always directory mode
    
    // check fail if file exists
    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.NO_OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.ALWAYS);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 1);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(exception);


    // check success if directory exists
    tmpOutPath.delete();
    Assert.assertTrue("Directory could not be created.", tmpOutPath.mkdir());


    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.NO_OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.ALWAYS);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 1);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(!exception);
    Assert.assertTrue(tmpOutPath.exists() && tmpOutPath.isDirectory());
    Assert.assertTrue(tmpOutFile.exists() && tmpOutFile.isFile());
    
    // check fail if file in directory exists
    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.NO_OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.ALWAYS);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 1);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(exception);
    
    // check success if no file exists
    // delete existing files
    (new File(tmpOutPath.getAbsoluteFile()+"/1")).delete();
    tmpOutPath.delete();
    
    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.NO_OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.ALWAYS);


    dfof.configure(new Configuration());

View Full Code Here

    
    String tmpFilePath = tmpOutPath.toURI().toString();


    // check fail if file exists
    DummyFileOutputFormat dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.NO_OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());
    
    boolean exception = false;
    try {
      dfof.open(0, 2);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(exception);


    // check success if directory exists
    tmpOutPath.delete();
    Assert.assertTrue("Directory could not be created.", tmpOutPath.mkdir());


    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.NO_OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 2);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(!exception);
    Assert.assertTrue(tmpOutPath.exists() && tmpOutPath.isDirectory());
    Assert.assertTrue(tmpOutFile.exists() && tmpOutFile.isFile());
    
    // check fail if file in directory exists
    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.NO_OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 2);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(exception);
    
    // check success if no file exists
    // delete existing files
    tmpOutFile.delete();
    tmpOutPath.delete();
    
    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.NO_OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());

View Full Code Here

    
    String tmpFilePath = tmpOutPath.toURI().toString();


    // check success if file exists
    DummyFileOutputFormat dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());
    
    boolean exception = false;
    try {
      dfof.open(0, 1);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(!exception);
    Assert.assertTrue(tmpOutPath.exists() && tmpOutPath.isFile());


    // check success if directory exists
    tmpOutPath.delete();
    Assert.assertTrue("Directory could not be created.", tmpOutPath.mkdir());


    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 1);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(!exception);
    Assert.assertTrue(tmpOutPath.exists() && tmpOutPath.isFile());
    
    // check success
    tmpOutPath.delete();
    
    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 1);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(!exception);
    Assert.assertTrue(tmpOutPath.exists() && tmpOutPath.isFile());
    
    // ----------- test again with always directory mode
    
    // check success if file exists
    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.ALWAYS);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 1);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(!exception);
    Assert.assertTrue(tmpOutPath.exists() && tmpOutPath.isDirectory());
    Assert.assertTrue(tmpOutFile.exists() && tmpOutFile.isFile());


    // check success if directory exists
    tmpOutFile.delete();
    tmpOutPath.delete();
    Assert.assertTrue("Directory could not be created.", tmpOutPath.mkdir());


    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.ALWAYS);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 1);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(!exception);
    Assert.assertTrue(tmpOutPath.exists() && tmpOutPath.isDirectory());
    Assert.assertTrue(tmpOutFile.exists() && tmpOutFile.isFile());
    
    // check success if file in directory exists
    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.ALWAYS);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 1);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(!exception);
    Assert.assertTrue(tmpOutPath.exists() && tmpOutPath.isDirectory());
    Assert.assertTrue(tmpOutFile.exists() && tmpOutFile.isFile());
    
    // check success if no file exists
    // delete existing files
    tmpOutFile.delete();
    tmpOutPath.delete();
    
    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.ALWAYS);


    dfof.configure(new Configuration());

View Full Code Here

    
    String tmpFilePath = tmpOutPath.toURI().toString();


    // check success if file exists
    DummyFileOutputFormat dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());
    
    boolean exception = false;
    try {
      dfof.open(0, 2);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(!exception);
    Assert.assertTrue(tmpOutPath.exists() && tmpOutPath.isDirectory());
    Assert.assertTrue(tmpOutFile.exists() && tmpOutFile.isFile());


    // check success if directory exists
    tmpOutFile.delete();
    tmpOutPath.delete();
    Assert.assertTrue("Directory could not be created.", tmpOutPath.mkdir());


    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 2);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(!exception);
    Assert.assertTrue(tmpOutPath.exists() && tmpOutPath.isDirectory());
    Assert.assertTrue(tmpOutFile.exists() && tmpOutFile.isFile());
    
    // check success if file in directory exists
    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());
    
    exception = false;
    try {
      dfof.open(0, 2);
      dfof.close();
    } catch (Exception e) {
      exception = true;
    }
    Assert.assertTrue(!exception);
    Assert.assertTrue(tmpOutPath.exists() && tmpOutPath.isDirectory());
    Assert.assertTrue(tmpOutFile.exists() && tmpOutFile.isFile());
    
    // check success if no file exists
    // delete existing files
    (new File(tmpOutPath.getAbsoluteFile()+"/1")).delete();
    tmpOutPath.delete();
    
    dfof = new DummyFileOutputFormat();
    dfof.setOutputFilePath(new Path(tmpFilePath));
    dfof.setWriteMode(WriteMode.OVERWRITE);
    dfof.setOutputDirectoryMode(OutputDirectoryMode.PARONLY);


    dfof.configure(new Configuration());

View Full Code Here

        URI u = new URI(filePath);
        if (!u.getPath().startsWith("/")) {
          u = new URI(new File(filePath).getAbsolutePath());
        }
        FileSystem fs = FileSystem.get(u);
        if (fs.exists(new Path(u.getPath()))) {
          this.cacheFile.put(name, u.toString());
        } else {
          throw new RuntimeException("File " + u.toString() + " doesn't exist.");
        }
      } catch (URISyntaxException ex) {

View Full Code Here

    Arrays.sort(inputSplits, new InputSplitSorter());


    int splitIndex = 0;
    for (int fileIndex = 0; fileIndex < this.degreeOfParallelism; fileIndex++) {
      List<FileInputSplit> sameFileSplits = new ArrayList<FileInputSplit>();
      Path lastPath = inputSplits[splitIndex].getPath();
      for (; splitIndex < inputSplits.length; splitIndex++) {
        if (!inputSplits[splitIndex].getPath().equals(lastPath)) {
          break;
        }
        sameFileSplits.add(inputSplits[splitIndex]);

View Full Code Here

    // store properties
    final long oldTimeout = this.openTimeout;
    final int oldBufferSize = this.bufferSize;
    final int oldLineLengthLimit = this.lineLengthLimit;
    try {
      final Path filePath = this.filePath;
    
      // get the filesystem
      final FileSystem fs = FileSystem.get(filePath.toUri());
      final ArrayList<FileStatus> allFiles = new ArrayList<FileStatus>(1);
      
      // let the file input format deal with the up-to-date check and the basic size
      final FileBaseStatistics stats = getFileStats(cachedFileStats, filePath, fs, allFiles);
      if (stats == null) {

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of eu.stratosphere.core.fs.Path

eu.stratosphere.api.avro.testjar.AvroExternalJarProgram

eu.stratosphere.api.common.cache.DistributedCache

eu.stratosphere.api.common.io.BinaryInputFormat

eu.stratosphere.api.common.io.DelimitedInputFormat

eu.stratosphere.api.common.io.DelimitedInputFormatTest

eu.stratosphere.api.common.io.FileInputFormat

eu.stratosphere.api.common.io.FileOutputFormat

eu.stratosphere.api.common.io.FileOutputFormat$OutputPathOpenThread

eu.stratosphere.api.common.io.FileOutputFormatTest

eu.stratosphere.api.common.io.FormatUtil

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.