Examples of eu.stratosphere.core.fs.Path

eu.stratosphere.core.fs.Path
Names a file or directory in a {@link FileSystem}. Path strings use slash as the directory separator. A path string is absolute if it begins with a slash.

      final JobGraph jg = new JobGraph("Job Graph 1");


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task 1", jg);
      t1.setTaskClass(ForwardTask.class);


      // task vertex 2
      final JobTaskVertex t2 = new JobTaskVertex("Task 2", jg);
      t2.setTaskClass(ForwardTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      t2.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      try {
        i1.connectTo(t1, ChannelType.NETWORK);
        t1.connectTo(t2, ChannelType.IN_MEMORY);
        t2.connectTo(o1, ChannelType.IN_MEMORY);
      } catch (JobGraphDefinitionException e) {
        e.printStackTrace();
      }


      // add jar
      jg.addJar(new Path(new File(ServerTestUtils.getTempDir() + File.separator + forwardClassName + ".jar")
        .toURI()));


      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);

View Full Code Here

      final JobGraph jg = new JobGraph("Job Graph for Double Connection Test");


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex("Input with two Outputs", jg);
      i1.setFileInputClass(DoubleSourceTask.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // task vertex 1
      final JobTaskVertex t1 = new JobTaskVertex("Task with two Inputs", jg);
      t1.setTaskClass(DoubleTargetTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output 1", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      t1.setVertexToShareInstancesWith(i1);
      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      i1.connectTo(t1, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      i1.connectTo(t1, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
      t1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(jarFile.toURI()));


      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);
      jobClient.submitJobAndWait();

View Full Code Here

      final JobGraph jg = new JobGraph();


      // input vertex
      final JobFileInputVertex i1 = new JobFileInputVertex(jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile.toURI()));


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex(jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      o1.setVertexToShareInstancesWith(i1);


      // connect vertices
      i1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(jarFile.toURI()));


      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);
      jobClient.submitJobAndWait();

View Full Code Here

      final JobGraph jg = new JobGraph("Union job " + limit);


      // input vertex 1
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile1.toURI()));


      // input vertex 2
      final JobFileInputVertex i2 = new JobFileInputVertex("Input 2", jg);
      i2.setFileInputClass(FileLineReader.class);
      i2.setFilePath(new Path(inputFile2.toURI()));


      // union task
      final JobTaskVertex u1 = new JobTaskVertex("Union", jg);
      u1.setTaskClass(UnionTask.class);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));


      i1.setVertexToShareInstancesWith(o1);
      i2.setVertexToShareInstancesWith(o1);
      u1.setVertexToShareInstancesWith(o1);


      // connect vertices
      i1.connectTo(u1, ChannelType.IN_MEMORY, DistributionPattern.POINTWISE);
      i2.connectTo(u1, ChannelType.IN_MEMORY);
      u1.connectTo(o1, ChannelType.IN_MEMORY);


      // add jar
      jg.addJar(new Path(jarFile.toURI()));


      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);


      try {

View Full Code Here

      final JobGraph jg = new JobGraph("Job with large DoP (" + numberOfSubtasks + ")");


      // input vertex 1
      final JobFileInputVertex i1 = new JobFileInputVertex("Input 1", jg);
      i1.setFileInputClass(FileLineReader.class);
      i1.setFilePath(new Path(inputFile1.toURI()));
      i1.setNumberOfSubtasks(numberOfSubtasks);
      i1.setNumberOfSubtasksPerInstance(numberOfSubtasks);


      // input vertex 2
      final JobFileInputVertex i2 = new JobFileInputVertex("Input 2", jg);
      i2.setFileInputClass(FileLineReader.class);
      i2.setFilePath(new Path(inputFile2.toURI()));
      i2.setNumberOfSubtasks(numberOfSubtasks);
      i2.setNumberOfSubtasksPerInstance(numberOfSubtasks);


      // union task
      final JobTaskVertex f1 = new JobTaskVertex("Forward 1", jg);
      f1.setTaskClass(DoubleTargetTask.class);
      f1.setNumberOfSubtasks(numberOfSubtasks);
      f1.setNumberOfSubtasksPerInstance(numberOfSubtasks);


      // output vertex
      JobFileOutputVertex o1 = new JobFileOutputVertex("Output", jg);
      o1.setFileOutputClass(FileLineWriter.class);
      o1.setFilePath(new Path(outputFile.toURI()));
      o1.setNumberOfSubtasks(numberOfSubtasks);
      o1.setNumberOfSubtasksPerInstance(numberOfSubtasks);


      i1.setVertexToShareInstancesWith(o1);
      i2.setVertexToShareInstancesWith(o1);
      f1.setVertexToShareInstancesWith(o1);


      // connect vertices
      i1.connectTo(f1, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
      i2.connectTo(f1, ChannelType.NETWORK, DistributionPattern.BIPARTITE);
      f1.connectTo(o1, ChannelType.NETWORK, DistributionPattern.BIPARTITE);


      // add jar
      jg.addJar(new Path(jarFile.toURI()));


      // Create job client and launch job
      jobClient = new JobClient(jg, configuration);
      
      // disable logging for the taskmanager and the client, as they will have many

View Full Code Here

   */
  @Test
  public void testLineReader() {
    final File testfile = new File(CommonTestUtils.getTempDir() + File.separator
      + CommonTestUtils.getRandomFilename());
    final Path pathtotestfile = new Path(testfile.toURI().getPath());


    try {
      PrintWriter pw = new PrintWriter(testfile, "UTF8");


      for (int i = 0; i < 100; i++) {

View Full Code Here

    if (!testActivated()) {
      return;
    }


    final String bucketName = getRandomName();
    final Path bucketPath = new Path(S3_BASE_URI + bucketName + Path.SEPARATOR);


    try {


      final FileSystem fs = bucketPath.getFileSystem();


      // Create directory
      fs.mkdirs(bucketPath);


      // Check if directory is correctly displayed in file system hierarchy
      final FileStatus[] content = fs.listStatus(new Path(S3_BASE_URI));
      boolean entryFound = false;
      for (final FileStatus entry : content) {
        if (bucketPath.equals(entry.getPath())) {
          entryFound = true;
          break;
        }
      }


      if (!entryFound) {
        fail("Cannot find entry " + bucketName + " in directory " + S3_BASE_URI);
      }


      // Check the concrete directory file status
      try {
        final FileStatus directoryFileStatus = fs.getFileStatus(bucketPath);
        assertTrue(directoryFileStatus.isDir());
        assertEquals(0L, directoryFileStatus.getAccessTime());
        assertTrue(directoryFileStatus.getModificationTime() > 0L);


      } catch (FileNotFoundException e) {
        fail(e.getMessage());
      }


      // Delete the bucket
      fs.delete(bucketPath, true);


      // Make sure the bucket no longer exists
      try {
        fs.getFileStatus(bucketPath);
        fail("Expected FileNotFoundException for " + bucketPath.toUri());
      } catch (FileNotFoundException e) {
        // This is an expected exception
      }


    } catch (IOException ioe) {

View Full Code Here


    final String dirName = getRandomName();
    final String subdirName = getRandomName();
    final String subsubdirName = getRandomName();
    final String fileName = getRandomName();
    final Path dir = new Path(S3_BASE_URI + dirName + Path.SEPARATOR);
    final Path subdir = new Path(S3_BASE_URI + dirName + Path.SEPARATOR + subdirName + Path.SEPARATOR);
    final Path subsubdir = new Path(S3_BASE_URI + dirName + Path.SEPARATOR + subdirName + Path.SEPARATOR
      + subsubdirName + Path.SEPARATOR);
    final Path file = new Path(S3_BASE_URI + dirName + Path.SEPARATOR + subdirName + Path.SEPARATOR + fileName);


    try {


      final FileSystem fs = dir.getFileSystem();

View Full Code Here

      return;
    }


    final String dirName = getRandomName();
    final String fileName = getRandomName();
    final Path dir = new Path(S3_BASE_URI + dirName + Path.SEPARATOR);
    final Path file = new Path(S3_BASE_URI + dirName + Path.SEPARATOR + fileName);


    try {


      final FileSystem fs = dir.getFileSystem();

View Full Code Here

      return;
    }


    final String bucketName = getRandomName();
    final String objectName = getRandomName();
    final Path bucketPath = new Path(S3_BASE_URI + bucketName + Path.SEPARATOR);
    final Path objectPath = new Path(S3_BASE_URI + bucketName + Path.SEPARATOR + objectName);


    FileSystem fs = bucketPath.getFileSystem();


    // Create test bucket
    fs.mkdirs(bucketPath);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of eu.stratosphere.core.fs.Path

eu.stratosphere.api.avro.testjar.AvroExternalJarProgram

eu.stratosphere.api.common.cache.DistributedCache

eu.stratosphere.api.common.io.BinaryInputFormat

eu.stratosphere.api.common.io.DelimitedInputFormat

eu.stratosphere.api.common.io.DelimitedInputFormatTest

eu.stratosphere.api.common.io.FileInputFormat

eu.stratosphere.api.common.io.FileOutputFormat

eu.stratosphere.api.common.io.FileOutputFormat$OutputPathOpenThread

eu.stratosphere.api.common.io.FileOutputFormatTest

eu.stratosphere.api.common.io.FormatUtil

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.