Package eu.stratosphere.core.fs

Examples of eu.stratosphere.core.fs.Path

105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
      this.name = name;
      this.filePath = filePath;
      this.jobID = jobID;
    }
    public Path call()  {
      Path tmp = getTempDir(jobID, name);
      try {
        if (!lfs.exists(tmp)) {
          FSDataOutputStream lfsOutput = lfs.create(tmp, false);
          Path distributedPath = new Path(filePath);
          FileSystem fs = distributedPath.getFileSystem();
          FSDataInputStream fsInput = fs.open(distributedPath);
          IOUtils.copyBytes(fsInput, lfsOutput);
        }
      } catch (IOException e1) {
        throw new RuntimeException("Error copying a file from hdfs to the local fs", e1);
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
      synchronized (count) {
        if (count.get(new ImmutablePair(jobID, name)) != oldCount) {
          return;
        }
      }
      Path tmp = getTempDir(jobID, name);
      try {
        if (lfs.exists(tmp)) {
          lfs.delete(tmp, true);
        }
      } catch (IOException e1) {
View Full Code Here
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
   
    final FileOutputFormat<?> fileOutputFormat = (FileOutputFormat<?>) this.format;
   
    // ----------------- This code applies only to file inputs ------------------
   
    final Path path = fileOutputFormat.getOutputFilePath();
    final WriteMode writeMode = fileOutputFormat.getWriteMode();
    final OutputDirectoryMode outDirMode = fileOutputFormat.getOutputDirectoryMode();

    // Prepare output path and determine max DOP   
    try {
     
      int dop = getTaskConfiguration().getInteger(DEGREE_OF_PARALLELISM_KEY, -1);
      final FileSystem fs = path.getFileSystem();
     
      if(dop == 1 && outDirMode == OutputDirectoryMode.PARONLY) {
        // output is not written in parallel and should be written to a single file.
       
        if(fs.isDistributedFS()) {
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
136
    if (path.length() == 0) {
      throw new InvalidProgramException("File path of FileDataSink is empty string.");
    }
   
    try {
      Path p = new Path(path);
      String scheme = p.toUri().getScheme();
     
      if (scheme == null) {
        throw new InvalidProgramException("File path \"" + path + "\" of FileDataSink has no file system scheme (like 'file:// or hdfs://').");
      }
    } catch (Exception e) {
View Full Code Here
154
155
156
157
158
159
160
161
162
163
164
165
    if (path.length() == 0) {
      throw new InvalidProgramException("File path of FileDataSource is empty string.");
    }
   
    try {
      Path p = new Path(path);
      String scheme = p.toUri().getScheme();
     
      if (scheme == null) {
        throw new InvalidProgramException("File path \"" + path + "\" of FileDataSource has no file system scheme (like 'file:// or hdfs://').");
      }
    } catch (Exception e) {
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
      fail();
      e.printStackTrace();
    }
    writer.invoke();

    final FileInputSplit split = new FileInputSplit(0, new Path(this.file.toURI().toString()), 0,
      this.file.length(), null);
    when(this.environment.getInputSplitProvider()).thenReturn(this.inputSplitProvider);
    when(this.inputSplitProvider.getNextInputSplit()).thenReturn(split, (FileInputSplit) null);

    FileLineReader reader = new FileLineReader();
View Full Code Here
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
    final String pathURI = getTaskConfiguration().getString(INPUT_PATH_CONFIG_KEY, null);
    if (pathURI == null) {
      throw new IOException("The path to the file was not found in the runtime configuration.");
    }

    final Path path;
    try {
      path = new Path(pathURI);
    } catch (Exception iaex) {
      throw new IOException("Invalid file path specifier: ", iaex);
    }

    final List<FileInputSplit> inputSplits = new ArrayList<FileInputSplit>();

    // get all the files that are involved in the splits
    final List<FileStatus> files = new ArrayList<FileStatus>();
    long totalLength = 0;

    final FileSystem fs = path.getFileSystem();
    final FileStatus pathFile = fs.getFileStatus(path);

    if (pathFile.isDir()) {
      // input is directory. list all contained files
      final FileStatus[] dir = fs.listStatus(path);
View Full Code Here
143
144
145
146
147
148
149
150
151
152
153
    // Use the File object to the convert the path to a proper URI
    final File path = new File(tmp + File.separator + LIBRARYCACHENAME + "-" + userName);
    final URI uri = path.toURI();

    this.libraryCachePath = new Path(uri);

    this.fs.mkdirs(this.libraryCachePath);

    // Create an MD5 message digest object we can use
    try {
View Full Code Here
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
      if (requiredJarFiles != null) {

        urls = new URL[requiredJarFiles.length];

        for (int i = 0; i < requiredJarFiles.length; i++) {
          final Path p = contains(requiredJarFiles[i]);
          if (p == null) {
            throw new IOException(requiredJarFiles[i] + " does not exist in the library cache");
          }

          // Add file to the URL array
          try {
            urls[i] = p.toUri().toURL();
          } catch (MalformedURLException e) {
            throw new IOException(StringUtils.stringifyException(e));
          }
        }
      }
View Full Code Here
392
393
394
395
396
397
398
399
400
401
402
   *         thrown if no access to the file system could be obtained
   */
  private Path containsInternal(final String cacheName) throws IOException {

    // Create a path object from the external name string
    final Path p = new Path(this.libraryCachePath + File.separator + cacheName);

    synchronized (this.fs) {
      if (fs.exists(p)) {
        return p;
      }
View Full Code Here
TOP

Related Classes of eu.stratosphere.core.fs.Path

  • eu.stratosphere.api.avro.testjar.AvroExternalJarProgram
  • eu.stratosphere.api.common.cache.DistributedCache
  • eu.stratosphere.api.common.io.BinaryInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormatTest
  • eu.stratosphere.api.common.io.FileInputFormat
  • eu.stratosphere.api.common.io.FileOutputFormat
  • eu.stratosphere.api.common.io.FileOutputFormat$OutputPathOpenThread
  • eu.stratosphere.api.common.io.FileOutputFormatTest
  • eu.stratosphere.api.common.io.FormatUtil
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.