Package eu.stratosphere.core.fs

Examples of eu.stratosphere.core.fs.Path

56
57
58
59
60
61
62
63
    LogUtils.initializeDefaultConsoleLogger(Level.WARN);
  }
 
  @Before
  public void setup() {
    this.format.setFilePath(new Path("file:///some/file/that/will/not/be/read"));
    this.config = new Configuration();
  }
View Full Code Here
185
186
187
188
189
190
191
   
    OutputStreamWriter wrt = new OutputStreamWriter(new FileOutputStream(this.tempFile));
    wrt.write(contents);
    wrt.close();
   
    return new FileInputSplit(0, new Path(this.tempFile.toURI().toString()), 0, this.tempFile.length(), new String[] {"localhost"});
  }
View Full Code Here
150
151
152
153
154
155
156

  @Override
  public Path getWorkingDirectory() {

    return new Path(this.s3Uri);
  }
View Full Code Here
743
744
745
746
747
748
749
750
751
752
    try {
      final URI extendedUri = new URI(parentUri.getScheme(),
        ((parentUri.getAuthority() != null) ? parentUri.getAuthority() : ""), extendedPath,
        parentUri.getQuery(), parentUri.getFragment());
      return new Path(extendedUri);
    } catch (URISyntaxException e) {
      throw new IOException(StringUtils.stringifyException(e));
    }
  }
View Full Code Here
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
      + CommonTestUtils.getRandomFilename());

    final File testfile2 = new File(tempdir.getAbsolutePath() + File.separator
      + CommonTestUtils.getRandomFilename());

    final Path pathtotestfile1 = new Path(testfile1.toURI().getPath());
    final Path pathtotestfile2 = new Path(testfile2.toURI().getPath());

    try {
      final LocalFileSystem lfs = new LocalFileSystem();

      final Path pathtotmpdir = new Path(tempdir.toURI().getPath());

      /*
       * check that lfs can see/create/delete/read directories
       */

 
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
      Class<F> inputFormatClass, String path, Configuration configuration)
    throws IOException
  {
    configuration = configuration == null ? new Configuration() : configuration;

    Path normalizedPath = normalizePath(new Path(path));
    final F inputFormat = ReflectionUtil.newInstance(inputFormatClass);

    inputFormat.setFilePath(normalizedPath);
    inputFormat.setOpenTimeout(0);
    inputFormat.configure(configuration);

    final FileSystem fs = FileSystem.get(normalizedPath.toUri());
    FileStatus fileStatus = fs.getFileStatus(normalizedPath);

    BlockLocation[] blocks = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
    inputFormat.open(new FileInputSplit(0, new Path(path), 0, fileStatus.getLen(), blocks[0].getHosts()));
    return inputFormat;
  }
View Full Code Here
90
91
92
93
94
95
96
97
98
99
100
101
   *         if an I/O error occurred while accessing the files or initializing the InputFormat.
   */
  @SuppressWarnings("unchecked")
  public static <T, F extends FileInputFormat<T>> List<F> openAllInputs(
      Class<F> inputFormatClass, String path, Configuration configuration) throws IOException {
    Path nephelePath = new Path(path);
    FileSystem fs = nephelePath.getFileSystem();
    FileStatus fileStatus = fs.getFileStatus(nephelePath);
    if (!fileStatus.isDir()) {
      return Arrays.asList(openInput(inputFormatClass, path, configuration));
    }
    FileStatus[] list = fs.listStatus(nephelePath);
View Full Code Here
150
151
152
153
154
155
156
157
158
159
160
  public static <T, F extends FileOutputFormat<? extends T>> F openOutput(
      Class<F> outputFormatClass, String path, Configuration configuration)
    throws IOException
  {
    final F outputFormat = ReflectionUtil.newInstance(outputFormatClass);
    outputFormat.setOutputFilePath(new Path(path));
    outputFormat.setOpenTimeout(0);
    outputFormat.setWriteMode(WriteMode.OVERWRITE);
 
    configuration = configuration == null ? new Configuration() : configuration;
   
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
  private static Path normalizePath(Path path) {
    URI uri = path.toUri();
    if (uri.getScheme() == null) {
      try {
        uri = new URI("file", uri.getHost(), uri.getPath(), uri.getFragment());
        path = new Path(uri.toString());
      } catch (URISyntaxException e) {
        throw new IllegalArgumentException("path is invalid", e);
      }
    }
    return path;
View Full Code Here
144
145
146
147
148
149
150
151
152
153
154
155
156
157
    final FileBaseStatistics cachedFileStats = (cachedStats != null && cachedStats instanceof FileBaseStatistics) ?
      (FileBaseStatistics) cachedStats : null;

    try {
      final Path filePath = this.filePath;

      // get the filesystem
      final FileSystem fs = FileSystem.get(filePath.toUri());
      final ArrayList<FileStatus> allFiles = new ArrayList<FileStatus>(1);

      // let the file input format deal with the up-to-date check and the basic size
      final FileBaseStatistics stats = getFileStats(cachedFileStats, filePath, fs, allFiles);
      if (stats == null) {
View Full Code Here
TOP

Related Classes of eu.stratosphere.core.fs.Path

  • eu.stratosphere.api.avro.testjar.AvroExternalJarProgram
  • eu.stratosphere.api.common.cache.DistributedCache
  • eu.stratosphere.api.common.io.BinaryInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormatTest
  • eu.stratosphere.api.common.io.FileInputFormat
  • eu.stratosphere.api.common.io.FileOutputFormat
  • eu.stratosphere.api.common.io.FileOutputFormat$OutputPathOpenThread
  • eu.stratosphere.api.common.io.FileOutputFormatTest
  • eu.stratosphere.api.common.io.FormatUtil
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.