Package eu.stratosphere.core.fs

Examples of eu.stratosphere.core.fs.Path

515
516
517
518
519
520
521
522
523
524
525
    if (libraryFileName == null) {
      throw new IOException("libraryName is null!");
    }

    final Path storePath = new Path(this.libraryCachePath + "/" + libraryFileName);

    synchronized (this.fs) {

      if (!fs.exists(storePath)) {
        throw new IOException(storePath + " does not exist!");
View Full Code Here
580
581
582
583
584
585
586
587
588
589
590
    }

    final byte[] buf = new byte[(int) length];
    in.readFully(buf);

    final Path storePath = new Path(this.libraryCachePath + "/" + libraryFileName);

    synchronized (this.fs) {

      // Check if file already exists in our library cache, if not write it to the cache directory
      if (!fs.exists(storePath)) {
View Full Code Here
648
649
650
651
652
653
654
655
656
657
658
    this.md.reset();
    this.md.update(buf);

    // Construct internal jar name from digest
    final String cacheName = StringUtils.byteToHexString(md.digest()) + ".jar";
    final Path storePath = new Path(this.libraryCachePath + "/" + cacheName);

    synchronized (this.fs) {

      // Check if file already exists in our library cache, if not write it to the cache directory
      if (!this.fs.exists(storePath)) {
View Full Code Here
292
293
294
295
296
297
298
299
300
301
302
303
304
   * Set minNumSplits to number of files.
   */
  @Override
  public FileInputSplit[] createInputSplits(int minNumSplits) throws IOException {
    int numAvroFiles = 0;
    final Path path = this.filePath;
    // get all the files that are involved in the splits
    final FileSystem fs = path.getFileSystem();
    final FileStatus pathFile = fs.getFileStatus(path);

    if (!acceptFile(pathFile)) {
      throw new IOException("The given file does not pass the file-filter");
    }
View Full Code Here
57
58
59
60
61
62
63
    this.path = filePath;
    this.executionContext = executionContext;
  }
 
  public CsvReader(String filePath, ExecutionEnvironment executionContext) {
    this(new Path(Validate.notNull(filePath, "The file path may not be null.")), executionContext);
  }
View Full Code Here
141
142
143
144
145
146
147
148
149
150
151
      TaskConfig taskConfig = new TaskConfig(output.getConfiguration());
      taskConfig.addInputToGroup(0);
      taskConfig.setInputSerializer(serializer, 0);

      PointOutFormat outFormat = new PointOutFormat();
      outFormat.setOutputFilePath(new Path(resultPath));
     
      taskConfig.setStubWrapper(new UserCodeObjectWrapper<PointOutFormat>(outFormat));
    }

    return output;
View Full Code Here
173
174
175
176
177
178
179
   * @return A DataSet that represents the data read from the given file as text lines.
   */
  public DataSource<String> readTextFile(String filePath) {
    Validate.notNull(filePath, "The file path may not be null.");
   
    return new DataSource<String>(this, new TextInputFormat(new Path(filePath)), BasicTypeInfo.STRING_TYPE_INFO );
  }
View Full Code Here
187
188
189
190
191
192
193
194
195
   * @return A DataSet that represents the data read from the given file as text lines.
   */
  public DataSource<String> readTextFile(String filePath, String charsetName) {
    Validate.notNull(filePath, "The file path may not be null.");

    TextInputFormat format = new TextInputFormat(new Path(filePath));
    format.setCharsetName(charsetName);
    return new DataSource<String>(this, format, BasicTypeInfo.STRING_TYPE_INFO );
  }
View Full Code Here
208
209
210
211
212
213
214
   * @return A DataSet that represents the data read from the given file as text lines.
   */
  public DataSource<StringValue> readTextFileWithValue(String filePath) {
    Validate.notNull(filePath, "The file path may not be null.");
   
    return new DataSource<StringValue>(this, new TextValueInputFormat(new Path(filePath)), new ValueTypeInfo<StringValue>(StringValue.class) );
  }
View Full Code Here
228
229
230
231
232
233
234
235
236
237
   * @return A DataSet that represents the data read from the given file as text lines.
   */
  public DataSource<StringValue> readTextFileWithValue(String filePath, String charsetName, boolean skipInvalidLines) {
    Validate.notNull(filePath, "The file path may not be null.");
   
    TextValueInputFormat format = new TextValueInputFormat(new Path(filePath));
    format.setCharsetName(charsetName);
    format.setSkipInvalidLines(skipInvalidLines);
    return new DataSource<StringValue>(this, format, new ValueTypeInfo<StringValue>(StringValue.class) );
  }
View Full Code Here
TOP

Related Classes of eu.stratosphere.core.fs.Path

  • eu.stratosphere.api.avro.testjar.AvroExternalJarProgram
  • eu.stratosphere.api.common.cache.DistributedCache
  • eu.stratosphere.api.common.io.BinaryInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormat
  • eu.stratosphere.api.common.io.DelimitedInputFormatTest
  • eu.stratosphere.api.common.io.FileInputFormat
  • eu.stratosphere.api.common.io.FileOutputFormat
  • eu.stratosphere.api.common.io.FileOutputFormat$OutputPathOpenThread
  • eu.stratosphere.api.common.io.FileOutputFormatTest
  • eu.stratosphere.api.common.io.FormatUtil
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.