Package org.apache.hadoop.hdfs.DFSClient

Examples of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
    switch(op.getValue()) {
    case OPEN:
    {
      final int b = bufferSize.getValue(conf);
      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
      DFSDataInputStream in = null;
      try {
        in = new DFSClient.DFSDataInputStream(
        dfsclient.open(fullpath, b, true, null));
        in.seek(offset.getValue());
      } catch(IOException ioe) {
        IOUtils.cleanup(LOG, in);
        IOUtils.cleanup(LOG, dfsclient);
        throw ioe;
      }
     
      final long n = length.getValue() != null? length.getValue()
          : in.getVisibleLength();
      return Response.ok(new OpenEntity(in, n, dfsclient)).type(
          MediaType.APPLICATION_OCTET_STREAM).build();
    }
    case GETFILECHECKSUM:
    {
View Full Code Here
249
250
251
252
253
254
255
256
257
258
    fs.delete(root, true);
    files = null;
  }
 
  public static Block getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
  } 
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
    UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username,
                                 new String[] {"supergroup"});
   
    final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
   
    final DFSDataInputStream in = (DFSDataInputStream)fs.open(p);

    //Check visible length
    Assert.assertTrue(in.getVisibleLength() >= expectedsize);

    //Able to read?
    for(int i = 0; i < expectedsize; i++) {
      Assert.assertEquals((byte)i, (byte)in.read()); 
    }

    in.close();
  }
View Full Code Here
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
    switch(op.getValue()) {
    case OPEN:
    {
      final int b = bufferSize.getValue(conf);
      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
      DFSDataInputStream in = null;
      try {
        in = new DFSClient.DFSDataInputStream(
            dfsclient.open(fullpath, b, true));
        in.seek(offset.getValue());
      } catch(IOException ioe) {
        IOUtils.cleanup(LOG, in);
        IOUtils.cleanup(LOG, dfsclient);
        throw ioe;
      }
      final DFSDataInputStream dis = in;
      final StreamingOutput streaming = new StreamingOutput() {
        @Override
        public void write(final OutputStream out) throws IOException {
          final Long n = length.getValue();
          DFSDataInputStream dfsin = dis;
          DFSClient client = dfsclient;
          try {
            if (n == null) {
              IOUtils.copyBytes(dfsin, out, b);
            } else {
              IOUtils.copyBytes(dfsin, out, n, false);
            }
            dfsin.close();
            dfsin = null;
            dfsclient.close();
            client = null;
          } finally {
            IOUtils.cleanup(LOG, dfsin);
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
      assertTrue(status.getLen() == fileSize);
      assertEquals(fs.makeQualified(file1).toString(),
          status.getPath().toString());

      // test getVisbileLen
      DFSDataInputStream fin = (DFSDataInputStream)fs.open(file1);
      assertEquals(status.getLen(), fin.getVisibleLength());
     
      // test listStatus on a file
      FileStatus[] stats = fs.listStatus(file1);
      assertEquals(1, stats.length);
      status = stats[0];
View Full Code Here
220
221
222
223
224
225
226
227
228
229
230
231
232
    switch(op.getValue()) {
    case OPEN:
    {
      final int b = bufferSize.getValue(conf);
      final DFSDataInputStream in = new DFSClient.DFSDataInputStream(
          dfsclient.open(fullpath, b, true, null));
      in.seek(offset.getValue());

      final StreamingOutput streaming = new StreamingOutput() {
        @Override
        public void write(final OutputStream out) throws IOException {
          final Long n = length.getValue();
View Full Code Here
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
    switch(op.getValue()) {
    case OPEN:
    {
      final int b = bufferSize.getValue(conf);
      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
      DFSDataInputStream in = null;
      try {
        in = new DFSClient.DFSDataInputStream(
        dfsclient.open(fullpath, b, true, null));
        in.seek(offset.getValue());
      } catch(IOException ioe) {
        IOUtils.cleanup(LOG, in);
        IOUtils.cleanup(LOG, dfsclient);
        throw ioe;
      }
      final DFSDataInputStream dis = in;
      final StreamingOutput streaming = new StreamingOutput() {
        @Override
        public void write(final OutputStream out) throws IOException {
          final Long n = length.getValue();
          DFSDataInputStream dfsin = dis;
          DFSClient client = dfsclient;
          try {
            if (n == null) {
              IOUtils.copyBytes(dfsin, out, b);
            } else {
              IOUtils.copyBytes(dfsin, out, n, b, false);
            }
            dfsin.close();
            dfsin = null;
            client.close();
            client = null;
          } finally {
            IOUtils.cleanup(LOG, dfsin);
View Full Code Here
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
    switch(op.getValue()) {
    case OPEN:
    {
      final int b = bufferSize.getValue(conf);
      final DFSClient dfsclient = new DFSClient(nnRpcAddr, conf);
      DFSDataInputStream in = null;
      try {
        in = new DFSClient.DFSDataInputStream(
            dfsclient.open(fullpath, b, true));
        in.seek(offset.getValue());
      } catch(IOException ioe) {
        IOUtils.cleanup(LOG, in);
        IOUtils.cleanup(LOG, dfsclient);
        throw ioe;
      }
     
      final long n = length.getValue() != null? length.getValue()
        : in.getVisibleLength() - offset.getValue();
      return Response.ok(new OpenEntity(in, n, dfsclient)).type(
          MediaType.APPLICATION_OCTET_STREAM).build();
    }
    case GETFILECHECKSUM:
    {
View Full Code Here
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
    UserGroupInformation ugi = UserGroupInformation.createUserForTesting(username,
                                 new String[] {"supergroup"});
   
    final FileSystem fs = DFSTestUtil.getFileSystemAs(ugi, conf);
   
    final DFSDataInputStream in = (DFSDataInputStream)fs.open(p);

    //Check visible length
    Assert.assertTrue(in.getVisibleLength() >= expectedsize);

    //Able to read?
    for(int i = 0; i < expectedsize; i++) {
      Assert.assertEquals((byte)i, (byte)in.read()); 
    }

    in.close();
  }
View Full Code Here
505
506
507
508
509
510
511
512
513
514
    fs.delete(root, true);
    files = null;
  }
 
  public static ExtendedBlock getFirstBlock(FileSystem fs, Path path) throws IOException {
    DFSDataInputStream in =
      (DFSDataInputStream) ((DistributedFileSystem)fs).open(path);
    in.readByte();
    return in.getCurrentBlock();
  } 
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.DFSClient.DFSDataInputStream

  • org.apache.hadoop.hdfs.DFSTestUtil
  • org.apache.hadoop.hdfs.server.datanode.web.resources.DatanodeWebHdfsMethods
  • org.apache.hadoop.hdfs.TestAvailableForUnderConstruction
  • org.apache.hadoop.hdfs.TestDFSClientMetrics
  • org.apache.hadoop.hdfs.TestDistributedFileSystem
  • org.apache.hadoop.hdfs.TestFileLengthOnClusterRestart
  • org.apache.hadoop.hdfs.TestFileStatus
  • org.apache.hadoop.hdfs.TestIoprio
  • org.apache.hadoop.hdfs.TestReadWhileWriting
  • org.apache.hadoop.hdfs.TestShortCircuitLocalRead
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.