Package org.apache.hadoop.io.file.tfile.TFile

Examples of org.apache.hadoop.io.file.tfile.TFile.Writer

330
331
332
333
334
335
336
337
338
339
340
341
342
  // unsorted with some codec
  void unsortedWithSomeCodec(String codec) throws IOException {
    Path uTfile = new Path(ROOT, "unsorted.tfile");
    FSDataOutputStream fout = createFSOutput(uTfile);
    Writer writer = new Writer(fout, minBlockSize, codec, null, conf);
    writeRecords(writer);
    writer.close();
    fout.close();
    FSDataInputStream fin = fs.open(uTfile);
    Reader reader =
        new Reader(fs.open(uTfile), fs.getFileStatus(uTfile).getLen(), conf);
View Full Code Here
415
416
417
418
419
420
421
422
423
424
425
426
427
  // test meta blocks for tfiles
  public void testMetaBlocks() throws IOException {
    Path mFile = new Path(ROOT, "meta.tfile");
    FSDataOutputStream fout = createFSOutput(mFile);
    Writer writer = new Writer(fout, minBlockSize, "none", null, conf);
    someTestingWithMetaBlock(writer, "none");
    writer.close();
    fout.close();
    FSDataInputStream fin = fs.open(mFile);
    Reader reader = new Reader(fin, fs.getFileStatus(mFile).getLen(), conf);
    someReadingWithMetaBlock(reader);
    fs.delete(mFile, true);
View Full Code Here
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
  private void createTFile() throws IOException {
    long totalBytes = 0;
    FSDataOutputStream fout = createFSOutput(path, fs);
    try {
      Writer writer =
          new Writer(fout, options.minBlockSize, options.compress, "memcmp",
              conf);
      try {
        BytesWritable key = new BytesWritable();
        BytesWritable val = new BytesWritable();
        timer.start();
        for (long i = 0; true; ++i) {
          if (i % 1000 == 0) { // test the size for every 1000 rows.
            if (fs.getFileStatus(path).getLen() >= options.fileSize) {
              break;
            }
          }
          kvGen.next(key, val, false);
          writer.append(key.get(), 0, key.getSize(), val.get(), 0, val
              .getSize());
          totalBytes += key.getSize();
          totalBytes += val.getSize();
        }
        timer.stop();
      }
      finally {
        writer.close();
      }
    }
    finally {
      fout.close();
    }
View Full Code Here
91
92
93
94
95
96
97
  @Before
  public void setUp() throws IOException {
    path = new Path(ROOT, outputFile);
    fs = path.getFileSystem(conf);
    out = fs.create(path);
    writer = new Writer(out, BLOCK_SIZE, compression, comparator, conf);
  }
View Full Code Here
365
366
367
368
369
370
371
372
373
374
375
    if (skip)
      return;
    closeOutput();
    out = fs.create(path);
    try {
      writer = new Writer(out, BLOCK_SIZE, "BAD", comparator, conf);
      Assert.fail("Error on handling invalid compression codecs.");
    } catch (Exception e) {
      // noop, expecting exceptions
      // e.printStackTrace();
    }
View Full Code Here
562
563
564
565
566
567
568
569
570
571
572
    closeOutput();
    out = fs.create(path);
    out.write(123);

    try {
      writer = new Writer(out, BLOCK_SIZE, compression, comparator, conf);
      Assert.fail("Failed to catch file write not at position 0.");
    } catch (Exception e) {
      // noop, expecting exceptions
    }
    closeOutput();
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
  void createFile(int count, String compress) throws IOException {
    conf = new Configuration();
    path = new Path(ROOT, outputFile + "." + compress);
    fs = path.getFileSystem(conf);
    FSDataOutputStream out = fs.create(path);
    Writer writer = new Writer(out, BLOCK_SIZE, compress, comparator, conf);

    int nx;
    for (nx = 0; nx < count; nx++) {
      byte[] key = composeSortedKey(KEY, count, nx).getBytes();
      byte[] value = (VALUE + nx).getBytes();
      writer.append(key, value);
    }
    writer.close();
    out.close();
  }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.file.tfile.TFile.Writer

  • org.apache.hadoop.io.file.tfile.TestTFile
  • org.apache.hadoop.io.file.tfile.TestTFileByteArrays
  • org.apache.hadoop.io.file.tfile.TestTFileComparator2
  • org.apache.hadoop.io.file.tfile.TestTFileComparators
  • org.apache.hadoop.io.file.tfile.TestTFileSeek
  • org.apache.hadoop.io.file.tfile.TestTFileSplit
  • org.apache.hadoop.io.file.tfile.TestTFileStreams
  • org.apache.hadoop.io.file.tfile.TestTFileUnsortedByteArrays
  • org.apache.pig.impl.io.TFileRecordWriter
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.