Package org.apache.drill.exec.cache

Examples of org.apache.drill.exec.cache.VectorAccessibleSerializable


    if (outputStream == null) {
      outputStream = fs.create(path);
    }
    int recordCount = newContainer.getRecordCount();
    WritableBatch batch = WritableBatch.getBatchNoHVWrap(recordCount, newContainer, false);
    VectorAccessibleSerializable outputBatch = new VectorAccessibleSerializable(batch, allocator);
    Stopwatch watch = new Stopwatch();
    watch.start();
    outputBatch.writeToStream(outputStream);
    newContainer.zeroVectors();
//    logger.debug("Took {} us to spill {} records", watch.elapsed(TimeUnit.MICROSECONDS), recordCount);
    spilledBatches++;
  }
View Full Code Here


    assert fs != null;
    assert path != null;
    if (inputStream == null) {
      inputStream = fs.open(path);
    }
    VectorAccessibleSerializable vas = new VectorAccessibleSerializable(allocator);
    Stopwatch watch = new Stopwatch();
    watch.start();
    vas.readFromStream(inputStream);
    VectorContainer c = (VectorContainer) vas.get();
//    logger.debug("Took {} us to read {} records", watch.elapsed(TimeUnit.MICROSECONDS), c.getRecordCount());
    spilledBatches--;
    return c;
  }
View Full Code Here

        FileSystem fs = FileSystem.get(conf);
      Path path = new Path(filename);
      assertTrue("Trace file does not exist", fs.exists(path));
      FSDataInputStream in = fs.open(path);

      VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(context.getAllocator());
      wrap.readFromStream(in);
      VectorAccessible container = wrap.get();

        /* Assert there are no selection vectors */
      assertTrue(wrap.getSv2() == null);

        /* Assert there is only one record */
        assertTrue(container.getRecordCount() == 1);

        /* Read the Integer value and ASSERT its Integer.MIN_VALUE */
 
View Full Code Here

    } else {
      sv = null;
    }
    WritableBatch batch = WritableBatch.getBatchNoHVWrap(incoming.getRecordCount(), incoming, incomingHasSv2 ? true
        : false);
    VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(batch, sv, oContext.getAllocator());

    try {
      wrap.writeToStreamAndRetain(fos);
    } catch (IOException e) {
      throw new RuntimeException(e);
    }
    batch.reconstructContainer(container);
    if (incomingHasSv2) {
      sv = wrap.getSv2();
    }
  }
View Full Code Here

    List<Integer> schemaChangeIdx = Lists.newArrayList();

    BatchMetaInfo aggBatchMetaInfo = new BatchMetaInfo();

    while (input.available() > 0) {
      VectorAccessibleSerializable vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
      vcSerializable.readFromStream(input);
       VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();

       aggBatchMetaInfo.add(getBatchMetaInfo(vcSerializable));

       if (vectorContainer.getRecordCount() == 0) {
         emptyBatchNum ++;
View Full Code Here

   * @throws Exception
   */
  protected void doBatch(FileInputStream input, int targetBatchNum, boolean showHeader) throws Exception {
    int batchNum = -1;

    VectorAccessibleSerializable vcSerializable = null;

    while (input.available() > 0 && batchNum ++ < targetBatchNum) {
      vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
      vcSerializable.readFromStream(input);

      if (batchNum != targetBatchNum) {
        VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
        vectorContainer.zeroVectors();
      }
    }

    if (batchNum < targetBatchNum) {
      System.out.println(String.format("Wrong input of batch # ! Total # of batch in the file is %d. Please input a number 0..%d as batch #", batchNum+1, batchNum));
      input.close();
      System.exit(-1);
    }

    if (vcSerializable != null) {
      showSingleBatch(vcSerializable, showHeader);
      VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
      vectorContainer.zeroVectors();
    }
  }
View Full Code Here

        FileSystem fs = FileSystem.get(conf);
      Path path = new Path(filename);
      assertTrue("Trace file does not exist", fs.exists(path));
      FSDataInputStream in = fs.open(path);

      VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(context.getAllocator());
      wrap.readFromStream(in);
      VectorAccessible container = wrap.get();

        /* Assert there are no selection vectors */
      assertTrue(wrap.getSv2() == null);

        /* Assert there is only one record */
        assertTrue(container.getRecordCount() == 1);

        /* Read the Integer value and ASSERT its Integer.MIN_VALUE */
 
View Full Code Here

    List<Integer> schemaChangeIdx = Lists.newArrayList();

    BatchMetaInfo aggBatchMetaInfo = new BatchMetaInfo();

    while (input.available() > 0) {
      VectorAccessibleSerializable vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
      vcSerializable.readFromStream(input);
      VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();

      aggBatchMetaInfo.add(getBatchMetaInfo(vcSerializable));

      if (vectorContainer.getRecordCount() == 0) {
        emptyBatchNum ++;
View Full Code Here

   * @throws Exception
   */
  protected void doBatch(FileInputStream input, int targetBatchNum, boolean showHeader) throws Exception {
    int batchNum = -1;

    VectorAccessibleSerializable vcSerializable = null;

    while (input.available() > 0 && batchNum ++ < targetBatchNum) {
      vcSerializable = new VectorAccessibleSerializable(DumpCat.allocator);
      vcSerializable.readFromStream(input);

      if (batchNum != targetBatchNum) {
        VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
        vectorContainer.zeroVectors();
      }
    }

    if (batchNum < targetBatchNum) {
      System.out.println(String.format("Wrong input of batch # ! Total # of batch in the file is %d. Please input a number 0..%d as batch #", batchNum+1, batchNum));
      input.close();
      System.exit(-1);
    }

    if (vcSerializable != null) {
      showSingleBatch(vcSerializable, showHeader);
      VectorContainer vectorContainer = (VectorContainer) vcSerializable.get();
      vectorContainer.zeroVectors();
    }
  }
View Full Code Here

    if (outputStream == null) {
      outputStream = fs.create(path);
    }
    int recordCount = newContainer.getRecordCount();
    WritableBatch batch = WritableBatch.getBatchNoHVWrap(recordCount, newContainer, false);
    VectorAccessibleSerializable outputBatch = new VectorAccessibleSerializable(batch, allocator);
    Stopwatch watch = new Stopwatch();
    watch.start();
    outputBatch.writeToStream(outputStream);
    newContainer.zeroVectors();
    logger.debug("Took {} us to spill {} records", watch.elapsed(TimeUnit.MICROSECONDS), recordCount);
    spilledBatches++;
  }
View Full Code Here

TOP

Related Classes of org.apache.drill.exec.cache.VectorAccessibleSerializable

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.