Package org.apache.drill.exec.physical.impl.sort

Examples of org.apache.drill.exec.physical.impl.sort.RecordBatchData

54
55
56
57
58
59
60
61
62
63
64
    if (batchBytes + runningBytes > Integer.MAX_VALUE) return false;      // TODO: 2GB is arbitrary
    if (runningBatches++ >= Character.MAX_VALUE) return false;            // allowed in batch.
    if (!svAllocator.preAllocate(batch.getRecordCount()*4)) return false; // sv allocation available.

    // transfer VVs to a new RecordBatchData
    RecordBatchData bd = new RecordBatchData(batch);
    runningBytes += batchBytes;
    queuedRightBatches.put(batch.getSchema(), bd);
    recordCount += bd.getRecordCount();
    return true;
  }
View Full Code Here
59
60
61
62
63
64
65
66
67
68
69
    if (batchBytes + runningBytes > Integer.MAX_VALUE) return false;      // TODO: 2GB is arbitrary
    if (runningBatches++ >= Character.MAX_VALUE) return false;            // allowed in batch.
    if (!svAllocator.preAllocate(batch.getRecordCount()*4)) return false; // sv allocation available.

    // transfer VVs to a new RecordBatchData
    RecordBatchData bd = new RecordBatchData(batch);
    runningBytes += batchBytes;
    queuedRightBatches.put(batch.getSchema(), bd);
    recordCount += bd.getRecordCount();
    return true;
  }
View Full Code Here
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
          sorter.setup(context, sv2, incoming);
          Stopwatch w = new Stopwatch();
          w.start();
          sorter.sort(sv2);
//          logger.debug("Took {} us to sort {} records", w.elapsed(TimeUnit.MICROSECONDS), count);
          RecordBatchData rbd = new RecordBatchData(incoming);
          if (incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.NONE) {
            rbd.setSv2(sv2);
          }
          batchGroups.add(new BatchGroup(rbd.getContainer(), rbd.getSv2()));
          batchesSinceLastSpill++;
          if (batchGroups.size() > SPILL_THRESHOLD && batchesSinceLastSpill >= SPILL_BATCH_GROUP_SIZE) {
            mergeAndSpill();
            batchesSinceLastSpill = 0;
          }
          long t = w.elapsed(TimeUnit.MICROSECONDS);
//          logger.debug("Took {} us to sort {} records", t, count);
          break;
        case OUT_OF_MEMORY:
          if (batchesSinceLastSpill > 2) mergeAndSpill();
          batchesSinceLastSpill = 0;
          break;
        default:
          throw new UnsupportedOperationException();
        }
      }

//      if (schema == null || totalcount == 0){
//        builder may be null at this point if the first incoming batch is empty
//        useIncomingSchema = true;
//        return IterOutcome.NONE;
//      }

      if (spillCount == 0) {
        Stopwatch watch = new Stopwatch();
        watch.start();
//        if (schema == null){
          // builder may be null at this point if the first incoming batch is empty
//          useIncomingSchema = true;
//          return IterOutcome.NONE;
//        }

        builder = new SortRecordBatchBuilder(oContext.getAllocator(), MAX_SORT_BYTES);

        for (BatchGroup group : batchGroups) {
          RecordBatchData rbd = new RecordBatchData(group.getFirstContainer());
          rbd.setSv2(group.getSv2());
          builder.add(rbd);
        }

        builder.build(context, container);
        sv4 = builder.getSv4();
View Full Code Here
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
                    /* Completed hashing all records in this batch. Transfer the batch
                     * to the hyper vector container. Will be used when we want to retrieve
                     * records that have matching keys on the probe side.
                     */
                    RecordBatchData nextBatch = new RecordBatchData(right);
                    if (hyperContainer == null) {
                        hyperContainer = new ExpandableHyperContainer(nextBatch.getContainer());
                    } else {
                        hyperContainer.addBatch(nextBatch.getContainer());
                    }

                    // completed processing a batch, increment batch index
                    buildBatchIndex++;
                    break;
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
174
175
176
          }
          // fall through.
        case OK:
          countSincePurge += incoming.getRecordCount();
          batchCount++;
          RecordBatchData batch = new RecordBatchData(incoming);
          batch.canonicalize();
          if (priorityQueue == null) {
            priorityQueue = createNewPriorityQueue(context, config.getOrderings(), new ExpandableHyperContainer(batch.getContainer()), MAIN_MAPPING, LEFT_MAPPING, RIGHT_MAPPING);
          }
          priorityQueue.add(context, batch);
          if (countSincePurge > config.getLimit() && batchCount > batchPurgeThreshold) {
            purge();
            countSincePurge = 0;
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
            return;
          case OK_NEW_SCHEMA:
          case OK:
            try {
              if (!stop) {
                wrapper = new RecordBatchDataWrapper(new RecordBatchData(incoming), false, false);
                queue.put(wrapper);
              }
            } catch (InterruptedException e) {
              if (!(context.isCancelled() || context.isFailed())) {
                context.fail(e);
View Full Code Here
185
186
187
188
189
190
191
192
193
194
195
196
197
198
          }
          // fall through.
        case OK:
          countSincePurge += incoming.getRecordCount();
          batchCount++;
          RecordBatchData batch = new RecordBatchData(incoming);
          batch.canonicalize();
          if (priorityQueue == null) {
            priorityQueue = createNewPriorityQueue(context, config.getOrderings(), new ExpandableHyperContainer(batch.getContainer()), MAIN_MAPPING, LEFT_MAPPING, RIGHT_MAPPING);
          }
          priorityQueue.add(context, batch);
          if (countSincePurge > config.getLimit() && batchCount > batchPurgeThreshold) {
            purge();
            countSincePurge = 0;
View Full Code Here
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
                    /* Completed hashing all records in this batch. Transfer the batch
                     * to the hyper vector container. Will be used when we want to retrieve
                     * records that have matching keys on the probe side.
                     */
                    RecordBatchData nextBatch = new RecordBatchData(right);
                    if (hyperContainer == null) {
                        hyperContainer = new ExpandableHyperContainer(nextBatch.getContainer());
                    } else {
                        hyperContainer.addBatch(nextBatch.getContainer());
                    }

                    // completed processing a batch, increment batch index
                    buildBatchIndex++;
                    break;
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
    if (!svAllocator.preAllocate(batch.getRecordCount()*4)) {
      return false;     // sv allocation available.
    }

    // transfer VVs to a new RecordBatchData
    RecordBatchData bd = new RecordBatchData(batch);
    runningBytes += batchBytes;
    queuedRightBatches.put(batch.getSchema(), bd);
    recordCount += bd.getRecordCount();
    return true;
  }
View Full Code Here
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
          sorter.setup(context, sv2, incoming);
          Stopwatch w = new Stopwatch();
          w.start();
          sorter.sort(sv2);
//          logger.debug("Took {} us to sort {} records", w.elapsed(TimeUnit.MICROSECONDS), count);
          RecordBatchData rbd = new RecordBatchData(incoming);
          if (incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.NONE) {
            rbd.setSv2(sv2);
          }
          batchGroups.add(new BatchGroup(rbd.getContainer(), rbd.getSv2()));
          batchesSinceLastSpill++;
          if ((spillCount > 0 && totalSizeInMemory > .75 * highWaterMark) ||
                  (totalSizeInMemory > .95 * popConfig.getMaxAllocation()) ||
                  (totalSizeInMemory > .95 * oContext.getAllocator().getFragmentLimit()) ||
                  (batchGroups.size() > SPILL_THRESHOLD && batchesSinceLastSpill >= SPILL_BATCH_GROUP_SIZE)) {
            mergeAndSpill();
            batchesSinceLastSpill = 0;
          }
          long t = w.elapsed(TimeUnit.MICROSECONDS);
//          logger.debug("Took {} us to sort {} records", t, count);
          break;
        case OUT_OF_MEMORY:
          highWaterMark = totalSizeInMemory;
          if (batchesSinceLastSpill > 2) {
            mergeAndSpill();
          }
          batchesSinceLastSpill = 0;
          break;
        default:
          throw new UnsupportedOperationException();
        }
      }

      if (spillCount == 0) {
        Stopwatch watch = new Stopwatch();
        watch.start();

        builder = new SortRecordBatchBuilder(oContext.getAllocator(), MAX_SORT_BYTES);

        for (BatchGroup group : batchGroups) {
          RecordBatchData rbd = new RecordBatchData(group.getContainer());
          rbd.setSv2(group.getSv2());
          builder.add(rbd);
        }

        builder.build(context, container);
        sv4 = builder.getSv4();
View Full Code Here
TOP

Related Classes of org.apache.drill.exec.physical.impl.sort.RecordBatchData

  • org.apache.drill.exec.physical.impl.join.HashJoinBatch
  • org.apache.drill.exec.physical.impl.join.MergeJoinBatchBuilder
  • org.apache.drill.exec.physical.impl.producer.ProducerConsumerBatch$Producer
  • org.apache.drill.exec.physical.impl.TopN.TopNBatch
  • org.apache.drill.exec.physical.impl.xsort.ExternalSortBatch
  • org.apache.drill.exec.record.BatchSchema.SelectionVectorMode
  • org.apache.drill.exec.record.TransferPair
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.