/***********************************************************************************************************************
* Copyright (C) 2010-2013 by the Stratosphere project (http://stratosphere.eu)
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
**********************************************************************************************************************/
package eu.stratosphere.pact.runtime.hash;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import eu.stratosphere.api.common.typeutils.TypeComparator;
import eu.stratosphere.api.common.typeutils.TypePairComparator;
import eu.stratosphere.api.common.typeutils.TypeSerializer;
import eu.stratosphere.core.memory.MemorySegment;
import eu.stratosphere.core.memory.MemorySegmentSource;
import eu.stratosphere.core.memory.SeekableDataOutputView;
import eu.stratosphere.nephele.services.iomanager.BlockChannelReader;
import eu.stratosphere.nephele.services.iomanager.BulkBlockChannelReader;
import eu.stratosphere.nephele.services.iomanager.Channel;
import eu.stratosphere.nephele.services.iomanager.ChannelReaderInputView;
import eu.stratosphere.nephele.services.iomanager.HeaderlessChannelReaderInputView;
import eu.stratosphere.nephele.services.iomanager.IOManager;
import eu.stratosphere.pact.runtime.io.ChannelReaderInputViewIterator;
import eu.stratosphere.pact.runtime.iterative.io.HashPartitionIterator;
import eu.stratosphere.pact.runtime.util.MathUtils;
import eu.stratosphere.util.MutableObjectIterator;
/**
* An implementation of a Hybrid Hash Join. The join starts operating in memory and gradually starts
* spilling contents to disk, when the memory is not sufficient. It does not need to know a priori
* how large the input will be.
* <p>
* The design of this class follows on many parts the design presented in
* "Hash joins and hash teams in Microsoft SQL Server", by Goetz Graefe et al. In its current state, the
* implementation lacks features like dynamic role reversal, partition tuning, or histogram guided partitioning.
*<p>
*
*
* <hr>
*
* The layout of the buckets inside a memory segment is as follows:
*
* <pre>
* +----------------------------- Bucket x ----------------------------
* |Partition (1 byte) | Status (1 byte) | element count (2 bytes) |
* | next-bucket-in-chain-pointer (8 bytes) | reserved (4 bytes) |
* |
* |hashCode 1 (4 bytes) | hashCode 2 (4 bytes) | hashCode 3 (4 bytes) |
* | ... hashCode n-1 (4 bytes) | hashCode n (4 bytes)
* |
* |pointer 1 (8 bytes) | pointer 2 (8 bytes) | pointer 3 (8 bytes) |
* | ... pointer n-1 (8 bytes) | pointer n (8 bytes)
* |
* +---------------------------- Bucket x + 1--------------------------
* |Partition (1 byte) | Status (1 byte) | element count (2 bytes) |
* | next-bucket-in-chain-pointer (8 bytes) | reserved (4 bytes) |
* |
* |hashCode 1 (4 bytes) | hashCode 2 (4 bytes) | hashCode 3 (4 bytes) |
* | ... hashCode n-1 (4 bytes) | hashCode n (4 bytes)
* |
* |pointer 1 (8 bytes) | pointer 2 (8 bytes) | pointer 3 (8 bytes) |
* | ... pointer n-1 (8 bytes) | pointer n (8 bytes)
* +-------------------------------------------------------------------
* | ...
* |
* </pre>
*
* @param <BT> The type of records from the build side that are stored in the hash table.
* @param <PT> The type of records from the probe side that are stored in the hash table.
*/
public class MutableHashTable<BT, PT> implements MemorySegmentSource {
private static final Log LOG = LogFactory.getLog(MutableHashTable.class);
// ------------------------------------------------------------------------
// Internal Constants
// ------------------------------------------------------------------------
/**
* The maximum number of recursive partitionings that the join does before giving up.
*/
private static final int MAX_RECURSION_DEPTH = 3;
/**
* The minimum number of memory segments the hash join needs to be supplied with in order to work.
*/
private static final int MIN_NUM_MEMORY_SEGMENTS = 33;
/**
* The maximum number of partitions, which defines the spilling granularity. Each recursion, the
* data is divided maximally into that many partitions, which are processed in one chuck.
*/
private static final int MAX_NUM_PARTITIONS = Byte.MAX_VALUE;
/**
* The default record width that is used when no width is given. The record width is
* used to determine the ratio of the number of memory segments intended for partition
* buffers and the number of memory segments in the hash-table structure.
*/
private static final int DEFAULT_RECORD_LEN = 24;
/**
* The length of the hash code stored in the bucket.
*/
private static final int HASH_CODE_LEN = 4;
/**
* The length of a pointer from a hash bucket to the record in the buffers.
*/
private static final int POINTER_LEN = 8;
/**
* The number of bytes for the serialized record length in the partition buffers.
*/
private static final int SERIALIZED_LENGTH_FIELD_BYTES = 0;
/**
* The number of bytes that the entry in the hash structure occupies, in bytes.
* It corresponds to a 4 byte hash value and an 8 byte pointer.
*/
private static final int RECORD_TABLE_BYTES = HASH_CODE_LEN + POINTER_LEN;
/**
* The total storage overhead per record, in bytes. This corresponds to the space in the
* actual hash table buckets, consisting of a 4 byte hash value and an 8 byte
* pointer, plus the overhead for the stored length field.
*/
private static final int RECORD_OVERHEAD_BYTES = RECORD_TABLE_BYTES + SERIALIZED_LENGTH_FIELD_BYTES;
// -------------------------- Bucket Size and Structure -------------------------------------
static final int NUM_INTRA_BUCKET_BITS = 7;
static final int HASH_BUCKET_SIZE = 0x1 << NUM_INTRA_BUCKET_BITS;
static final int BUCKET_HEADER_LENGTH = 16;
private static final int NUM_ENTRIES_PER_BUCKET = (HASH_BUCKET_SIZE - BUCKET_HEADER_LENGTH) / RECORD_OVERHEAD_BYTES;
private static final int BUCKET_POINTER_START_OFFSET = BUCKET_HEADER_LENGTH + (NUM_ENTRIES_PER_BUCKET * HASH_CODE_LEN);
// ------------------------------ Bucket Header Fields ------------------------------
/**
* Offset of the field in the bucket header indicating the bucket's partition.
*/
private static final int HEADER_PARTITION_OFFSET = 0;
/**
* Offset of the field in the bucket header indicating the bucket's status (spilled or in-memory).
*/
private static final int HEADER_STATUS_OFFSET = 1;
/**
* Offset of the field in the bucket header indicating the bucket's element count.
*/
private static final int HEADER_COUNT_OFFSET = 2;
/**
* Offset of the field in the bucket header that holds the forward pointer to its
* first overflow bucket.
*/
private static final int HEADER_FORWARD_OFFSET = 4;
/**
* Constant for the forward pointer, indicating that the pointer is not set.
*/
private static final long BUCKET_FORWARD_POINTER_NOT_SET = ~0x0L;
// private static final byte BUCKET_STATUS_SPILLED = 1;
/**
* Constant for the bucket status, indicating that the bucket is in memory.
*/
private static final byte BUCKET_STATUS_IN_MEMORY = 0;
// ------------------------------------------------------------------------
// Members
// ------------------------------------------------------------------------
/**
* The utilities to serialize the build side data types.
*/
protected final TypeSerializer<BT> buildSideSerializer;
/**
* The utilities to serialize the probe side data types.
*/
protected final TypeSerializer<PT> probeSideSerializer;
/**
* The utilities to hash and compare the build side data types.
*/
protected final TypeComparator<BT> buildSideComparator;
/**
* The utilities to hash and compare the probe side data types.
*/
private final TypeComparator<PT> probeSideComparator;
/**
* The comparator used to determine (in)equality between probe side and build side records.
*/
private final TypePairComparator<PT, BT> recordComparator;
/**
* The free memory segments currently available to the hash join.
*/
protected final List<MemorySegment> availableMemory;
/**
* The queue of buffers that can be used for write-behind. Any buffer that is written
* asynchronously to disk is returned through this queue. hence, it may sometimes contain more
*/
protected final LinkedBlockingQueue<MemorySegment> writeBehindBuffers;
/**
* The I/O manager used to instantiate writers for the spilled partitions.
*/
protected final IOManager ioManager;
/**
* The size of the segments used by the hash join buckets. All segments must be of equal size to ease offset computations.
*/
protected final int segmentSize;
/**
* The total number of memory segments available to the hash join.
*/
private final int totalNumBuffers;
/**
* The number of write-behind buffers used.
*/
private final int numWriteBehindBuffers;
/**
* The number of hash table buckets in a single memory segment - 1.
* Because memory segments can be comparatively large, we fit multiple buckets into one memory segment.
* This variable is a mask that is 1 in the lower bits that define the number of a bucket
* in a segment.
*/
protected final int bucketsPerSegmentMask;
/**
* The number of bits that describe the position of a bucket in a memory segment. Computed as log2(bucketsPerSegment).
*/
protected final int bucketsPerSegmentBits;
/**
* An estimate for the average record length.
*/
private final int avgRecordLen;
// ------------------------------------------------------------------------
/**
* The partitions that are built by processing the current partition.
*/
protected final ArrayList<HashPartition<BT, PT>> partitionsBeingBuilt;
/**
* The partitions that have been spilled previously and are pending to be processed.
*/
private final ArrayList<HashPartition<BT, PT>> partitionsPending;
/**
* Iterator over the elements in the hash table.
*/
private HashBucketIterator<BT, PT> bucketIterator;
// private LazyHashBucketIterator<BT, PT> lazyBucketIterator;
/**
* Iterator over the elements from the probe side.
*/
protected ProbeIterator<PT> probeIterator;
/**
* The reader for the spilled-file of the probe partition that is currently read.
*/
private BlockChannelReader currentSpilledProbeSide;
/**
* The channel enumerator that is used while processing the current partition to create
* channels for the spill partitions it requires.
*/
protected Channel.Enumerator currentEnumerator;
/**
* The array of memory segments that contain the buckets which form the actual hash-table
* of hash-codes and pointers to the elements.
*/
protected MemorySegment[] buckets;
/**
* The number of buckets in the current table. The bucket array is not necessarily fully
* used, when not all buckets that would fit into the last segment are actually used.
*/
protected int numBuckets;
/**
* The number of buffers in the write behind queue that are actually not write behind buffers,
* but regular buffers that only have not yet returned. This is part of an optimization that the
* spilling code needs not wait until the partition is completely spilled before proceeding.
*/
protected int writeBehindBuffersAvailable;
/**
* The recursion depth of the partition that is currently processed. The initial table
* has a recursion depth of 0. Partitions spilled from a table that is built for a partition
* with recursion depth <i>n</i> have a recursion depth of <i>n+1</i>.
*/
protected int currentRecursionDepth;
/**
* Flag indicating that the closing logic has been invoked.
*/
protected AtomicBoolean closed = new AtomicBoolean();
/**
* If true, build side partitions are kept for multiple probe steps.
*/
protected boolean keepBuildSidePartitions = false;
protected boolean furtherPartitioning = false;
private boolean running = true;
// ------------------------------------------------------------------------
// Construction and Teardown
// ------------------------------------------------------------------------
public MutableHashTable(TypeSerializer<BT> buildSideSerializer, TypeSerializer<PT> probeSideSerializer,
TypeComparator<BT> buildSideComparator, TypeComparator<PT> probeSideComparator,
TypePairComparator<PT, BT> comparator, List<MemorySegment> memorySegments, IOManager ioManager)
{
this(buildSideSerializer, probeSideSerializer, buildSideComparator, probeSideComparator, comparator,
memorySegments, ioManager, DEFAULT_RECORD_LEN);
}
public MutableHashTable(TypeSerializer<BT> buildSideSerializer, TypeSerializer<PT> probeSideSerializer,
TypeComparator<BT> buildSideComparator, TypeComparator<PT> probeSideComparator,
TypePairComparator<PT, BT> comparator, List<MemorySegment> memorySegments,
IOManager ioManager, int avgRecordLen)
{
// some sanity checks first
if (memorySegments == null) {
throw new NullPointerException();
}
if (memorySegments.size() < MIN_NUM_MEMORY_SEGMENTS) {
throw new IllegalArgumentException("Too few memory segments provided. Hash Join needs at least " +
MIN_NUM_MEMORY_SEGMENTS + " memory segments.");
}
// assign the members
this.buildSideSerializer = buildSideSerializer;
this.probeSideSerializer = probeSideSerializer;
this.buildSideComparator = buildSideComparator;
this.probeSideComparator = probeSideComparator;
this.recordComparator = comparator;
this.availableMemory = memorySegments;
this.ioManager = ioManager;
this.avgRecordLen = avgRecordLen > 0 ? avgRecordLen :
buildSideSerializer.getLength() == -1 ? DEFAULT_RECORD_LEN : buildSideSerializer.getLength();
// check the size of the first buffer and record it. all further buffers must have the same size.
// the size must also be a power of 2
this.totalNumBuffers = memorySegments.size();
this.segmentSize = memorySegments.get(0).size();
if ( (this.segmentSize & this.segmentSize - 1) != 0) {
throw new IllegalArgumentException("Hash Table requires buffers whose size is a power of 2.");
}
int bucketsPerSegment = this.segmentSize >> NUM_INTRA_BUCKET_BITS;
if (bucketsPerSegment == 0) {
throw new IllegalArgumentException("Hash Table requires buffers of at least " + HASH_BUCKET_SIZE + " bytes.");
}
this.bucketsPerSegmentMask = bucketsPerSegment - 1;
this.bucketsPerSegmentBits = MathUtils.log2strict(bucketsPerSegment);
// take away the write behind buffers
this.writeBehindBuffers = new LinkedBlockingQueue<MemorySegment>();
this.numWriteBehindBuffers = getNumWriteBehindBuffers(memorySegments.size());
this.partitionsBeingBuilt = new ArrayList<HashPartition<BT, PT>>();
this.partitionsPending = new ArrayList<HashPartition<BT, PT>>();
// because we allow to open and close multiple times, the state is initially closed
this.closed.set(true);
}
// ------------------------------------------------------------------------
// Life-Cycle
// ------------------------------------------------------------------------
/**
* Opens the hash join. This method reads the build-side input and constructs the initial
* hash table, gradually spilling partitions that do not fit into memory.
*
* @throws IOException Thrown, if an I/O problem occurs while spilling a partition.
*/
public void open(final MutableObjectIterator<BT> buildSide, final MutableObjectIterator<PT> probeSide)
throws IOException
{
// sanity checks
if (!this.closed.compareAndSet(true, false)) {
throw new IllegalStateException("Hash Join cannot be opened, because it is currently not closed.");
}
// grab the write behind buffers first
for (int i = this.numWriteBehindBuffers; i > 0; --i)
{
this.writeBehindBuffers.add(this.availableMemory.remove(this.availableMemory.size() - 1));
}
// open builds the initial table by consuming the build-side input
this.currentRecursionDepth = 0;
buildInitialTable(buildSide);
// the first prober is the probe-side input
this.probeIterator = new ProbeIterator<PT>(probeSide, this.probeSideSerializer.createInstance());
// the bucket iterator can remain constant over the time
this.bucketIterator = new HashBucketIterator<BT, PT>(this.buildSideSerializer, this.recordComparator);
}
protected boolean processProbeIter() throws IOException{
final ProbeIterator<PT> probeIter = this.probeIterator;
final TypeComparator<PT> probeAccessors = this.probeSideComparator;
PT next;
while ((next = probeIter.next()) != null) {
final int hash = hash(probeAccessors.hash(next), this.currentRecursionDepth);
final int posHashCode = hash % this.numBuckets;
// get the bucket for the given hash code
final int bucketArrayPos = posHashCode >> this.bucketsPerSegmentBits;
final int bucketInSegmentOffset = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS;
final MemorySegment bucket = this.buckets[bucketArrayPos];
// get the basic characteristics of the bucket
final int partitionNumber = bucket.get(bucketInSegmentOffset + HEADER_PARTITION_OFFSET);
final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(partitionNumber);
// for an in-memory partition, process set the return iterators, else spill the probe records
if (p.isInMemory()) {
this.recordComparator.setReference(next);
this.bucketIterator.set(bucket, p.overflowSegments, p, hash, bucketInSegmentOffset);
return true;
}
else {
p.insertIntoProbeBuffer(next);
}
}
// -------------- partition done ---------------
return false;
}
protected boolean prepareNextPartition() throws IOException {
// finalize and cleanup the partitions of the current table
int buffersAvailable = 0;
for (int i = 0; i < this.partitionsBeingBuilt.size(); i++) {
final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(i);
p.setFurtherPatitioning(this.furtherPartitioning);
buffersAvailable += p.finalizeProbePhase(this.availableMemory, this.partitionsPending);
}
this.partitionsBeingBuilt.clear();
this.writeBehindBuffersAvailable += buffersAvailable;
releaseTable();
if (this.currentSpilledProbeSide != null) {
this.currentSpilledProbeSide.closeAndDelete();
this.currentSpilledProbeSide = null;
}
// check if there are pending partitions
if (!this.partitionsPending.isEmpty())
{
final HashPartition<BT, PT> p = this.partitionsPending.get(0);
// build the next table
buildTableFromSpilledPartition(p);
// set the probe side - gather memory segments for reading
LinkedBlockingQueue<MemorySegment> returnQueue = new LinkedBlockingQueue<MemorySegment>();
this.currentSpilledProbeSide = this.ioManager.createBlockChannelReader(p.getProbeSideChannel().getChannelID(), returnQueue);
List<MemorySegment> memory = new ArrayList<MemorySegment>();
memory.add(getNextBuffer());
memory.add(getNextBuffer());
ChannelReaderInputViewIterator<PT> probeReader = new ChannelReaderInputViewIterator<PT>(this.currentSpilledProbeSide,
returnQueue, memory, this.availableMemory, this.probeSideSerializer, p.getProbeSideBlockCount());
this.probeIterator.set(probeReader);
// unregister the pending partition
this.partitionsPending.remove(0);
this.currentRecursionDepth = p.getRecursionLevel() + 1;
// recursively get the next
return nextRecord();
}
else {
// no more data
return false;
}
}
/**
* @return
* @throws IOException
*/
public boolean nextRecord() throws IOException {
final boolean probeProcessing = processProbeIter();
if(probeProcessing) {
return true;
}
return prepareNextPartition();
}
public HashBucketIterator<BT, PT> getMatchesFor(PT record) throws IOException
{
final TypeComparator<PT> probeAccessors = this.probeSideComparator;
final int hash = hash(probeAccessors.hash(record), this.currentRecursionDepth);
final int posHashCode = hash % this.numBuckets;
// get the bucket for the given hash code
final int bucketArrayPos = posHashCode >> this.bucketsPerSegmentBits;
final int bucketInSegmentOffset = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS;
final MemorySegment bucket = this.buckets[bucketArrayPos];
// get the basic characteristics of the bucket
final int partitionNumber = bucket.get(bucketInSegmentOffset + HEADER_PARTITION_OFFSET);
final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(partitionNumber);
// for an in-memory partition, process set the return iterators, else spill the probe records
if (p.isInMemory()) {
this.recordComparator.setReference(record);
this.bucketIterator.set(bucket, p.overflowSegments, p, hash, bucketInSegmentOffset);
return this.bucketIterator;
}
else {
throw new IllegalStateException("Method is not applicable to partially spilled hash tables.");
}
}
// public LazyHashBucketIterator<BT, PT> getLazyMatchesFor(PT record) throws IOException
// {
// final TypeComparator<PT> probeAccessors = this.probeSideComparator;
// final int hash = hash(probeAccessors.hash(record), this.currentRecursionDepth);
// final int posHashCode = hash % this.numBuckets;
//
// // get the bucket for the given hash code
// final int bucketArrayPos = posHashCode >> this.bucketsPerSegmentBits;
// final int bucketInSegmentOffset = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS;
// final MemorySegment bucket = this.buckets[bucketArrayPos];
//
// // get the basic characteristics of the bucket
// final int partitionNumber = bucket.get(bucketInSegmentOffset + HEADER_PARTITION_OFFSET);
// final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(partitionNumber);
//
// // for an in-memory partition, process set the return iterators, else spill the probe records
// if (p.isInMemory()) {
// this.recordComparator.setReference(record);
// this.lazyBucketIterator.set(bucket, p.overflowSegments, p, hash, bucketInSegmentOffset);
// return this.lazyBucketIterator;
// }
// else {
// throw new IllegalStateException("Method is not applicable to partially spilled hash tables.");
// }
// }
/**
* @return
*/
public PT getCurrentProbeRecord() {
return this.probeIterator.getCurrent();
}
/**
* @return
*/
public HashBucketIterator<BT, PT> getBuildSideIterator() {
return this.bucketIterator;
}
public MutableObjectIterator<BT> getPartitionEntryIterator() {
return new HashPartitionIterator<BT, PT>(this.partitionsBeingBuilt.iterator(), this.buildSideSerializer);
}
/**
* Closes the hash table. This effectively releases all internal structures and closes all
* open files and removes them. The call to this method is valid both as a cleanup after the
* complete inputs were properly processed, and as an cancellation call, which cleans up
* all resources that are currently held by the hash join.
*/
public void close() {
// make sure that we close only once
if (!this.closed.compareAndSet(false, true)) {
return;
}
// clear the iterators, so the next call to next() will notice
this.bucketIterator = null;
this.probeIterator = null;
// release the table structure
releaseTable();
// clear the memory in the partitions
clearPartitions();
// clear the current probe side channel, if there is one
if (this.currentSpilledProbeSide != null) {
try {
this.currentSpilledProbeSide.closeAndDelete();
}
catch (Throwable t) {
LOG.warn("Could not close and delete the temp file for the current spilled partition probe side.", t);
}
}
// clear the partitions that are still to be done (that have files on disk)
for (int i = 0; i < this.partitionsPending.size(); i++) {
final HashPartition<BT, PT> p = this.partitionsPending.get(i);
p.clearAllMemory(this.availableMemory);
}
// return the write-behind buffers
for (int i = 0; i < this.numWriteBehindBuffers + this.writeBehindBuffersAvailable; i++) {
try {
this.availableMemory.add(this.writeBehindBuffers.take());
}
catch (InterruptedException iex) {
throw new RuntimeException("Hashtable closing was interrupted");
}
}
}
public void abort() {
this.running = false;
}
public List<MemorySegment> getFreedMemory() {
if (!this.closed.get()) {
throw new IllegalStateException("Cannot return memory while join is open.");
}
return this.availableMemory;
}
// ------------------------------------------------------------------------
// Hash Table Building
// ------------------------------------------------------------------------
/**
* @param input
* @throws IOException
*/
protected void buildInitialTable(final MutableObjectIterator<BT> input)
throws IOException
{
// create the partitions
final int partitionFanOut = getPartitioningFanOutNoEstimates(this.availableMemory.size());
if (partitionFanOut > MAX_NUM_PARTITIONS) {
throw new RuntimeException("Hash join partitions estimate exeeds maximum number of partitions.");
}
createPartitions(partitionFanOut, 0);
// set up the table structure. the write behind buffers are taken away, as are one buffer per partition
final int numBuckets = getInitialTableSize(this.availableMemory.size(), this.segmentSize,
partitionFanOut, this.avgRecordLen);
initTable(numBuckets, (byte) partitionFanOut);
final TypeComparator<BT> buildTypeComparator = this.buildSideComparator;
BT record = this.buildSideSerializer.createInstance();
// go over the complete input and insert every element into the hash table
while (this.running && ((record = input.next(record)) != null)) {
final int hashCode = hash(buildTypeComparator.hash(record), 0);
insertIntoTable(record, hashCode);
}
if (!this.running) {
return;
}
// finalize the partitions
for (int i = 0; i < this.partitionsBeingBuilt.size(); i++) {
HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(i);
p.finalizeBuildPhase(this.ioManager, this.currentEnumerator, this.writeBehindBuffers);
}
}
/**
* @param p
* @throws IOException
*/
protected void buildTableFromSpilledPartition(final HashPartition<BT, PT> p) throws IOException {
final int nextRecursionLevel = p.getRecursionLevel() + 1;
if (nextRecursionLevel > MAX_RECURSION_DEPTH) {
throw new RuntimeException("Hash join exceeded maximum number of recursions, without reducing "
+ "partitions enough to be memory resident. Probably cause: Too many duplicate keys.");
}
// we distinguish two cases here:
// 1) The partition fits entirely into main memory. That is the case if we have enough buffers for
// all partition segments, plus enough buffers to hold the table structure.
// --> We read the partition in as it is and create a hashtable that references only
// that single partition.
// 2) We can not guarantee that enough memory segments are available and read the partition
// in, distributing its data among newly created partitions.
final int totalBuffersAvailable = this.availableMemory.size() + this.writeBehindBuffersAvailable;
if (totalBuffersAvailable != this.totalNumBuffers - this.numWriteBehindBuffers) {
throw new RuntimeException("Hash Join bug in memory management: Memory buffers leaked.");
}
long numBuckets = (p.getBuildSideRecordCount() * RECORD_TABLE_BYTES) / (HASH_BUCKET_SIZE - BUCKET_HEADER_LENGTH) + 1;
// we need to consider the worst case where everything hashes to one bucket which needs to overflow by the same
// number of total buckets again.
final long totalBuffersNeeded = (numBuckets * 2) / (this.bucketsPerSegmentMask + 1) + p.getBuildSideBlockCount() + 1;
if (totalBuffersNeeded < totalBuffersAvailable) {
// we are guaranteed to stay in memory
ensureNumBuffersReturned(p.getBuildSideBlockCount());
// first read the partition in
final BulkBlockChannelReader reader = this.ioManager.createBulkBlockChannelReader(p.getBuildSideChannel().getChannelID(),
this.availableMemory, p.getBuildSideBlockCount());
// call waits until all is read
if (keepBuildSidePartitions && p.recursionLevel == 0) {
reader.close(); // keep the partitions
} else {
reader.closeAndDelete();
}
final List<MemorySegment> partitionBuffers = reader.getFullSegments();
final HashPartition<BT, PT> newPart = new HashPartition<BT, PT>(this.buildSideSerializer, this.probeSideSerializer,
0, nextRecursionLevel, partitionBuffers, p.getBuildSideRecordCount(), this.segmentSize, p.getLastSegmentLimit());
this.partitionsBeingBuilt.add(newPart);
// erect the buckets
initTable((int) numBuckets, (byte) 1);
// now, index the partition through a hash table
final HashPartition<BT, PT>.PartitionIterator pIter = newPart.getPartitionIterator(this.buildSideComparator);
BT record = this.buildSideSerializer.createInstance();
while ((record = pIter.next(record)) != null) {
final int hashCode = hash(pIter.getCurrentHashCode(), nextRecursionLevel);
final int posHashCode = hashCode % this.numBuckets;
final long pointer = pIter.getPointer();
// get the bucket for the given hash code
final int bucketArrayPos = posHashCode >> this.bucketsPerSegmentBits;
final int bucketInSegmentPos = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS;
final MemorySegment bucket = this.buckets[bucketArrayPos];
insertBucketEntry(newPart, bucket, bucketInSegmentPos, hashCode, pointer);
}
}
else {
// we need to partition and partially spill
final int avgRecordLenPartition = (int) (((long) p.getBuildSideBlockCount()) *
this.segmentSize / p.getBuildSideRecordCount());
final int bucketCount = (int) (((long) totalBuffersAvailable) * RECORD_TABLE_BYTES /
(avgRecordLenPartition + RECORD_OVERHEAD_BYTES));
// compute in how many splits, we'd need to partition the result
final int splits = (int) (totalBuffersNeeded / totalBuffersAvailable) + 1;
final int partitionFanOut = Math.min(10 * splits /* being conservative */, MAX_NUM_PARTITIONS);
createPartitions(partitionFanOut, nextRecursionLevel);
// set up the table structure. the write behind buffers are taken away, as are one buffer per partition
initTable(bucketCount, (byte) partitionFanOut);
// go over the complete input and insert every element into the hash table
// first set up the reader with some memory.
final List<MemorySegment> segments = new ArrayList<MemorySegment>(2);
segments.add(getNextBuffer());
segments.add(getNextBuffer());
final BlockChannelReader inReader = this.ioManager.createBlockChannelReader(p.getBuildSideChannel().getChannelID());
final ChannelReaderInputView inView = new HeaderlessChannelReaderInputView(inReader, segments,
p.getBuildSideBlockCount(), p.getLastSegmentLimit(), false);
final ChannelReaderInputViewIterator<BT> inIter = new ChannelReaderInputViewIterator<BT>(inView,
this.availableMemory, this.buildSideSerializer);
final TypeComparator<BT> btComparator = this.buildSideComparator;
BT rec = this.buildSideSerializer.createInstance();
while ((rec = inIter.next(rec)) != null)
{
final int hashCode = hash(btComparator.hash(rec), nextRecursionLevel);
insertIntoTable(rec, hashCode);
}
// finalize the partitions
for (int i = 0; i < this.partitionsBeingBuilt.size(); i++) {
HashPartition<BT, PT> part = this.partitionsBeingBuilt.get(i);
part.finalizeBuildPhase(this.ioManager, this.currentEnumerator, this.writeBehindBuffers);
}
}
}
/**
* @param record
* @param hashCode
* @throws IOException
*/
protected final void insertIntoTable(final BT record, final int hashCode) throws IOException {
final int posHashCode = hashCode % this.numBuckets;
// get the bucket for the given hash code
final int bucketArrayPos = posHashCode >> this.bucketsPerSegmentBits;
final int bucketInSegmentPos = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS;
final MemorySegment bucket = this.buckets[bucketArrayPos];
// get the basic characteristics of the bucket
final int partitionNumber = bucket.get(bucketInSegmentPos + HEADER_PARTITION_OFFSET);
// get the partition descriptor for the bucket
if (partitionNumber < 0 || partitionNumber >= this.partitionsBeingBuilt.size()) {
throw new RuntimeException("Error: Hash structures in Hash-Join are corrupt. Invalid partition number for bucket.");
}
final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(partitionNumber);
// --------- Step 1: Get the partition for this pair and put the pair into the buffer ---------
long pointer = p.insertIntoBuildBuffer(record);
if (pointer != -1) {
// record was inserted into an in-memory partition. a pointer must be inserted into the buckets
insertBucketEntry(p, bucket, bucketInSegmentPos, hashCode, pointer);
}
}
/**
* @param p
* @param bucket
* @param bucketInSegmentPos
* @param hashCode
* @param pointer
* @throws IOException
*/
final void insertBucketEntry(final HashPartition<BT, PT> p, final MemorySegment bucket,
final int bucketInSegmentPos, final int hashCode, final long pointer)
throws IOException
{
// find the position to put the hash code and pointer
final int count = bucket.getShort(bucketInSegmentPos + HEADER_COUNT_OFFSET);
if (count < NUM_ENTRIES_PER_BUCKET)
{
// we are good in our current bucket, put the values
bucket.putInt(bucketInSegmentPos + BUCKET_HEADER_LENGTH + (count * HASH_CODE_LEN), hashCode); // hash code
bucket.putLong(bucketInSegmentPos + BUCKET_POINTER_START_OFFSET + (count * POINTER_LEN), pointer); // pointer
bucket.putShort(bucketInSegmentPos + HEADER_COUNT_OFFSET, (short) (count + 1)); // update count
}
else {
// we need to go to the overflow buckets
final long originalForwardPointer = bucket.getLong(bucketInSegmentPos + HEADER_FORWARD_OFFSET);
final long forwardForNewBucket;
if (originalForwardPointer != BUCKET_FORWARD_POINTER_NOT_SET) {
// forward pointer set
final int overflowSegNum = (int) (originalForwardPointer >>> 32);
final int segOffset = (int) (originalForwardPointer & 0xffffffff);
final MemorySegment seg = p.overflowSegments[overflowSegNum];
final short obCount = seg.getShort(segOffset + HEADER_COUNT_OFFSET);
// check if there is space in this overflow bucket
if (obCount < NUM_ENTRIES_PER_BUCKET) {
// space in this bucket and we are done
seg.putInt(segOffset + BUCKET_HEADER_LENGTH + (obCount * HASH_CODE_LEN), hashCode); // hash code
seg.putLong(segOffset + BUCKET_POINTER_START_OFFSET + (obCount * POINTER_LEN), pointer); // pointer
seg.putShort(segOffset + HEADER_COUNT_OFFSET, (short) (obCount + 1)); // update count
return;
}
else {
// no space here, we need a new bucket. this current overflow bucket will be the
// target of the new overflow bucket
forwardForNewBucket = originalForwardPointer;
}
}
else {
// no overflow bucket yet, so we need a first one
forwardForNewBucket = BUCKET_FORWARD_POINTER_NOT_SET;
}
// we need a new overflow bucket
MemorySegment overflowSeg;
final int overflowBucketNum;
final int overflowBucketOffset;
// first, see if there is space for an overflow bucket remaining in the last overflow segment
if (p.nextOverflowBucket == 0) {
// no space left in last bucket, or no bucket yet, so create an overflow segment
overflowSeg = getNextBuffer();
if (overflowSeg == null) {
// no memory available to create overflow bucket. we need to spill a partition
final int spilledPart = spillPartition();
if (spilledPart == p.getPartitionNumber()) {
// this bucket is no longer in-memory
return;
}
overflowSeg = getNextBuffer();
if (overflowSeg == null) {
throw new RuntimeException("Bug in HybridHashJoin: No memory became available after spilling a partition.");
}
}
overflowBucketOffset = 0;
overflowBucketNum = p.numOverflowSegments;
// add the new overflow segment
if (p.overflowSegments.length <= p.numOverflowSegments) {
MemorySegment[] newSegsArray = new MemorySegment[p.overflowSegments.length * 2];
System.arraycopy(p.overflowSegments, 0, newSegsArray, 0, p.overflowSegments.length);
p.overflowSegments = newSegsArray;
}
p.overflowSegments[p.numOverflowSegments] = overflowSeg;
p.numOverflowSegments++;
}
else {
// there is space in the last overflow bucket
overflowBucketNum = p.numOverflowSegments - 1;
overflowSeg = p.overflowSegments[overflowBucketNum];
overflowBucketOffset = p.nextOverflowBucket << NUM_INTRA_BUCKET_BITS;
}
// next overflow bucket is one ahead. if the segment is full, the next will be at the beginning
// of a new segment
p.nextOverflowBucket = (p.nextOverflowBucket == this.bucketsPerSegmentMask ? 0 : p.nextOverflowBucket + 1);
// insert the new overflow bucket in the chain of buckets
// 1) set the old forward pointer
// 2) let the bucket in the main table point to this one
overflowSeg.putLong(overflowBucketOffset + HEADER_FORWARD_OFFSET, forwardForNewBucket);
final long pointerToNewBucket = (((long) overflowBucketNum) << 32) | ((long) overflowBucketOffset);
bucket.putLong(bucketInSegmentPos + HEADER_FORWARD_OFFSET, pointerToNewBucket);
// finally, insert the values into the overflow buckets
overflowSeg.putInt(overflowBucketOffset + BUCKET_HEADER_LENGTH, hashCode); // hash code
overflowSeg.putLong(overflowBucketOffset + BUCKET_POINTER_START_OFFSET, pointer); // pointer
// set the count to one
overflowSeg.putShort(overflowBucketOffset + HEADER_COUNT_OFFSET, (short) 1);
}
}
// --------------------------------------------------------------------------------------------
// Setup and Tear Down of Structures
// --------------------------------------------------------------------------------------------
/**
* Returns a new inMemoryPartition object.
* This is required as a plug for ReOpenableMutableHashTable.
*/
protected HashPartition<BT, PT> getNewInMemoryPartition(int number, int recursionLevel) {
return new HashPartition<BT, PT>(this.buildSideSerializer, this.probeSideSerializer,
number, recursionLevel, this.availableMemory.remove(this.availableMemory.size() - 1),
this, this.segmentSize);
}
/**
* @param numPartitions
*/
protected void createPartitions(int numPartitions, int recursionLevel) {
// sanity check
ensureNumBuffersReturned(numPartitions);
this.currentEnumerator = this.ioManager.createChannelEnumerator();
this.partitionsBeingBuilt.clear();
for (int i = 0; i < numPartitions; i++) {
HashPartition<BT, PT> p = getNewInMemoryPartition(i, recursionLevel);
this.partitionsBeingBuilt.add(p);
}
}
/**
* This method clears all partitions currently residing (partially) in memory. It releases all memory
* and deletes all spilled partitions.
* <p>
* This method is intended for a hard cleanup in the case that the join is aborted.
*/
protected void clearPartitions() {
for (int i = this.partitionsBeingBuilt.size() - 1; i >= 0; --i) {
final HashPartition<BT, PT> p = this.partitionsBeingBuilt.get(i);
try {
p.clearAllMemory(this.availableMemory);
} catch (Exception e) {
LOG.error("Error during partition cleanup.", e);
}
}
this.partitionsBeingBuilt.clear();
}
/**
* @param numBuckets
* @param numPartitions
* @return
*/
protected void initTable(int numBuckets, byte numPartitions) {
final int bucketsPerSegment = this.bucketsPerSegmentMask + 1;
final int numSegs = (numBuckets >>> this.bucketsPerSegmentBits) + ( (numBuckets & this.bucketsPerSegmentMask) == 0 ? 0 : 1);
final MemorySegment[] table = new MemorySegment[numSegs];
ensureNumBuffersReturned(numSegs);
// go over all segments that are part of the table
for (int i = 0, bucket = 0; i < numSegs && bucket < numBuckets; i++) {
final MemorySegment seg = getNextBuffer();
// go over all buckets in the segment
for (int k = 0; k < bucketsPerSegment && bucket < numBuckets; k++, bucket++) {
final int bucketOffset = k * HASH_BUCKET_SIZE;
// compute the partition that the bucket corresponds to
final byte partition = assignPartition(bucket, numPartitions);
// initialize the header fields
seg.put(bucketOffset + HEADER_PARTITION_OFFSET, partition);
seg.put(bucketOffset + HEADER_STATUS_OFFSET, BUCKET_STATUS_IN_MEMORY);
seg.putShort(bucketOffset + HEADER_COUNT_OFFSET, (short) 0);
seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET);
}
table[i] = seg;
}
this.buckets = table;
this.numBuckets = numBuckets;
}
/**
* Releases the table (the array of buckets) and returns the occupied memory segments to the list of free segments.
*/
protected void releaseTable() {
// set the counters back
this.numBuckets = 0;
if (this.buckets != null) {
for (int i = 0; i < this.buckets.length; i++) {
this.availableMemory.add(this.buckets[i]);
}
this.buckets = null;
}
}
// --------------------------------------------------------------------------------------------
// Memory Handling
// --------------------------------------------------------------------------------------------
/**
* Selects a partition and spills it. The number of the spilled partition is returned.
*
* @return The number of the spilled partition.
*/
protected int spillPartition() throws IOException {
// find the largest partition
ArrayList<HashPartition<BT, PT>> partitions = this.partitionsBeingBuilt;
int largestNumBlocks = 0;
int largestPartNum = -1;
for (int i = 0; i < partitions.size(); i++) {
HashPartition<BT, PT> p = partitions.get(i);
if (p.isInMemory() && p.getBuildSideBlockCount() > largestNumBlocks) {
largestNumBlocks = p.getBuildSideBlockCount();
largestPartNum = i;
}
}
final HashPartition<BT, PT> p = partitions.get(largestPartNum);
// spill the partition
int numBuffersFreed = p.spillPartition(this.availableMemory, this.ioManager,
this.currentEnumerator.next(), this.writeBehindBuffers);
this.writeBehindBuffersAvailable += numBuffersFreed;
// grab as many buffers as are available directly
MemorySegment currBuff = null;
while (this.writeBehindBuffersAvailable > 0 && (currBuff = this.writeBehindBuffers.poll()) != null) {
this.availableMemory.add(currBuff);
this.writeBehindBuffersAvailable--;
}
return largestPartNum;
}
/**
* This method makes sure that at least a certain number of memory segments is in the list of free segments.
* Free memory can be in the list of free segments, or in the return-queue where segments used to write behind are
* put. The number of segments that are in that return-queue, but are actually reclaimable is tracked. This method
* makes sure at least a certain number of buffers is reclaimed.
*
* @param minRequiredAvailable The minimum number of buffers that needs to be reclaimed.
*/
final void ensureNumBuffersReturned(final int minRequiredAvailable) {
if (minRequiredAvailable > this.availableMemory.size() + this.writeBehindBuffersAvailable) {
throw new IllegalArgumentException("More buffers requested available than totally available.");
}
try {
while (this.availableMemory.size() < minRequiredAvailable) {
this.availableMemory.add(this.writeBehindBuffers.take());
this.writeBehindBuffersAvailable--;
}
}
catch (InterruptedException iex) {
throw new RuntimeException("Hash Join was interrupted.");
}
}
/**
* Gets the next buffer to be used with the hash-table, either for an in-memory partition, or for the
* table buckets. This method returns <tt>null</tt>, if no more buffer is available. Spilling a partition
* may free new buffers then.
*
* @return The next buffer to be used by the hash-table, or null, if no buffer remains.
* @throws IOException Thrown, if the thread is interrupted while grabbing the next buffer. The I/O
* exception replaces the <tt>InterruptedException</tt> to consolidate the exception
* signatures.
*/
final MemorySegment getNextBuffer() {
// check if the list directly offers memory
int s = this.availableMemory.size();
if (s > 0) {
return this.availableMemory.remove(s-1);
}
// check if there are write behind buffers that actually are to be used for the hash table
if (this.writeBehindBuffersAvailable > 0) {
// grab at least one, no matter what
MemorySegment toReturn;
try {
toReturn = this.writeBehindBuffers.take();
}
catch (InterruptedException iex) {
throw new RuntimeException("Hybrid Hash Join was interrupted while taking a buffer.");
}
this.writeBehindBuffersAvailable--;
// grab as many more buffers as are available directly
MemorySegment currBuff = null;
while (this.writeBehindBuffersAvailable > 0 && (currBuff = this.writeBehindBuffers.poll()) != null) {
this.availableMemory.add(currBuff);
this.writeBehindBuffersAvailable--;
}
return toReturn;
} else {
// no memory available
return null;
}
}
@Override
public MemorySegment nextSegment() {
final MemorySegment seg = getNextBuffer();
if (seg == null) {
try {
spillPartition();
} catch (IOException ioex) {
throw new RuntimeException("Error spilling Hash Join Partition" + (ioex.getMessage() == null ?
"." : ": " + ioex.getMessage()), ioex);
}
MemorySegment fromSpill = getNextBuffer();
if (fromSpill == null) {
throw new RuntimeException("BUG in Hybrid Hash Join: Spilling did not free a buffer.");
} else {
return fromSpill;
}
} else {
return seg;
}
}
// --------------------------------------------------------------------------------------------
// Utility Computational Functions
// --------------------------------------------------------------------------------------------
/**
* Determines the number of buffers to be used for asynchronous write behind. It is currently
* computed as the logarithm of the number of buffers to the base 4, rounded up, minus 2.
* The upper limit for the number of write behind buffers is however set to six.
*
* @param numBuffers The number of available buffers.
* @return The number
*/
public static final int getNumWriteBehindBuffers(int numBuffers) {
int numIOBufs = (int) (Math.log(numBuffers) / Math.log(4) - 1.5);
return numIOBufs > 6 ? 6 : numIOBufs;
}
/**
* Gets the number of partitions to be used for an initial hash-table, when no estimates are
* available.
* <p>
* The current logic makes sure that there are always between 10 and 127 partitions, and close
* to 0.1 of the number of buffers.
*
* @param numBuffers The number of buffers available.
* @return The number of partitions to use.
*/
public static final int getPartitioningFanOutNoEstimates(int numBuffers) {
return Math.max(10, Math.min(numBuffers / 10, MAX_NUM_PARTITIONS));
}
public static final int getInitialTableSize(int numBuffers, int bufferSize, int numPartitions, int recordLenBytes) {
// ----------------------------------------------------------------------------------------
// the following observations hold:
// 1) If the records are assumed to be very large, then many buffers need to go to the partitions
// and fewer to the table
// 2) If the records are small, then comparatively many have to go to the buckets, and fewer to the
// partitions
// 3) If the bucket-table is chosen too small, we will eventually get many collisions and will grow the
// hash table, incrementally adding buffers.
// 4) If the bucket-table is chosen to be large and we actually need more buffers for the partitions, we
// cannot subtract them afterwards from the table
//
// ==> We start with a comparatively small hash-table. We aim for a 200% utilization of the bucket table
// when all the partition buffers are full. Most likely, that will cause some buckets to be re-hashed
// and grab additional buffers away from the partitions.
// NOTE: This decision may be subject to changes after conclusive experiments!
// ----------------------------------------------------------------------------------------
final long totalSize = ((long) bufferSize) * numBuffers;
final long numRecordsStorable = totalSize / (recordLenBytes + RECORD_OVERHEAD_BYTES);
final long bucketBytes = numRecordsStorable * RECORD_TABLE_BYTES;
final long numBuckets = bucketBytes / (2 * HASH_BUCKET_SIZE) + 1;
return numBuckets > Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) numBuckets;
}
/**
* Assigns a partition to a bucket.
*
* @param bucket
* @param numPartitions
* @return The hash code for the integer.
*/
public static final byte assignPartition(int bucket, byte numPartitions) {
return (byte) (bucket % numPartitions);
}
/**
* This function hashes an integer value. It is adapted from Bob Jenkins' website
* <a href="http://www.burtleburtle.net/bob/hash/integer.html">http://www.burtleburtle.net/bob/hash/integer.html</a>.
* The hash function has the <i>full avalanche</i> property, meaning that every bit of the value to be hashed
* affects every bit of the hash value.
*
* @param code The integer to be hashed.
* @return The hash code for the integer.
*/
public static final int hash(int code, int level) {
final int rotation = level * 11;
code = (code << rotation) | (code >>> -rotation);
code = (code + 0x7ed55d16) + (code << 12);
code = (code ^ 0xc761c23c) ^ (code >>> 19);
code = (code + 0x165667b1) + (code << 5);
code = (code + 0xd3a2646c) ^ (code << 9);
code = (code + 0xfd7046c5) + (code << 3);
code = (code ^ 0xb55a4f09) ^ (code >>> 16);
return code >= 0 ? code : -(code + 1);
}
public TypeComparator<PT> getProbeSideComparator () {
return this.probeSideComparator;
}
// ======================================================================================================
/**
*
*/
public static class HashBucketIterator<BT, PT> implements MutableObjectIterator<BT> {
private final TypeSerializer<BT> accessor;
private final TypePairComparator<PT, BT> comparator;
private MemorySegment bucket;
private MemorySegment[] overflowSegments;
private HashPartition<BT, PT> partition;
private int bucketInSegmentOffset;
private int searchHashCode;
private int posInSegment;
private int countInSegment;
private int numInSegment;
private int originalBucketInSegmentOffset;
private MemorySegment originalBucket;
private long lastPointer;
HashBucketIterator(TypeSerializer<BT> accessor, TypePairComparator<PT, BT> comparator) {
this.accessor = accessor;
this.comparator = comparator;
}
void set(MemorySegment bucket, MemorySegment[] overflowSegments, HashPartition<BT, PT> partition,
int searchHashCode, int bucketInSegmentOffset)
{
this.bucket = bucket;
this.originalBucket = bucket;
this.overflowSegments = overflowSegments;
this.partition = partition;
this.searchHashCode = searchHashCode;
this.bucketInSegmentOffset = bucketInSegmentOffset;
this.originalBucketInSegmentOffset = bucketInSegmentOffset;
this.posInSegment = this.bucketInSegmentOffset + BUCKET_HEADER_LENGTH;
this.countInSegment = bucket.getShort(bucketInSegmentOffset + HEADER_COUNT_OFFSET);
this.numInSegment = 0;
}
public BT next(BT reuse) {
// loop over all segments that are involved in the bucket (original bucket plus overflow buckets)
while (true) {
while (this.numInSegment < this.countInSegment) {
final int thisCode = this.bucket.getInt(this.posInSegment);
this.posInSegment += HASH_CODE_LEN;
// check if the hash code matches
if (thisCode == this.searchHashCode) {
// get the pointer to the pair
final long pointer = this.bucket.getLong(this.bucketInSegmentOffset +
BUCKET_POINTER_START_OFFSET + (this.numInSegment * POINTER_LEN));
this.numInSegment++;
// deserialize the key to check whether it is really equal, or whether we had only a hash collision
try {
this.partition.setReadPosition(pointer);
reuse = this.accessor.deserialize(reuse, this.partition);
if (this.comparator.equalToReference(reuse)) {
this.lastPointer = pointer;
return reuse;
}
}
catch (IOException ioex) {
throw new RuntimeException("Error deserializing key or value from the hashtable: " +
ioex.getMessage(), ioex);
}
}
else {
this.numInSegment++;
}
}
// this segment is done. check if there is another chained bucket
final long forwardPointer = this.bucket.getLong(this.bucketInSegmentOffset + HEADER_FORWARD_OFFSET);
if (forwardPointer == BUCKET_FORWARD_POINTER_NOT_SET) {
return null;
}
final int overflowSegNum = (int) (forwardPointer >>> 32);
this.bucket = this.overflowSegments[overflowSegNum];
this.bucketInSegmentOffset = (int) (forwardPointer & 0xffffffff);
this.countInSegment = this.bucket.getShort(this.bucketInSegmentOffset + HEADER_COUNT_OFFSET);
this.posInSegment = this.bucketInSegmentOffset + BUCKET_HEADER_LENGTH;
this.numInSegment = 0;
}
}
public void writeBack(BT value) throws IOException {
final SeekableDataOutputView outView = this.partition.getWriteView();
outView.setWritePosition(this.lastPointer);
this.accessor.serialize(value, outView);
}
public void reset() {
this.bucket = this.originalBucket;
this.bucketInSegmentOffset = this.originalBucketInSegmentOffset;
this.posInSegment = this.bucketInSegmentOffset + BUCKET_HEADER_LENGTH;
this.countInSegment = bucket.getShort(bucketInSegmentOffset + HEADER_COUNT_OFFSET);
this.numInSegment = 0;
}
} // end HashBucketIterator
// ======================================================================================================
// public static final class LazyHashBucketIterator<BT, PT> {
//
// private final TypePairComparator<PT, BT> comparator;
//
// private MemorySegment bucket;
//
// private MemorySegment[] overflowSegments;
//
// private HashPartition<BT, PT> partition;
//
// private int bucketInSegmentOffset;
//
// private int searchHashCode;
//
// private int posInSegment;
//
// private int countInSegment;
//
// private int numInSegment;
//
// private LazyHashBucketIterator(TypePairComparator<PT, BT> comparator) {
// this.comparator = comparator;
// }
//
//
// void set(MemorySegment bucket, MemorySegment[] overflowSegments, HashPartition<BT, PT> partition,
// int searchHashCode, int bucketInSegmentOffset) {
//
// this.bucket = bucket;
// this.overflowSegments = overflowSegments;
// this.partition = partition;
// this.searchHashCode = searchHashCode;
// this.bucketInSegmentOffset = bucketInSegmentOffset;
//
// this.posInSegment = this.bucketInSegmentOffset + BUCKET_HEADER_LENGTH;
// this.countInSegment = bucket.getShort(bucketInSegmentOffset + HEADER_COUNT_OFFSET);
// this.numInSegment = 0;
// }
//
// public boolean next(BT target) {
// // loop over all segments that are involved in the bucket (original bucket plus overflow buckets)
// while (true) {
//
// while (this.numInSegment < this.countInSegment) {
//
// final int thisCode = this.bucket.getInt(this.posInSegment);
// this.posInSegment += HASH_CODE_LEN;
//
// // check if the hash code matches
// if (thisCode == this.searchHashCode) {
// // get the pointer to the pair
// final long pointer = this.bucket.getLong(this.bucketInSegmentOffset +
// BUCKET_POINTER_START_OFFSET + (this.numInSegment * POINTER_LEN));
// this.numInSegment++;
//
// // check whether it is really equal, or whether we had only a hash collision
// LazyDeSerializable lds = (LazyDeSerializable) target;
// lds.setDeSerializer(this.partition, this.partition.getWriteView(), pointer);
// if (this.comparator.equalToReference(target)) {
// return true;
// }
// }
// else {
// this.numInSegment++;
// }
// }
//
// // this segment is done. check if there is another chained bucket
// final long forwardPointer = this.bucket.getLong(this.bucketInSegmentOffset + HEADER_FORWARD_OFFSET);
// if (forwardPointer == BUCKET_FORWARD_POINTER_NOT_SET) {
// return false;
// }
//
// final int overflowSegNum = (int) (forwardPointer >>> 32);
// this.bucket = this.overflowSegments[overflowSegNum];
// this.bucketInSegmentOffset = (int) (forwardPointer & 0xffffffff);
// this.countInSegment = this.bucket.getShort(this.bucketInSegmentOffset + HEADER_COUNT_OFFSET);
// this.posInSegment = this.bucketInSegmentOffset + BUCKET_HEADER_LENGTH;
// this.numInSegment = 0;
// }
// }
// }
// ======================================================================================================
public static final class ProbeIterator<PT> {
private MutableObjectIterator<PT> source;
private PT instance;
ProbeIterator(MutableObjectIterator<PT> source, PT instance) {
this.instance = instance;
set(source);
}
void set(MutableObjectIterator<PT> source) {
this.source = source;
}
public PT next() throws IOException {
PT retVal = this.source.next(this.instance);
if (retVal != null) {
this.instance = retVal;
return retVal;
} else {
return null;
}
}
public PT getCurrent() {
return this.instance;
}
}
}