Package org.apache.flink.runtime.util

Examples of org.apache.flink.runtime.util.LongArrayList


      int hashOffset = 0;
      int hash = 0;
      int pointerOffset = 0;
      long pointer = 0;
      IntArrayList hashList = new IntArrayList(NUM_ENTRIES_PER_BUCKET);
      LongArrayList pointerList = new LongArrayList(NUM_ENTRIES_PER_BUCKET);
      IntArrayList overflowHashes = new IntArrayList(64);
      LongArrayList overflowPointers = new LongArrayList(64);
      // go over all buckets and split them between old and new buckets
      for(int i = 0; i < numPartitions; i++) {
        InMemoryPartition<T> partition = this.partitions.get(i);
        final MemorySegment[] overflowSegments = partition.overflowSegments;
        int posHashCode = 0;
        for (int j = 0, bucket = i; j < this.buckets.length && bucket < oldNumBuckets; j++) {
          MemorySegment segment = this.buckets[j];
          // go over all buckets in the segment belonging to the partition
          for (int k = bucket % bucketsPerSegment; k < bucketsPerSegment && bucket < oldNumBuckets; k += numPartitions, bucket += numPartitions) {
            int bucketOffset = k * HASH_BUCKET_SIZE;
            if((int)segment.get(bucketOffset + HEADER_PARTITION_OFFSET) != i) {
              throw new IOException("Accessed wrong bucket! wanted: " + i + " got: " + segment.get(bucketOffset + HEADER_PARTITION_OFFSET));
            }
            // loop over all segments that are involved in the bucket (original bucket plus overflow buckets)
            int countInSegment = segment.getInt(bucketOffset + HEADER_COUNT_OFFSET);
            int numInSegment = 0;
            pointerOffset = bucketOffset + BUCKET_POINTER_START_OFFSET;
            hashOffset = bucketOffset + BUCKET_HEADER_LENGTH;
            while (true) {
              while (numInSegment < countInSegment) {
                hash = segment.getInt(hashOffset);
                if((hash % this.numBuckets) != bucket && (hash % this.numBuckets) != (bucket+oldNumBuckets)) {
                  throw new IOException("wanted: " + bucket + " or " + (bucket + oldNumBuckets) + " got: " + hash%this.numBuckets);
                }
                pointer = segment.getLong(pointerOffset);
                hashList.add(hash);
                pointerList.add(pointer);
                pointerOffset += POINTER_LEN;
                hashOffset += HASH_CODE_LEN;
                numInSegment++;
              }
              // this segment is done. check if there is another chained bucket
              final long forwardPointer = segment.getLong(bucketOffset + HEADER_FORWARD_OFFSET);
              if (forwardPointer == BUCKET_FORWARD_POINTER_NOT_SET) {
                break;
              }
              final int overflowSegNum = (int) (forwardPointer >>> 32);
              segment = overflowSegments[overflowSegNum];
              bucketOffset = (int)(forwardPointer & 0xffffffff);
              countInSegment = segment.getInt(bucketOffset + HEADER_COUNT_OFFSET);
              pointerOffset = bucketOffset + BUCKET_POINTER_START_OFFSET;
              hashOffset = bucketOffset + BUCKET_HEADER_LENGTH;
              numInSegment = 0;
            }
            segment = this.buckets[j];
            bucketOffset = k * HASH_BUCKET_SIZE;
            // reset bucket for re-insertion
            segment.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0);
            segment.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET);
            // refill table
            if(hashList.size() != pointerList.size()) {
              throw new IOException("Pointer and hash counts do not match. hashes: " + hashList.size() + " pointer: " + pointerList.size());
            }
            int newSegmentIndex = (bucket + oldNumBuckets) / bucketsPerSegment;
            MemorySegment newSegment = this.buckets[newSegmentIndex];
            // we need to avoid overflows in the first run
            int oldBucketCount = 0;
            int newBucketCount = 0;
            while(!hashList.isEmpty()) {
              hash = hashList.removeInt(hashList.size()-1);
              pointer = pointerList.removeLong(pointerList.size()-1);
              posHashCode = hash % this.numBuckets;
              if(posHashCode == bucket && oldBucketCount < NUM_ENTRIES_PER_BUCKET) {
                bucketOffset = (bucket % bucketsPerSegment) * HASH_BUCKET_SIZE;
                insertBucketEntryFromStart(partition, segment, bucketOffset, hash, pointer);
                oldBucketCount++;
              } else if(posHashCode == (bucket + oldNumBuckets) && newBucketCount < NUM_ENTRIES_PER_BUCKET) {
                bucketOffset = ((bucket + oldNumBuckets) % bucketsPerSegment) * HASH_BUCKET_SIZE;
                insertBucketEntryFromStart(partition, newSegment, bucketOffset, hash, pointer);
                newBucketCount++;
              } else if(posHashCode == (bucket + oldNumBuckets) || posHashCode == bucket) {
                overflowHashes.add(hash);
                overflowPointers.add(pointer);
              } else {
                throw new IOException("Accessed wrong bucket. Target: " + bucket + " or " + (bucket + oldNumBuckets) + " Hit: " + posHashCode);
              }
            }
            hashList.clear();
            pointerList.clear();
          }
        }
        // reset partition's overflow buckets and reclaim their memory
        this.availableMemory.addAll(partition.resetOverflowBuckets());
        // clear overflow lists
        int bucketArrayPos = 0;
        int bucketInSegmentPos = 0;
        MemorySegment bucket = null;
        while(!overflowHashes.isEmpty()) {
          hash = overflowHashes.removeInt(overflowHashes.size()-1);
          pointer = overflowPointers.removeLong(overflowPointers.size()-1);
          posHashCode = hash % this.numBuckets;
          bucketArrayPos = posHashCode >>> this.bucketsPerSegmentBits;
          bucketInSegmentPos = (posHashCode & this.bucketsPerSegmentMask) << NUM_INTRA_BUCKET_BITS;
          bucket = this.buckets[bucketArrayPos];
          insertBucketEntryFromStart(partition, bucket, bucketInSegmentPos, hash, pointer);
        }
        overflowHashes.clear();
        overflowPointers.clear();
      }
      this.isResizing = false;
      return true;
    }
  }
View Full Code Here

TOP

Related Classes of org.apache.flink.runtime.util.LongArrayList

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.