Package eu.stratosphere.core.memory

Examples of eu.stratosphere.core.memory.MemorySegment


    final int numSegs = (numBuckets >>> this.bucketsPerSegmentBits) + ( (numBuckets & this.bucketsPerSegmentMask) == 0 ? 0 : 1);
    final MemorySegment[] table = new MemorySegment[numSegs];
   
    // go over all segments that are part of the table
    for (int i = 0, bucket = 0; i < numSegs && bucket < numBuckets; i++) {
      final MemorySegment seg = getNextBuffer();
     
      // go over all buckets in the segment
      for (int k = 0; k < bucketsPerSegment && bucket < numBuckets; k++, bucket++) {
        final int bucketOffset = k * HASH_BUCKET_SIZE; 
       
        // compute the partition that the bucket corresponds to
        final byte partition = assignPartition(bucket, numPartitions);
       
        // initialize the header fields
        seg.put(bucketOffset + HEADER_PARTITION_OFFSET, partition);
        seg.putInt(bucketOffset + HEADER_COUNT_OFFSET, 0);
        seg.putLong(bucketOffset + HEADER_FORWARD_OFFSET, BUCKET_FORWARD_POINTER_NOT_SET);
      }
     
      table[i] = seg;
    }
    this.buckets = table;
View Full Code Here


    long pointer = 0L;
    int pointerOffset = 0;
    int bucketOffset = 0;
    final int bucketsPerSegment = this.bucketsPerSegmentMask + 1;
    for (int i = 0, bucket = partitionNumber; i < this.buckets.length && bucket < this.numBuckets; i++) {
      MemorySegment segment = this.buckets[i];
      // go over all buckets in the segment belonging to the partition
      for (int k = bucket % bucketsPerSegment; k < bucketsPerSegment && bucket < this.numBuckets; k += numPartitions, bucket += numPartitions) {
        bucketOffset = k * HASH_BUCKET_SIZE;
        if((int)segment.get(bucketOffset + HEADER_PARTITION_OFFSET) != partitionNumber) {
          throw new IOException("Accessed wrong bucket! ");
        }
        int count = segment.getInt(bucketOffset + HEADER_COUNT_OFFSET);
        for (int j = 0; j < NUM_ENTRIES_PER_BUCKET && j < count; j++) {
          pointerOffset = bucketOffset + BUCKET_POINTER_START_OFFSET + (j * POINTER_LEN);
          pointer = segment.getLong(pointerOffset);
          partition.readRecordAt(pointer, tempHolder);
          pointer = this.compactionMemory.appendRecord(tempHolder);
          segment.putLong(pointerOffset, pointer);
        }
        long overflowPointer = segment.getLong(bucketOffset + HEADER_FORWARD_OFFSET);
        if(overflowPointer != BUCKET_FORWARD_POINTER_NOT_SET) {
          // scan overflow buckets
          int current = NUM_ENTRIES_PER_BUCKET;
          bucketOffset = (int) (overflowPointer & 0xffffffff);
          pointerOffset = ((int) (overflowPointer & 0xffffffff)) + BUCKET_POINTER_START_OFFSET;
View Full Code Here

    private boolean fillCache() throws IOException {
      if(currentBucketIndex >= table.numBuckets) {
        return false;
      }
      MemorySegment bucket = table.buckets[currentSegmentIndex];
      // get the basic characteristics of the bucket
      final int partitionNumber = bucket.get(currentBucketOffset + HEADER_PARTITION_OFFSET);
      final InMemoryPartition<T> partition = table.partitions.get(partitionNumber);
      final MemorySegment[] overflowSegments = partition.overflowSegments;
     
      int countInSegment = bucket.getInt(currentBucketOffset + HEADER_COUNT_OFFSET);
      int numInSegment = 0;
      int posInSegment = currentBucketOffset + BUCKET_POINTER_START_OFFSET;
      int bucketOffset = currentBucketOffset;

      // loop over all segments that are involved in the bucket (original bucket plus overflow buckets)
      while (true) {
        while (numInSegment < countInSegment) {
          long pointer = bucket.getLong(posInSegment);
          posInSegment += POINTER_LEN;
          numInSegment++;
          T target = table.buildSideSerializer.createInstance();
          try {
            partition.readRecordAt(pointer, target);
            cache.add(target);
          } catch (IOException e) {
              throw new RuntimeException("Error deserializing record from the hashtable: " + e.getMessage(), e);
          }
        }
        // this segment is done. check if there is another chained bucket
        final long forwardPointer = bucket.getLong(bucketOffset + HEADER_FORWARD_OFFSET);
        if (forwardPointer == BUCKET_FORWARD_POINTER_NOT_SET) {
          break;
        }
        final int overflowSegNum = (int) (forwardPointer >>> 32);
        bucket = overflowSegments[overflowSegNum];
        bucketOffset = (int)(forwardPointer & 0xffffffff);
        countInSegment = bucket.getInt(bucketOffset + HEADER_COUNT_OFFSET);
        posInSegment = bucketOffset + BUCKET_POINTER_START_OFFSET;
        numInSegment = 0;
      }
      currentBucketIndex++;
      if(currentBucketIndex % bucketsPerSegment == 0) {
View Full Code Here

   * @see eu.stratosphere.nephele.services.memorymanager.AbstractPagedOutputView#nextSegment(eu.stratosphere.nephele.services.memorymanager.MemorySegment, int)
   */
  @Override
  protected MemorySegment nextSegment(MemorySegment current, int positionInCurrent) throws EOFException
 
    final MemorySegment next = this.memorySource.nextSegment();
    if (next != null) {
      this.fullSegments.add(next);
      this.segmentNum++;
      return next;
    } else {
View Full Code Here

  protected MemorySegment nextSegment(MemorySegment current, int positionInCurrent) throws IOException {
    // check if we are still in memory
    if (this.writer == null) {
      this.fullSegments.add(current);
     
      final MemorySegment nextSeg = this.memorySource.nextSegment();
      if (nextSeg != null) {
        return nextSeg;
      } else {
        // out of memory, need to spill: create a writer
        this.writer = this.ioManager.createBlockChannelWriter(this.ioManager.createChannel());
       
        // add all segments to the writer
        this.blockCount = this.fullSegments.size();
        this.numMemorySegmentsInWriter = this.blockCount;
        for (int i = 0; i < this.fullSegments.size(); i++) {
          this.writer.writeBlock(this.fullSegments.get(i));
        }
        this.fullSegments.clear();
        final MemorySegment seg = this.writer.getNextReturnedSegment();
        this.numMemorySegmentsInWriter--;
        return seg;
      }
    } else {
      // spilling
View Full Code Here

 
  private static List<MemorySegment> getMemory(int numPages, int pageSize) {
    List<MemorySegment> memory = new ArrayList<MemorySegment>();
   
    for (int i = 0; i < numPages; i++) {
      memory.add(new MemorySegment(new byte[pageSize]));
    }
   
    return memory;
  }
View Full Code Here

 
  private static List<MemorySegment> getMemory(int numPages, int pageSize) {
    List<MemorySegment> memory = new ArrayList<MemorySegment>();
   
    for (int i = 0; i < numPages; i++) {
      memory.add(new MemorySegment(new byte[pageSize]));
    }
   
    return memory;
  }
View Full Code Here

  @Override
  public Buffer requestBuffer(int sizeOfBuffer) throws IOException {
    if (rnd.nextFloat() < this.probabilityForNoneAvailable) {
      return null;
    } else {
      MemorySegment segment = new MemorySegment(new byte[this.sizeOfMemorySegments]);
      return new Buffer(segment, sizeOfBuffer, this.recycler);
    }
  }
View Full Code Here

    }
  }

  @Override
  public Buffer requestBufferBlocking(int sizeOfBuffer) throws IOException, InterruptedException {
    MemorySegment segment = new MemorySegment(new byte[this.sizeOfMemorySegments]);
    return new Buffer(segment, sizeOfBuffer, this.recycler);
  }
View Full Code Here

    final int SERIALIZATION_OVERHEAD = 4; // length encoding

    final RecordSerializer<SerializationTestType> serializer = new SpanningRecordSerializer<SerializationTestType>();
    final RecordDeserializer<SerializationTestType> deserializer = new AdaptiveSpanningRecordDeserializer<SerializationTestType>();

    final Buffer buffer = new Buffer(new MemorySegment(new byte[segmentSize]), segmentSize, null);

    final ArrayDeque<SerializationTestType> serializedRecords = new ArrayDeque<SerializationTestType>();

    // -------------------------------------------------------------------------------------------------------------
View Full Code Here

TOP

Related Classes of eu.stratosphere.core.memory.MemorySegment

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.