Package org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm

Examples of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot

73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
        factory.createDescriptor("testAllocateSlots", 4096);
    ShortCircuitShm shm = new ShortCircuitShm(ShmId.createRandom(), stream);
    int numSlots = 0;
    ArrayList<Slot> slots = new ArrayList<Slot>();
    while (!shm.isFull()) {
      Slot slot = shm.allocAndRegisterSlot(new ExtendedBlockId(123L, "test_bp1"));
      slots.add(slot);
      numSlots++;
    }
    LOG.info("allocated " + numSlots + " slots before running out.");
    int slotIdx = 0;
    for (Iterator<Slot> iter = shm.slotIterator();
        iter.hasNext(); ) {
      Assert.assertTrue(slots.contains(iter.next()));
    }
    for (Slot slot : slots) {
      Assert.assertFalse(slot.addAnchor());
      Assert.assertEquals(slotIdx++, slot.getSlotIdx());
    }
    for (Slot slot : slots) {
      slot.makeAnchorable();
    }
    for (Slot slot : slots) {
      Assert.assertTrue(slot.addAnchor());
    }
    for (Slot slot : slots) {
      slot.removeAnchor();
    }
    for (Slot slot : slots) {
      shm.unregisterSlot(slot.getSlotIdx());
      slot.makeInvalid();
    }
    shm.free();
    stream.close();
    FileUtil.fullyDelete(path);
  }
View Full Code Here
110
111
112
113
114
115
116
117
118
119
120
121
122
    RegisteredShm removedShm = segments.remove(shm.getShmId());
    Preconditions.checkState(removedShm == shm,
        "failed to remove " + shm.getShmId());
    // Stop tracking the slots.
    for (Iterator<Slot> iter = shm.slotIterator(); iter.hasNext(); ) {
      Slot slot = iter.next();
      boolean removed = slots.remove(slot.getBlockId(), slot);
      Preconditions.checkState(removed);
      slot.makeInvalid();
    }
    // De-allocate the memory map and close the shared file.
    shm.free();
  }
View Full Code Here
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
    RegisteredShm shm = segments.get(shmId);
    if (shm == null) {
      throw new InvalidRequestException("there is no shared memory segment " +
          "registered with shmId " + shmId);
    }
    Slot slot = shm.registerSlot(slotId.getSlotIdx(), blockId);
    if (isCached) {
      slot.makeAnchorable();
    } else {
      slot.makeUnanchorable();
    }
    boolean added = slots.put(blockId, slot);
    Preconditions.checkState(added);
    if (LOG.isTraceEnabled()) {
      LOG.trace(this + ": registered " + blockId + " with slot " +
View Full Code Here
325
326
327
328
329
330
331
332
333
334
    RegisteredShm shm = segments.get(shmId);
    if (shm == null) {
      throw new InvalidRequestException("there is no shared memory segment " +
          "registered with shmId " + shmId);
    }
    Slot slot = shm.getSlot(slotId.getSlotIdx());
    slot.makeInvalid();
    shm.unregisterSlot(slotId.getSlotIdx());
    slots.remove(slot.getBlockId(), slot);
  }
View Full Code Here
419
420
421
422
423
424
425
426
427
428
429
    MutableBoolean usedPeer = new MutableBoolean(false);
    ExtendedBlockId blockId = new ExtendedBlockId(123, "xyz");
    final DatanodeInfo datanode =
        new DatanodeInfo(cluster.getDataNodes().get(0).getDatanodeId());
    // Allocating the first shm slot requires using up a peer.
    Slot slot = cache.allocShmSlot(datanode, peer, usedPeer,
                    blockId, "testAllocShm_client");
    Assert.assertNotNull(slot);
    Assert.assertTrue(usedPeer.booleanValue());
    cache.getDfsClientShmManager().visit(new Visitor() {
      @Override
View Full Code Here
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
        return null;
      }
      Entry<ShmId, DfsClientShm> entry = notFull.firstEntry();
      DfsClientShm shm = entry.getValue();
      ShmId shmId = shm.getShmId();
      Slot slot = shm.allocAndRegisterSlot(blockId);
      if (shm.isFull()) {
        if (LOG.isTraceEnabled()) {
          LOG.trace(this + ": pulled the last slot " + slot.getSlotIdx() +
              " out of " + shm);
        }
        DfsClientShm removedShm = notFull.remove(shmId);
        Preconditions.checkState(removedShm == shm);
        full.put(shmId, shm);
      } else {
        if (LOG.isTraceEnabled()) {
          LOG.trace(this + ": pulled slot " + slot.getSlotIdx() +
              " out of " + shm);
        }
      }
      return slot;
    }
View Full Code Here
239
240
241
242
243
244
245
246
247
248
249
            LOG.trace(this + ": shared memory segment access is disabled.");
          }
          return null;
        }
        // Try to use an existing slot.
        Slot slot = allocSlotFromExistingShm(blockId);
        if (slot != null) {
          return slot;
        }
        // There are no free slots.  If someone is loading more slots, wait
        // for that to finish.
View Full Code Here
441
442
443
444
445
446
447
448
449
450
451
    while (true) {
      curPeer = nextDomainPeer();
      if (curPeer == null) break;
      if (curPeer.fromCache) remainingCacheTries--;
      DomainPeer peer = (DomainPeer)curPeer.peer;
      Slot slot = null;
      ShortCircuitCache cache = clientContext.getShortCircuitCache();
      try {
        MutableBoolean usedPeer = new MutableBoolean(false);
        slot = cache.allocShmSlot(datanode, peer, usedPeer,
            new ExtendedBlockId(block.getBlockId(), block.getBlockPoolId()),
View Full Code Here
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
              Map<Long, ShortCircuitReplica> evictableMmapped) {
            Assert.assertEquals(expectedOutstandingMmaps, numOutstandingMmaps);
            ShortCircuitReplica replica =
                replicas.get(ExtendedBlockId.fromExtendedBlock(block));
            Assert.assertNotNull(replica);
            Slot slot = replica.getSlot();
            if ((expectedIsAnchorable != slot.isAnchorable()) ||
                (expectedIsAnchored != slot.isAnchored())) {
              LOG.info("replica " + replica + " has isAnchorable = " +
                slot.isAnchorable() + ", isAnchored = " + slot.isAnchored() +
                ".  Waiting for isAnchorable = " + expectedIsAnchorable +
                ", isAnchored = " + expectedIsAnchored);
              return;
            }
            result.setValue(true);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot

  • org.apache.hadoop.fs.TestEnhancedByteBufferAccess
  • org.apache.hadoop.hdfs.BlockReaderFactory
  • org.apache.hadoop.hdfs.server.datanode.ShortCircuitRegistry
  • org.apache.hadoop.hdfs.shortcircuit.DfsClientShmManager$EndpointShmManager
  • org.apache.hadoop.hdfs.shortcircuit.TestShortCircuitCache
  • org.apache.hadoop.hdfs.shortcircuit.TestShortCircuitShm
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.