Package org.apache.bookkeeper.meta

Examples of org.apache.bookkeeper.meta.LedgerUnderreplicationManager


     */
    @Test(timeout = 30000)
    public void testMultipleManagersShouldBeAbleToTakeAndReleaseLock()
            throws Exception {
        String missingReplica1 = "localhost:3181";
        final LedgerUnderreplicationManager m1 = lmf1
                .newLedgerUnderreplicationManager();
        final LedgerUnderreplicationManager m2 = lmf2
                .newLedgerUnderreplicationManager();
        Long ledgerA = 0xfeadeefdacL;
        m1.markLedgerUnderreplicated(ledgerA, missingReplica1);
        final int iterationCount = 100;
        final CountDownLatch latch1 = new CountDownLatch(iterationCount);
        final CountDownLatch latch2 = new CountDownLatch(iterationCount);
        Thread thread1 = new Thread() {
            @Override
            public void run() {
                takeLedgerAndRelease(m1, latch1, iterationCount);
            }
        };

        Thread thread2 = new Thread() {
            @Override
            public void run() {
                takeLedgerAndRelease(m2, latch2, iterationCount);
            }
        };
        thread1.start();
        thread2.start();

        // wait until at least one thread completed
        while (!latch1.await(50, TimeUnit.MILLISECONDS)
                && !latch2.await(50, TimeUnit.MILLISECONDS)) {
            Thread.sleep(50);
        }

        m1.close();
        m2.close();

        // After completing 'lock acquire,release' job, it should notify below
        // wait
        latch1.await();
        latch2.await();
View Full Code Here


     * able to getLedgerToRereplicate(). This calls will enter into infinite
     * waiting until enabling rereplication process
     */
    @Test(timeout = 20000)
    public void testDisableLedegerReplication() throws Exception {
        final LedgerUnderreplicationManager replicaMgr = lmf1
                .newLedgerUnderreplicationManager();

        // simulate few urLedgers before disabling
        final Long ledgerA = 0xfeadeefdacL;
        final String missingReplica = "localhost:3181";

        // disabling replication
        replicaMgr.disableLedgerReplication();
        LOG.info("Disabled Ledeger Replication");

        try {
            replicaMgr.markLedgerUnderreplicated(ledgerA, missingReplica);
        } catch (UnavailableException e) {
            LOG.debug("Unexpected exception while marking urLedger", e);
            fail("Unexpected exception while marking urLedger" + e.getMessage());
        }

View Full Code Here

     * should continue getLedgerToRereplicate() task
     */
    @Test(timeout = 20000)
    public void testEnableLedegerReplication() throws Exception {
        isLedgerReplicationDisabled = true;
        final LedgerUnderreplicationManager replicaMgr = lmf1
                .newLedgerUnderreplicationManager();

        // simulate few urLedgers before disabling
        final Long ledgerA = 0xfeadeefdacL;
        final String missingReplica = "localhost:3181";
        try {
            replicaMgr.markLedgerUnderreplicated(ledgerA, missingReplica);
        } catch (UnavailableException e) {
            LOG.debug("Unexpected exception while marking urLedger", e);
            fail("Unexpected exception while marking urLedger" + e.getMessage());
        }

        // disabling replication
        replicaMgr.disableLedgerReplication();
        LOG.debug("Disabled Ledeger Replication");

        String znodeA = getUrLedgerZnode(ledgerA);
        final CountDownLatch znodeLatch = new CountDownLatch(2);
        String urledgerA = StringUtils.substringAfterLast(znodeA, "/");
        String urLockLedgerA = basePath + "/locks/" + urledgerA;
        zkc1.exists(urLockLedgerA, new Watcher(){
            @Override
            public void process(WatchedEvent event) {
                if (event.getType() == EventType.NodeCreated) {
                    znodeLatch.countDown();
                    LOG.debug("Recieved node creation event for the zNodePath:"
                            + event.getPath());
                }
               
            }});
        // getLedgerToRereplicate is waiting until enable rereplication
        Thread thread1 = new Thread() {
            @Override
            public void run() {
                try {
                    Long lA = replicaMgr.getLedgerToRereplicate();
                    assertEquals("Should be the ledger I just marked", lA,
                            ledgerA);
                    isLedgerReplicationDisabled = false;
                    znodeLatch.countDown();
                } catch (UnavailableException e) {
                    LOG.debug("Unexpected exception while marking urLedger", e);
                    isLedgerReplicationDisabled = false;
                }
            }
        };
        thread1.start();

        try {
            znodeLatch.await(5, TimeUnit.SECONDS);
            assertTrue("Ledger replication is not disabled!",
                    isLedgerReplicationDisabled);
            assertEquals("Failed to disable ledger replication!", 2, znodeLatch
                    .getCount());

            replicaMgr.enableLedgerReplication();
            znodeLatch.await(5, TimeUnit.SECONDS);
            LOG.debug("Enabled Ledeger Replication");
            assertTrue("Ledger replication is not disabled!",
                    !isLedgerReplicationDisabled);
            assertEquals("Failed to disable ledger replication!", 0, znodeLatch
View Full Code Here

     * Test that the hierarchy gets cleaned up as ledgers
     * are marked as fully replicated
     */
    @Test(timeout=60000)
    public void testHierarchyCleanup() throws Exception {
        final LedgerUnderreplicationManager replicaMgr = lmf1
            .newLedgerUnderreplicationManager();
        // 4 ledgers, 2 in the same hierarchy
        long[] ledgers = { 0x00000000deadbeefL, 0x00000000deadbeeeL,
                           0x00000000beefcafeL, 0x00000000cafed00dL };

        for (long l : ledgers) {
            replicaMgr.markLedgerUnderreplicated(l, "localhost:3181");
        }
        // can't simply test top level as we are limited to ledger
        // ids no larger than an int
        String testPath = urLedgerPath + "/0000/0000";
        List<String> children = zkc1.getChildren(testPath, false);
        assertEquals("Wrong number of hierarchies", 3, children.size());

        int marked = 0;
        while (marked < 3) {
            long l = replicaMgr.getLedgerToRereplicate();
            if (l != ledgers[0]) {
                replicaMgr.markLedgerReplicated(l);
                marked++;
            } else {
                replicaMgr.releaseUnderreplicatedLedger(l);
            }
        }
        children = zkc1.getChildren(testPath, false);
        assertEquals("Wrong number of hierarchies", 1, children.size());

        long l = replicaMgr.getLedgerToRereplicate();
        assertEquals("Got wrong ledger", ledgers[0], l);
        replicaMgr.markLedgerReplicated(l);

        children = zkc1.getChildren(urLedgerPath, false);
        assertEquals("All hierarchies should be cleaned up", 0, children.size());
    }
View Full Code Here

     * Test that as the hierarchy gets cleaned up, it doesn't interfere
     * with the marking of other ledgers as underreplicated
     */
    @Test(timeout = 90000)
    public void testHierarchyCleanupInterference() throws Exception {
        final LedgerUnderreplicationManager replicaMgr1 = lmf1
            .newLedgerUnderreplicationManager();
        final LedgerUnderreplicationManager replicaMgr2 = lmf2
            .newLedgerUnderreplicationManager();

        final int iterations = 1000;
        final AtomicBoolean threadFailed = new AtomicBoolean(false);
        Thread markUnder = new Thread() {
                public void run() {
                    long l = 1;
                    try {
                        for (int i = 0; i < iterations; i++) {
                            replicaMgr1.markLedgerUnderreplicated(l, "localhost:3181");
                            l += 10000;
                        }
                    } catch (Exception e) {
                        LOG.error("markUnder Thread failed with exception", e);
                        threadFailed.set(true);
                        return;
                    }
                }
            };
        final AtomicInteger processed = new AtomicInteger(0);
        Thread markRepl = new Thread() {
                public void run() {
                    try {
                        for (int i = 0; i < iterations; i++) {
                            long l = replicaMgr2.getLedgerToRereplicate();
                            replicaMgr2.markLedgerReplicated(l);
                            processed.incrementAndGet();
                        }
                    } catch (Exception e) {
                        LOG.error("markRepl Thread failed with exception", e);
                        threadFailed.set(true);
View Full Code Here

    private void verifyMarkLedgerUnderreplicated(Collection<String> missingReplica)
            throws KeeperException, InterruptedException,
            CompatibilityException, UnavailableException {
        Long ledgerA = 0xfeadeefdacL;
        String znodeA = getUrLedgerZnode(ledgerA);
        LedgerUnderreplicationManager replicaMgr = lmf1
                .newLedgerUnderreplicationManager();
        for (String replica : missingReplica) {
            replicaMgr.markLedgerUnderreplicated(ledgerA, replica);
        }

        String urLedgerA = getData(znodeA);
        UnderreplicatedLedgerFormat.Builder builderA = UnderreplicatedLedgerFormat
                .newBuilder();
View Full Code Here

     */
    @Test(timeout=30000)
    public void testPeriodicBookieCheckInterval() throws Exception {
        LedgerManagerFactory mFactory = LedgerManagerFactory.newLedgerManagerFactory(bsConfs.get(0), zkc);
        LedgerManager ledgerManager = mFactory.newLedgerManager();
        final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
        final int numLedgers = 1;

        LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
        LedgerMetadata md = LedgerHandleAdapter.getLedgerMetadata(lh);

        List<InetSocketAddress> ensemble = md.getEnsembles().get(0L);
        ensemble.set(0, new InetSocketAddress("1.1.1.1", 1000));

        TestCallbacks.GenericCallbackFuture<Void> cb = new TestCallbacks.GenericCallbackFuture<Void>();
        ledgerManager.writeLedgerMetadata(lh.getId(), md, cb);
        cb.get();

        long underReplicatedLedger = -1;
        for (int i = 0; i < 10; i++) {
            underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
            if (underReplicatedLedger != -1) {
                break;
            }
            Thread.sleep(CHECK_INTERVAL*1000);
        }
View Full Code Here

     * the bookie entry log
     */
    @Test(timeout=30000)
    public void testEntryLogCorruption() throws Exception {
        LedgerManagerFactory mFactory = LedgerManagerFactory.newLedgerManagerFactory(bsConfs.get(0), zkc);
        LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
        underReplicationManager.disableLedgerReplication();

        LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
        long ledgerId = lh.getId();
        for (int i = 0; i < 100; i++) {
            lh.addEntry("testdata".getBytes());
        }
        lh.close();

        BookieAccessor.forceFlush(bs.get(0).getBookie());


        File ledgerDir = bsConfs.get(0).getLedgerDirs()[0];
        ledgerDir = Bookie.getCurrentDirectory(ledgerDir);
        // corrupt of entryLogs
        File[] entryLogs = ledgerDir.listFiles(new FilenameFilter() {
                public boolean accept(File dir, String name) {
                    return name.endsWith(".log");
                }
            });
        ByteBuffer junk = ByteBuffer.allocate(1024*1024);
        for (File f : entryLogs) {
            FileOutputStream out = new FileOutputStream(f);
            out.getChannel().write(junk);
            out.close();
        }
        restartBookies(); // restart to clear read buffers

        underReplicationManager.enableLedgerReplication();
        long underReplicatedLedger = -1;
        for (int i = 0; i < 10; i++) {
            underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
            if (underReplicatedLedger != -1) {
                break;
            }
            Thread.sleep(CHECK_INTERVAL * 1000);
        }
        assertEquals("Ledger should be under replicated", ledgerId, underReplicatedLedger);
        underReplicationManager.close();
    }
View Full Code Here

     * the bookie index files
     */
    @Test(timeout=30000)
    public void testIndexCorruption() throws Exception {
        LedgerManagerFactory mFactory = LedgerManagerFactory.newLedgerManagerFactory(bsConfs.get(0), zkc);
        LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();

        LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
        long ledgerToCorrupt = lh.getId();
        for (int i = 0; i < 100; i++) {
            lh.addEntry("testdata".getBytes());
        }
        lh.close();

        // push ledgerToCorrupt out of page cache (bookie is configured to only use 1 page)
        lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());
        for (int i = 0; i < 100; i++) {
            lh.addEntry("testdata".getBytes());
        }
        lh.close();

        BookieAccessor.forceFlush(bs.get(0).getBookie());

        File ledgerDir = bsConfs.get(0).getLedgerDirs()[0];
        ledgerDir = Bookie.getCurrentDirectory(ledgerDir);

        // corrupt of entryLogs
        File index = new File(ledgerDir, LedgerCacheImpl.getLedgerName(ledgerToCorrupt));
        LOG.info("file to corrupt{}" , index);
        ByteBuffer junk = ByteBuffer.allocate(1024*1024);
        FileOutputStream out = new FileOutputStream(index);
        out.getChannel().write(junk);
        out.close();

        long underReplicatedLedger = -1;
        for (int i = 0; i < 10; i++) {
            underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
            if (underReplicatedLedger != -1) {
                break;
            }
            Thread.sleep(CHECK_INTERVAL * 1000);
        }
        assertEquals("Ledger should be under replicated", ledgerToCorrupt, underReplicatedLedger);
        underReplicationManager.close();
    }
View Full Code Here

     * Test that the period checker will not run when auto replication has been disabled
     */
    @Test(timeout=60000)
    public void testPeriodicCheckWhenDisabled() throws Exception {
        LedgerManagerFactory mFactory = LedgerManagerFactory.newLedgerManagerFactory(bsConfs.get(0), zkc);
        final LedgerUnderreplicationManager underReplicationManager = mFactory.newLedgerUnderreplicationManager();
        final int numLedgers = 100;

        for (int i = 0; i < numLedgers; i++) {
            LedgerHandle lh = bkc.createLedger(3, 3, DigestType.CRC32, "passwd".getBytes());

            for (int j = 0; j < 100; j++) {
                lh.addEntry("testdata".getBytes());
            }
            lh.close();
        }
        underReplicationManager.disableLedgerReplication();

        final AtomicInteger numReads = new AtomicInteger(0);
        ServerConfiguration conf = killBookie(0);

        Bookie deadBookie = new Bookie(conf) {
            @Override
            public ByteBuffer readEntry(long ledgerId, long entryId)
                    throws IOException, NoLedgerException {
                // we want to disable during checking
                numReads.incrementAndGet();
                throw new IOException("Fake I/O exception");
            }
        };
        bsConfs.add(conf);
        bs.add(startBookie(conf, deadBookie));

        Thread.sleep(CHECK_INTERVAL * 2000);
        assertEquals("Nothing should have tried to read", 0, numReads.get());
        underReplicationManager.enableLedgerReplication();
        Thread.sleep(CHECK_INTERVAL * 2000); // give it time to run

        underReplicationManager.disableLedgerReplication();
        // give it time to stop, from this point nothing new should be marked
        Thread.sleep(CHECK_INTERVAL * 2000);

        int numUnderreplicated = 0;
        long underReplicatedLedger = -1;
        do {
            underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
            if (underReplicatedLedger == -1) {
                break;
            }
            numUnderreplicated++;

            underReplicationManager.markLedgerReplicated(underReplicatedLedger);
        } while (underReplicatedLedger != -1);

        Thread.sleep(CHECK_INTERVAL * 2000); // give a chance to run again (it shouldn't, it's disabled)

        // ensure that nothing is marked as underreplicated
        underReplicatedLedger = underReplicationManager.pollLedgerToRereplicate();
        assertEquals("There should be no underreplicated ledgers", -1, underReplicatedLedger);

        LOG.info("{} of {} ledgers underreplicated", numUnderreplicated, numUnderreplicated);
        assertTrue("All should be underreplicated",
                numUnderreplicated <= numLedgers && numUnderreplicated > 0);
View Full Code Here

TOP

Related Classes of org.apache.bookkeeper.meta.LedgerUnderreplicationManager

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.