Package com.ngdata.hbaseindexer.impl

Source Code of com.ngdata.hbaseindexer.impl.IndexerModelImplTest

/*
* Copyright 2013 NGDATA nv
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.ngdata.hbaseindexer.impl;

import com.google.common.base.Charsets;
import com.ngdata.hbaseindexer.model.api.IndexerDefinition;
import com.ngdata.hbaseindexer.model.api.IndexerDefinitionBuilder;
import com.ngdata.hbaseindexer.model.api.IndexerModelEvent;
import com.ngdata.hbaseindexer.model.api.IndexerModelEventType;
import com.ngdata.hbaseindexer.model.api.IndexerModelListener;
import com.ngdata.hbaseindexer.model.api.IndexerUpdateException;
import com.ngdata.hbaseindexer.model.api.WriteableIndexerModel;
import com.ngdata.hbaseindexer.model.impl.IndexerModelImpl;
import com.ngdata.sep.util.io.Closer;
import com.ngdata.sep.util.zookeeper.ZkUtil;
import com.ngdata.sep.util.zookeeper.ZooKeeperItf;

import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

import javax.management.MBeanServerConnection;
import javax.management.ObjectName;

import java.io.File;
import java.io.IOException;
import java.net.ServerSocket;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;

import static com.ngdata.hbaseindexer.model.api.IndexerDefinition.IncrementalIndexingState;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

public class IndexerModelImplTest {
    private static MiniZooKeeperCluster ZK_CLUSTER;
    private static File ZK_DIR;
    private static int ZK_CLIENT_PORT;

    @BeforeClass
    public static void setUpBeforeClass() throws Exception {
        ZK_DIR = new File(System.getProperty("java.io.tmpdir") + File.separator + "hbaseindexer.zklocktest");
        FileUtils.deleteDirectory(ZK_DIR);
        ZK_CLIENT_PORT = getFreePort();

        ZK_CLUSTER = new MiniZooKeeperCluster();
        ZK_CLUSTER.setDefaultClientPort(ZK_CLIENT_PORT);
        ZK_CLUSTER.startup(ZK_DIR);
    }

    @AfterClass
    public static void tearDownAfterClass() throws Exception {
        if (ZK_CLUSTER != null) {
            ZK_CLUSTER.shutdown();
        }
    }

    @Test
    public void testEvents() throws Exception {
        ZooKeeperItf zk1 = ZkUtil.connect("localhost:" + ZK_CLIENT_PORT, 15000);
        ZooKeeperItf zk2 = ZkUtil.connect("localhost:" + ZK_CLIENT_PORT, 15000);
        WriteableIndexerModel model1 = null;
        WriteableIndexerModel model2 = null;
        try {
            TestListener listener = new TestListener();

            model1 = new IndexerModelImpl(zk1, "/test");
            model1.registerListener(listener);

            // Create an indexer -- verify INDEXER_ADDED event
            IndexerDefinition indexer1 = new IndexerDefinitionBuilder()
                    .name("indexer1")
                    .configuration("my-conf".getBytes("UTF-8"))
                    .build();
            model1.addIndexer(indexer1);

            listener.waitForEvents(1);
            listener.verifyEvents(new IndexerModelEvent(IndexerModelEventType.INDEXER_ADDED, "indexer1"));

            // Verify that a fresh indexer model has the index
            model2 = new IndexerModelImpl(zk2, "/test");
            Collection<IndexerDefinition> indexers = model2.getIndexers();
            assertEquals("Expected indexer1, got " + indexers, 1, indexers.size());
            assertTrue(model2.hasIndexer("indexer1"));

            // Update the indexer -- verify INDEXER_UPDATED event
            indexer1 = new IndexerDefinitionBuilder()
                    .startFrom(indexer1)
                    .incrementalIndexingState(IncrementalIndexingState.SUBSCRIBE_DO_NOT_CONSUME)
                    .build();
            String lock = model1.lockIndexer("indexer1");
            model1.updateIndexer(indexer1, lock);

            listener.waitForEvents(1);
            listener.verifyEvents(new IndexerModelEvent(IndexerModelEventType.INDEXER_UPDATED, "indexer1"));
            model1.unlockIndexer(lock);

            // Delete the indexer -- verify INDEXER_DELETED event
            model1.deleteIndexerInternal("indexer1");
            listener.waitForEvents(1);
            listener.verifyEvents(new IndexerModelEvent(IndexerModelEventType.INDEXER_DELETED, "indexer1"));

            // Create some more indexes and verify we get the correct number of INDEXER_ADDED events
            IndexerModelEvent[] expectedEvents = new IndexerModelEvent[9];
            for (int i = 2; i <= 10; i++) {
                String name = "indexer" + i;
                IndexerDefinition indexer = new IndexerDefinitionBuilder()
                        .name(name)
                        .configuration("my-conf".getBytes("UTF-8"))
                        .build();
                model1.addIndexer(indexer);
                expectedEvents[i - 2] = new IndexerModelEvent(IndexerModelEventType.INDEXER_ADDED, name);
            }

            listener.waitForEvents(9);
            listener.verifyEvents(expectedEvents);

            // Terminate ZK connections: clients should automatically re-establish the connection and things
            // should work as before
            assertEquals(2, terminateZooKeeperConnections());

            // Do another index update and check we get an event
            IndexerDefinition indexer2 = new IndexerDefinitionBuilder()
                    .name("indexer2")
                    .incrementalIndexingState(IncrementalIndexingState.DO_NOT_SUBSCRIBE)
                    .configuration("my-conf".getBytes(Charsets.UTF_8))
                    .build();
            lock = model1.lockIndexer("indexer2");
            model1.updateIndexer(indexer2, lock);
            model1.unlockIndexer(lock);

            listener.waitForEvents(1);
            listener.verifyEvents(new IndexerModelEvent(IndexerModelEventType.INDEXER_UPDATED, "indexer2"));
        } finally {
            Closer.close(model1);
            Closer.close(model2);
            Closer.close(zk1);
            Closer.close(zk2);
        }
    }

    @Test
    public void testLocking() throws Exception {
        ZooKeeperItf zk1 = ZkUtil.connect("localhost:" + ZK_CLIENT_PORT, 15000);
        ZooKeeperItf zk2 = ZkUtil.connect("localhost:" + ZK_CLIENT_PORT, 15000);
        WriteableIndexerModel model1 = null;
        WriteableIndexerModel model2 = null;
        String indexerName = "lock_test_indexer";
        try {
            model1 = new IndexerModelImpl(zk1, "/test");
            model2 = new IndexerModelImpl(zk2, "/test");

            // Create an index
            IndexerDefinition indexer1 = new IndexerDefinitionBuilder()
                    .name(indexerName)
                    .configuration("foo".getBytes(Charsets.UTF_8))
                    .build();
            model1.addIndexer(indexer1);

            // Lock the index via the first client
            String lock = model1.lockIndexer(indexerName);

            // Try to update it via the second client
            indexer1 = new IndexerDefinitionBuilder()
                    .startFrom(indexer1)
                    .configuration("foo1".getBytes(Charsets.UTF_8))
                    .build();

            try {
                model2.updateIndexer(indexer1, lock + "foo");
                fail("Expected exception");
            } catch (IndexerUpdateException e) {
                // verify the exception says something about locks
                assertTrue(e.getMessage().contains("lock"));
            }

            // First client should be able to do the update though
            model1.updateIndexer(indexer1, lock);

            model1.unlockIndexer(lock);
           
            model1.deleteIndexerInternal(indexerName);
        } finally {
            Closer.close(model1);
            Closer.close(model2);
            Closer.close(zk1);
            Closer.close(zk2);
        }
    }

    private class TestListener implements IndexerModelListener {
        private Set<IndexerModelEvent> events = new HashSet<IndexerModelEvent>();

        @Override
        public void process(IndexerModelEvent event) {
            synchronized (this) {
                events.add(event);
                notifyAll();
            }
        }

        public void waitForEvents(int count) throws InterruptedException {
            long timeout = 1000;
            long now = System.currentTimeMillis();
            synchronized (this) {
                while (events.size() < count && System.currentTimeMillis() - now < timeout) {
                    wait(timeout);
                }
            }
        }

        public void verifyEvents(IndexerModelEvent... expectedEvents) {
            if (events.size() != expectedEvents.length) {
                if (events.size() > 0) {
                    System.out.println("The events are:");
                    for (IndexerModelEvent item : events) {
                        System.out.println(item.getType() + " - " + item.getIndexerName());
                    }
                } else {
                    System.out.println("There are no events.");
                }

                assertEquals("Expected number of events", expectedEvents.length, events.size());
            }

            Set<IndexerModelEvent> expectedEventsSet  = new HashSet<IndexerModelEvent>(Arrays.asList(expectedEvents));

            for (IndexerModelEvent event : expectedEvents) {
                if (!events.contains(event)) {
                    fail("Expected event not present among events: " + event);
                }
            }

            for (IndexerModelEvent event : events) {
                if (!expectedEventsSet.contains(event)) {
                    fail("Got an event which is not among the expected events: " + event);
                }
            }

            events.clear();
        }
    }

    public static int getFreePort() {
        ServerSocket socket = null;
        try {
            socket = new ServerSocket(0);
            return socket.getLocalPort();
        } catch (IOException e) {
            throw new RuntimeException("Error finding a free port", e);
        } finally {
            if (socket != null) {
                try {
                    socket.close();
                } catch (IOException e) {
                    throw new RuntimeException("Error closing ServerSocket used to detect a free port.", e);
                }
            }
        }
    }

    public int terminateZooKeeperConnections() throws Exception {
        MBeanServerConnection connection = java.lang.management.ManagementFactory.getPlatformMBeanServer();
        ObjectName replicationSources = new ObjectName("org.apache.ZooKeeperService:name0=*,name1=Connections,name2=*,name3=*");
        Set<ObjectName> mbeans = connection.queryNames(replicationSources, null);
        int connectionCount = mbeans.size();

        for (ObjectName name : mbeans) {
            connection.invoke(name, "terminateConnection", new Object[] {}, new String[] {});
        }

        return connectionCount;
    }
}
TOP

Related Classes of com.ngdata.hbaseindexer.impl.IndexerModelImplTest

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.