Examples of org.apache.hadoop.hbase.client.replication.ReplicationAdmin

Package org.apache.hadoop.hbase.client.replication

Examples of org.apache.hadoop.hbase.client.replication.ReplicationAdmin

org.apache.hadoop.hbase.client.replication.ReplicationAdmin

This class provides the administrative interface to HBase cluster replication. In order to use it, the cluster and the client using ReplicationAdmin must be configured with hbase.replication set to true.

Adding a new peer results in creating new outbound connections from every region server to a subset of region servers on the slave cluster. Each new stream of replication will start replicating from the beginning of the current HLog, meaning that edits from that past will be replicated.

Removing a peer is a destructive and irreversible operation that stops all the replication streams for the given cluster and deletes the metadata used to keep track of the replication state.

Enabling and disabling peers is currently not supported.

As cluster replication is still experimental, a kill switch is provided in order to stop all replication-related operations, see {@link #setReplicating(boolean)}. When setting it back to true, the new state of all the replication streams will be unknown and may have holes. Use at your own risk.

To see which commands are available in the shell, type replication.

            }
            admin.close();


            // Delete all replication peers
            System.out.println(">>> Deleting all replication peers from HBase");
            ReplicationAdmin replAdmin = new ReplicationAdmin(conf);
            for (String peerId : replAdmin.listPeers().keySet()) {
                replAdmin.removePeer(peerId);
            }
            replAdmin.close();
            SepTestUtil.waitOnAllReplicationPeersStopped();


            // Clear Solr indexes
            System.out.println(">>> Clearing Solr indexes");
            collection1.deleteByQuery("*:*");

View Full Code Here

        }
    }


    @Override
    public boolean addSubscriptionSilent(String name) throws InterruptedException, KeeperException, IOException {
        ReplicationAdmin replicationAdmin = new ReplicationAdmin(hbaseConf);
        try {
            String internalName = toInternalSubscriptionName(name);
            if (replicationAdmin.listPeers().containsKey(internalName)) {
                return false;
            }


            String basePath = baseZkPath + "/" + internalName;
            UUID uuid = UUID.nameUUIDFromBytes(Bytes.toBytes(internalName)); // always gives the same uuid for the same name
            ZkUtil.createPath(zk, basePath + "/hbaseid", Bytes.toBytes(uuid.toString()));
            ZkUtil.createPath(zk, basePath + "/rs");




            try {
                replicationAdmin.addPeer(internalName, zkQuorumString + ":" + zkClientPort + ":" + basePath);
            } catch (IllegalArgumentException e) {
                if (e.getMessage().equals("Cannot add existing peer")) {
                    return false;
                }
                throw e;

View Full Code Here

        }
    }


    @Override
    public boolean removeSubscriptionSilent(String name) throws IOException {
        ReplicationAdmin replicationAdmin = new ReplicationAdmin(hbaseConf);
        try {
            String internalName = toInternalSubscriptionName(name);
            if (!replicationAdmin.listPeers().containsKey(internalName)) {
                log.error("Requested to remove a subscription which does not exist, skipping silently: '" + name + "'");
                return false;
            } else {
                try {
                    replicationAdmin.removePeer(internalName);
                } catch (IllegalArgumentException e) {
                    if (e.getMessage().equals("Cannot remove inexisting peer")) { // see ReplicationZookeeper
                        return false;
                    }
                    throw e;

View Full Code Here

        }
    }


    @Override
    public boolean hasSubscription(String name) throws IOException {
        ReplicationAdmin replicationAdmin = new ReplicationAdmin(hbaseConf);
        try {
            String internalName = toInternalSubscriptionName(name);
            return replicationAdmin.listPeers().containsKey(internalName);
        } finally {
            Closer.close(replicationAdmin);
        }
    }

View Full Code Here

     * Removes nay HBase replication peers. This method does not wait for the actual related processes to stop,
     * for this {@link ReplicationPeerUtil#waitOnReplicationPeerStopped(String)} can be used on the list of
     * returned peer id's.
     */
    public List<String> cleanHBaseReplicas() throws Exception {
        ReplicationAdmin repliAdmin = new ReplicationAdmin(conf);
        List<String> removedPeers = new ArrayList<String>();
        try {
            for (String peerId : repliAdmin.listPeers().keySet()) {
                repliAdmin.removePeer(peerId);
                removedPeers.add(peerId);
            }
        } finally {
            Closer.close(repliAdmin);
        }

View Full Code Here

 }


  @Test(timeout=300000)
  public void testPerTableCFReplication() throws Exception {
    LOG.info("testPerTableCFReplication");
    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);


    new HBaseAdmin(conf1).createTable(tabA);
    new HBaseAdmin(conf1).createTable(tabB);
    new HBaseAdmin(conf1).createTable(tabC);
    new HBaseAdmin(conf2).createTable(tabA);
    new HBaseAdmin(conf2).createTable(tabB);
    new HBaseAdmin(conf2).createTable(tabC);
    new HBaseAdmin(conf3).createTable(tabA);
    new HBaseAdmin(conf3).createTable(tabB);
    new HBaseAdmin(conf3).createTable(tabC);


    HTable htab1A = new HTable(conf1, tabAName);
    HTable htab2A = new HTable(conf2, tabAName);
    HTable htab3A = new HTable(conf3, tabAName);


    HTable htab1B = new HTable(conf1, tabBName);
    HTable htab2B = new HTable(conf2, tabBName);
    HTable htab3B = new HTable(conf3, tabBName);


    HTable htab1C = new HTable(conf1, tabCName);
    HTable htab2C = new HTable(conf2, tabCName);
    HTable htab3C = new HTable(conf3, tabCName);


    // A. add cluster2/cluster3 as peers to cluster1
    admin1.addPeer("2", utility2.getClusterKey(), "TC;TB:f1,f3");
    admin1.addPeer("3", utility3.getClusterKey(), "TA;TB:f1,f2");


    // A1. tableA can only replicated to cluster3
    putAndWaitWithFamily(row1, f1Name, htab1A, htab3A);
    ensureRowNotReplicated(row1, f1Name, htab2A);
    deleteAndWaitWithFamily(row1, f1Name, htab1A, htab3A);


    putAndWaitWithFamily(row1, f2Name, htab1A, htab3A);
    ensureRowNotReplicated(row1, f2Name, htab2A);
    deleteAndWaitWithFamily(row1, f2Name, htab1A, htab3A);


    putAndWaitWithFamily(row1, f3Name, htab1A, htab3A);
    ensureRowNotReplicated(row1, f3Name, htab2A);
    deleteAndWaitWithFamily(row1, f3Name, htab1A, htab3A);


    // A2. cf 'f1' of tableB can replicated to both cluster2 and cluster3
    putAndWaitWithFamily(row1, f1Name, htab1B, htab2B, htab3B);
    deleteAndWaitWithFamily(row1, f1Name, htab1B, htab2B, htab3B);


    //  cf 'f2' of tableB can only replicated to cluster3
    putAndWaitWithFamily(row1, f2Name, htab1B, htab3B);
    ensureRowNotReplicated(row1, f2Name, htab2B);
    deleteAndWaitWithFamily(row1, f2Name, htab1B, htab3B);


    //  cf 'f3' of tableB can only replicated to cluster2
    putAndWaitWithFamily(row1, f3Name, htab1B, htab2B);
    ensureRowNotReplicated(row1, f3Name, htab3B);
    deleteAndWaitWithFamily(row1, f3Name, htab1B, htab2B);


    // A3. tableC can only replicated to cluster2
    putAndWaitWithFamily(row1, f1Name, htab1C, htab2C);
    ensureRowNotReplicated(row1, f1Name, htab3C);
    deleteAndWaitWithFamily(row1, f1Name, htab1C, htab2C);


    putAndWaitWithFamily(row1, f2Name, htab1C, htab2C);
    ensureRowNotReplicated(row1, f2Name, htab3C);
    deleteAndWaitWithFamily(row1, f2Name, htab1C, htab2C);


    putAndWaitWithFamily(row1, f3Name, htab1C, htab2C);
    ensureRowNotReplicated(row1, f3Name, htab3C);
    deleteAndWaitWithFamily(row1, f3Name, htab1C, htab2C);


    // B. change peers' replicable table-cf config
    admin1.setPeerTableCFs("2", "TA:f1,f2; TC:f2,f3");
    admin1.setPeerTableCFs("3", "TB; TC:f3");


    // B1. cf 'f1' of tableA can only replicated to cluster2
    putAndWaitWithFamily(row2, f1Name, htab1A, htab2A);
    ensureRowNotReplicated(row2, f1Name, htab3A);
    deleteAndWaitWithFamily(row2, f1Name, htab1A, htab2A);

View Full Code Here

    utility1.startMiniZKCluster();
    MiniZooKeeperCluster miniZK = utility1.getZkCluster();
    // Have to reget conf1 in case zk cluster location different
    // than default
    conf1 = utility1.getConfiguration();
    replicationAdmin = new ReplicationAdmin(conf1);
    LOG.info("Setup first Zk");


    // Base conf2 on conf1 so it gets the right zk cluster.
    conf2 = HBaseConfiguration.create(conf1);
    conf2.setInt("hfile.format.version", 3);

View Full Code Here

  public void testMultiSlaveReplication() throws Exception {
    LOG.info("testCyclicReplication");
    MiniHBaseCluster master = utility1.startMiniCluster();
    utility2.startMiniCluster();
    utility3.startMiniCluster();
    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);


    new HBaseAdmin(conf1).createTable(table);
    new HBaseAdmin(conf2).createTable(table);
    new HBaseAdmin(conf3).createTable(table);
    HTable htable1 = new HTable(conf1, tableName);
    htable1.setWriteBufferSize(1024);
    HTable htable2 = new HTable(conf2, tableName);
    htable2.setWriteBufferSize(1024);
    HTable htable3 = new HTable(conf3, tableName);
    htable3.setWriteBufferSize(1024);
    
    admin1.addPeer("1", utility2.getClusterKey());


    // put "row" and wait 'til it got around, then delete
    putAndWait(row, famName, htable1, htable2);
    deleteAndWait(row, htable1, htable2);
    // check it wasn't replication to cluster 3
    checkRow(row,0,htable3);


    putAndWait(row2, famName, htable1, htable2);


    // now roll the region server's logs
    new HBaseAdmin(conf1).rollHLogWriter(master.getRegionServer(0).getServerName().toString());
    // after the log was rolled put a new row
    putAndWait(row3, famName, htable1, htable2);


    admin1.addPeer("2", utility3.getClusterKey());


    // put a row, check it was replicated to all clusters
    putAndWait(row1, famName, htable1, htable2, htable3);
    // delete and verify
    deleteAndWait(row1, htable1, htable2, htable3);

View Full Code Here

    mimicSyncUpAfterPut();


  }


  private void setupReplication() throws Exception {
    ReplicationAdmin admin1 = new ReplicationAdmin(conf1);
    ReplicationAdmin admin2 = new ReplicationAdmin(conf2);


    HBaseAdmin ha = new HBaseAdmin(conf1);
    ha.createTable(t1_syncupSource);
    ha.createTable(t2_syncupSource);
    ha.close();


    ha = new HBaseAdmin(conf2);
    ha.createTable(t1_syncupTarget);
    ha.createTable(t2_syncupTarget);
    ha.close();


    // Get HTable from Master
    ht1Source = new HTable(conf1, t1_su);
    ht1Source.setWriteBufferSize(1024);
    ht2Source = new HTable(conf1, t2_su);
    ht1Source.setWriteBufferSize(1024);


    // Get HTable from Peer1
    ht1TargetAtPeer1 = new HTable(conf2, t1_su);
    ht1TargetAtPeer1.setWriteBufferSize(1024);
    ht2TargetAtPeer1 = new HTable(conf2, t2_su);
    ht2TargetAtPeer1.setWriteBufferSize(1024);


    /**
     * set M-S : Master: utility1 Slave1: utility2
     */
    admin1.addPeer("1", utility2.getClusterKey());


    admin1.close();
    admin2.close();
  }

View Full Code Here

    MiniZooKeeperCluster miniZK = utility1.getZkCluster();
    // Have to reget conf1 in case zk cluster location different
    // than default
    conf1 = utility1.getConfiguration();  
    zkw1 = new ZooKeeperWatcher(conf1, "cluster1", null, true);
    admin = new ReplicationAdmin(conf1);
    LOG.info("Setup first Zk");


    // Base conf2 on conf1 so it gets the right zk cluster.
    conf2 = HBaseConfiguration.create(conf1);
    conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");

View Full Code Here

0 1 2 3 4 5 6 7

TOP

Related Classes of org.apache.hadoop.hbase.client.replication.ReplicationAdmin

com.ngdata.hbaseindexer.indexer.IndexerIT

com.ngdata.sep.impl.SepModelImpl

org.apache.hadoop.hbase.Abortable

org.apache.hadoop.hbase.client.TestReplicaWithCluster

org.apache.hadoop.hbase.replication.regionserver.TestRegionReplicaReplicationEndpoint

org.apache.hadoop.hbase.replication.ReplicationPeerConfig

org.apache.hadoop.hbase.replication.ReplicationZookeeper

org.apache.hadoop.hbase.replication.TestMasterReplication

org.apache.hadoop.hbase.replication.TestMultiSlaveReplication

org.apache.hadoop.hbase.replication.TestPerTableCFReplication

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.