Package com.ngdata.hbaseindexer.model.api

Examples of com.ngdata.hbaseindexer.model.api.IndexerDefinitionBuilder


        // Create a table in HBase
        createTable("table1", "family1");

        // Add an indexer
        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(
                        Bytes.toBytes("<indexer table='regex:table\\d+'><field name='field1_s' " +
                                              "value='family1:qualifier1'/></indexer>"))
                .connectionType("solr")
View Full Code Here


    @Test
    public void testIndexerDefinitionChangesPickedUp() throws Exception {
        createTable("table1", "family1");

        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(
                        Bytes.toBytes(("<indexer table='table1'><field name='field1_s' " +
                                          "value='family1:qualifier1'/></indexer>")))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                        "solr.collection", "collection1"))
                .build();

        indexerModel.addIndexer(indexerDef);

        HTable table = new HTable(conf, "table1");
        Put put = new Put(b("row1"));
        put.add(b("family1"), b("qualifier1"), b("value1"));
        table.put(put);

        // Also put a value which should not be processed by the current config
        put = new Put(b("row2"));
        put.add(b("family1"), b("qualifier2"), b("value1"));
        table.put(put);

        SepTestUtil.waitOnReplicationPeerReady(peerId("indexer1"));
        SepTestUtil.waitOnReplication(conf, 60000L);

        collection1.commit();
        QueryResponse response = collection1.query(new SolrQuery("*:*"));
        assertEquals(1, response.getResults().size());

        // update indexer model
        markEventCounts();
        String lock = indexerModel.lockIndexer("indexer1");
        indexerDef = new IndexerDefinitionBuilder()
                .startFrom(indexerModel.getFreshIndexer("indexer1"))
                .configuration(Bytes.toBytes("<indexer table='table1'>" +
                        "<field name='field1_s' value='family1:qualifier1'/>" +
                        "<field name='field2_s' value='family1:qualifier2'/>" +
                        "</indexer>"))
View Full Code Here

        assertEquals(0, collection1.query(new SolrQuery("*:*")).getResults().size());
        assertEquals(0, collection2.query(new SolrQuery("*:*")).getResults().size());

        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(Bytes.toBytes("<indexer table='table1'>" +
                        "<field name='field1_s' value='family1:qualifier1'/>" +
                        "</indexer>"))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                        "solr.collection", "collection1"))
                .build();

        indexerModel.addIndexer(indexerDef);

        indexerDef = new IndexerDefinitionBuilder()
                .name("indexer2")
                .configuration(Bytes.toBytes("<indexer table='table2'>" +
                        "<field name='field1_s' value='family1:qualifier1'/>" +
                        "</indexer>"))
                .connectionType("solr")
View Full Code Here

        createTable("table1", "family1");

        // First create an index in the default incremental indexing state (SUBSCRIBE_AND_CONSUME)
        markEventCounts();
        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(Bytes.toBytes("<indexer table='table1'>" +
                        "<field name='field1_s' value='family1:qualifier1'/>" +
                        "</indexer>"))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                        "solr.collection", "collection1"))
                .build();
        indexerModel.addIndexer(indexerDef);
        // wait for 2 events because: first the indexer is added (= first event), then IndexerMaster
        // updates it to assign subscription (= second event), and only then IndexerSupervisor will start the indexer
        waitOnEventsProcessed(2);

        // Make sure the SEP and indexer processes were started
        assertEquals(1, main.getIndexerSupervisor().getRunningIndexers().size());
        SepTestUtil.waitOnReplicationPeerReady(peerId("indexer1"));

        // Now change to DO_NOT_SUBSCRIBE_STATE
        markEventCounts();
        String lock = indexerModel.lockIndexer("indexer1");
        indexerDef = new IndexerDefinitionBuilder()
                .startFrom(indexerModel.getFreshIndexer("indexer1"))
                .incrementalIndexingState(IncrementalIndexingState.DO_NOT_SUBSCRIBE)
                .build();
        indexerModel.updateIndexer(indexerDef, lock);
        indexerModel.unlockIndexer(lock);
        waitOnEventsProcessed(1);

        // Verify master removed the SEP subscription and unassigned the subscription ID
        SepTestUtil.waitOnReplicationPeerStopped(peerId("indexer1"));
        assertNull(indexerModel.getFreshIndexer("indexer1").getSubscriptionId());

        // Verify supervisor stopped the indexer
        assertEquals(0, main.getIndexerSupervisor().getRunningIndexers().size());

        // Change to SUBSCRIBE_DO_NOT_CONSUME
        markEventCounts();
        lock = indexerModel.lockIndexer("indexer1");
        indexerDef = new IndexerDefinitionBuilder()
                .startFrom(indexerModel.getFreshIndexer("indexer1"))
                .incrementalIndexingState(IncrementalIndexingState.SUBSCRIBE_DO_NOT_CONSUME)
                .build();
        indexerModel.updateIndexer(indexerDef, lock);
        indexerModel.unlockIndexer(lock);
        waitOnEventsProcessed(1);

        // Verify master registered the SEP subscription and assigned the subscription ID
        SepTestUtil.waitOnReplicationPeerReady(peerId("indexer1"));
        assertNotNull(indexerModel.getFreshIndexer("indexer1").getSubscriptionId());

        // Verify supervisor didn't start the indexer (because in state "do no consume")
        assertEquals(0, main.getIndexerSupervisor().getRunningIndexers().size());

        // Change again to default SUBSCRIBE_AND_CONSUME
        markEventCounts();
        lock = indexerModel.lockIndexer("indexer1");
        indexerDef = new IndexerDefinitionBuilder()
                .startFrom(indexerModel.getFreshIndexer("indexer1"))
                .incrementalIndexingState(IncrementalIndexingState.SUBSCRIBE_AND_CONSUME)
                .build();
        indexerModel.updateIndexer(indexerDef, lock);
        indexerModel.unlockIndexer(lock);
View Full Code Here

        createTable("table1", "family1");

        // Create an index
        markEventCounts();
        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(Bytes.toBytes("<indexer table='table1'>" +
                        "<field name='field1_s' value='family1:qualifier1'/>" +
                        "</indexer>"))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                        "solr.collection", "collection1"))
                .build();

        indexerModel.addIndexer(indexerDef);
        waitOnEventsProcessed(2);

        // Make sure the SEP and indexer processes were started
        assertEquals(1, main.getIndexerSupervisor().getRunningIndexers().size());
        SepTestUtil.waitOnReplicationPeerReady(peerId("indexer1"));

        // Now request to delete the indexer, this is done by changing the lifecycle state to DELETE_REQUESTED
        markEventCounts();
        String lock = indexerModel.lockIndexer("indexer1");
        indexerDef = new IndexerDefinitionBuilder()
                .startFrom(indexerModel.getFreshIndexer("indexer1"))
                .lifecycleState(IndexerDefinition.LifecycleState.DELETE_REQUESTED)
                .build();
        indexerModel.updateIndexer(indexerDef, lock);
        indexerModel.unlockIndexer(lock);
View Full Code Here

       
        // Ensure that the index is added on a different timestamp than the last put
        Thread.sleep(5);

        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(Bytes.toBytes("<indexer table='table1'>" +
                        "<field name='field1_s' value='family1:qualifier1'/>" +
                        "</indexer>"))
                .connectionType("solr")
View Full Code Here

        indexerConf.append("  <field name='field1_s' value='family1:field1' type='string'/>");
        indexerConf.append("</indexer>");

        checkLifecycleEvents(0,0,0,0, indexerLifecycleListener);
        createIndexer1(indexerConf.toString());
        IndexerDefinitionBuilder indexerDefinitionBuilder;

        SepTestUtil.waitOnReplicationPeerReady(peerId("indexer1"));
        checkLifecycleEvents(1,0,0,0, indexerLifecycleListener);
        Mockito.reset(indexerLifecycleListener);

        IndexerDefinition indexerDefinition = main.getIndexerModel().getIndexer("indexer1");

        markEventCounts();
        indexerDefinitionBuilder = new IndexerDefinitionBuilder().startFrom(indexerDefinition);
        indexerDefinitionBuilder.incrementalIndexingState(IncrementalIndexingState.DO_NOT_SUBSCRIBE);
        String lock = main.getIndexerModel().lockIndexer(indexerDefinition.getName());
        main.getIndexerModel().updateIndexer(indexerDefinitionBuilder.build(), lock);
        main.getIndexerModel().unlockIndexer(lock);
        waitOnEventsProcessed(1);

        checkLifecycleEvents(0, 1, 0, 0, indexerLifecycleListener);
        Mockito.reset(indexerLifecycleListener);

        markEventCounts();
        indexerDefinition = main.getIndexerModel().getIndexer("indexer1");
        indexerDefinitionBuilder = new IndexerDefinitionBuilder().startFrom(indexerDefinition);
        indexerDefinitionBuilder.batchIndexingState(IndexerDefinition.BatchIndexingState.BUILD_REQUESTED);
        lock = main.getIndexerModel().lockIndexer(indexerDefinition.getName());
        main.getIndexerModel().updateIndexer(indexerDefinitionBuilder.build(), lock);
        main.getIndexerModel().unlockIndexer(lock);
        waitOnEventsProcessed(1);

        checkLifecycleEvents(0, 0, 0, 1, indexerLifecycleListener);
        Mockito.reset(indexerLifecycleListener);

        markEventCounts();
        indexerDefinition = main.getIndexerModel().getIndexer("indexer1");
        indexerDefinitionBuilder = new IndexerDefinitionBuilder().startFrom(indexerDefinition);
        indexerDefinitionBuilder.lifecycleState(IndexerDefinition.LifecycleState.DELETE_REQUESTED);
        lock = main.getIndexerModel().lockIndexer(indexerDefinition.getName());
        main.getIndexerModel().updateIndexer(indexerDefinitionBuilder.build(), lock);
        main.getIndexerModel().unlockIndexer(lock);
        waitOnEventsProcessed(1);

        checkLifecycleEvents(0, 0, 1, 0, indexerLifecycleListener);
    }
View Full Code Here

    }


    private void createIndexer1(String indexerConf) throws Exception {
        WriteableIndexerModel indexerModel = main.getIndexerModel();
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                .name("indexer1")
                .configuration(Bytes.toBytes(indexerConf.toString()))
                .connectionType("solr")
                .connectionParams(ImmutableMap.of("solr.zk", solrTestingUtility.getZkConnectString(),
                                                  "solr.collection", "collection1"))
View Full Code Here

        SOLR_ZK = "127.0.0.1:" + zkClientPort + "/solr";
        INDEXER_ZK = "localhost:" + zkClientPort;
        ZooKeeperItf zkItf = ZkUtil.connect(INDEXER_ZK, 15000);
        INDEXER_MODEL = new IndexerModelImpl(zkItf, "/ngdata/hbaseindexer");
        IndexerDefinition indexerDef = new IndexerDefinitionBuilder()
                                                .name("zkindexerdef")
                                                .indexerComponentFactory(DefaultIndexerComponentFactory.class.getName())
                                                .configuration(Resources.toByteArray(Resources.getResource(
                                                        HBaseMapReduceIndexerToolGoLiveTest.class, "user_indexer.xml")))
                                                .connectionParams(ImmutableMap.of(
View Full Code Here

     * an initial state.
     */
    protected IndexerDefinitionBuilder buildIndexerDefinition(OptionSet options, IndexerDefinition oldIndexerDef)
            throws IOException {

        IndexerDefinitionBuilder builder = new IndexerDefinitionBuilder();
        if (oldIndexerDef != null)
            builder.startFrom(oldIndexerDef);

        // name option is always required, so don't need to check for nulliness
        String indexerName = nameOption.value(options);
        IndexerNameValidator.validate(indexerName);
        builder.name(indexerName);

        LifecycleState lifecycleState = lifecycleStateOption.value(options);
        if (lifecycleState != null)
            builder.lifecycleState(lifecycleState);

        IncrementalIndexingState incrementalIdxState = incrementalIdxStateOption.value(options);
        if (incrementalIdxState != null)
            builder.incrementalIndexingState(incrementalIdxState);

        BatchIndexingState batchIdxState = batchIdxStateOption.value(options);
        if (batchIdxState != null)
            builder.batchIndexingState(batchIdxState);

        // connection type is a hardcoded setting
        builder.connectionType("solr");

        Map<String, String> connectionParams = getConnectionParams(options,
                oldIndexerDef != null ? oldIndexerDef.getConnectionParams() : null);
        if (connectionParams != null)
            builder.connectionParams(connectionParams);

        if (oldIndexerDef == null || oldIndexerDef.getIndexerComponentFactory() == null)
            builder.indexerComponentFactory(indexerComponentFactoryOption.value(options));

        byte[] indexerConf = getIndexerConf(options, indexerComponentFactoryOption, indexerConfOption, connectionParams);
        if (indexerConf != null)
            builder.configuration(indexerConf);

        String[] defaultBatchIndexCliArguments = getBatchIndexingCliArguments(options, defaultBatchIndexCliArgumentsOption);
        if (defaultBatchIndexCliArguments != null) {
            if (defaultBatchIndexCliArguments.length == 0) {
                builder.defaultBatchIndexCliArguments(null);
            } else {
                builder.defaultBatchIndexCliArguments(defaultBatchIndexCliArguments);
            }
        }

        String[] batchIndexCliArguments = getBatchIndexingCliArguments(options, batchIndexCliArgumentsOption);
        if (batchIndexCliArguments != null) {
            if (batchIndexCliArguments.length == 0) {
                builder.batchIndexCliArguments(null);
            } else {
                builder.batchIndexCliArguments(batchIndexCliArguments);
            }
        }

        return builder;
    }
View Full Code Here

TOP

Related Classes of com.ngdata.hbaseindexer.model.api.IndexerDefinitionBuilder

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.