Examples of com.ngdata.hbaseindexer.conf.IndexerConf

com.ngdata.hbaseindexer.conf.IndexerConf
The configuration for an indexer, i.e. this defines the behavior of the {@link Indexer} and of the parser/mappercalled by the indexer.
Instances of IndexerConf can be created using {@link IndexerConfBuilder} or from XML using an{@link IndexerComponentFactory}.


    @VisibleForTesting
    void evaluateScan() {
        this.scans = Lists.newArrayList();
        IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(hbaseIndexingSpecification.getIndexerComponentFactory(), new ByteArrayInputStream(hbaseIndexingSpecification.getConfiguration()), hbaseIndexingSpecification.getIndexConnectionParams());
        IndexerConf indexerConf = factory.createIndexerConf();
        applyMorphLineParams(indexerConf);
        List<byte[]> tableNames = Lists.newArrayList();
        String tableNameSpec = indexerConf.getTable();
        if (indexerConf.tableNameIsRegex()) {
            HTableDescriptor[] tables;
            try {
                HBaseAdmin admin = getHbaseAdmin();
                tables = admin.listTables(tableNameSpec);
            } catch (IOException e) {
                throw new RuntimeException("Error occurred fetching hbase tables", e);
            }
            for (HTableDescriptor descriptor : tables) {
                tableNames.add(descriptor.getName());
            }
        } else {
            tableNames.add(Bytes.toBytesBinary(tableNameSpec));
        }
        
        for (byte[] tableName : tableNames) {
            Scan hbaseScan = new Scan();
            hbaseScan.setCacheBlocks(false);
            hbaseScan.setCaching(conf.getInt("hbase.client.scanner.caching", 200));


            if (hbaseStartRow != null) {
                hbaseScan.setStartRow(Bytes.toBytesBinary(hbaseStartRow));
                LOG.debug("Starting row scan at " + hbaseStartRow);
            }


            if (hbaseEndRow != null) {
                hbaseScan.setStopRow(Bytes.toBytesBinary(hbaseEndRow));
                LOG.debug("Stopping row scan at " + hbaseEndRow);
            }


            Long startTime = evaluateTimestamp(hbaseStartTimeString, hbaseTimestampFormat);
            Long endTime = evaluateTimestamp(hbaseEndTimeString, hbaseTimestampFormat);


            if (startTime != null || endTime != null) {
                long scanStartTime = 0L;
                long scanEndTime = Long.MAX_VALUE;
                if (startTime != null) {
                    scanStartTime = startTime;
                    LOG.debug("Setting scan start of time range to " + startTime);
                }
                if (endTime != null) {
                    scanEndTime = endTime;
                    LOG.debug("Setting scan end of time range to " + endTime);
                }
                try {
                    hbaseScan.setTimeRange(scanStartTime, scanEndTime);
                } catch (IOException e) {
                    // In reality an IOE will never be thrown here
                    throw new RuntimeException(e);
                }
            }
            // Only scan the column families and/or cells that the indexer requires
            // if we're running in row-indexing mode        
            if (indexerConf.getMappingType() == MappingType.ROW) {
                MorphlineClasspathUtil.setupJavaCompilerClasspath();


                ResultToSolrMapper resultToSolrMapper = factory.createMapper(
                        hbaseIndexingSpecification.getIndexerName()
                );

View Full Code Here

            }
        }


        ByteArrayInputStream is = new ByteArrayInputStream(configuration);
        IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(hbaseIndexerComponentFactory, is, indexConnectionParams);
        IndexerConf indexerConf = factory.createIndexerConf();
        applyMorphLineParams(indexerConf);


        if (hbaseTableName != null) {
            tableName = hbaseTableName;
        } else {
            tableName = indexerConf.getTable();
        }


        if (hbaseIndexerName == null) {
            hbaseIndexerName = DEFAULT_INDEXER_NAME;
        }

View Full Code Here

        conf.set(HBaseIndexerMapper.INDEX_NAME_CONF_KEY, indexingSpec.getIndexerName());
        conf.set(HBaseIndexerMapper.TABLE_NAME_CONF_KEY, indexingSpec.getTableName());
        HBaseIndexerMapper.configureIndexConnectionParams(conf, indexingSpec.getIndexConnectionParams());


        IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexingSpec.getIndexerComponentFactory(), new ByteArrayInputStream(indexingSpec.getConfiguration()), indexingSpec.getIndexConnectionParams());
        IndexerConf indexerConf = factory.createIndexerConf();


        Map<String, String> params = indexerConf.getGlobalParams();
        String morphlineFile = params.get(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM);
        if (hbaseIndexingOpts.morphlineFile != null) {
            morphlineFile = hbaseIndexingOpts.morphlineFile.getPath();
        }
        if (morphlineFile != null) {

View Full Code Here

      
        long programStartTime = System.currentTimeMillis();
        IndexingSpecification indexingSpec = indexingOpts.getIndexingSpecification();


        IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexingSpec.getIndexerComponentFactory(), new ByteArrayInputStream(indexingSpec.getConfiguration()), Maps.<String, String>newHashMap());
        IndexerConf indexerConf = factory.createIndexerConf();


        if (indexerConf.getRowReadMode() != RowReadMode.NEVER) {
            LOG.warn("Changing row read mode from " + indexerConf.getRowReadMode() + " to " + RowReadMode.NEVER);
            indexerConf = new IndexerConfBuilder(indexerConf).rowReadMode(RowReadMode.NEVER).build();
        }


        Map<String, String> params = indexerConf.getGlobalParams();
        if (indexingOpts.morphlineFile != null) {
            params.put(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, indexingOpts.morphlineFile.getPath());
        }
        if (indexingOpts.morphlineId != null) {
            params.put(MorphlineResultToSolrMapper.MORPHLINE_ID_PARAM, indexingOpts.morphlineId);

View Full Code Here

            throw new IllegalStateException("No configuration value supplied for " + TABLE_NAME_CONF_KEY);
        }


        Map<String, String> indexConnectionParams = getIndexConnectionParams(context.getConfiguration());
        IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexerComponentFactory, new ByteArrayInputStream(indexConfiguration.getBytes(Charsets.UTF_8)), indexConnectionParams);
        IndexerConf indexerConf = factory.createIndexerConf();


        String morphlineFile = context.getConfiguration().get(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM);
        Map<String, String> params = indexerConf.getGlobalParams();
        if (morphlineFile != null) {
            params.put(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, morphlineFile);
        }


        String morphlineId = context.getConfiguration().get(MorphlineResultToSolrMapper.MORPHLINE_ID_PARAM);
        if (morphlineId != null) {
            params.put(MorphlineResultToSolrMapper.MORPHLINE_ID_PARAM, morphlineId);
        }


        for (Map.Entry<String, String> entry : context.getConfiguration()) {
          if (entry.getKey().startsWith(MorphlineResultToSolrMapper.MORPHLINE_VARIABLE_PARAM + ".")) {
              params.put(entry.getKey(), entry.getValue());
          }
          if (entry.getKey().startsWith(MorphlineResultToSolrMapper.MORPHLINE_FIELD_PARAM + ".")) {
              params.put(entry.getKey(), entry.getValue());
          }
        }


        ResultToSolrMapper mapper = factory.createMapper(indexName);


        // TODO This would be better-placed in the top-level job setup -- however, there isn't currently any
        // infrastructure to handle converting an in-memory model into XML (we can only interpret an
        // XML doc into the internal model), so we need to do this here for now
        if (indexerConf.getRowReadMode() != RowReadMode.NEVER) {
            LOG.warn("Changing row read mode from " + indexerConf.getRowReadMode() + " to " + RowReadMode.NEVER);
            indexerConf = new IndexerConfBuilder(indexerConf).rowReadMode(RowReadMode.NEVER).build();
        }
        indexerConf.setGlobalParams(params);


        try {
            indexer = createIndexer(indexName, context, indexerConf, tableName, mapper, indexConnectionParams);
        } catch (SharderException e) {
            throw new RuntimeException(e);

View Full Code Here

     * When receiving an event for a different table than the one specified in the IndexerConf, the Indexer should not
     * interact with HBase or Solr.
     */
    @Test
    public void testNonmatchedTable() {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).build();


        Indexer indexer = Indexer.createIndexer("index name", conf, TABLE_A, null, tablePool, null, solrDocumentWriter);
        IndexingEventListener indexingEventListener = new IndexingEventListener(indexer, TABLE_A, false);


        SepEvent event = new SepEvent(Bytes.toBytes(TABLE_B), null, null, null);

View Full Code Here

     * When we get a sep event for a row which does not exist anymore (or at least, doesn't contain any relevant
     * columns), then we expect a delete on Solr.
     */
    @Test
    public void testNonExistingRow() throws Exception {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).build();


        when(tableA.get(any(Get.class))).thenReturn(new Result());


        ResultToSolrMapper mapper = mock(ResultToSolrMapper.class);
        when(mapper.isRelevantKV(any(KeyValue.class))).thenReturn(true);

View Full Code Here

        };
    }


    @Test
    public void testRowBasedIndexing_RowReadModeNever() throws SolrServerException, IOException {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).rowReadMode(RowReadMode.NEVER).build();


        ResultToSolrMapper mapper = createHbaseToSolrMapper(true);


        Indexer indexer = Indexer.createIndexer("index name", conf, "record", mapper, tablePool, null, solrDocumentWriter);
        IndexingEventListener indexingEventListener = new IndexingEventListener(indexer, TABLE_A, false);

View Full Code Here

        verifyZeroInteractions(tableA, tableB);
    }


    @Test
    public void testRowBasedIndexing_RowReadModeDynamic_RereadRequired() throws IOException, SolrServerException {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).rowReadMode(RowReadMode.DYNAMIC).build();


        ResultToSolrMapper mapper = createHbaseToSolrMapper(false);


        when(tableA.get(any(Get.class))).thenReturn(newResult(Lists.newArrayList(new KeyValue())));

View Full Code Here

        verify(tableA).get(any(Get.class));
    }


    @Test
    public void testRowBasedIndexing_RowReadModeDynamic_NoRereadRequired() throws SolrServerException, IOException {
        IndexerConf conf = new IndexerConfBuilder().table(TABLE_A).rowReadMode(RowReadMode.DYNAMIC).build();


        ResultToSolrMapper mapper = createHbaseToSolrMapper(true);


        Indexer indexer = Indexer.createIndexer("index name", conf, "record", mapper, tablePool, null, solrDocumentWriter);
        IndexingEventListener indexingEventListener = new IndexingEventListener(indexer, TABLE_A, false);

View Full Code Here

0 1

TOP

Related Classes of com.ngdata.hbaseindexer.conf.IndexerConf

com.ngdata.hbaseindexer.indexer.IndexingEventListenerTest

com.ngdata.hbaseindexer.mr.HBaseIndexerMapper

com.ngdata.hbaseindexer.mr.HBaseIndexingOptions

com.ngdata.hbaseindexer.mr.HBaseMapReduceIndexerTool

com.ngdata.hbaseindexer.mr.IndexerDryRun

com.ngdata.hbaseindexer.supervisor.IndexerSupervisor

org.lilyproject.indexer.model.util.IndexesInfoImpl

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.