Package org.apache.stanbol.entityhub.indexing.core.config

Examples of org.apache.stanbol.entityhub.indexing.core.config.IndexingConfig


        trimEntityId = DEFAULT_TRIM_ENTITY;
        trimLine = DEFAULT_TRIM_LINE;
    }
    @Override
    public void setConfiguration(Map<String,Object> config) {
        IndexingConfig indexingConfig = (IndexingConfig)config.get(IndexingConfig.KEY_INDEXING_CONFIG);
        log.info("Configure {} :",getClass().getSimpleName());
        Object value = config.get(PARAM_CHARSET);
        if(value != null && value.toString() != null){
            this.charset = value.toString();
            log.info("Set charset to '{}'",charset);
        }
        //parse encode/decode EntityIDs
        value = config.get(PARAM_URL_ENCODE_ENTITY_IDS);
        boolean encodeIds;
        if(value != null){
            encodeIds = Boolean.parseBoolean(value.toString());
        } else if (config.containsKey(PARAM_URL_ENCODE_ENTITY_IDS)){
            encodeIds = true;
        } else {
            encodeIds = false;
        }
        value = config.get(PARAM_URL_DECODE_ENTITY_IDS);
        boolean decodeIds;
        if(value != null){
            decodeIds = Boolean.parseBoolean(value.toString());
        } else if (config.containsKey(PARAM_URL_DECODE_ENTITY_IDS)){
            decodeIds = true;
        } else {
            decodeIds = false;
        }
        if(encodeIds && decodeIds){
            throw new IllegalArgumentException(String.format(
                "One can not enable both Parameters '{}' and '{}'!",
                PARAM_URL_DECODE_ENTITY_IDS,PARAM_URL_DECODE_ENTITY_IDS));
        } else if(encodeIds){
            this.encodeEntityIds = 1;
            log.info("activate URL encoding of Entity IDs");
        } else if(decodeIds){
            this.encodeEntityIds = -1;
            log.info("activate URL decoding of Entity IDs");
        }
        value = config.get(PARAM_ENTITY_SCORE_FILE);
        if(reader == null){
            if(value == null || value.toString().isEmpty()){
                scoreFile = indexingConfig.getSourceFile(DEFAULT_ENTITY_SCORE_FILE);
            } else {
                scoreFile = indexingConfig.getSourceFile(value.toString());
            }
            log.info("Set Source File to '"+this.scoreFile+"'");
        } //else reader parsed in the constructor ... nothing todo
        //now done in the initialise() method
//        try {
View Full Code Here


    }
    @Test
    public void testEntityDataIterable(){
        log.info(" --- testEntityDataIterable ---");
        String testName = "iterable";
        IndexingConfig config = new IndexingConfig(CONFIG_ROOT+File.separatorChar+testName,
            CONFIG_ROOT+'/'+testName){};
        EntityDataIterable iterable = config.getDataIterable();
        assertNotNull(iterable);
        assertEquals(iterable.getClass(), RdfIndexingSource.class);
        assertTrue(iterable.needsInitialisation());
        iterable.initialise();
        EntityDataIterator it = iterable.entityDataIterator();
View Full Code Here

    }
    @Test
    public void testEntityDataProvider(){
        log.info(" --- testEntityDataProvider ---");
        String testName = "provider";
        IndexingConfig config = new IndexingConfig(CONFIG_ROOT+File.separatorChar+testName,
            CONFIG_ROOT+'/'+testName){};
        EntityIterator entityIdIterator = config.getEntityIdIterator();
        assertNotNull("Unable to perform test whithout EntityIterator",entityIdIterator);
        if(entityIdIterator.needsInitialisation()){
            entityIdIterator.initialise();
        }
        EntityDataProvider dataProvider = config.getEntityDataProvider();
        assertNotNull(dataProvider);
        assertTrue(dataProvider.needsInitialisation());//there are test data to load
        dataProvider.initialise();
        assertEquals(dataProvider.getClass(), RdfIndexingSource.class);
        long count = 0;
View Full Code Here

     */
    @Test
    public void testQuadsImport(){
        log.info(" --- testQuadsImport ---");
        String testName = "quads";
        IndexingConfig config = new IndexingConfig(CONFIG_ROOT+File.separatorChar+testName,
            CONFIG_ROOT+'/'+testName){};
        EntityIterator entityIdIterator = config.getEntityIdIterator();
        assertNotNull("Unable to perform test whithout EntityIterator",entityIdIterator);
        if(entityIdIterator.needsInitialisation()){
            entityIdIterator.initialise();
        }
        EntityDataProvider dataProvider = config.getEntityDataProvider();
        assertNotNull(dataProvider);
        assertTrue(dataProvider.needsInitialisation());//there are test data to load
        dataProvider.initialise();
        assertEquals(dataProvider.getClass(), RdfIndexingSource.class);
        long count = 0;
View Full Code Here

    }
    @Test
    public void testBNodeSupport(){
        log.info(" --- testBNodeSupport ---");
        String testName = "bnode";
        IndexingConfig config = new IndexingConfig(CONFIG_ROOT+File.separatorChar+testName,
            CONFIG_ROOT+'/'+testName){};
        EntityDataIterable iterable = config.getDataIterable();
        assertNotNull(iterable);
        assertEquals(iterable.getClass(), RdfIndexingSource.class);
        assertTrue(iterable.needsInitialisation());
        iterable.initialise();
        //((RdfIndexingSource)iterable).debug();
View Full Code Here

     * @param classpathOffset
     * @return
     */
    protected Indexer create(String dir,String classpathOffset){
        Indexer indexer;
        IndexingConfig config;
        if(classpathOffset != null){
            config= new IndexingConfig(dir,classpathOffset){};
        } else {
            config= new IndexingConfig(dir);
        }
        //get the mode based on the configured IndexingComponents
        EntityDataIterable dataIterable = config.getDataIterable();
        EntityIterator idIterator = config.getEntityIdIterator();
        EntityDataProvider dataProvider = config.getEntityDataProvider();
        EntityScoreProvider scoreProvider = config.getEntityScoreProvider();
       
       
        IndexingDestination destination = config.getIndexingDestination();
        if(destination == null){
            log.error("The indexing configuration does not provide an " +
                "indexing destination. This needs to be configured by the key " +
                "'{}' in the indexing.properties within the directory {}",
                IndexingConstants.KEY_INDEXING_DESTINATION,config.getConfigFolder());
            throw new IllegalArgumentException("No IndexingDestination present");
        }
        List<EntityProcessor> processors = config.getEntityProcessors();
        if(processors == null){
            log.error("The indexing configuration does not provide an " +
                "entity processor. This needs to be configured by the key " +
                "'{}' in the indexing.properties within the directory {}",
                IndexingConstants.KEY_ENTITY_PROCESSOR,config.getConfigFolder());
        }
        List<EntityProcessor> postProcessors = config.getEntityPostProcessors();
        log.info("Present Source Configuration:");
        log.info(" - EntityDataIterable: {}",dataIterable);
        log.info(" - EntityIterator: {}",idIterator);
        log.info(" - EntityDataProvider: {}",dataProvider);
        log.info(" - EntityScoreProvider: {}",scoreProvider);
        log.info(" - EntityProcessors ({}):",processors.size());
        if(postProcessors != null){
            log.info(" - EntityPostProcessors ({}):",postProcessors.size());
        }
        int i=0;
        for(EntityProcessor processor : processors){
            i++;
            log.info("    {}) {}",i,processor);
        }
        if(dataIterable != null && scoreProvider != null){
            // iterate over data and lookup scores
            indexer = new IndexerImpl(dataIterable, scoreProvider,
                config.getNormaliser(),destination, processors,
                config.getIndexedEntitiesIdsFile(),postProcessors);
        } else if(idIterator != null && dataProvider != null){
            // iterate over id and lookup data
            indexer = new IndexerImpl(idIterator,dataProvider,
                config.getNormaliser(),destination, processors,
                config.getIndexedEntitiesIdsFile(),postProcessors);
        } else if(dataIterable != null && idIterator != null){
            // create an EntityIterator to EntityScoreProvider adapter
            log.info(
                "Create Adapter from the configured EntityIterator '{}' to the " +
                "required EntityScoreProvider as needed together with the " +
              "configured EntityDataIterable '{}'",
              idIterator.getClass(), dataIterable.getClass());
            indexer = new IndexerImpl(dataIterable,
                new EntityIneratorToScoreProviderAdapter(idIterator),
                config.getNormaliser(),destination, processors,
                config.getIndexedEntitiesIdsFile(),postProcessors);
        } else {
            log.error("Invalid Indexing Source configuration: ");
            log.error(" - To iterate over the data and lookup scores one need to " +
                "configure an EntityDataIterable and an EntityScoreProvider ");
            log.error(" - To iterate over the Id and and lookup data one need to " +
View Full Code Here

        StandaloneManagedSolrServer.shutdownManagedServer();
    }
    @Test(expected=IllegalArgumentException.class)
    public void testMissingBoostConfig(){
        String testName = "missingBoostConfig";
        IndexingConfig config = new IndexingConfig(CONFIG_ROOT+File.separatorChar+testName,
            CONFIG_ROOT+'/'+testName){};
        config.getIndexingDestination();
    }
View Full Code Here

        config.getIndexingDestination();
    }
    @Test(expected=IllegalArgumentException.class)
    public void testInvalidBoostConfig(){
        String testName = "invalidBoostConfig";
        IndexingConfig config = new IndexingConfig(CONFIG_ROOT+File.separatorChar+testName,
            CONFIG_ROOT+'/'+testName){};
        config.getIndexingDestination();
    }
View Full Code Here

     * file is the default. The referenced directory is missing
     */
    @Test(expected=IllegalArgumentException.class)
    public void testMissingDefaultSolrSchemaConfig(){
        String testName = "missingDefaultSolrConf";
        IndexingConfig config = new IndexingConfig(CONFIG_ROOT+File.separatorChar+testName,
            CONFIG_ROOT+'/'+testName){};
        config.getIndexingDestination();
    }
View Full Code Here

     * file is specified. The referenced directory is missing
     */
    @Test(expected=IllegalArgumentException.class)
    public void testMissingSolrSchemaConfig(){
        String testName = "missingSolrConf";
        IndexingConfig config = new IndexingConfig(CONFIG_ROOT+File.separatorChar+testName,
            CONFIG_ROOT+'/'+testName){};
        config.getIndexingDestination();
    }
View Full Code Here

TOP

Related Classes of org.apache.stanbol.entityhub.indexing.core.config.IndexingConfig

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.