* Licensed to the Apache Software Foundation (ASF) under one or more

* contributor license agreements.  See the NOTICE file distributed with

* this work for additional information regarding copyright ownership.

* The ASF licenses this file to You under the Apache License, Version 2.0

* (the "License"); you may not use this file except in compliance with

* the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* Implementation of the {@link Yard} interface based on a Solr Server.

* <p>

* This Yard implementation supports to store data of multiple yard instances within the same Solr index. The

* {@link FieldMapper#getDocumentDomainField()} with the value of the Yard ID ({@link #getId()}) is used to

* mark documents stored by the different Yards using the same Index. Queries are also restricted to documents

* stored by the actual Yard by adding a <a

* href="http://wiki.apache.org/solr/CommonQueryParameters#fq">FilterQuery</a>

* <code>fq=fieldMapper.getDocumentDomainField()+":"+getId()</code> to all queries. This feature can be

* activated by setting the {@link #MULTI_YARD_INDEX_LAYOUT} in the configuration. However this requires, that

* the documents in the index are already marked with the ID of the Yard. So setting this property makes

* usually only sense when the Solr index do not contain any data.

* <p>

* Also note, that the different Yards using the same index MUST NOT store Representations with the same ID.

* If that happens, that the Yard writing the Representation last will win and the Representation will be

* deleted for the other Yard!

* <p>

* The SolrJ library is used for the communication with the SolrServer.

* <p>

* TODO: There is still some refactoring needed, because a lot of the code within this bundle is more generic

* and usable regardless what kind of "document based" store is used. Currently the Solr specific stuff is in

* the impl and the default packages. All the other classes are intended to be generally useful. However there

* might be still some unwanted dependencies.

* <p>

* TODO: It would be possible to support for multi cores (see http://wiki.apache.org/solr/CoreAdmin for more

* Information)<br>

* However it is not possible to create cores on the fly (at least not directly; one would need to create

* first the needed directories and than call CREATE via the CoreAdmin). As soon as Solr is directly started

* via OSGI and we do know the Solr home, than it would be possible to implement "on the fly" generation of

* new cores. this would also allow a configuration where - as default - a new core is created automatically

* on the integrated Solr Server for any configured SolrYard.

*

* @author Rupert Westenthaler

*

*/

    metatype = true,

    immediate = true,

    configurationFactory = true,

    policy = ConfigurationPolicy.REQUIRE,

    specVersion = "1.1")

@Properties(

    value = {

         // NOTE: Added propertied from AbstractYard to fix ordering!

         @Property(name = Yard.ID),

         @Property(name = Yard.NAME),

         @Property(name = Yard.DESCRIPTION),

         @Property(name = AbstractYard.DEFAULT_QUERY_RESULT_NUMBER, intValue = -1),

         @Property(name = AbstractYard.MAX_QUERY_RESULT_NUMBER, intValue = -1),

         // BEGIN SolrYard specific Properties

         @Property(name = SolrYard.SOLR_SERVER_LOCATION),

         @Property(name = SolrYard.SOLR_INDEX_DEFAULT_CONFIG,boolValue=SolrYard.DEFAULT_SOLR_INDEX_DEFAULT_CONFIG_STATE),

         @Property(name = SolrYard.MULTI_YARD_INDEX_LAYOUT,boolValue=false),

         @Property(name = SolrYard.MAX_BOOLEAN_CLAUSES, intValue = SolrYard.defaultMaxBooleanClauses)})

    /**

     * The key used to configure the URL for the SolrServer

     */

    public static final String SOLR_SERVER_LOCATION = "org.apache.stanbol.entityhub.yard.solr.solrUri";

    /**

     * The key used to configure if data of multiple Yards are stored within the same index (

     * <code>default=false</code>)

     */

    public static final String MULTI_YARD_INDEX_LAYOUT = "org.apache.stanbol.entityhub.yard.solr.multiYardIndexLayout";

    /**

     * The maximum boolean clauses as configured in the solrconfig.xml of the SolrServer. The default value

     * for this config in Solr 1.4 is 1024.

     * <p>

     * This value is important for generating queries that search for multiple documents, because it

     * determines the maximum number of OR combination for the searched document ids.

     */

    public static final String MAX_BOOLEAN_CLAUSES = "org.apache.stanbol.entityhub.yard.solr.maxBooleanClauses";

    /**

     * This property allows to define a field that is used to parse the boost for the parsed representation.

     * Typically this will be the pageRank of that entity within the referenced site (e.g.

     * {@link Math#log1p(double)} of the number of incoming links

     */

    public static final String DOCUMENT_BOOST_FIELD = "org.apache.stanbol.entityhub.yard.solr.documentBoost";

    /**

     * Key used to configure {@link Entry Entry&lt;String,Float&gt;} for fields with the boost. If no Map is

     * configured or a field is not present in the Map, than 1.0f is used as Boost. If a Document boost is

     * present than the boost of a Field is documentBoost*fieldBoost.

     */

    public static final String FIELD_BOOST_MAPPINGS = "org.apache.stanbol.entityhub.yard.solr.fieldBoosts";

//     * Key used to configure the implementation of the {@link SolrServer} to be used by this SolrYard

//     * implementation. The default value is determined by the type of the value configured by the

//     * {@link #SOLR_SERVER_LOCATION}. In case a path of a File URI is used, the type is set to

//     * {@link SolrServerTypeEnum#EMBEDDED} otherwise {@link SolrServerTypeEnum#HTTP} is used as default.

//     */

//    public static final String SOLR_SERVER_TYPE = "org.apache.stanbol.entityhub.yard.solr.solrServerType";

    /**

     * Key used to to enable/disable the default configuration. If this is enabled,

     * that the index will get initialised with the Default configuration.<p>

     * Notes:<ul>

     * <li> Configuration is only supported for EmbeddedSolrServers that use a

     * relative path

     * <li> If this property is enabled the value of the

     * {@link #SOLR_INDEX_CONFIGURATION_NAME} will be ignored.

     * </ul>

     * Only applies in case a EmbeddedSolrServer is used.

     * @see SolrYardConfig#isDefaultInitialisation()

     * @see SolrYardConfig#setDefaultInitialisation(Boolean)

     */

    public static final String SOLR_INDEX_DEFAULT_CONFIG = "org.apache.stanbol.entityhub.yard.solr.useDefaultConfig";

    /**

     * By default the use of an default configuration is disabled!

     */

    public static final boolean DEFAULT_SOLR_INDEX_DEFAULT_CONFIG_STATE = false;

    /**

     * The name of the configuration use as default.

     */

    public static final String DEFAULT_SOLR_INDEX_CONFIGURATION_NAME = "default";

    /**

     * Allows to configure the name of the index used for the configuration of the Solr Core.

     * Only applies in case of using an EmbeddedSolrServer and

     * {@link #SOLR_INDEX_DEFAULT_CONFIG} is disabled.

     * As default the value of the {@link #SOLR_SERVER_LOCATION} is used.

     * @see SolrYardConfig#getIndexConfigurationName()

     * @see SolrYardConfig#setIndexConfigurationName(String)

     */

    public static final String SOLR_INDEX_CONFIGURATION_NAME = "org.apache.stanbol.entityhub.yard.solr.configName";

    /**

     * The default value for the maxBooleanClauses of SolrQueries. Set to {@value #defaultMaxBooleanClauses}

     * the default of Slor 1.4

     */

    protected static final int defaultMaxBooleanClauses = 1024;

    /**

     * Key used to enable/disable committing of update(..) and store(..) operations. Enabling this ensures

     * that indexed documents are immediately available for searches, but it will also decrease the

     * performance for updates.

     */

    public static final String IMMEDIATE_COMMIT = "org.apache.stanbol.entityhub.yard.solr.immediateCommit";

    /**

     * By default {@link #IMMEDIATE_COMMIT} is enabled

     */

    public static final boolean DEFAULT_IMMEDIATE_COMMIT_STATE = true;

    /**

     * If {@link #IMMEDIATE_COMMIT} is deactivated, than this time is parsed to update(..) and store(..)

     * operations as the maximum time (in ms) until a commit.

     */

    public static final String COMMIT_WITHIN_DURATION = "org.apache.stanbol.entityhub.yard.solr.commitWithinDuration";

    /**

     * The default value for the {@link #COMMIT_WITHIN_DURATION} parameter is 10 sec.

     */

    public static final int DEFAULT_COMMIT_WITHIN_DURATION = 1000 * 10;

    /**

     * What a surprise it's the logger!

     */

    private Logger log = LoggerFactory.getLogger(SolrYard.class);

    /**

     * The SolrServer used for this Yard. Initialisation is done based on the configured parameters in

     * {@link #activate(ComponentContext)}.

     */

    private SolrServer _server;

   

    /**

     * In case the {@link SolrServer} changes during normal operation the

     * {@link SolrFieldMapper} needs to be reinitialised for the new Core.

     * This lock is used to avoid NPE when setting the {@link #_fieldMapper}

     * variable back to <code>null</code> in those cases. While this will

     * trigger automatic re-initialisation on the next call to

     * {@link #getFieldMapper()} there would be the possibility that calls to

     * {@link #getFieldMapper()} return <code>null</code>.

     */

    private final ReadWriteLock fieldMapperLock = new ReentrantReadWriteLock();

    /**

     * The {@link FieldMapper} is responsible for converting fields of {@link Representation} to fields in the

     * {@link SolrInputDocument} and vice versa

     */

    private SolrFieldMapper _fieldMapper;

    /**

     * The {@link IndexValueFactory} is responsible for converting values of fields in the

     * {@link Representation} to the according {@link IndexValue}. One should note, that some properties of

     * the {@link IndexValue} such as the language ({@link IndexValue#getLanguage()}) and the dataType (

     * {@link IndexValue#getType()}) are encoded within the field name inside the {@link SolrInputDocument}

     * and {@link SolrDocument}. This is done by the configured {@link FieldMapper}.

     */

    private IndexValueFactory indexValueFactory;

    /**

     * The {@link SolrQueryFactory} is responsible for converting the {@link Constraint}s of a query to

     * constraints in the index. This requires usually that a single {@link Constraint} is described by

     * several constraints in the index (see {@link IndexConstraintTypeEnum}).

     * <p>

     * TODO: The encoding of such constraints is already designed correctly, the {@link SolrQueryFactory} that

     * implements logic of converting the Incoming {@link Constraint}s and generating the {@link SolrQuery}

     * needs to undergo some refactoring!

     *

     */

    private SolrQueryFactory _solrQueryFactoy;

    /**

     * Used to store the name of the field used to get the {@link SolrInputDocument#setDocumentBoost(float)}

     * for a Representation. This name is available via {@link SolrYardConfig#getDocumentBoostFieldName()}

     * however it is stored here to prevent lookups for field of every stored {@link Representation}.

     */

    private String documentBoostFieldName;

    /**

     * Map used to store boost values for fields. The default Boost for fields is 1.0f. This is used if this

     * map is <code>null</code>, a field is not a key in this map, the value of a field in that map is

     * <code>null</code> or lower equals zero. Also NOTE that the boost for fields is multiplied with the

     * boost for the Document if present.

     */

    private Map<String,Float> fieldBoostMap;

    /**

     * Optionally a {@link ManagedSolrServer} that is used to create new

     * Solr indexes based on parsed configurations.

     */

    @Reference(cardinality=ReferenceCardinality.OPTIONAL_UNARY,

        bind="bindManagedSolrServer",

        unbind="unbindManagedSolrServer",

        strategy=ReferenceStrategy.EVENT,

        policy=ReferencePolicy.DYNAMIC)

    private ManagedSolrServer managedSolrServer;

   

    @Reference(cardinality=ReferenceCardinality.OPTIONAL_UNARY)

    private NamespacePrefixService nsPrefixService;

    /**

     * If update(..) and store(..) calls should be immediately committed.

     */

    private boolean immediateCommit = DEFAULT_IMMEDIATE_COMMIT_STATE;

    /**

     * If <code>{@link #immediateCommit} == false</code> this is the time in ms parsed to Solr until the

     * documents parsed to update(..) and store(..) need to be committed.

     */

    private int commitWithin = DEFAULT_COMMIT_WITHIN_DURATION;

    /**

     * the {@link ComponentContext}. Will be <code>null</code> if not running

     * within OSGI

     */

    private ComponentContext context;

    private RegisteredSolrServerTracker _registeredServerTracker;

    /**

     * Default constructor as used by the OSGI environment.

     * <p>

     * DO NOT USE to manually create instances! The SolrYard instances do need to be configured. YOU NEED TO

     * USE {@link #SolrYard(SolrYardConfig)} to parse the configuration and the initialise the Yard if running

     * outside a OSGI environment.

     */

    public SolrYard() {

        super();

    }

    /**

     * Constructor to be used outside of an OSGI environment

     *

     * @param config

     *            the configuration for the SolrYard

     * @throws IllegalArgumentException

     *             if the configuration is not valid

     * @throws YardException

     *             on any Error while initialising the Solr Server for this Yard

     */

    public SolrYard(SolrYardConfig config) throws IllegalArgumentException, YardException {

        // init via java.util.ServiceLoader

        Iterator<ManagedSolrServer> providerIt = ServiceLoader.load(ManagedSolrServer.class,

            ManagedSolrServer.class.getClassLoader()).iterator();

        if (providerIt.hasNext()) {

            managedSolrServer = providerIt.next();

//                File location = ConfigUtils.toFile(config.getSolrServerLocation());

//                if(!location.isAbsolute()){

//                    location = solrDirectoryManager.getSolrIndexDirectory(location.toString());

//                    config.setSolrServerLocation(location.getAbsolutePath());

//                }

//            }

        } else {

            throw new IllegalStateException("Unable to instantiate "

                                            + ManagedSolrServer.class.getSimpleName()

                                            + " service by using " + ServiceLoader.class.getName() + "!");

        }

        // we need to change the exceptions, because this will be called outside

        // of an OSGI environment!

        try {

            activate(config);

        } catch (IOException e) {

            new YardException("Unable to access SolrServer" + config.getSolrServerLocation());

        } catch (SolrServerException e) {

            new YardException("Unable to initialize SolrServer" + config.getSolrServerLocation());

        } catch (ConfigurationException e) {

            new IllegalArgumentException("Unable to initialise SolrYard with the provided configuration", e);

        }

    }

//        this.solrServerProviderManager = manager;

//    }

//    protected void unbindSolrServerProviderManager(SolrServerProviderManager manager){

//        this.solrServerProviderManager = null;

//    }

    protected void bindManagedSolrServer(ManagedSolrServer manager){

        SolrYardConfig config = (SolrYardConfig) this.getConfig();

        log.info(" ... bind ManagedSolrServer '{}' to SolrYard '{}'",

            manager.getServerName(),config != null ? config.getId() : "<not yet activated>");

        this.managedSolrServer = manager;

        if(config != null){ //if activated

            try {

                checkManagedSolrIndex(manager, config);

            } catch (Exception e) {

                log.error("Exception while checking SolrIndex '"+ config.getSolrServerLocation()

                    +"' on ManagedSolrServer '"+manager.getServerName()+"'!",e);

            }

        }

    }

   

    protected void unbindManagedSolrServer(ManagedSolrServer manager){

        log.info(" ... unbind ManagedSolrServer '{}' from SolrYard '{}'",

            manager.getServerName(),getConfig() != null ? getConfig().getId() : "<not yet activated>");

        this.managedSolrServer = null;

    }

    /**

     * Builds an {@link SolrYardConfig} instance based on the parsed {@link ComponentContext} and forwards to

     * {@link #activate(SolrYardConfig)}.

     *

     * @param context

     *            The component context only used to create the {@link SolrYardConfig} based on

     *            {@link ComponentContext#getProperties()}.

     * @throws ConfigurationException

     *             If the configuration is not valid

     * @throws IOException

     *             In case the initialisation of the Solr index was not possible

     * @throws SolrServerException

     *             Indicates that the referenced SolrServer has some problems (usually an invalid

     *             configuration).

     */

    @SuppressWarnings("unchecked")

    @Activate

    protected final void activate(ComponentContext context) throws ConfigurationException,

                                                           IOException,

                                                           SolrServerException {

        log.info("in {}.activate(..) with config {}",SolrYard.class.getSimpleName(), context.getProperties());

        this.context = context;

        activate(new SolrYardConfig((Dictionary<String,Object>) context.getProperties()));

    }

    /**

     * Internally used to configure an instance (within and without an OSGI container

     *

     * @param config

     *            The configuration

     * @throws ConfigurationException

     *             If the configuration is not valid

     * @throws IOException

     *             In case the initialisation of the Solr index was not possible

     * @throws SolrServerException

     *             Indicates that the referenced SolrServer has some problems (usually an invalid

     *             configuration).

     */

    private void activate(SolrYardConfig config) throws ConfigurationException,

                                                IOException,

                                                SolrServerException {

        // init with the default implementations of the ValueFactory and the QueryFactory

        super.activate(InMemoryValueFactory.getInstance(), DefaultQueryFactory.getInstance(), config);

        // mayby the super activate has updated the configuration

        config = (SolrYardConfig) this.getConfig();

        // check if immediateCommit is enable or disabled

        if (config.isImmediateCommit() != null) {

            immediateCommit = config.isImmediateCommit().booleanValue();

        } else {

            immediateCommit = DEFAULT_IMMEDIATE_COMMIT_STATE;

        }

        // check the maximum duration until changes are commited

        if (config.getCommitWithinDuration() != null) {

            commitWithin = config.getCommitWithinDuration().intValue();

        } else {

            commitWithin = DEFAULT_COMMIT_WITHIN_DURATION;

        }

        this.indexValueFactory = IndexValueFactory.getInstance();

        this.documentBoostFieldName = config.getDocumentBoostFieldName();

        this.fieldBoostMap = config.getFieldBoosts();

        //try to initialise the SolrServer

        ManagedSolrServer managedSolrServer = this.managedSolrServer;

        if(managedSolrServer != null){

            //check also if we need to create/init an SolrServer on startup

            try {

                checkManagedSolrIndex(managedSolrServer, config);

            } catch (Exception e) {

                log.error("Exception while checking SolrIndex '"+ config.getSolrServerLocation()

                    +"' on ManagedSolrServer '"+managedSolrServer.getServerName()+"'!",e);

            }

        }

        log.info("Activated SolrYard {}",config.getId());

    }

    /**

     * @param config

     */

    private SolrQueryFactory getSolrQueryFactory() throws YardException {

        SolrYardConfig config = (SolrYardConfig)getConfig();

        if(_solrQueryFactoy == null){

            _solrQueryFactoy = new SolrQueryFactory(getValueFactory(), indexValueFactory, getFieldMapper());

            if (config.isMultiYardIndexLayout()) { // set the yardID as domain if multiYardLayout is activated

                _solrQueryFactoy.setDomain(config.getId());

            }

            _solrQueryFactoy.setDefaultQueryResults(config.getDefaultQueryResultNumber());

            _solrQueryFactoy.setMaxQueryResults(config.getMaxQueryResultNumber());

        }

        return _solrQueryFactoy;

    }

    /**

     * Getter for the {@link SolrServer} used by this {@link SolrYard}.<p>

     * This method tries to {@link #initSolrServer()} if both {@link #_server}

     * and {@link #_registeredServerTracker} are <code>null</code>. The

     * {@link #_server} is used for remote SolrServers or if the component is

     * not running within an OSGI Environment. THe {@link #_registeredServerTracker}

     * is used for tracking SolrServers that do run in the same JVM.

     * @throws YardException if the {@link SolrServer} is currently not

     * active.

     */

    private SolrServer getServer() throws YardException {

        SolrServer server = null;

        if(_server == null && _registeredServerTracker == null){

            initSolrServer();

        }

        //when an internally managed Solr server is used by this SolrYard

        //we dynamically return the tracked version

        if(_registeredServerTracker != null){

            server = _registeredServerTracker.getService();

            //TODO: remove and replace with a setting where the SolrYard does not

            //      not activate until the SolrServer is available.

            if(server == null){

                for(int i = 0;i<5;i++){//waiting for a maximum of 5sec

                    try {

                        log.info(" ... waiting 1sec for SolrServer");

                       

                        server = (SolrServer)_registeredServerTracker.waitForService(1000);

                    } catch (InterruptedException e) {}

                }

            }

            if(server != null && !server.equals(this._server)){

                //reset the fieldMapper so that it is reinitialised for the new one

                //STANBOL-519

                _server = server;

                Lock writeLock = fieldMapperLock.writeLock();

                writeLock.lock();

                try {

                    _fieldMapper = null;

                }finally{

                    writeLock.unlock();

                }

            }

        } else {

            //for remove servers and when running outside OSGI

            server = _server;

        }

        //the server is not available -> throw an exception!

        if(server != null){

            return server;

        } else {

            _server = null;

            Lock writeLock = fieldMapperLock.writeLock();

            writeLock.lock();

            try {

                _fieldMapper = null;

            }finally{

                writeLock.unlock();

            }

            throw new YardException(String.format("The SolrIndex '%s' for SolrYard '%s' is currently not active!",

                ((SolrYardConfig)getConfig()).getSolrServerLocation(),getName()));

           

        }

    }

    /**

     * If the {@link SolrServer} of this SolrYard is managed on the

     * {@link #managedSolrServer}, than this method deactivates it. If a

     * remote server is used than calling this method does not have any effect.

     */

    private void deactivateSolrServer(){

        SolrYardConfig config = (SolrYardConfig) this.getConfig();

        String indexLocation = config.getSolrServerLocation();

        if(!(indexLocation.startsWith("http") && indexLocation.indexOf("://") > 0)){

            IndexReference indexReference = IndexReference.parse(indexLocation);

            ManagedSolrServer managedSolrServer = this.managedSolrServer;

            if(indexReference.isName() && managedSolrServer != null

                    && (indexReference.getServer() == null ||

                            indexReference.getServer().equals(managedSolrServer.getServerName()))){

                managedSolrServer.deactivateIndex(indexReference.getIndex());

            } //else no managed index or index not managed on #managedSolrServer

        }//an remote server

    }

    /**

     * Assumes that this method is only called if {@link #_server} and

     * {@link #_registeredServerTracker} is <code>null</code>

     * @throws YardException

     */

    private void initSolrServer() throws YardException {

        SolrYardConfig config = (SolrYardConfig) this.getConfig();

        String indexLocation = config.getSolrServerLocation();

        if(indexLocation.startsWith("http") && indexLocation.indexOf("://") > 0){

            //init remote server

            try {

                _server = new HttpSolrServer(indexLocation);

                _server.ping(); //test if remove service is available

            } catch (RuntimeException e) {

                throw new YardException("Unable to connect to remote SolrServer '"+

                    config.getSolrServerLocation() +"' because of "+e.getMessage(),e);

            } catch (SolrServerException e) {

                throw new YardException("Unable to initialise to remote SolrServer '"+

                    config.getSolrServerLocation() +"' because of "+e.getMessage(),e);

            } catch (IOException e) {

                throw new YardException("Unable to connect to remote SolrServer '"+

                    config.getSolrServerLocation() +"' because of "+e.getMessage(),e);

            }

        } else { //locally managed Server

            //(1) check if available (also tries to create if not available and

            //    create is allowed based on the configuration)

            IndexReference indexReference = checkManagedSolrIndex(managedSolrServer,config);

            if(indexReference != null){

                if(context == null){ // outside OSGI

                    try {

                        _server = StandaloneEmbeddedSolrServerProvider.getInstance().getSolrServer(indexReference);

                    } catch (RuntimeException e) {

                        throw new YardException("Unable to initialise configured SolrServer'"+

                            config.getSolrServerLocation() +"'!",e);

                    }

                } else { //within OSGI dynamically track the service

                    try {

                        _registeredServerTracker = new RegisteredSolrServerTracker(

                            context.getBundleContext(), indexReference, null);

                        log.info(" ... start tracking for SolrCore based on {}",indexReference);

                        _registeredServerTracker.open(); //start tracking

                    } catch (InvalidSyntaxException e) {

                        throw new YardException("Unable to track configured SolrServer'"+

                            config.getSolrServerLocation() +"'!",e);

                    } catch (RuntimeException e) {

                        throw new YardException("Unable to initialise configured SolrServer'"+

                            config.getSolrServerLocation() +"'!",e);

                    }

                }

            }

        }

    }

    /**

     * Checks if the SolrYard is active on the {@link #managedSolrServer}

     * @param config

     * @param indexReference

     * @throws YardException

     */

    private IndexReference checkManagedSolrIndex(ManagedSolrServer managedSolrServer, SolrYardConfig config) throws YardException {

        IndexReference indexReference = IndexReference.parse(config.getSolrServerLocation());

        if(indexReference.isName()){

            if(managedSolrServer == null){

                log.warn("Unable to init SolrIndex '{}' because ManagedSolrServer service is not available",

                    config.getSolrServerLocation());

                return null; //indicate that the server is not yet active

            } else if(indexReference.getServer() == null ||

                        indexReference.getServer().equals(managedSolrServer.getServerName())){

                //check if the referenced Index is Managed

                IndexMetadata indexMetadata = managedSolrServer.getIndexMetadata(indexReference.getIndex());

                if(indexMetadata == null){

                    // not managed -> try to create

                    IndexReference createdIndexRef = createSolrIndex(managedSolrServer,config, indexReference.getIndex());

                    if(context == null){

                        //in that case we need to replace the parse SolrServerLocation

                        //with the name of the created solr index

                        config.setSolrServerLocation(createdIndexRef.getIndex());

                    }

                    //return the created IndexReference

                    return createdIndexRef;

                } else if(!indexMetadata.isActive()){ //already managed, but not active

                    //try to activate

                    try {

                        IndexMetadata activatedMetadata = managedSolrServer.activateIndex(indexReference.getIndex());

                        if(activatedMetadata == null){

                            throw new YardException(String.format(

                                "Unable to actiate SolrIndex '%s' for SolrYard '%s" +

                                "on MnagedSolrServer '%s'!",indexReference,

                                getConfig().getName(),managedSolrServer.getServerName()));

                        } else {

                            return activatedMetadata.getIndexReference();

                        }

                    } catch (IOException e) {

                        throw new YardException("Unable to actiate SolrIndex for SolrYard "+getConfig().getName(),e);

                    } catch (SAXException e) {

                        throw new YardException("Unable to actiate SolrIndex for SolrYard "+getConfig().getName(),e);

                    } catch (RuntimeException e){

                        throw new YardException("Unable to actiate SolrIndex for SolrYard "+getConfig().getName(),e);

                    }

                } else{ //already active ... noting todo

                    return indexReference;

                }

            } else { //indexReference.getServer() != managedSolrServer.getServerName

                //TODO we would need to track all active ManagedSolrServer and

                //     check/create the core on the requested one!

                throw new IllegalStateException("Support for multiple ManagedSolrServer" +

                    "is not yet implemented (defaultServerName: '"+

                    managedSolrServer.getServerName()+"' requested: '"+

                    indexReference.getServer()+"')!");

            }

        } else { //a path was parsed

            // -> create not supported. Users are responsible to provide the server

            return indexReference;

        }

    }

    /**

     * Creates a new SolrIndex based on this Yards configuration by using the

     * {@link ManagedSolrServer} service

     * @param config the configuration of this SolrYard

     * @param solrIndexLocation the name of the index to create

     * @throws YardException On any Exception while creating the index

     */

    private IndexReference createSolrIndex(ManagedSolrServer managedServer, SolrYardConfig config, final String solrIndexLocation) throws YardException {

        String configName;

        if(config.isDefaultInitialisation()){

            configName = SolrYard.DEFAULT_SOLR_INDEX_CONFIGURATION_NAME;

        } else {

            configName = config.getIndexConfigurationName();

        }

        log.info(" ... initialise new SolrDirectory Index with name {} by using Index Configuration {}",

            solrIndexLocation,configName);

        try {

            IndexMetadata metadata = managedServer.createSolrIndex(solrIndexLocation,configName,null);

            if (metadata == null) {

                throw new YardException("SolrIndex "+ config.getSolrServerLocation() +

                        " is not available" + (config.isDefaultInitialisation() ?

                                " and could not be initialised!" :

                                    ". The necessary Index is not yet installed."));

            } else {

                log.info(" ... created IndexDirectory {} for SolrIndex {} by using config {}",

                    new Object[]{metadata.getDirectory(),solrIndexLocation,configName});

            }

            return metadata.getIndexReference();

        } catch (IOException e) {

            throw new YardException("SolrIndex "+ config.getSolrServerLocation() +

                    " could not be initialised!",e);

        }

    }

    private SolrFieldMapper getFieldMapper() throws YardException {

        Lock readLock = fieldMapperLock.readLock();

        readLock.lock();

        try {

            if(_fieldMapper != null){

                return _fieldMapper;

            }

        } finally {

            readLock.unlock();

        }

        Lock writeLock = fieldMapperLock.writeLock();

        writeLock.lock();

        try {

            if(_fieldMapper == null){ //might be init by an other thread

                _fieldMapper = new SolrFieldMapper(getServer(), nsPrefixService);

            }

            return _fieldMapper;

        } finally {

            writeLock.unlock();

        }

    }

   

    /**

     * Deactivates this SolrYard instance after committing remaining changes

     *

     * @param context

     */

    @Deactivate

    protected final void deactivate(ComponentContext context) {

        SolrYardConfig config = (SolrYardConfig) getConfig();

        log.info("... deactivating SolrYard " + config.getName() + " (id=" + config.getId() + ")");

        //close the RegisteredSolrServer tracker if used

        if(_registeredServerTracker != null){

            _registeredServerTracker.close();

            _registeredServerTracker = null;

        }

        this._server = null;

        this._fieldMapper = null; //in this case we can directly access the lazy field

        this.indexValueFactory = null;

        this._solrQueryFactoy = null;  //in this case we can directly access the lazy field

        this.documentBoostFieldName = null;

        this.fieldBoostMap = null;

        // reset the commitWithin and immediateCommit to the defaults

        this.commitWithin = DEFAULT_COMMIT_WITHIN_DURATION;

        this.immediateCommit = DEFAULT_IMMEDIATE_COMMIT_STATE;

        //deactivates the SolrCore used by this Yard if running in the local JVM

        deactivateSolrServer();

        super.deactivate(); // deactivate the super implementation

        this.context = null;

    }

    /**

     * This will case the SolrIndex to be optimised

     * @throws YardException on any error while optimising

     */

    public final void optimize() throws YardException{

        SolrServer server = getServer();

        try {

            server.optimize();

        } catch (SolrServerException e) {

            throw new YardException("Unable to optimise SolrIndex!", e);

        } catch (IOException e) {

            throw new YardException("Unable to optimise SolrIndex!", e);

        }

    }

    /**

     * can be used outside of the OSGI environment to deactivate this instance. Thiw will cause the SolrIndex

     * to be committed and optimised.

     */

    public void close() {

        deactivate(null);

    }

    /**

     * Calls the {@link #deactivate(ComponentContext)} with <code>null</code> as component context

     */

    @Override

    protected void finalize() throws Throwable {

        deactivate(null);

        super.finalize();

    }

    @Override

    public final QueryResultList<Representation> find(final FieldQuery parsedQuery) throws YardException {

        return find(getFieldMapper(),parsedQuery, SELECT.QUERY);

    }

    private QueryResultList<Representation> find(final FieldMapper fieldMapper,final FieldQuery parsedQuery, SELECT select) throws YardException {

        //create a clone of the query, because we need to refine it because the

        //query (as executed) needs to be included in the result set

        FieldQuery fieldQuery = parsedQuery.clone();

        log.debug("find " + fieldQuery);

        long start = System.currentTimeMillis();

        final Set<String> selected;

        if (select == SELECT.QUERY) {

            // if query set the fields to add to the result Representations

            selected = new HashSet<String>(fieldQuery.getSelectedFields());

            // add the score to query results!

            selected.add(RdfResourceEnum.resultScore.getUri());

        } else {

            // otherwise add all fields

            selected = null;

        }

        final SolrQuery query = getSolrQueryFactory().parseFieldQuery(fieldQuery, select);

        long queryGeneration = System.currentTimeMillis();

        final SolrServer server = getServer();

        QueryResponse response;

        try {

            response = AccessController.doPrivileged(new PrivilegedExceptionAction<QueryResponse>() {

                public QueryResponse run() throws IOException, SolrServerException {

                    StreamQueryRequest request = new StreamQueryRequest(query);

                     return request.process(server);

                }

            });

        } catch (PrivilegedActionException pae) {

            Exception e = pae.getException();

            if(e instanceof SolrServerException){

                if ("unknown handler: /mlt".equals(e.getCause().getMessage())) {

                    throw new YardException("Solr is missing '<requestHandler name=\"/mlt\""

                        + " class=\"solr.MoreLikeThisHandler\" startup=\"lazy\" />'"

                        + " in 'solrconfig.xml'", e);

                }

                throw new YardException("Error while performing Query on SolrServer: " + query.getQuery(), e);

            } else if(e instanceof IOException){

                throw new YardException("Unable to access SolrServer",e);

            } else {

                throw RuntimeException.class.cast(e);

            }

        }

        if(query.getQueryType() == SolrQueryFactory.MLT_QUERY_TYPE){

            log.info("{}",response);

        }

        long queryTime = System.currentTimeMillis();

        // return a queryResultList

        QueryResultListImpl<Representation> resultList = new QueryResultListImpl<Representation>(fieldQuery,

        // by adapting SolrDocuments to Representations

                new AdaptingIterator<SolrDocument,Representation>(response.getResults().iterator(),

                // inline Adapter Implementation

                        new AdaptingIterator.Adapter<SolrDocument,Representation>() {

                            @Override

                            public Representation adapt(SolrDocument doc, Class<Representation> type) {

                                // use this method for the conversion!

                                return createRepresentation(fieldMapper, doc, selected);

                            }

                        }, Representation.class), Representation.class);

        long resultProcessing = System.currentTimeMillis();

        log.debug(String.format(

            "  ... done [queryGeneration=%dms|queryTime=%dms|resultProcessing=%dms|sum=%dms]",

            (queryGeneration - start), (queryTime - queryGeneration), (resultProcessing - queryTime),

            (resultProcessing - start)));

        return resultList;

    }

    @Override

    public final QueryResultList<String> findReferences(FieldQuery parsedQuery) throws YardException {

        //create a clone of the query, because we need to refine it because the

        //query (as executed) needs to be included in the result set

        FieldQuery fieldQuery = parsedQuery.clone();

        final SolrQuery query = getSolrQueryFactory().parseFieldQuery(fieldQuery, SELECT.ID);

        QueryResponse response;

        final SolrServer server = getServer();

        try {

            response = AccessController.doPrivileged(new PrivilegedExceptionAction<QueryResponse>() {

                public QueryResponse run() throws IOException, SolrServerException {

                        return server.query(query, METHOD.POST);

                }

            });

        } catch (PrivilegedActionException pae) {

            Exception e = pae.getException();

            if(e instanceof SolrServerException){

                throw new YardException("Error while performing query on the SolrServer (query: "

                        + query.getQuery()+")!", e);

            } else if(e instanceof IOException){

                throw new YardException("Unable to access SolrServer",e);

            } else {

                throw RuntimeException.class.cast(e);

            }

        }

        final FieldMapper fieldMapper = getFieldMapper();

        // return a queryResultList

        return new QueryResultListImpl<String>(fieldQuery,

        // by adapting SolrDocuments to Representations

                new AdaptingIterator<SolrDocument,String>(response.getResults().iterator(),

                // inline Adapter Implementation

                        new AdaptingIterator.Adapter<SolrDocument,String>() {

                            @Override

                            public String adapt(SolrDocument doc, Class<String> type) {

                                // use this method for the conversion!

                                return doc.getFirstValue(fieldMapper.getDocumentIdField()).toString();

                            }

                        }, String.class), String.class);

    }

    @Override

    public final QueryResultList<Representation> findRepresentation(FieldQuery parsedQuery) throws YardException {

        return find(getFieldMapper(),parsedQuery, SELECT.ALL);

    }

    @Override

    public final Representation getRepresentation(String id) throws YardException {

        if (id == null) {

            throw new IllegalArgumentException("The parsed Representation id MUST NOT be NULL!");

        }

        if (id.isEmpty()) {

            throw new IllegalArgumentException("The parsed Representation id MUST NOT be empty!");

        }

        SolrServer server = getServer();

        FieldMapper fieldMapper = getFieldMapper();

        SolrDocument doc;

        long start = System.currentTimeMillis();

        try {

            doc = getSolrDocument(server,fieldMapper,id);

        } catch (SolrServerException e) {

            throw new YardException("Error while getting SolrDocument for id" + id, e);

        } catch (IOException e) {

            throw new YardException("Unable to access SolrServer", e);

        }

        long retrieve = System.currentTimeMillis();

        Representation rep;

        if (doc != null) {

            // create an Representation for the Doc! retrieve

            log.debug(String.format("Create Representation %s from SolrDocument",

                doc.getFirstValue(fieldMapper.getDocumentIdField())));

            rep = createRepresentation(fieldMapper,doc, null);

        } else {

            rep = null;

        }

        long create = System.currentTimeMillis();

        log.debug(String.format("  ... %s [retrieve=%dms|create=%dms|sum=%dms]", rep == null ? "not found"

                : "done", (retrieve - start), (create - retrieve), (create - start)));

        return rep;

    }

    /**

     * Creates the Representation for the parsed SolrDocument!

     *

     * @param doc

     *            The Solr Document to convert

     * @param fields

     *            if NOT NULL only this fields are added to the Representation

     * @return the Representation

     */

    protected final Representation createRepresentation(FieldMapper fieldMapper, SolrDocument doc, Set<String> fields) {

        if(fieldMapper == null){

            throw new IllegalArgumentException("The parsed FieldMapper MUST NOT be NULL!");

        }

        if(doc == null){

            throw new IllegalArgumentException("The parsed SolrDocument MUST NOT be NULL!");

        }

        Object id = doc.getFirstValue(fieldMapper.getDocumentIdField());

        if (id == null) {

            throw new IllegalStateException(String.format(

                "The parsed Solr Document does not contain a value for the %s Field!",

                fieldMapper.getDocumentIdField()));

        }

        Representation rep = getValueFactory().createRepresentation(id.toString());

        for (String fieldName : doc.getFieldNames()) {

            IndexField indexField = fieldMapper.getField(fieldName);

            if (indexField != null && indexField.getPath().size() == 1) {

                String lang = indexField.getLanguages().isEmpty() ? null : indexField.getLanguages()

                        .iterator().next();

                if (fields == null || fields.contains(indexField.getPath().get(0))) {

                    for (Object value : doc.getFieldValues(fieldName)) {

                        if (value != null) {

                            IndexDataTypeEnum dataTypeEnumEntry = IndexDataTypeEnum.forIndexType(indexField

                                    .getDataType());

                            if (dataTypeEnumEntry != null) {

                                Object javaValue = indexValueFactory.createValue(

                                    dataTypeEnumEntry.getJavaType(), indexField.getDataType(), value, lang);

                                if (javaValue != null) {

                                    rep.add(indexField.getPath().iterator().next(), javaValue);

                                } else {

                                    log.warn(String.format("java value=null for index value %s", value));

                                }

                            } else {

                                log.warn(String.format(

                                    "No DataType Configuration found for Index Data Type %s!",

                                    indexField.getDataType()));

                            }

                        } // else index value == null -> ignore

                    } // end for all values

                }

            } else {

                if (indexField != null) {

                    log.warn(String.format("Unable to prozess Index Field %s (for IndexDocument Field: %s)",

                        indexField, fieldName));

                }

            }

        } // end for all fields

        return rep;

    }

    @Override

    public final boolean isRepresentation(String id) throws YardException {

        if (id == null) {

            throw new IllegalArgumentException("The parsed Representation id MUST NOT be NULL!");

        }

        if (id.isEmpty()) {

            throw new IllegalArgumentException("The parsed Representation id MUST NOT be empty!");

        }

        final SolrServer server = getServer();

        final FieldMapper fieldMapper = getFieldMapper();

        try {

            return getSolrDocument(server,fieldMapper,id, Arrays.asList(fieldMapper.getDocumentIdField())) != null;

        } catch (SolrServerException e) {

            throw new YardException("Error while performing getDocumentByID request for id " + id, e);

        } catch (IOException e) {

            throw new YardException("Unable to access SolrServer", e);

        }

    }

    /**

     * Checks what of the documents referenced by the parsed IDs are present in the Solr Server

     *

     * @param ids

     *            the ids of the documents to check

     * @return the ids of the found documents

     * @throws SolrServerException

     *             on any exception of the SolrServer

     * @throws IOException

     *             an any IO exception while accessing the SolrServer

     */

    protected final Set<String> checkRepresentations(SolrServer server,

                                                     FieldMapper fieldMapper,

                                                     Set<String> ids)

                                                     throws SolrServerException, IOException {

        Set<String> found = new HashSet<String>();

        String field = fieldMapper.getDocumentIdField();

        for (SolrDocument foundDoc : getSolrDocuments(server,fieldMapper,ids, Arrays.asList(field))) {

            Object value = foundDoc.getFirstValue(field);

            if (value != null) {

                found.add(value.toString());

            }

        }

        return found;

    }

    @Override

    public final void remove(final String id) throws YardException, IllegalArgumentException {

        if (id == null) {

            throw new IllegalArgumentException("The parsed Representation id MUST NOT be NULL!");

        }

        if (id.isEmpty()) {

            throw new IllegalArgumentException("The parsed Representation id MUST NOT be empty!");

        }

        final SolrServer server = getServer();

        final SolrYardConfig config = (SolrYardConfig)getConfig();

        final SolrFieldMapper fieldMapper = getFieldMapper();

        try {

            AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {

                public Object run() throws IOException, SolrServerException {

                    if(config.isMultiYardIndexLayout()){

                        //make sure we only delete the Entity only if it is  managed by

                        //this Yard. Entities of other Yards MUST NOT be deleted!

                        server.deleteByQuery(String.format("%s:%s AND %s:%s",

                            fieldMapper.getDocumentDomainField(),

                            SolrUtil.escapeSolrSpecialChars(getId()),

                            fieldMapper.getDocumentIdField(),

                            SolrUtil.escapeSolrSpecialChars(id)));

                    } else {

                        server.deleteById(id);

                    }

                    server.commit();

                    return null;

                }

            });

        } catch (PrivilegedActionException pae) {

            Exception e = pae.getException();

            if(e instanceof SolrServerException){

                throw new YardException("Error while deleting document " + id + " from the Solr server", e);

            } else if(e instanceof IOException){

                throw new YardException("Unable to access SolrServer",e);

            } else {

                throw RuntimeException.class.cast(e);

            }

        }

        // NOTE: We do not need to update all Documents that refer this ID, because

        // only the representation of the Entity is deleted and not the

        // Entity itself. So even that we do no longer have an representation

        // the entity still exists and might be referenced by others!

    }

    @Override

    public final void remove(Iterable<String> ids) throws IllegalArgumentException, YardException {

        if (ids == null) {

            throw new IllegalArgumentException("The parsed IDs MUST NOT be NULL");

        }

        final List<String> toRemove = new ArrayList<String>();

        for (String id : ids) {

            if (id != null && !id.isEmpty()) {

                toRemove.add(id);

            }

        }

        final SolrServer server = getServer();

        final SolrYardConfig config = (SolrYardConfig)getConfig();

        final SolrFieldMapper fieldMapper = getFieldMapper();

        try {

            AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {

                public Object run() throws IOException, SolrServerException {

                    if(config.isMultiYardIndexLayout()){

                        //make sure we only delete Entities managed by this Yard

                        //if someone parses an ID managed by an other yard we MUST NOT

                        //delete it!

                        for(String id : toRemove){

                            server.deleteByQuery(String.format("%s:%s AND %s:%s",

                                fieldMapper.getDocumentDomainField(),

                                SolrUtil.escapeSolrSpecialChars(getId()),

                                fieldMapper.getDocumentIdField(),

                                SolrUtil.escapeSolrSpecialChars(id)));

                        }

                    } else {

                        server.deleteById(toRemove);

                    }

                    server.commit();

                    return null;

                }

            });

        } catch (PrivilegedActionException pae) {

            Exception e = pae.getException();

            if(e instanceof SolrServerException){

                throw new YardException("Error while deleting documents from the Solr server", e);

            } else if(e instanceof IOException){

                throw new YardException("Unable to access SolrServer",e);

            } else {

                throw RuntimeException.class.cast(e);

            }

        }

        // NOTE: We do not need to update all Documents that refer this ID, because

        // only the representation of the Entity is deleted and not the

        // Entity itself. So even that we do no longer have an representation

        // the entity still exists and might be referenced by others!

    }

    @Override

    public void removeAll() throws YardException {

        final SolrServer server = getServer();

        final SolrYardConfig config = (SolrYardConfig)getConfig();

        try {

            //delete all documents

            AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {

                public Object run() throws IOException, SolrServerException, YardException {

                    //ensures that the fildMapper is initialised and reads the

                    //namespace config before deleting all documents

                    FieldMapper fieldMapper = getFieldMapper();

                    if(config.isMultiYardIndexLayout()){

                        //only delete entities of this referenced site

                        server.deleteByQuery(String.format("%s:%s",

                            fieldMapper.getDocumentDomainField(),

                            SolrUtil.escapeSolrSpecialChars(getId())));

                    } else { //we can delete all

                        server.deleteByQuery("*:*");

                    }

                    //ensure that the namespace config is stored again after deleting

                    //all documents

                    getFieldMapper().saveNamespaceConfig(false);

                    server.commit();

                    return null;

                }

            });

        } catch (PrivilegedActionException pae) {

            Exception e = pae.getException();

            if(e instanceof SolrServerException){

                throw new YardException("Error while deleting documents from the Solr server", e);

            } else if(e instanceof IOException){

                throw new YardException("Unable to access SolrServer",e);

            } else if(e instanceof YardException){

                throw (YardException)e;

            } else {

                throw RuntimeException.class.cast(e);

            }

        }       

    }

    @Override

    public final Representation store(Representation representation) throws YardException,

                                                                    IllegalArgumentException {

        log.debug("Store {}", representation != null ? representation.getId() : null);

        if (representation == null) {

            throw new IllegalArgumentException("The parsed Representation MUST NOT be NULL!");

        }

        final SolrServer server = getServer();

        FieldMapper fieldMapper = getFieldMapper();

        long start = System.currentTimeMillis();

        final SolrInputDocument inputDocument = createSolrInputDocument(fieldMapper,representation);

        long create = System.currentTimeMillis();

        try {

            final UpdateRequest update = new UpdateRequest();

            if (!immediateCommit) {

                update.setCommitWithin(commitWithin);

            }

            update.add(inputDocument);

            AccessController.doPrivileged(new PrivilegedExceptionAction<Object>() {

                public Object run() throws IOException, SolrServerException {

                    update.process(server);

                    if (immediateCommit) {

                        server.commit();

                    }

                    return null; // nothing to return

                }

            });

            long stored = System.currentTimeMillis();