* Licensed to the Apache Software Foundation (ASF) under one or more

* contributor license agreements. See the NOTICE file distributed with

* this work for additional information regarding copyright ownership.

* The ASF licenses this file to You under the Apache License, Version 2.0

* (the "License"); you may not use this file except in compliance with

* the License. You may obtain a copy of the License at

*

* http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

*/

  public static final String _rcsid = "@(#)$Id: SolrConnector.java 998081 2010-09-17 11:33:15Z kwright $";

  // Activities we log

  /** Ingestion activity */

  public final static String INGEST_ACTIVITY = "document ingest";

  /** Document removal activity */

  public final static String REMOVE_ACTIVITY = "document deletion";

  /** Local connection */

  protected HttpPoster poster = null;

 

  /** Expiration */

  protected long expirationTime = -1L;

 

  /** The allow attribute name */

  protected String allowAttributeName = "allow_token_";

  /** The deny attribute name */

  protected String denyAttributeName = "deny_token_";

  /** The maximum document length */

  protected Long maxDocumentLength = null;

  /** Included mime types string */

  protected String includedMimeTypesString = null;

  /** Included mime types */

  protected Map<String,String> includedMimeTypes = null;

  /** Excluded mime types string */

  protected String excludedMimeTypesString = null;

  /** Excluded mime types */

  protected Map<String,String> excludedMimeTypes = null;

 

  // Attributes going into Solr

  protected String idAttributeName = null;

  protected String modifiedDateAttributeName = null;

  protected String createdDateAttributeName = null;

  protected String indexedDateAttributeName = null;

  protected String fileNameAttributeName = null;

  protected String mimeTypeAttributeName = null;

  protected String contentAttributeName = null;

  /** Use extractiing update handler? */

  protected boolean useExtractUpdateHandler = true;

  /** Use URL encoding for field names? */

  protected boolean useUrlEncoding = true;

 

  /** Whether or not to commit */

  protected boolean doCommits = false;

 

  /** Idle connection expiration interval */

  protected final static long EXPIRATION_INTERVAL = 300000L;

  static

  {

    System.setProperty("jdk.xml.maxGeneralEntitySizeLimit",new Integer(Integer.MAX_VALUE).toString());

  }

 

  /** Constructor.

  */

  public SolrConnector()

  {

  }

  /** Return the list of activities that this connector supports (i.e. writes into the log).

  *@return the list.

  */

  @Override

  public String[] getActivitiesList()

  {

    return new String[]{INGEST_ACTIVITY,REMOVE_ACTIVITY};

  }

  /** Connect.

  *@param configParameters is the set of configuration parameters, which

  * in this case describe the target appliance, basic auth configuration, etc.  (This formerly came

  * out of the ini file.)

  */

  @Override

  public void connect(ConfigParams configParameters)

  {

    super.connect(configParameters);

  }

  /** This method is periodically called for all connectors that are connected but not

  * in active use.

  */

  @Override

  public void poll()

    throws ManifoldCFException

  {

    if (poster != null)

    {

      if (expirationTime <= System.currentTimeMillis())

      {

        // Expire connection

        poster.shutdown();

        poster = null;

        expirationTime = -1L;

      }

    }

  }

  /** This method is called to assess whether to count this connector instance should

  * actually be counted as being connected.

  *@return true if the connector instance is actually connected.

  */

  @Override

  public boolean isConnected()

  {

    return poster != null;

  }

  /** Close the connection.  Call this before discarding the connection.

  */

  @Override

  public void disconnect()

    throws ManifoldCFException

  {

    if (poster != null)

    {

      poster.shutdown();

      poster = null;

      expirationTime = -1L;

    }

    maxDocumentLength = null;

    includedMimeTypesString = null;

    includedMimeTypes = null;

    excludedMimeTypesString = null;

    excludedMimeTypes = null;

    idAttributeName = null;

    modifiedDateAttributeName = null;

    createdDateAttributeName = null;

    indexedDateAttributeName = null;

    fileNameAttributeName = null;

    mimeTypeAttributeName = null;

    contentAttributeName = null;

    useExtractUpdateHandler = true;

    useUrlEncoding = true;

    super.disconnect();

  }

  /** Set up a session */

  protected void getSession()

    throws ManifoldCFException

  {

    if (poster == null)

    {

      String updatePath = params.getParameter(SolrConfig.PARAM_UPDATEPATH);

      if (updatePath == null || updatePath.length() == 0)

        updatePath = "";

      String removePath = params.getParameter(SolrConfig.PARAM_REMOVEPATH);

      if (removePath == null || removePath.length() == 0)

        removePath = "";

      String statusPath = params.getParameter(SolrConfig.PARAM_STATUSPATH);

      if (statusPath == null || statusPath.length() == 0)

        statusPath = "";

      idAttributeName = params.getParameter(SolrConfig.PARAM_IDFIELD);

      if (idAttributeName == null || idAttributeName.length() == 0)

        idAttributeName = "id";

      modifiedDateAttributeName = params.getParameter(SolrConfig.PARAM_MODIFIEDDATEFIELD);

      if (modifiedDateAttributeName == null || modifiedDateAttributeName.length() == 0)

        modifiedDateAttributeName = null;

      createdDateAttributeName = params.getParameter(SolrConfig.PARAM_CREATEDDATEFIELD);

      if (createdDateAttributeName == null || createdDateAttributeName.length() == 0)

        createdDateAttributeName = null;

 

      indexedDateAttributeName = params.getParameter(SolrConfig.PARAM_INDEXEDDATEFIELD);

      if (indexedDateAttributeName == null || indexedDateAttributeName.length() == 0)

        indexedDateAttributeName = null;

      fileNameAttributeName = params.getParameter(SolrConfig.PARAM_FILENAMEFIELD);

      if (fileNameAttributeName == null || fileNameAttributeName.length() == 0)

        fileNameAttributeName = null;

      mimeTypeAttributeName = params.getParameter(SolrConfig.PARAM_MIMETYPEFIELD);

      if (mimeTypeAttributeName == null || mimeTypeAttributeName.length() == 0)

        mimeTypeAttributeName = null;

      contentAttributeName = params.getParameter(SolrConfig.PARAM_CONTENTFIELD);

      if (contentAttributeName == null || contentAttributeName.length() == 0)

        contentAttributeName = null;

     

      String useExtractUpdateHandlerValue = params.getParameter(SolrConfig.PARAM_EXTRACTUPDATE);

      if (useExtractUpdateHandlerValue == null || useExtractUpdateHandlerValue.length() == 0)

        useExtractUpdateHandler = true;

      else

        useExtractUpdateHandler = !useExtractUpdateHandlerValue.equals("false");

      if (contentAttributeName == null && !useExtractUpdateHandler)

        throw new ManifoldCFException("Content attribute name required for non-extract-update indexing");

      String useUrlEncodingValue = params.getParameter(SolrConfig.PARAM_URLENCODING);

      if (useUrlEncodingValue == null || useUrlEncodingValue.length() == 0)

        useUrlEncoding = true;

      else

        useUrlEncoding = !useUrlEncodingValue.equals("false");

      String commits = params.getParameter(SolrConfig.PARAM_COMMITS);

      if (commits == null || commits.length() == 0)

        commits = "true";

     

      doCommits = commits.equals("true");

     

      String commitWithin = params.getParameter(SolrConfig.PARAM_COMMITWITHIN);

      if (commitWithin == null || commitWithin.length() == 0)

        commitWithin = null;

     

      String docMax = params.getParameter(SolrConfig.PARAM_MAXLENGTH);

      if (docMax == null || docMax.length() == 0)

        maxDocumentLength = null;

      else

        maxDocumentLength = new Long(docMax);

      if (maxDocumentLength == null && !useExtractUpdateHandler)

        throw new ManifoldCFException("Maximum document length required for non-extract-update indexing");

     

      includedMimeTypesString = params.getParameter(SolrConfig.PARAM_INCLUDEDMIMETYPES);

      if (includedMimeTypesString == null || includedMimeTypesString.length() == 0)

      {

        includedMimeTypesString = null;

        includedMimeTypes = null;

      }

      else

      {

        // Parse the included mime types

        includedMimeTypes = parseMimeTypes(includedMimeTypesString);

        if (includedMimeTypes.size() == 0)

        {

          includedMimeTypesString = null;

          includedMimeTypes = null;

        }

      }

      excludedMimeTypesString = params.getParameter(SolrConfig.PARAM_EXCLUDEDMIMETYPES);

      if (excludedMimeTypesString == null || excludedMimeTypesString.length() == 0)

      {

        excludedMimeTypesString = null;

        excludedMimeTypes = null;

      }

      else

      {

        // Parse the included mime types

        excludedMimeTypes = parseMimeTypes(excludedMimeTypesString);

        if (excludedMimeTypes.size() == 0)

        {

          excludedMimeTypesString = null;

          excludedMimeTypes = null;

        }

      }

     

      // Now, initialize Solr-j

      String solrType = params.getParameter(SolrConfig.PARAM_SOLR_TYPE);

      if (solrType == null)

        solrType = SolrConfig.SOLR_TYPE_STANDARD;

      if (solrType.equals(SolrConfig.SOLR_TYPE_STANDARD))

      {

        String userID = params.getParameter(SolrConfig.PARAM_USERID);

        String password = params.getObfuscatedParameter(SolrConfig.PARAM_PASSWORD);

        String realm = params.getParameter(SolrConfig.PARAM_REALM);

        String keystoreData = params.getParameter(SolrConfig.PARAM_KEYSTORE);

        IKeystoreManager keystoreManager;

        if (keystoreData != null)

          keystoreManager = KeystoreManagerFactory.make("",keystoreData);

        else

          keystoreManager = null;

        String protocol = params.getParameter(SolrConfig.PARAM_PROTOCOL);

        if (protocol == null || protocol.length() == 0)

          throw new ManifoldCFException("Missing parameter: "+SolrConfig.PARAM_PROTOCOL);

        String server = params.getParameter(SolrConfig.PARAM_SERVER);

        if (server == null || server.length() == 0)

          throw new ManifoldCFException("Missing parameter: "+SolrConfig.PARAM_SERVER);

        String port = params.getParameter(SolrConfig.PARAM_PORT);

        if (port == null || port.length() == 0)

          port = "80";

        String webapp = params.getParameter(SolrConfig.PARAM_WEBAPPNAME);

        if (webapp != null && webapp.length() == 0)

          webapp = null;

        String core = params.getParameter(SolrConfig.PARAM_CORE);

        if (core != null && core.length() == 0)

          core = null;

        // Pick up timeouts

        String socketTimeoutString = params.getParameter(SolrConfig.PARAM_SOCKET_TIMEOUT);

        if (socketTimeoutString == null)

          socketTimeoutString = "900";

        String connectTimeoutString = params.getParameter(SolrConfig.PARAM_CONNECTION_TIMEOUT);

        if (connectTimeoutString == null)

          connectTimeoutString = "60";

       

        try

        {

          int socketTimeout = Integer.parseInt(socketTimeoutString) * 1000;

          int connectTimeout = Integer.parseInt(connectTimeoutString) * 1000;

         

          poster = new HttpPoster(protocol,server,Integer.parseInt(port),webapp,core,

            connectTimeout,socketTimeout,

            updatePath,removePath,statusPath,realm,userID,password,

            allowAttributeName,denyAttributeName,idAttributeName,

            modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,

            fileNameAttributeName,mimeTypeAttributeName,contentAttributeName,

            keystoreManager,maxDocumentLength,commitWithin,useExtractUpdateHandler,

            useUrlEncoding);

         

        }

        catch (NumberFormatException e)

        {

          throw new ManifoldCFException(e.getMessage());

        }

      }

      else if (solrType.equals(SolrConfig.SOLR_TYPE_SOLRCLOUD))

      {

        StringBuilder zookeeperString = new StringBuilder();

        // Pull together the zookeeper string describing the zookeeper nodes

        for (int i = 0; i < params.getChildCount(); i++)

        {

          ConfigurationNode cn = params.getChild(i);

          if (cn.getType().equals(SolrConfig.NODE_ZOOKEEPER))

          {

            if (zookeeperString.length() > 0)

              zookeeperString.append(",");

            zookeeperString.append(cn.getAttributeValue(SolrConfig.ATTR_HOST)).append(":").append(cn.getAttributeValue(SolrConfig.ATTR_PORT));

          }

        }

       

        String znodePath = params.getParameter(SolrConfig.PARAM_ZOOKEEPER_ZNODE_PATH);

        if (znodePath == null)

          znodePath = "";

       

        String zookeeperHost = zookeeperString.toString() + znodePath;

       

        // Get collection

        String collection = params.getParameter(SolrConfig.PARAM_COLLECTION);

        if (collection == null)

          collection = "collection1";

        // Pick up timeouts

        String zkClientTimeoutString = params.getParameter(SolrConfig.PARAM_ZOOKEEPER_CLIENT_TIMEOUT);

        if (zkClientTimeoutString == null)

          zkClientTimeoutString = "60";

        String zkConnectTimeoutString = params.getParameter(SolrConfig.PARAM_ZOOKEEPER_CONNECT_TIMEOUT);

        if (zkConnectTimeoutString == null)

          zkConnectTimeoutString = "60";

       

        // Create an httpposter

        try

        {

          int zkClientTimeout = Integer.parseInt(zkClientTimeoutString) * 1000;

          int zkConnectTimeout = Integer.parseInt(zkConnectTimeoutString) * 1000;

         

          poster = new HttpPoster(zookeeperHost,collection,

            zkClientTimeout,zkConnectTimeout,

            updatePath,removePath,statusPath,

            allowAttributeName,denyAttributeName,idAttributeName,

            modifiedDateAttributeName,createdDateAttributeName,indexedDateAttributeName,

            fileNameAttributeName,mimeTypeAttributeName,contentAttributeName,

            maxDocumentLength,commitWithin,useExtractUpdateHandler,

            useUrlEncoding);

         

        }

        catch (NumberFormatException e)

        {

          throw new ManifoldCFException(e.getMessage());

        }

      }

      else

        throw new ManifoldCFException("Illegal value for parameter '"+SolrConfig.PARAM_SOLR_TYPE+"': '"+solrType+"'");

     

    }

    expirationTime = System.currentTimeMillis() + EXPIRATION_INTERVAL;

  }

  /** Parse a mime type field into individual mime types in a hash */

  protected static Map<String,String> parseMimeTypes(String mimeTypes)

    throws ManifoldCFException

  {

    Map<String,String> rval = new HashMap<String,String>();

    try

    {

      java.io.Reader str = new java.io.StringReader(mimeTypes);

      try

      {

        java.io.BufferedReader is = new java.io.BufferedReader(str);

        try

        {

          while (true)

          {

            String nextString = is.readLine();

            if (nextString == null)

              break;

            if (nextString.length() == 0)

              continue;

            rval.put(nextString,nextString);

          }

          return rval;

        }

        finally

        {

          is.close();

        }

      }

      finally

      {

        str.close();

      }

    }

    catch (java.io.IOException e)

    {

      throw new ManifoldCFException("IO error: "+e.getMessage(),e);

    }

  }

  /** Test the connection.  Returns a string describing the connection integrity.

  *@return the connection's status as a displayable string.

  */

  @Override

  public String check()

    throws ManifoldCFException

  {

    try

    {

      getSession();

      poster.checkPost();

      return super.check();

    }

    catch (ServiceInterruption e)

    {

      return "Transient error: "+e.getMessage();

    }

  }

  /** Get an output version string, given an output specification.  The output version string is used to uniquely describe the pertinent details of

  * the output specification and the configuration, to allow the Connector Framework to determine whether a document will need to be output again.

  * Note that the contents of the document cannot be considered by this method, and that a different version string (defined in IRepositoryConnector)

  * is used to describe the version of the actual document.

  *

  * This method presumes that the connector object has been configured, and it is thus able to communicate with the output data store should that be

  * necessary.

  *@param spec is the current output specification for the job that is doing the crawling.

  *@return a string, of unlimited length, which uniquely describes output configuration and specification in such a way that if two such strings are equal,

  * the document will not need to be sent again to the output data store.

  */

  @Override

  public VersionContext getPipelineDescription(Specification spec)

    throws ManifoldCFException, ServiceInterruption

  {

    getSession();

    SpecPacker sp = new SpecPacker(spec);

    return new VersionContext(sp.toPackedString(),params,spec);

  }

  private final static Set<String> acceptableMimeTypes = new HashSet<String>();

  static

  {

    acceptableMimeTypes.add("text/plain;charset=utf-8");

    acceptableMimeTypes.add("text/plain;charset=ascii");

    acceptableMimeTypes.add("text/plain;charset=us-ascii");

    acceptableMimeTypes.add("text/plain");

  }

  /** Detect if a mime type is indexable or not.  This method is used by participating repository connectors to pre-filter the number of

  * unusable documents that will be passed to this output connector.

  *@param outputDescription is the document's output version.

  *@param mimeType is the mime type of the document.

  *@return true if the mime type is indexable by this connector.

  */

  @Override

  public boolean checkMimeTypeIndexable(String outputDescription, String mimeType)

    throws ManifoldCFException, ServiceInterruption

  {

    getSession();

    if (useExtractUpdateHandler)

    {

      if (includedMimeTypes != null && includedMimeTypes.get(mimeType) == null)

        return false;

      if (excludedMimeTypes != null && excludedMimeTypes.get(mimeType) != null)

        return false;

      return super.checkMimeTypeIndexable(outputDescription,mimeType);

    }

    return acceptableMimeTypes.contains(mimeType.toLowerCase(Locale.ROOT));

  }

  /** Pre-determine whether a document's length is indexable by this connector.  This method is used by participating repository connectors

  * to help filter out documents that are too long to be indexable.

  *@param outputDescription is the document's output version.

  *@param length is the length of the document.

  *@return true if the file is indexable.

  */

  @Override

  public boolean checkLengthIndexable(String outputDescription, long length)

    throws ManifoldCFException, ServiceInterruption

  {

    getSession();

    if (maxDocumentLength != null && length > maxDocumentLength.longValue())

      return false;

    return super.checkLengthIndexable(outputDescription,length);

  }

  /** Add (or replace) a document in the output data store using the connector.

  * This method presumes that the connector object has been configured, and it is thus able to communicate with the output data store should that be

  * necessary.

  * The OutputSpecification is *not* provided to this method, because the goal is consistency, and if output is done it must be consistent with the

  * output description, since that was what was partly used to determine if output should be taking place.  So it may be necessary for this method to decode

  * an output description string in order to determine what should be done.

  *@param documentURI is the URI of the document.  The URI is presumed to be the unique identifier which the output data store will use to process

  * and serve the document.  This URI is constructed by the repository connector which fetches the document, and is thus universal across all output connectors.

  *@param outputDescription is the description string that was constructed for this document by the getOutputDescription() method.

  *@param document is the document data to be processed (handed to the output data store).

  *@param authorityNameString is the name of the authority responsible for authorizing any access tokens passed in with the repository document.  May be null.

  *@param activities is the handle to an object that the implementer of an output connector may use to perform operations, such as logging processing activity.

  *@return the document status (accepted or permanently rejected).

  */

  @Override

  public int addOrReplaceDocument(String documentURI, String outputDescription, RepositoryDocument document, String authorityNameString, IOutputAddActivity activities)

    throws ManifoldCFException, ServiceInterruption

  {

    SpecPacker sp = new SpecPacker(outputDescription);

    // Establish a session

    getSession();

    // Now, go off and call the ingest API.

    if (poster.indexPost(documentURI,document,sp.getArgs(),sp.getMappings(),sp.keepAllMetadata(),authorityNameString,activities))

      return DOCUMENTSTATUS_ACCEPTED;

    return DOCUMENTSTATUS_REJECTED;

  }

  /** Remove a document using the connector.

  * Note that the last outputDescription is included, since it may be necessary for the connector to use such information to know how to properly remove the document.

  *@param documentURI is the URI of the document.  The URI is presumed to be the unique identifier which the output data store will use to process

  * and serve the document.  This URI is constructed by the repository connector which fetches the document, and is thus universal across all output connectors.

  *@param outputDescription is the last description string that was constructed for this document by the getOutputDescription() method above.

  *@param activities is the handle to an object that the implementer of an output connector may use to perform operations, such as logging processing activity.

  */

  @Override

  public void removeDocument(String documentURI, String outputDescription, IOutputRemoveActivity activities)

    throws ManifoldCFException, ServiceInterruption

  {

    // Establish a session

    getSession();

    poster.deletePost(documentURI,activities);

  }

  /** Notify the connector of a completed job.

  * This is meant to allow the connector to flush any internal data structures it has been keeping around, or to tell the output repository that this

  * is a good time to synchronize things.  It is called whenever a job is either completed or aborted.

  *@param activities is the handle to an object that the implementer of an output connector may use to perform operations, such as logging processing activity.

  */

  @Override

  public void noteJobComplete(IOutputNotifyActivity activities)

    throws ManifoldCFException, ServiceInterruption

  {

    // Establish a session

    getSession();

   

    // Do a commit post

    if (doCommits)

    {

      poster.commitPost();

    }

  }

  // UI support methods.

  //