Package com.dotcms.content.elasticsearch.business

Source Code of com.dotcms.content.elasticsearch.business.ESContentletIndexAPI

package com.dotcms.content.elasticsearch.business;

import java.io.File;
import java.io.IOException;
import java.net.URL;
import java.sql.Connection;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.List;
import java.util.Map;

import com.dotcms.repackage.org.elasticsearch.ElasticSearchException;
import com.dotcms.repackage.org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import com.dotcms.repackage.org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import com.dotcms.repackage.org.elasticsearch.action.admin.indices.status.IndexStatus;
import com.dotcms.repackage.org.elasticsearch.action.bulk.BulkRequestBuilder;
import com.dotcms.repackage.org.elasticsearch.action.index.IndexRequest;
import com.dotcms.repackage.org.elasticsearch.client.Client;
import com.dotcms.repackage.org.elasticsearch.client.IndicesAdminClient;
import com.dotcms.repackage.org.elasticsearch.index.query.QueryBuilders;

import com.dotcms.content.business.DotMappingException;
import com.dotcms.content.elasticsearch.business.IndiciesAPI.IndiciesInfo;
import com.dotcms.content.elasticsearch.util.ESClient;
import com.dotmarketing.business.APILocator;
import com.dotmarketing.business.DotStateException;
import com.dotmarketing.cache.StructureCache;
import com.dotmarketing.common.db.DotConnect;
import com.dotmarketing.db.DbConnectionFactory;
import com.dotmarketing.db.HibernateUtil;
import com.dotmarketing.exception.DotDataException;
import com.dotmarketing.exception.DotHibernateException;
import com.dotmarketing.exception.DotSecurityException;
import com.dotmarketing.portlets.contentlet.model.Contentlet;
import com.dotmarketing.portlets.structure.factories.RelationshipFactory;
import com.dotmarketing.portlets.structure.model.Relationship;
import com.dotmarketing.util.Logger;
import com.dotmarketing.util.UtilMethods;
import com.dotcms.repackage.com.google.gson.Gson;

public class ESContentletIndexAPI implements ContentletIndexAPI{
  private static final ESIndexAPI iapi  = new ESIndexAPI();
    private static final ESMappingAPIImpl mappingAPI = new ESMappingAPIImpl();

    public static final SimpleDateFormat timestampFormatter=new SimpleDateFormat("yyyyMMddHHmmss");

  public synchronized void getRidOfOldIndex() throws DotDataException {
      IndiciesInfo idxs=APILocator.getIndiciesAPI().loadIndicies();
      if(idxs.working!=null)
          delete(idxs.working);
      if(idxs.live!=null)
          delete(idxs.live);
      if(idxs.reindex_working!=null)
          delete(idxs.reindex_working);
      if(idxs.reindex_live!=null)
          delete(idxs.reindex_live);
  }

  /**
   * Tells if at least we have a "working_XXXXXX" index
   * @return
   * @throws DotDataException
   */
  private synchronized boolean indexReady() throws DotDataException {
     IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies();
     return info.working!=null && info.live!=null;
  }



  /**
   * Inits the indexs
   */
  public  synchronized void checkAndInitialiazeIndex() {
    new ESClient().getClient(); // this will call initNode
    try {
      // if we don't have a working index, create it
      if (!indexReady())
        initIndex();
    } catch (Exception e) {
      Logger.fatal("ESUil.checkAndInitialiazeIndex", e.getMessage());

    }
  }




  public synchronized boolean createContentIndex(String indexName) throws ElasticSearchException, IOException {
    return createContentIndex(indexName, 0);
  }
  @Override
  public synchronized boolean createContentIndex(String indexName, int shards) throws ElasticSearchException, IOException {

    CreateIndexResponse cir = iapi.createIndex(indexName, null, shards);
    int i = 0;
    while(!cir.isAcknowledged()){

      try {
        Thread.sleep(100);
      } catch (InterruptedException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }

      if(i++ > 300){
        throw new ElasticSearchException("index timed out creating");
      }
    }


    ClassLoader classLoader = null;
    URL url = null;
    classLoader = Thread.currentThread().getContextClassLoader();
    url = classLoader.getResource("es-content-mapping.json");
        // create actual index
    String mapping = new String(com.liferay.util.FileUtil.getBytes(new File(url.getPath())));

    mappingAPI.putMapping(indexName, "content", mapping);



    return true;
  }

  /**
   * Creates new indexes /working_TIMESTAMP (aliases working_read, working_write and workinglive)
   * and /live_TIMESTAMP with (aliases live_read, live_write, workinglive)
   *
   * @return the timestamp string used as suffix for indices
   * @throws ElasticSearchException if Murphy comes arround
   * @throws DotDataException
   */
  private synchronized String initIndex() throws ElasticSearchException, DotDataException {
      if(indexReady()) return "";
    try {
        final String timeStamp=timestampFormatter.format(new Date());

        final String workingIndex=ES_WORKING_INDEX_NAME+"_"+timeStamp;
        final String liveIndex=ES_LIVE_INDEX_NAME+ "_" + timeStamp;

            final IndicesAdminClient iac = new ESClient().getClient().admin().indices();

            createContentIndex(workingIndex,0);
            createContentIndex(liveIndex,0);

            IndiciesInfo info=new IndiciesInfo();
            info.working=workingIndex;
            info.live=liveIndex;
            APILocator.getIndiciesAPI().point(info);

            return timeStamp;
    } catch (Exception e) {
      throw new ElasticSearchException(e.getMessage(), e);
    }

  }

  /**
   * creates new working and live indexes with reading aliases pointing to old index
   * and write aliases pointing to both old and new indexes
   * @return the timestamp string used as suffix for indices
   * @throws DotDataException
   * @throws ElasticSearchException
   */
  public synchronized String setUpFullReindex() throws ElasticSearchException, DotDataException {
      if(indexReady()) {
          try {

                final String timeStamp=timestampFormatter.format(new Date());

                // index names for new index
                final String workingIndex=ES_WORKING_INDEX_NAME + "_" + timeStamp;
                final String liveIndex=ES_LIVE_INDEX_NAME + "_" + timeStamp;

                final IndicesAdminClient iac = new ESClient().getClient().admin().indices();

                createContentIndex(workingIndex);
                createContentIndex(liveIndex);

                IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies();
                IndiciesInfo newinfo=new IndiciesInfo();
                newinfo.working=info.working;
                newinfo.live=info.live;
                newinfo.reindex_working=workingIndex;
                newinfo.reindex_live=liveIndex;
                APILocator.getIndiciesAPI().point(newinfo);

                iapi.moveIndexToLocalNode(workingIndex);
                iapi.moveIndexToLocalNode(liveIndex);

                return timeStamp;
            } catch (Exception e) {
                throw new ElasticSearchException(e.getMessage(), e);
            }
      }
      else
          return initIndex();
  }
 
  public boolean isInFullReindex() throws DotDataException {
      return isInFullReindex(DbConnectionFactory.getConnection());
  }
 
  public boolean isInFullReindex(Connection conn) throws DotDataException {
      IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies(conn);
      return info.reindex_working!=null && info.reindex_live!=null;
  }

  public synchronized void fullReindexSwitchover() {
      fullReindexSwitchover(DbConnectionFactory.getConnection());
  }
 
  /**
   * This will drop old index and will point read aliases to new index.
   * This method should be called after call to {@link #setUpFullReindex()}
   * @return
   */
  public synchronized void fullReindexSwitchover(Connection conn) {
      try {
          if(!isInFullReindex()) return;

            IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies(conn);

            Logger.info(this, "Executing switchover from old index ["
                   +info.working+","+info.live+"] and new index ["
                   +info.reindex_working+","+info.reindex_live+"]");

            final String oldw=info.working;
            final String oldl=info.live;

            IndiciesInfo newinfo=new IndiciesInfo();
            newinfo.working=info.reindex_working;
            newinfo.live=info.reindex_live;
            APILocator.getIndiciesAPI().point(conn,newinfo);

            iapi.moveIndexBackToCluster(newinfo.working);
            iapi.moveIndexBackToCluster(newinfo.live);

            ArrayList<String> list=new ArrayList<String>();
            list.add(newinfo.working);
            list.add(newinfo.live);
            optimize(list);

      } catch (Exception e) {
            throw new ElasticSearchException(e.getMessage(), e);
        }
  }

  public boolean delete(String indexName) {
    return iapi.delete(indexName);
  }

  public boolean optimize(List<String> indexNames) {
    return iapi.optimize(indexNames);
  }

  public void addContentToIndex(final Contentlet content) throws DotHibernateException {
      addContentToIndex(content, true);
  }

  public void addContentToIndex(final Contentlet content, final boolean deps) throws DotHibernateException {
      addContentToIndex(content,deps,false);
  }

  public void addContentToIndex(final Contentlet content, final boolean deps, boolean indexBeforeCommit) throws DotHibernateException {
      addContentToIndex(content,deps,indexBeforeCommit,false);
  }

  public void addContentToIndex(final Contentlet content, final boolean deps, boolean indexBeforeCommit, final boolean reindexOnly) throws DotHibernateException {
      addContentToIndex(content,deps,indexBeforeCommit,reindexOnly,null);
  }

  public void addContentToIndex(final Contentlet content, final boolean deps, boolean indexBeforeCommit, final boolean reindexOnly, final BulkRequestBuilder bulk) throws DotHibernateException {

      if(content==null || !UtilMethods.isSet(content.getIdentifier())) return;

      Runnable indexAction=new Runnable() {
            public void run() {
                try {
                    Client client=new ESClient().getClient();
                    BulkRequestBuilder req = (bulk==null) ? client.prepareBulk() : bulk;

                    // http://jira.dotmarketing.net/browse/DOTCMS-6886
                    // check for related content to reindex
                    List<Contentlet> contentToIndex=new ArrayList<Contentlet>();
                    contentToIndex.add(content);
                    if(deps)
                        contentToIndex.addAll(loadDeps(content));
                   
                    indexContentletList(req, contentToIndex,reindexOnly);
                                       
                    if(bulk==null && req.numberOfActions()>0)
                        req.execute().actionGet();

                } catch (Exception e) {
                    Logger.error(ESContentFactoryImpl.class, e.getMessage(), e);
                }
            }
        };

      if(bulk!=null || indexBeforeCommit) {
          indexAction.run();
      }
      else {
            // add a commit listener to index the contentlet if the entire
            // transaction finish clean
            HibernateUtil.addCommitListener(content.getInode()+HibernateUtil.addToIndex,indexAction);
      }     
  }

  private void indexContentletList(BulkRequestBuilder req, List<Contentlet> contentToIndex, boolean reindexOnly) throws DotStateException, DotDataException, DotSecurityException, DotMappingException {

    for(Contentlet con : contentToIndex) {
            String id=con.getIdentifier()+"_"+con.getLanguageId();
            IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies();
            Gson gson=new Gson();
            String mapping=null;
            try {
                if(con.isWorking()) {
                    mapping=gson.toJson(mappingAPI.toMap(con));
                   
                    if(!reindexOnly)
                        req.add(new IndexRequest(info.working, "content", id)
                                    .source(mapping));
                    if(info.reindex_working!=null)
                        req.add(new IndexRequest(info.reindex_working, "content", id)
                                    .source(mapping));
                }
   
                if(con.isLive()) {
                    if(mapping==null)
                        mapping=gson.toJson(mappingAPI.toMap(con));
                   
                    if(!reindexOnly)
                        req.add(new IndexRequest(info.live, "content", id)
                                .source(mapping));
                    if(info.reindex_live!=null)
                        req.add(new IndexRequest(info.reindex_live, "content", id)
                                .source(mapping));
                }
            }
            catch(DotMappingException ex) {
                Logger.error(this, "Can't get a mapping for contentlet with id_lang:"+id+" Content data: "+con.getMap(), ex);
            }
        }
   
  }

  @SuppressWarnings("unchecked")
  private List<Contentlet> loadDeps(Contentlet content) throws DotDataException, DotSecurityException {
      List<Contentlet> contentToIndex=new ArrayList<Contentlet>();
      List<String> depsIdentifiers=mappingAPI.dependenciesLeftToReindex(content);
        for(String ident : depsIdentifiers) {
            // get working and live version for all languages based on the identifier
//            String sql = "select distinct inode from contentlet join contentlet_version_info " +
//                    " on (inode=live_inode or inode=working_inode) and contentlet.identifier=?";
            String sql = "select working_inode,live_inode from contentlet_version_info where identifier=?";
         
            DotConnect dc = new DotConnect();
            dc.setSQL(sql);
            dc.addParam(ident);
            List<Map<String,String>> ret = dc.loadResults();
            List<String> inodes = new ArrayList<String>();
            for(Map<String,String> m : ret) {
              String workingInode = m.get("working_inode");
              String liveInode = m.get("live_inode");
              inodes.add(workingInode);
              if(UtilMethods.isSet(liveInode) && !workingInode.equals(liveInode)){
                inodes.add(liveInode);
              }
            }
           
            for(String inode : inodes) {
                Contentlet con=APILocator.getContentletAPI().find(inode, APILocator.getUserAPI().getSystemUser(), false);
                contentToIndex.add(con);
            }
        }
        return contentToIndex;
  }

  public void removeContentFromIndex(final Contentlet content) throws DotHibernateException {
      removeContentFromIndex(content, false);
  }

  private void removeContentFromIndex(final Contentlet content, final boolean onlyLive, final List<Relationship> relationships) throws DotHibernateException {
     Runnable indexRunner = new Runnable() {
              public void run() {
                try {
                    String id=content.getIdentifier()+"_"+content.getLanguageId();
                    Client client=new ESClient().getClient();
                    BulkRequestBuilder bulk=client.prepareBulk();
                    IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies();

                    bulk.add(client.prepareDelete(info.live, "content", id));
                    if(info.reindex_live!=null)
                        bulk.add(client.prepareDelete(info.reindex_live, "content", id));

                    if(!onlyLive) {

                        // here we search for relationship fields pointing to this
                        // content to be deleted. Those contentlets are reindexed
                        // to avoid left those fields making noise in the index
                        for(Relationship rel : relationships) {
                            String q = "";
                            boolean isSameStructRelationship = rel.getParentStructureInode().equalsIgnoreCase(rel.getChildStructureInode());

                            if(isSameStructRelationship)
                                q = "+type:content +(" + rel.getRelationTypeValue() + "-parent:" + content.getIdentifier() + " " +
                                    rel.getRelationTypeValue() + "-child:" + content.getIdentifier() + ") ";
                            else
                                q = "+type:content +" + rel.getRelationTypeValue() + ":" + content.getIdentifier();

                            List<Contentlet> related = APILocator.getContentletAPI().search(q, -1, 0, null, APILocator.getUserAPI().getSystemUser(), false);
                            indexContentletList(bulk, related, false);
                        }

                        bulk.add(client.prepareDelete(info.working, "content", id));
                        if(info.reindex_working!=null)
                            bulk.add(client.prepareDelete(info.reindex_working, "content", id));
                    }

                      bulk.execute().actionGet();

                }
                catch(Exception ex) {
                    throw new ElasticSearchException(ex.getMessage(),ex);
                }
              }
          };
          HibernateUtil.addCommitListener(content.getInode()+HibernateUtil.removeFromIndex,indexRunner);
  }
 
  public void removeContentFromIndex(final Contentlet content, final boolean onlyLive) throws DotHibernateException {

      if(content==null || !UtilMethods.isSet(content.getIdentifier())) return;

      List<Relationship> relationships = RelationshipFactory.getAllRelationshipsByStructure(content.getStructure());
      // add a commit listener to index the contentlet if the entire
        // transaction finish clean
        removeContentFromIndex(content, onlyLive, relationships);
      
  }

  public void removeContentFromLiveIndex(final Contentlet content) throws DotHibernateException {
        removeContentFromIndex(content, true);
    }

  public void removeContentFromIndexByStructureInode(String structureInode) throws DotDataException {
      String structureName=StructureCache.getStructureByInode(structureInode).getVelocityVarName();
      IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies();

      // collecting indexes
      List<String> idxs=new ArrayList<String>();
      idxs.add(info.working);
      idxs.add(info.live);
      if(info.reindex_working!=null)
          idxs.add(info.reindex_working);
      if(info.reindex_live!=null)
          idxs.add(info.reindex_live);
      String[] idxsArr=new String[idxs.size()];
      idxsArr=idxs.toArray(idxsArr);

      // deleting those with the specified structure inode
      new ESClient().getClient().prepareDeleteByQuery()
              .setIndices(idxsArr)
              .setQuery(QueryBuilders.queryString("+structurename:"+structureName))
              .execute().actionGet();
  }

    public void fullReindexAbort() {
        try {
            if(!isInFullReindex()) return;

            IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies();

            final String rew=info.reindex_working;
            final String rel=info.reindex_live;

            IndiciesInfo newinfo=new IndiciesInfo();
            newinfo.working=info.working;
            newinfo.live=info.live;
            APILocator.getIndiciesAPI().point(newinfo);

            iapi.moveIndexBackToCluster(rew);
            iapi.moveIndexBackToCluster(rel);

        } catch (Exception e) {
            throw new ElasticSearchException(e.getMessage(), e);
        }
    }

    public boolean isDotCMSIndexName(String indexName) {
        return indexName.startsWith(ES_WORKING_INDEX_NAME+"_") || indexName.startsWith(ES_LIVE_INDEX_NAME+"_");
    }

    public List<String> listDotCMSClosedIndices() {
        List<String> indexNames=new ArrayList<String>();
        List<String> list=APILocator.getESIndexAPI().getClosedIndexes();
        for(String idx : list)
            if(isDotCMSIndexName(idx))
                indexNames.add(idx);
        return indexNames;
    }

    /**
     * Returns a list of dotcms working and live indices.
     * @return
     */
    public List<String> listDotCMSIndices() {
        Client client=new ESClient().getClient();
        Map<String,IndexStatus> indices=APILocator.getESIndexAPI().getIndicesAndStatus();
        List<String> indexNames=new ArrayList<String>();

        for(String idx : indices.keySet())
            if(isDotCMSIndexName(idx))
                indexNames.add(idx);

        List<String> existingIndex=new ArrayList<String>();
        for(String idx : indexNames)
            if(client.admin().indices().exists(new IndicesExistsRequest(idx)).actionGet().isExists())
                existingIndex.add(idx);
        indexNames=existingIndex;

        List<String> indexes = new ArrayList<String>();
        indexes.addAll(indexNames);
        Collections.sort(indexes, new IndexSortByDate());

        return indexes;
    }

    public void activateIndex(String indexName) throws DotDataException {
        IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies();
        IndiciesInfo newinfo=new IndiciesInfo();
        newinfo.working=info.working;
        newinfo.live=info.live;
        newinfo.reindex_working=info.reindex_working;
        newinfo.reindex_live=info.reindex_live;
        newinfo.site_search=info.site_search;
        if(indexName.startsWith(ES_WORKING_INDEX_NAME)) {
            newinfo.working=indexName;
        }
        else if(indexName.startsWith(ES_LIVE_INDEX_NAME)) {
            newinfo.live=indexName;
        }
        APILocator.getIndiciesAPI().point(newinfo);
    }

    public void deactivateIndex(String indexName) throws DotDataException, IOException {
        IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies();
        IndiciesInfo newinfo=new IndiciesInfo();
        newinfo.working=info.working;
        newinfo.live=info.live;
        newinfo.reindex_working=info.reindex_working;
        newinfo.reindex_live=info.reindex_live;
        newinfo.site_search=info.site_search;
        if(indexName.equals(info.working)) {
            newinfo.working=null;
        }
        else if(indexName.equals(info.live)) {
            newinfo.live=null;
        }
        else if(indexName.equals(info.reindex_working)) {
            iapi.moveIndexBackToCluster(info.reindex_working);
            newinfo.reindex_working=null;
        }
        else if(indexName.equals(info.reindex_live)) {
            iapi.moveIndexBackToCluster(info.reindex_live);
            newinfo.reindex_live=null;
        }
        APILocator.getIndiciesAPI().point(newinfo);
    }



    public synchronized List<String> getCurrentIndex() throws DotDataException {
        List<String> newIdx = new ArrayList<String>();
        IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies();
        newIdx.add(info.working);
        newIdx.add(info.live);
        return newIdx;
    }

    public synchronized List<String> getNewIndex() throws DotDataException {
        List<String> newIdx = new ArrayList<String>();
        IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies();
        if(info.reindex_working!=null)
            newIdx.add(info.reindex_working);
        if(info.reindex_live!=null)
            newIdx.add(info.reindex_live);
        return newIdx;
    }

    private class IndexSortByDate implements Comparator<String> {
        public int compare(String o1, String o2) {
            if(o1 == null || o2==null ){
                return 0;
            }
            if(o1.indexOf("_") <0 ){
                return 1;
            }
            if(o2.indexOf("_") <0 ){
                return -1;
            }
            String one = o1.split("_")[1];
            String two = o2.split("_")[1];
            return two.compareTo(one);
        }
    }

    public String getActiveIndexName(String type) throws DotDataException {
        IndiciesInfo info=APILocator.getIndiciesAPI().loadIndicies();
        if(type.equalsIgnoreCase(ES_WORKING_INDEX_NAME)) {
           return info.working;
        }
        else if(type.equalsIgnoreCase(ES_LIVE_INDEX_NAME)) {
           return info.live;
        }
        return null;
    }

}
TOP

Related Classes of com.dotcms.content.elasticsearch.business.ESContentletIndexAPI

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.