Package com.sun.enterprise.ee.web.authenticator

Source Code of com.sun.enterprise.ee.web.authenticator.ReplicationSingleSignOn

/*
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
*
* Copyright 1997-2007 Sun Microsystems, Inc. All rights reserved.
*
* The contents of this file are subject to the terms of either the GNU
* General Public License Version 2 only ("GPL") or the Common Development
* and Distribution License("CDDL") (collectively, the "License").  You
* may not use this file except in compliance with the License. You can obtain
* a copy of the License at https://glassfish.dev.java.net/public/CDDL+GPL.html
* or glassfish/bootstrap/legal/LICENSE.txt.  See the License for the specific
* language governing permissions and limitations under the License.
*
* When distributing the software, include this License Header Notice in each
* file and include the License file at glassfish/bootstrap/legal/LICENSE.txt.
* Sun designates this particular file as subject to the "Classpath" exception
* as provided by Sun in the GPL Version 2 section of the License file that
* accompanied this code.  If applicable, add the following below the License
* Header, with the fields enclosed by brackets [] replaced by your own
* identifying information: "Portions Copyrighted [year]
* [name of copyright owner]"
*
* Contributor(s):
*
* If you wish your version of this file to be governed by only the CDDL or
* only the GPL Version 2, indicate your decision by adding "[Contributor]
* elects to include this software in this distribution under the [CDDL or GPL
* Version 2] license."  If you don't indicate a single choice of license, a
* recipient has the option to distribute your version of this file under
* either the CDDL, the GPL Version 2 or to extend the choice of license to
* its licensees as provided above.  However, if you add GPL Version 2 code
* and therefore, elected the GPL Version 2 license, then the option applies
* only if the new code is made subject to such option by the copyright
* holder.
*/
/*
* ReplicationSingleSignOn.java
*
* Created on June 19, 2006, 10:12 AM
*
*/

package com.sun.enterprise.ee.web.authenticator;

import com.sun.appserv.ha.spi.BackingStore;
import com.sun.appserv.ha.spi.BackingStoreException;
import com.sun.appserv.ha.spi.BackingStoreFactory;
import com.sun.appserv.ha.spi.BackingStoreRegistry;
import com.sun.appserv.ha.util.SimpleMetadata;
import com.sun.appserv.util.cache.BaseCache;
import com.sun.enterprise.ee.web.sessmgmt.*;
import com.sun.enterprise.security.web.SingleSignOn;
import com.sun.enterprise.security.web.SingleSignOnEntry;
import com.sun.enterprise.web.ServerConfigLookup;
import com.sun.logging.LogDomains;
import com.sun.web.security.RealmAdapter;
import org.apache.catalina.*;
import org.apache.catalina.authenticator.*;
import org.apache.catalina.session.StandardSession;

import javax.servlet.ServletException;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.lang.reflect.InvocationTargetException;
import java.security.AccessController;
import java.security.Principal;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;

/**
*
* @author Larry White
*/
public class ReplicationSingleSignOn extends SingleSignOn
        implements HASSO, ReplicationManager {
   
    /**
     * The logger to use for logging ALL web container related messages.
     */
    private static Logger _logger = null;
   
    final static String MODE_SSO = ReplicationState.MODE_SSO;

    protected static java.util.concurrent.atomic.AtomicLong unique =
        new AtomicLong(0);   
   
    final static String DUPLICATE_IDS_SEMANTICS_PROPERTY
        = ReplicationState.DUPLICATE_IDS_SEMANTICS_PROPERTY;
    private final static String MESSAGE_BROADCAST_LOAD_RECEIVED
        = ReplicationState.MESSAGE_BROADCAST_LOAD_RECEIVED;   
    public final static String MESSAGE_BROADCAST_SSO_ENTRY_ID_QUERY
        = "broadcastfindssoentryids";
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_RESPONSE
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_RESPONSE;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_RESPONSE
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_RESPONSE;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_RECONCILIATION_COMPLETE_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_RECONCILIATION_COMPLETE_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_RESPONSE
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_RESPONSE;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_ADVISORY
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_ADVISORY;
    public final static String MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_RESPONSE
        = RollingUpgradeHandler.MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_RESPONSE;
    private final static String SESSION_MANAGER_PROPERTY =
            ReplicationState.SESSION_MANAGER_PROPERTY; // used during jxtabackingstore creation to store a reference to this session manager in jxtabackingstore.

    protected static int _maxBaseCacheSize = 4096;
    protected static float _loadFactor = 0.75f;
   
    /**
     * the list of method names that are broadcasts or unicast
     */
    private static List broadcastMethods
        = Arrays.asList(            
            MESSAGE_BROADCAST_SSO_ENTRY_ID_QUERY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_RESPONSE,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_RESPONSE,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_RESPONSE,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_ADVISORY,
            MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_RESPONSE,           
            MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_RECONCILIATION_COMPLETE_ADVISORY);

    /**
     * the list of method names that are removes
     */
    private static List removeMethods
        = Arrays.asList(ReplicationState.REMOVE_COMMAND,
            ReplicationState.REMOVE_SYNCHRONIZED_COMMAND);
    private static List saveMethods
  = Arrays.asList(ReplicationState.SAVE_COMMAND);
   
    static
  {
            checkSessionCacheProperties();
            registerBroadcastMethods();
            registerRemoveMethods();
            registerSaveMethods();
  }
   
    private static ReplicationMessageRouter getRouter() {
        ReplicationMessageRouter receiver = null;
        if (Globals.IS_SECURITY_ENABLED) {
            receiver = (ReplicationMessageRouter)
                AccessController.doPrivileged(
                    new PrivilegedGetReplicationMessageRouter());
        } else {
            receiver = ReplicationMessageRouter.createInstance();
        }
        return receiver;
    }

    protected static void registerBroadcastMethods() {
        ReplicationMessageRouter router = getRouter();
        router.registerBroadcastMethodList(broadcastMethods);
    }

    protected static void registerRemoveMethods() {
        ReplicationMessageRouter router = getRouter();
        router.registerRemoveMethodList(removeMethods);
    }

    protected static void registerSaveMethods() {
        ReplicationMessageRouter router = getRouter();
        router.registerSaveMethodList(saveMethods);
    }

    protected static boolean checkSessionCacheProperties() {
        boolean result = false;
  try
        {
            Properties props = System.getProperties();
            String cacheSize=props.getProperty("HTTP_SESSION_CACHE_MAX_BASE_CACHE_SIZE");
            if(null!=cacheSize) {
                _maxBaseCacheSize = (new Integer (cacheSize).intValue());
           
            String loadFactor=props.getProperty("HTTP_SESSION_CACHE_MAX_BASE_LOAD_FACTOR");
            if(null!=loadFactor) {
                _loadFactor = (new Float (loadFactor).floatValue());
            }
            if (_logger.isLoggable(Level.FINER)) {
                _logger.finer("_maxBaseCacheSize=" + _maxBaseCacheSize +
                              " _loadFactor=" + _loadFactor);
            }
        } catch(Exception e)
        {
            //do nothing accept defaults
        }
        return result;
    }

    protected String _rollingUpgradeBackupDirectory = null;

    protected ReplicationSessionMonitors replicatedSessionMonitors = null;

    /**
     * get the rollingUpgradeBackupDirectory
     */
    public String getRollingUpgradeBackupDirectory() {
        return _rollingUpgradeBackupDirectory;
    }

    /**
     * set the rollingUpgradeBackupDirectory
     * @param value
     */
    public void setRollingUpgradeBackupDirectory(String value) {
        _rollingUpgradeBackupDirectory = value;
    }

    public <V> V __load(String id, String version, JxtaBackingStoreImpl jxtaBackingStore)
            throws BackingStoreException {
        ReplicationSSOStore store = getSingletonSSOStore();
        return (V)store.__load(id, this);
    }

    /**
     * The store pool.
     */   
    protected StorePool _pool = null;
   
    /**
     * The singleton store.
     */   
    protected ReplicationSSOStore _store = null;   
   
    /**
     * The background thread.
     */
    private Thread thread = null;


    /**
     * The background thread completion semaphore.
     */
    private boolean threadDone = false;
   
    /**
     * The virtual server name
     */
    private String virtualServerName = null;
   
    public String getPassedInPersistenceType() {
        return _passedInPersistenceType;
    }   
   
    public void setPassedInPersistenceType(String persistenceType) {
        _passedInPersistenceType = persistenceType;
    }

    ReplicaCache getReplicaCache() {
        return replicaCache;
    }

    public void setReplicaCache(ReplicaCache replicaCache) {
        this.replicaCache = replicaCache;
    }   

    /**
    * Our ReplicaCache - the receiving side for replicas
    */
    protected ReplicaCache replicaCache = null;

    /**
    * the passed in persistence type may be replicated or extension type
    */   
    protected String _passedInPersistenceType = null;
   
    protected ConcurrentHashMap stillOwnedSingleSignOnEntryIdsForActiveCacheReconciliation = null;
   
    protected AtomicBoolean _activeCacheReconciliationOngoing
        = new AtomicBoolean(false);  
   
    public boolean isActiveCacheReconciliationOngoing() {
        return _activeCacheReconciliationOngoing.get();
    }   
   
    public void setActiveCacheReconciliationOngoing(boolean value) {
        _activeCacheReconciliationOngoing.set(value);
    }

    protected ConcurrentHashMap<String, ReplicationState> ownedReplicaListsReceivedForActiveCacheRestoration = null;
    protected ConcurrentHashMap<String, ReplicationState> ownedReplicaListsReceivedForReplicaCacheRestoration = null;
    protected ConcurrentHashMap<String, ReplicationState> ownedIdsForReplicaCacheReconciliation = null;
    protected ConcurrentHashMap<String, ReplicationState> ownedIdsForActiveCacheReconciliation = null;

    private volatile CountDownLatch reconcileReplicaCacheDoneSignal = null;
    private volatile CountDownLatch getIdsForReplicaCacheReconciliationDoneSignal = null;
    private volatile CountDownLatch getIdsForActiveCacheReconciliationDoneSignal = null;
    private volatile CountDownLatch restoreActiveCacheDoneSignal = null;
    private volatile CountDownLatch restoreReplicaCacheDoneSignal = null;
   
    /**
    * Our Replicator instance
    */
    protected BackingStore backingStore = null;
   
    /**
    * get the backingStore
    */
    public BackingStore getBackingStore() {
        if(backingStore == null) {
            this.createBackingStore();
        }
        return backingStore;
    }
   
    /**
    * set the backing store
    * @param aBackingStore
    */
    public void setBackingStore(BackingStore aBackingStore) {
        backingStore = aBackingStore;
    }
   
    void createBackingStore() {
        //BackingStoreFactory storeFactory = new JxtaBackingStoreFactory();
        BackingStoreFactory storeFactory = getBackingStoreFactory();
        BackingStoreRegistry backingStoreRegistry
            = BackingStoreRegistry.getInstance();
        Properties inputEnv
            = backingStoreRegistry.getFactoryClassEnv(getPassedInPersistenceType());
        Properties env = (Properties)inputEnv.clone();       
        //this is always true for sso
        env.put(DUPLICATE_IDS_SEMANTICS_PROPERTY, true);       
        BackingStore backingStore = null;
        env.put(SESSION_MANAGER_PROPERTY, this);
        try {
            //FIXED: backingStore = storeFactory.createBackingStore(
            //FIXED:            SimpleMetadata.class,     //type
            //FIXED:            this.getApplicationId(), //appid
            //FIXED:            env);
            backingStore = storeFactory.createBackingStore(
                        this.getApplicationId(), //appid
                        String.class,
                        SimpleMetadata.class,     //type
                        env);
        } catch (BackingStoreException ex) {
            //deliberate no-op
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("backingStore: " + backingStore);
        }        
        if(backingStore != null) {
            if(backingStore instanceof JxtaBackingStoreImpl) {
                ((JxtaBackingStoreImpl)backingStore).setMode(MODE_SSO);
            }               
            this.setBackingStore(backingStore);
        }
    }
   
    protected BackingStoreFactory getBackingStoreFactory() {
        BackingStoreFactory backingStoreFactory = new JxtaBackingStoreFactory();
        BackingStoreRegistry backingStoreRegistry
            = BackingStoreRegistry.getInstance();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicationSingleSignOn>>getBackingStoreFactory:passedInPersistenceType=" + getPassedInPersistenceType());
        }        
        if(getPassedInPersistenceType() == null) {
            return backingStoreFactory;
        }
        String factoryClassName
            = backingStoreRegistry.getFactoryClassName(this.getPassedInPersistenceType());
        return getBackingStoreFactoryFromName(factoryClassName);
    }
   
    private BackingStoreFactory getBackingStoreFactoryFromName(String className) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicationSingleSignOn:className: " + className);
        }          
        BackingStoreFactory backingStoreFactory = new JxtaBackingStoreFactory();
        try {
            backingStoreFactory =
                (BackingStoreFactory) (Class.forName(className)).newInstance();
        } catch (Exception ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("unable to create backing store factory");
            }           
        }
        return backingStoreFactory;
    }         
   
    /** Creates a new instance of ReplicationSingleSignOn */
    public ReplicationSingleSignOn(String theVirtualServerName) {
        if (_logger == null) {
            _logger = LogDomains.getLogger(LogDomains.WEB_LOGGER);
        }
        virtualServerName = theVirtualServerName;
        replicatedSessionMonitors = new ReplicationSessionMonitors(_logger, _maxBaseCacheSize, _loadFactor);
    }

    public Object getReplicationSessionMonitor(String id) {
        return replicatedSessionMonitors.get(id);
    }

    /**
    * get the replicated ssoEntries cache
    */
    public BaseCache getReplicatedSSOEntries() {
        return replicaCache.getReplicatedSessions();
    }
   
    protected void putInReplicationCache(ReplicationState state) {
        replicaCache.putInReplicationCache(state);
    }
   
    protected ReplicationState getFromReplicationCache(String id) {
        return replicaCache.getFromReplicationCache(id);
    }

    /**
     * remove sso State from replica cache based on the id of sessionState
     * @param state sso replica state
     */   
    protected void removeFromReplicationCache(ReplicationState state) {
        if(state == null) {
            return;
        }
        removeFromReplicationCache((String)state.getId());
    }
   
    /**
     * remove sso State from replica cache based on the id of sessionState
     * @param id id of sso replica state
     */   
    protected void removeFromReplicationCache(String id) {
        replicaCache.removeFromReplicationCache(id);
    }   
   
    protected synchronized ReplicationState transferFromReplicationCache(String id) {
        ReplicationState result = this.getFromReplicationCache(id);
        removeFromReplicationCache(result);
        return result;
    }
   
    /** Returns a store from the pool This method intializes the store with right parameters
     * @return returns ReplicationSSOStorePoolElement
     */
    private ReplicationSSOStorePoolElement getSSOStore() {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationSingleSignOn", "getSSOStore");
        }
        if(ReplicationUtil.isSynchronousReplicationConfigured()) {
            return (ReplicationSSOStore)this.getSingletonSSOStore();
        }       
        ReplicationSSOStorePoolElement store = null;
        try {
            store = (ReplicationSSOStorePoolElement) _pool.take();
            store.setContainer(this.getContainer());
            store.setParent(this);
            store.setApplicationId(this.getApplicationId());
            if(_logger.isLoggable(Level.FINEST)) {
                _logger.log(Level.FINEST,
                    "ReplicationSingleSignOn.getSSOStore returning   " + store);
            }
            return store;
        }
        catch (Exception e) {
            if (_logger.isLoggable(Level.FINE)) {
                _logger.log(Level.FINE, "exception occurred in getSSOStore", e);
            }
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationSingleSignOn", "getSSOStore", store);
        }
        return store;
    }
   
    /**
     *  Returns (puts) a store back to the pool
     */
    private void putSSOStore(ReplicationSSOStorePoolElement store) {
        if(ReplicationUtil.isSynchronousReplicationConfigured()) {
            return;
        }
        if (store != null) {
            store.setContainer(null);       
            try {
                StorePool storePool = this.getSSOStorePool();
                if(storePool != null) {
                    storePool.put( (StorePoolElement) store);
                }
            }
            catch (InterruptedException ex1) {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.log(Level.FINE, "exception occurred in putSSOStore", ex1);
                }
            }
        }
    }       
   
    // store pool was set during initialization of this valve
    public void setSSOStorePool(StorePool pool) {
        _pool = pool;
    }

    public StorePool getSSOStorePool() {
        return _pool;
    }
   
    // singleton store was set during initialization of this valve
    public void setSingletonSSOStore(ReplicationSSOStore store) {
        _store = store;
        _store.setContainer(this.getContainer());
        _store.setParent(this);
        _store.setApplicationId(this.getApplicationId());       
    }

    public ReplicationSSOStore getSingletonSSOStore() {
        return _store;
    }   
   
    public String getVirtualServerName() {
        return virtualServerName;
    }
   
    public void setVirtualServerName(String value) {
        virtualServerName = value;
    }
           
    /**
    * The application id
    */ 
    protected String applicationId = null;       
   
    public String getApplicationId() {
        if(applicationId != null)
            return applicationId;
        Container container = this.getContainer();
        StringBuffer sb = new StringBuffer(50);
        //sb.append(this.getClusterId());
        ArrayList list = new ArrayList();
        while (container != null) {
            if(container.getName() != null) {
                list.add(":" + container.getName());
            }
            container = container.getParent();
        }
        sb.append("SSO");
        for(int i=(list.size() -1); i>-1; i--) {
            String nextString = (String) list.get(i);
            sb.append(nextString);
        }
        sb.append(":" + this.getVirtualServerName());
        applicationId = sb.toString();
        return applicationId;

    }   
   
   /**
     * Perform single-sign-on support processing for this request.
     *
     * @param request The servlet request we are processing
     * @param response The servlet response we are creating
     *
     * @exception IOException if an input/output error occurs
     * @exception ServletException if a servlet error occurs
     */
    public int invoke(Request request, Response response)
            throws IOException, ServletException {

        if (_logger.isLoggable(Level.FINEST)) {
            _logger.finest("ReplicationSingleSignOn.invoke()");
        }

        // If this is not an HTTP request and response, just pass them on
        if (!(request instanceof HttpRequest) ||
            !(response instanceof HttpResponse)) {
            // START OF IASRI 4665318
            // context.invokeNext(request, response);
            // return;
            return INVOKE_NEXT;
            //return 0;
            // END OF IASRI 4665318
        }

        HttpServletRequest hreq = (HttpServletRequest) request.getRequest();
        HttpServletResponse hres = (HttpServletResponse) response.getResponse();
        request.removeNote(Constants.REQ_SSOID_NOTE);

        // Has a valid user already been authenticated?
        if (hreq.getUserPrincipal() != null) {
            if (_logger.isLoggable(Level.FINEST)) {
                _logger.finest("Principal '"
                               + hreq.getUserPrincipal().getName()
                               + "' has already been authenticated");
            }
            return INVOKE_NEXT;
            //return 0;
        }

        if (_logger.isLoggable(Level.FINEST)) {
      _logger.finest("Checking for SSO cookie");
        }
        Cookie cookie = null;
        Cookie cookies[] = hreq.getCookies();
        if (cookies == null)
            cookies = new Cookie[0];
        for (int i = 0; i < cookies.length; i++) {
            if (Constants.SINGLE_SIGN_ON_COOKIE.equals(cookies[i].getName())) {
                cookie = cookies[i];
                break;
            }
        }
        if (cookie == null) {
            if (_logger.isLoggable(Level.FINEST)) {
                _logger.finest("SSO cookie not present");
            }
            return INVOKE_NEXT;
        }

        Realm realm = request.getContext().getRealm();
        if (realm == null) {
            if (_logger.isLoggable(Level.FINE)) {
                _logger.fine(" No realm configured for this application, SSO "
                             + "does not apply.");
            }
            return INVOKE_NEXT;
        }

        String realmName = realm.getRealmName();
        if (realmName == null) {
            if (_logger.isLoggable(Level.FINE)) {
                _logger.fine(" No realm configured for this application, SSO "
                             + "does not apply.");
            }
            return INVOKE_NEXT;
        }

        // Look up the cached Principal associated with this cookie value
        if (_logger.isLoggable(Level.FINEST)) {
            _logger.finest("Checking for cached principal for "
                           + cookie.getValue());
        }

        SingleSignOnEntry entry = lookupEntry(cookie.getValue());
        if (entry != null) {

            if (_logger.isLoggable(Level.FINEST)) {
                _logger.finest("Cached SingleSignOnEntry: " + entry);
            }

            // only use this SSO identity if it was set in the same realm
            if (!realmName.equals(entry.realmName)) {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine(" Ignoring SSO entry which does not match "
                                 + "application realm '" + realmName + "'");
                }
                return INVOKE_NEXT;
            }

            if (_logger.isLoggable(Level.FINEST)) {
                if (entry.principal != null) {
                    _logger.finest("Found cached principal '"
                                   + entry.principal.getName()
                                   + "' with auth type '" + entry.authType
                                   + "'");
                } else {
                    _logger.finest("No cached principal found");
                }
            }

            if ((entry.principal == null)
                    && (entry.username != null)) {
                entry.principal = ((RealmAdapter)request.getContext().getRealm()).createFailOveredPrincipal(entry.username);
                if (_logger.isLoggable(Level.FINEST)) {
                    if (entry.principal != null) {
                        _logger.finest("Found failed over principal '"
                                        + entry.principal
                                        + "' with auth type '" + entry.authType
                                        + "'");
                    } else {
                        _logger.finest("No failed over principal found");
                    }
                }
            }
           

            request.setNote(Constants.REQ_SSOID_NOTE, cookie.getValue());
            ((HttpRequest) request).setAuthType(entry.authType);
            ((HttpRequest) request).setUserPrincipal(entry.principal);
            // Touch the SSO entry access time
            entry.lastAccessTime = System.currentTimeMillis();
            ((HASingleSignOnEntry)entry).dirty = true;

        } else {
            if (_logger.isLoggable(Level.FINEST)) {
                _logger.finest("No cached principal found, erasing SSO cookie");
            }
            cookie.setMaxAge(0);
      //Bug : 4833387 
      cookie.setPath("/");
            hres.addCookie(cookie);
        }

        // Invoke the next Valve in our pipeline
        return INVOKE_NEXT;
        //return 0;
    }
   
    /**
     * Gets the SingleSignOnEntry with the given id
     * from the active cache only
     *
     * @param id the id
     * @return The SingleSignOnEntry with the given id, or null if not
     * found
     */
    public SingleSignOnEntry findSSOEntryFromCacheOnly(String id) {       
        if (id == null)
            return (null);
        HASingleSignOnEntry result
            = (HASingleSignOnEntry)super.lookupEntry(id);
        if(!this.isActiveCacheReconciliationOngoing()) {
            return result;
        } else {
            if(result == null || !result.isSuspect()) {
                //if null just return it
                //if not suspect it has already been checked
                return result;
            }           
            synchronized(result) {
                if(checkSuspectSSOEntry(result)) {
                    //if it passes the check return it
                    return result;
                } else {
                    return null;
                }
            }           
        }         
    }
   
    protected boolean checkSuspectSSOEntry(HASingleSignOnEntry haSSOEntry) {
        if(stillOwnedSingleSignOnEntryIdsForActiveCacheReconciliation.get(haSSOEntry.getId()) == null) {
            clearFromManagerCache(haSSOEntry.getId());
            return false;
        } else {
            haSSOEntry.setSuspect(false);
            return true;
        }
    } 
   
    /** Look up and return the cached SingleSignOn entry associated with this
     * sso id value, if there is one; otherwise return <code>null</code>.
     *
     * @param ssoId Single sign on identifier to look up
     * @return SingleSignOnEntry
     */
    public SingleSignOnEntry lookupEntry(String ssoId) {
        if (_logger.isLoggable(Level.FINEST)) {
            _logger.finest("ReplicationSingleSignOn.lookupEntry(): ssoId=" + ssoId);
        }
        SingleSignOnEntry ssoEntry=null;
       
        ReplicationSSOStore store = null;
        try {
            store = (ReplicationSSOStore)getSSOStore();                       
            ssoEntry = findSSOEntryFromCacheOnly(ssoId);
            if (ssoEntry != null) {
                return ssoEntry; //return if the sso is in cache
            }
            ssoEntry = store.loadSSO(ssoId, this);
            if(ssoEntry != null) {
                if (_logger.isLoggable(Level.FINEST)) {
                    _logger.finest("lookup before :- ssoID="+ssoId+"   "
                                   +ssoEntry);
                    _logger.finest("container= "+container+" realm= "
                                   +container.getRealm());
                    _logger.finest("lookup after if :- ssoID="+ssoId+"   "
                                   +ssoEntry);
                }
                registerInMemory(ssoId, ssoEntry);
            }                       
        } catch (Exception e) {
            if (_logger.isLoggable(Level.FINE)) {
                _logger.log(Level.FINE, "exception occurred in ReplicationSSO>>lookupEntry() ", e);
            }
        } finally {
            this.putSSOStore((ReplicationSSOStorePoolElement)store);
        }                
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationSingleSignOn", "lookupEntry");
        }
        return ssoEntry;
    }
   
     /**
     * Register the specified SingleSignOnEntry as being associated with the specified
     * value for the single sign on identifier.
     *
     * @param ssoId Single sign on identifier to register
     * @param sso Single sign on entry
     */
     void registerInMemory(String ssoId, SingleSignOnEntry sso) {
        
       synchronized (cache) {
            cache.put(ssoId, sso);
        }
     }
    
     /**
     * Register the specified Principal as being associated with the specified
     * value for the single sign on identifier.
     *
     * @param ssoId Single sign on identifier to register
     * @param principal Associated user principal that is identified
     * @param authType Authentication type used to authenticate this
     *  user principal
     * @param username Username used to authenticate this user
     * @param password Password used to authenticate this user
     */
    protected void register(String ssoId,
                             Principal principal,
                             String authType,
                             String username,
                             String password,
                             String realmName) {
        
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicationSingleSignOn>>register: ssoId=" + ssoId
            + " authType=" + authType + " username=" + username
            + " password=" + password + " realmName=" + realmName);
        }        
        ReplicationSSOStorePoolElement store = null;

        try {
            store = (ReplicationSSOStore)getSSOStore();
            SingleSignOnEntry ssoEntry = new HASingleSignOnEntry(ssoId,
                                                                 principal,
                                                                 authType,
                                                                 username,
                                                                 password,
                                                                 realmName);
            registerInMemory(ssoId, ssoEntry);

            if (!authType.equals(org.apache.catalina.authenticator.Constants.FORM_METHOD)
                    && !authType.equals(org.apache.catalina.authenticator.Constants.BASIC_METHOD)) {
                return;
            }

            if(_logger.isLoggable(Level.FINEST)) {
                _logger.finest("ReplicationSingleSignOn.register(): "
                               + "About to save: ssoId=" + ssoId
                               + " ssoEntry=" + ssoEntry);
            }
            store.save(ssoId, ssoEntry);
        } catch (Exception e) {
            if (_logger.isLoggable(Level.FINE)) {
                _logger.log(Level.FINE, "exception occurred in register ssoId=" + ssoId, e);
            }
        } finally {
            this.putSSOStore((ReplicationSSOStorePoolElement)store);
        }
    }
    
    /**
     * Persists the given HASingleSignOnEntry.
     *
     * @param ssoEntry The HASingleSignOnEntry to be persisted
     */
    public void saveSSOEntry(HASingleSignOnEntry ssoEntry)
            throws IOException {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationSingleSignOn", "saveSSOEntry", ssoEntry);
        }       
        ReplicationSSOStore store = null;
        try {
            store = (ReplicationSSOStore)getSSOStore();           
            store.save(ssoEntry.getId(), ssoEntry);
        }
        finally {
            this.putSSOStore((ReplicationSSOStorePoolElement)store);
        }                
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationSingleSignOn", "saveSSOEntry");
        }
    }
   
    /**
     * Removes the given HASingleSignOnEntry based on id
     *
     * @param ssoId The id of the HASingleSignOnEntry to be removed
     */
    public void removeSSOEntry(String ssoId)
            throws IOException {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationSingleSignOn", "removeSSOEntry", ssoId);
        }       
        ReplicationSSOStore store = null;
        try {
            store = (ReplicationSSOStore)getSSOStore();           
            store.remove(ssoId);
        }
        finally {
            this.putSSOStore((ReplicationSSOStorePoolElement)store);
        }                
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationSingleSignOn", "removeSSOEntry");
        }
    }      
    
   /**
     * Deregister the specified single sign on identifier, and invalidate
     * any associated sessions.
     *
     * @param ssoId Single sign on identifier to deregister
     */
    protected void deregister(String ssoId) {
        deregister(ssoId, false);
    }
   
    protected void deregister(String ssoId, boolean bgCall) {
        if (_logger.isLoggable(Level.FINEST)) {
            if (bgCall) {
                _logger.finest("BackGround : Deregistering ssoId '"
                               + ssoId + "'");
            } else {
                _logger.finest("ForeGround : Deregistering ssoId '"
                               + ssoId + "'");
            }
        }
        // Look up and remove the corresponding SingleSignOnEntry
        SingleSignOnEntry sso = null;
        ReplicationSSOStore store = null;
        long startTime = 0L;
        if (_logger.isLoggable(Level.FINEST)) {
            startTime = System.currentTimeMillis();
        }
       
        try {
            store = (ReplicationSSOStore)getSSOStore();

            synchronized (cache) {
                sso = (SingleSignOnEntry) cache.remove(ssoId);
            }
            try {
                if(!bgCall)
                    store.remove(ssoId);//remove from ssotable
                else
                    //FIXME remove after test
                    //store.remove(ssoId,null);
                    store.remove(ssoId);
            } catch (Exception e) {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.log(Level.FINE, "exception occurred in deregister ssoId=" + ssoId, e);
                }
            }

            if (sso == null)
                return;

            // Expire any associated sessions
            Session sessions[] = sso.findSessions();
            for (int i = 0; i < sessions.length; i++) {
                if (_logger.isLoggable(Level.FINEST)) {
                    _logger.finest("Invalidating session " + sessions[i]);
                }
                // Remove from reverse cache first to avoid recursion
                synchronized (reverse) {
                    reverse.remove(sessions[i]);
                }
                // Invalidate this session
                sessions[i].expire();
            }
            try {
                if(!bgCall)
                    store.removeInActiveSessions(ssoId);
                else
                    //FIXME remove after test
                    //store.removeInActiveSessions(ssoId,null);
                    store.removeInActiveSessions(ssoId);              

            } catch (Exception e) {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.log(Level.FINE, "exception occurred in deregister ssoId=" + ssoId, e);
                }
            }
        } finally {
            this.putSSOStore((ReplicationSSOStorePoolElement)store);
            if (_logger.isLoggable(Level.FINEST)) {
                long endTime = System.currentTimeMillis();
                _logger.finest("deregister_TIME MILLIS = "
                               + (endTime - startTime));
            }           
        }
        // NOTE:  Clients may still possess the old single sign on cookie,
        // but it will be removed on the next request since it is no longer
        // in the cache       
    }
   
    /**
     * Associate the specified single sign on identifier with the
     * specified Session.
     *
     * @param ssoId Single sign on identifier
     * @param session Session to be associated
     */
    public void associate(String ssoId, Session session) {
        if (!started) {
            return;
        }
        if (_logger.isLoggable(Level.FINEST)) {
            _logger.finest("ReplicationSingleSignOn.associate(): "
                           + "Associate sso id " + ssoId
                           + " with session " + session);
        }
        long startTime = 0L;
        if (_logger.isLoggable(Level.FINEST)) {
            startTime = System.currentTimeMillis();
        }
        ReplicationSSOStore store = null;
        try {
            store = (ReplicationSSOStore)getSSOStore();           
            SingleSignOnEntry sso = lookupEntry(ssoId);
            if (sso != null)
                sso.addSession(this, session);
            synchronized (reverse) {
                reverse.put(session, ssoId);
            }
            if((session != null)&&(session instanceof HASession)){
                ((HASession)session).setSsoId(ssoId);
                store.associate((StandardSession)session,ssoId);
            }        
        } catch (Exception e) {
            _logger.log(Level.WARNING,
                "Exception in ReplicationSingleSignOn.associate()",
                e);
        } finally {
            this.putSSOStore((ReplicationSSOStorePoolElement)store);
            if (_logger.isLoggable(Level.FINEST)) {
                long endTime = System.currentTimeMillis();
                _logger.finest("associate_TIME MILLIS = "
                               + (endTime - startTime));
            }           
        }                
    }
   
   /**
     * Prepare for the beginning of active use of the public methods of this
     * component.  This method should be called after <code>configure()</code>,
     * and before any of the public methods of the component are utilized.
     *
     * @exception IllegalStateException if this component has already been
     *  started
     * @exception LifecycleException if this component detects a fatal error
     *  that prevents this component from being used
     */
    public void start() throws LifecycleException {

        // Validate and update our current component state
        if (started) {
            throw new LifecycleException
                (sm.getString("authenticator.alreadyStarted"));
        }

        lifecycle.fireLifecycleEvent(START_EVENT, null);
        started = true;
       
        ReplicationMessageRouter router = null;
        if (Globals.IS_SECURITY_ENABLED) {
            router = (ReplicationMessageRouter)
                AccessController.doPrivileged(
                    new PrivilegedGetReplicationMessageRouter());
        } else {
            router = ReplicationMessageRouter.createInstance();
        }       
        if(router != null) {
            router.addReplicationManager(getApplicationId(), this);
        }
        //FIXME:this is for testing
        //will eventually be triggered by separate admin command
        /*
        if(ReplicationUtil.isRollingUpgradeEnabled()) {
            CountDownLatch doneSignal = new CountDownLatch(1);
            doRollingUpgradePostStartupProcessing(System.currentTimeMillis(), doneSignal);
        }
        */
        // BEGIN IASRI 4705699
        // Start the background reaper thread
        threadStart();
        // END IASRI 4705699
    }
   
    /**
     * Gracefully terminate the active use of the public methods of this
     * component.  This method should be the last one called on a given
     * instance of this component.
     *
     * @exception IllegalStateException if this component has not been started
     * @exception LifecycleException if this component detects a fatal error
     *  that needs to be reported
     */
    public void stop() throws LifecycleException {
        // Validate and update our current component state
        if (!started) {
            throw new LifecycleException
                (sm.getString("authenticator.notStarted"));
        }

        long startTime = 0L;
        if(_logger.isLoggable(Level.FINEST)) {
            startTime = System.currentTimeMillis();
        }
        ReplicationSSOStore store = null;
        lifecycle.fireLifecycleEvent(STOP_EVENT, null);
        started = false;       
       
        try {
            store = (ReplicationSSOStore)getSSOStore();           
            synchronized (cache) {

                Iterator it = cache.keySet().iterator();
                while (it.hasNext()) {
                    String key = (String) it.next();
                    SingleSignOnEntry sso = (SingleSignOnEntry) cache.get(key);
                    if (((HASingleSignOnEntry)sso).dirty) {
                        if(_logger.isLoggable(Level.FINEST)) {
                            _logger.finest("Stop: updating the SSO session "
                                           +key);
                        }                        
                      store.save(key,sso);
                        ((HASingleSignOnEntry)sso).dirty = false;
                    }
                }
            }           
        } catch (Exception ex){
            if (_logger.isLoggable(Level.FINE)) {
                _logger.log(Level.FINE, "exception occurred in stop", ex);
            }
        } finally {
            this.putSSOStore((ReplicationSSOStorePoolElement)store);
            if(_logger.isLoggable(Level.FINEST)) {                   
                long endTime = System.currentTimeMillis();
                _logger.finest("stop_TIME MILLIS = "
                               + (endTime - startTime));
            }           
        }
        threadStop();
        ReplicationMessageRouter router = null;
        if (Globals.IS_SECURITY_ENABLED) {
            router = (ReplicationMessageRouter)
                AccessController.doPrivileged(
                    new PrivilegedGetReplicationMessageRouter());
        } else {
            router = ReplicationMessageRouter.createInstance();
        }        
        if(router != null) {
            router.removeReplicationManager(getApplicationId());
        }       
    }
   
    public void repair(long repairStartTime) {
        if (_logger.isLoggable(Level.FINEST)) {
            _logger.finest("ReplicationSingleSignOn>>repair");
        }       
        /*
        if (!started)
            return;
         */

        if(ReplicationHealthChecker.isStopping()) {
            return;
        }

        ReplicationState ssoEntries[] = getReplicatedSsoEntriesArray();
        for (int i = 0; i < ssoEntries.length; i++) {
            ReplicationState ssoEntry = (ReplicationState) ssoEntries[i];
            try {
                repairSave(ssoEntry);
            } catch(Exception ex) {
                // FIXME evaluate log level
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.log(Level.FINE, "exception occurred in repair", ex);
                }
            }
        }
    }
   
    public void repair(long repairStartTime, boolean checkForStopping) {
        if (_logger.isLoggable(Level.FINEST)) {
            _logger.finest("ReplicationSingleSignOn>>repair");
        }       
        /*
        if (!started)
            return;
         */

        if(checkForStopping && ReplicationHealthChecker.isStopping()) {
            return;
        }

        ReplicationState ssoEntries[] = getReplicatedSsoEntriesArray();
        for (int i = 0; i < ssoEntries.length; i++) {
            ReplicationState ssoEntry = (ReplicationState) ssoEntries[i];
            try {
                repairSave(ssoEntry);
            } catch(Exception ex) {
                // FIXME evaluate log level
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.log(Level.FINE,
                    "Exception in ReplicationSingleSignOn.repair()",
                    ex);
                }
            } catch(Throwable t) {
                // FIXME evaluate log level
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.log(Level.FINE,
                        "Exception in ReplicationSingleSignOn.repair()",
                        t);
                }
                break;
            }
        }
    }
   
    public void respondToFailure(String instanceName, boolean checkForStopping) {
        ; //no-op
    }
   
    /**
     * Saves the state
     * @param state ReplicationState
     *
     */
    public void repairSave(ReplicationState state) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationSingleSignOn", "repairSave", state);
        }       
        ReplicationSSOStore store = null;
        try {
            store = (ReplicationSSOStore)getSSOStore();           
            store.saveForRepair(state);
        } catch (Exception e) {
            _logger.log(Level.WARNING,
                "Exception in ReplicationSingleSignOn.repairSave()",
                e);
        } finally {
            this.putSSOStore((ReplicationSSOStorePoolElement)store);
        }                
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationSingleSignOn", "repairSave");
        }
    }

    public ReplicationState[] getReplicatedSsoEntriesArray() {
        BaseCache replicatedSSOEntriesCache = getReplicatedSSOEntries();
        ReplicationState[] ssoEntries = null;
        int numberOfIds = replicatedSSOEntriesCache.getEntryCount();
        ArrayList valuesList = new ArrayList(numberOfIds);
        Iterator valuesIter = replicatedSSOEntriesCache.values();
        while(valuesIter.hasNext()) {
            valuesList.add((ReplicationState)valuesIter.next());
        }
        ReplicationState[] template = new ReplicationState[valuesList.size()];
        ssoEntries = (ReplicationState[])valuesList.toArray(template);
        return ssoEntries;

    }    
   
//---------------------------Background thread methods-----------------   

    public int processExpiredSessions() {
        processExpires();
        return 0;
    }
   /**
     * Invalidate all SSO cache entries that have expired.
     */
    private void processExpires() {

        long tooOld = System.currentTimeMillis() - getMaxInactive() * 1000;

        if (_logger.isLoggable(Level.FINEST)) {
            _logger.finest(tooOld
                           + "SSO Expiration thread started. Current entries: "
                           + cache.size());
        }

        ArrayList removals = new ArrayList(cache.size()/2);

        // build list of removal targets

        // Note that only those SSO entries which are NOT associated with
        // any session are elegible for removal here.
        // Currently no session association ever happens so this covers all
        // SSO entries. However, this should be addressed separately.

        try {
            synchronized (cache) {

                Iterator it = cache.keySet().iterator();
                while (it.hasNext()) {
                    String key = (String) it.next();
                    SingleSignOnEntry sso = (SingleSignOnEntry) cache.get(key);
                    if(_logger.isLoggable(Level.FINEST)) {
                        _logger.finest(tooOld
                                       + "*******************  "
                                       + sso.lastAccessTime
                                       + "   SSO Expiration thread started. Current entries: "
                                       + cache.size());
                    }
                    if (sso.sessions.length == 0 &&
                        sso.lastAccessTime < tooOld) {
                        removals.add(key);
                    }
                }
            }
            int removalCount = removals.size();

            if (_logger.isLoggable(Level.FINEST)) {
                _logger.finest("SSO cache will expire " + removalCount
                               + " entries.");
            }

            // deregister any elegible sso entries
            for (int i=0; i < removalCount; i++) {
              if(_logger.isLoggable(Level.FINEST)) {
                    _logger.finest("SSO Expiration removing entry: "
                                   + removals.get(i));
                }
                deregister((String)removals.get(i),true);
            }

        } catch (Throwable e) { // don't let thread die
            _logger.log(Level.WARNING,
                        "Exception in ReplicationSingleSignOn.processExpires()",
                        e);
        }
    }
   
   /**
     * Update all SSO enytries whose LAT has changed
     */
    private void processUpdateLat() {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationSingleSignOn", "processUpdateLat");
        }       
        ReplicationSSOStore store = null;
        Hashtable updatedLats = new Hashtable(cache.size()/2);
       
        try {
            synchronized (cache) {

              Iterator it = cache.keySet().iterator();
                while (it.hasNext()) {

                    String key = (String) it.next();
                    SingleSignOnEntry sso = (SingleSignOnEntry) cache.get(key);
                    if(_logger.isLoggable(Level.FINEST)) {
                        _logger.finest("===="
                                       + sso.lastAccessTime
                                       + "   SSO Expiration/Updation thread started. Current cache entries: "
                                       + cache.size());
                    }
                    if (((HASingleSignOnEntry)sso).dirty) {
                        updatedLats.put(key,sso);
                    }
                }
            }

            int updatedLatsCount = updatedLats.size();

            if(_logger.isLoggable(Level.FINEST)) {
                _logger.finest("SSO table will updated " + updatedLatsCount
                               + " entries.");
            }

            store = (ReplicationSSOStore)getSSOStore();

            // update all elegible sso entries
            Iterator it = updatedLats.keySet().iterator();
            while (it.hasNext()) {
                String ssoId = (String) it.next();
                if(_logger.isLoggable(Level.FINEST)) {
                    _logger.finest("SSO LATupdation updating entry: "
                                   + ssoId);
                }

                store.updateLastAccessTime(
                    ssoId,
                    ((SingleSignOnEntry)updatedLats.get(ssoId)).lastAccessTime);
                ((HASingleSignOnEntry)updatedLats.get(ssoId)).dirty = false;
            }       
       
        } catch (Throwable e) { // don't let thread die
            _logger.log(Level.WARNING,
                        "Exception in ReplicationSingleSignOn.processUpdateLat()",
                        e);
        } finally {
            this.putSSOStore((ReplicationSSOStorePoolElement)store);
        }                
        if (_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationSingleSignOn", "processUpdateLat");
        }
    }
   
    // begin message processing methods

    public void processMessage(ReplicationState message) {
        //handle broadcast methods
        if(ReplicationState.isBroadcastState(message)) {
            processBroadcastMessage(message);
            return;
        }
       
        //handle non-void methods
        ReplicationStateQueryResponse queryResult = null;
        //do process non-void message (and cannot be response either)
        if(!message.isResponseState() && !message.isVoidMethodReturnState()) {
            //do non-void processing including sending response
            queryResult = this.doProcessQueryMessage(message);
            ReplicationState qResponse = queryResult.getState();
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("ReplicationSingleSignOn:qResponse=" + qResponse);
            }            
            if(qResponse != null) {
                //sourceInstanceName is preserved in the response
                ReplicationState response =
                    ReplicationState.createResponseFrom(qResponse);
                if(_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationSingleSignOn:responseState=" + response);
                }               
                this.doSendResponse(response);
            }
            return;
        }
        //end do process non-void message
       
        /*
        ReplicationState response =
            ReplicationState.createResponseFrom(message);
         */
        //send a response before further processing only if processed
        //msg is not itself a response and if method is a void return type
        //FIXME this send will be removed if upstream ack send works
        /* removing this for test
        if(!message.isReturnMessage() && message.isVoidMethodReturnState()) {
            this.doSendResponse(response);
        }
        */       

        boolean isResponse = this.doProcessMessage(message);
       
        //send a response only if processed msg is not itself a response
        //and if method is not void return type (in that case ack was
        //already sent)
        /*
        if(!isResponse && !message.isVoidMethodReturnState()) {
            //ReplicationState response =
            //    ReplicationState.createResponseFrom(message);
            this.doSendResponse(response);
        }
         */
    }   
   
    public void processMessagePrevious(ReplicationState message) {
        //FIXME complete
        if(ReplicationState.isBroadcastState(message)) {
            processBroadcastMessage(message);
            return;
        }
       
        ReplicationState response =
            ReplicationState.createResponseFrom(message);
        //send a response before further processing only if processed
        //msg is not itself a response and if method is a void return type
        //FIXME this send will be removed if upstream ack send works
        /* removing this for test
        if(!message.isReturnMessage() && message.isVoidMethodReturnState()) {
            this.doSendResponse(response);
        }
         */        

        boolean isResponse = this.doProcessMessage(message);
        //send a response only if processed msg is not itself a response
        //and if method is not void return type (in that case ack was
        //already sent)
        if(!isResponse && !message.isVoidMethodReturnState()) {
            /*
            ReplicationState response =
                ReplicationState.createResponseFrom(message);
             */
            this.doSendResponse(response);
        }
    }
   
    //return true if message is processResponse
    public boolean doProcessMessage(ReplicationState message) {
        boolean result = false;
        String methodName = ReplicationUtil.getProcessMethodName(message);
        if (_logger.isLoggable(Level.FINEST)) {
            _logger.finest("in ReplicationSingleSignOn>>doProcessMessage:methodName=" + methodName);
        }
        try {
            Class myClass = this.getClass();
            myClass.getMethod(
                methodName,
                    new Class[]{ message.getClass() }).invoke(
                        this, new Object[]{ message });
        } catch (IllegalAccessException ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in ReplicationSingleSignOn>>doProcessMessage:methodName=" + methodName + "illegalAccessException");             
            }           
        } catch (NoSuchMethodException ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in ReplicationSingleSignOn>>doProcessMessage:methodName=" + methodName + "noSuchMethodException");             
            }            
        } catch (InvocationTargetException ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in ReplicationSingleSignOn>>doProcessMessage:methodName=" + methodName + "invocationTargetException");
                _logger.log(Level.FINE, "invocationException.getCause()= " + ex.getCause(), ex);
            }                     
        }             
        if(methodName.equals("processResponse")) {
            result = true;
        }
        return result;
    }
   
    /**
    * send the response
    *
    * @param ssoEntryState
    *   The replication state response
    */   
    public void doSendResponse(ReplicationState ssoEntryState) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationSingleSignOn",
                             "doSendResponse");
        }               
        ReplicationSSOStore store = null;
        try {
            store = (ReplicationSSOStore)getSSOStore();
            store.sendResponse(ssoEntryState);
        } catch (Exception ex) {
            if (_logger.isLoggable(Level.FINE)) {
                _logger.log(Level.FINE, "exception occurred in doSendResponse id=" + ssoEntryState.getId(), ex);
            }
        } finally {
            this.putSSOStore(store);           
        }                
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationSingleSignOn",
                            "doSendResponse");
        }
    }
   
    public void processQueryMessage(ReplicationState message, String returnInstance) {
        //FIXME complete
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicatedSingleSignOn>>processQueryMessage:returnInstance= " + returnInstance);
        }       
        ReplicationStateQueryResponse response = this.doProcessQueryMessage(message);
        boolean isResponse = response.isResponse();
        ReplicationState responseState = response.getState();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("RepSSO:processQueryMessage:after doProcessQueryMessage:response=" + isResponse);
            _logger.fine("RepSSO:processQueryMessage:after doProcessQueryMessage:responseState=" + responseState);           
        }       
        //don't send a response to a response
        if(!isResponse && responseState != null) {
            //point-to-point response back to sender
            //doSendQueryResponse(responseState, this.getInstanceName());
            doSendQueryResponse(responseState, returnInstance);
        }
    }
   
    public void processBroadcastMessage(ReplicationState message) {
        ReplicationStateQueryResponse response = this.doProcessQueryMessage(message);
        boolean isResponse = response.isResponse();
        ReplicationState responseState = response.getState();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:response=" + isResponse);
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:responseState=" + responseState);
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:responseStateTrunk=" + responseState.getTrunkState());
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:responseStateAttr=" + responseState.getState());
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:responseStateVer=" + responseState.getVersion());
        }
       
        //don't send a response to a response
        if(!isResponse) {
            //point-to-point response back to sender
            //doSendQueryResponse(responseState, this.getInstanceName());
            doSendQueryResponse(responseState, message.getInstanceName());
            /*
            ReplicationState response =
                ReplicationState.createResponseFrom(message);
            this.doSendResponse(response);
             */
        }
    }   
   
    public void processBroadcastMessagePrevious(ReplicationState message) {
        //FIXME complete
        ReplicationStateQueryResponse response = this.doProcessQueryMessage(message);
        boolean isResponse = response.isResponse();
        ReplicationState responseState = response.getState();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:response=" + isResponse);
            _logger.fine("processBroadcastMessage:after doProcessQueryMessage:responseState=" + responseState);                     
        }
        //don't send a response to a response
        if(!isResponse) {
            //point-to-point response back to sender
            doSendQueryResponse(responseState, ReplicationUtil.getInstanceName());
            /*
            ReplicationState response =
                ReplicationState.createResponseFrom(message);
            this.doSendResponse(response);
             */
        }
    }
   
    /**
    * send the response
    *
    * @param ssoEntryState
    *   The replication state response
    * @param instanceName  the name of the target instance
    */   
    public void doSendQueryResponse(ReplicationState ssoEntryState, String instanceName) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationSingleSignOn",
                             "doSendQueryResponse");
        }               
        ReplicationSSOStore store = null;
        try {
            store = (ReplicationSSOStore)getSSOStore();
            store.sendQueryResponse(ssoEntryState, instanceName);
        } catch (Exception ex) {
            if (_logger.isLoggable(Level.FINE)) {
                _logger.log(Level.FINE, "exception occurred in doSendQuerytyResponse ssoId=" + ssoEntryState.getId() +
                                        " instanceName=" + instanceName, ex);
            }
        } finally {
            this.putSSOStore(store);           
        }                
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationSingleSignOn",
                            "doSendQueryResponse");
        }
    }   
   
    //return true if message is processQueryResponse
    public ReplicationStateQueryResponse doProcessQueryMessage(ReplicationState message) {
        //FIXME
        ReplicationState resultState = null;
        String methodName = ReplicationUtil.getProcessMethodName(message);
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in ReplicationSingleSignOn>>doProcessQueryMessage:methodName=" + methodName);
        }       
        try {
            Class myClass = this.getClass();
            resultState = (ReplicationState) myClass.getMethod(
                methodName,
                    new Class[]{ message.getClass() }).invoke(
                        this, new Object[]{ message });          
        } catch (IllegalAccessException ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in ReplicationSingleSignOn>>doProcessQueryMessage:methodName=" + methodName + "illegalAccessException");             
            }                       
        } catch (NoSuchMethodException ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in ReplicationSingleSignOn>>doProcessQueryMessage:methodName=" + methodName + "noSuchMethodException");             
            }                       
        } catch (InvocationTargetException ex) {
            if(_logger.isLoggable(Level.FINE)) {
                _logger.fine("in ReplicationSingleSignOn>>doProcessQueryMessage:methodName=" + methodName + "invocationTargetException");
                _logger.log(Level.FINE, "invocationException.getCause()= " + ex.getCause(), ex);
            }            
        }           
        boolean isResponse = methodName.equals("processBroadcastresponse");
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("ReplicationSingleSignOn>>doProcessQueryMessage:resultState=" + resultState);
        }
        if (resultState != null) {
            resultState.setRouteAdvertisement(message.getRouteAdvertisement());
        }       
        return new ReplicationStateQueryResponse(resultState, isResponse);
    }
   
    public void processValvesave(ReplicationState message) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in ReplicationSingleSignOn>>processValvesave");
        }       
        this.putInReplicationCache(message);
    }
   
    public void processRemove(ReplicationState message) {
        if(message.getState() != null) {
            processBulkRemove(message);
            return;
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in ReplicationSingleSignOn>>processRemove");
        }       
        this.removeFromReplicationCache(message);
    }   

    public void processBulkRemove(ReplicationState message) {
        replicaCache.removeFromReplicationCache(message.getState());
    }
   
    public void processUpdatelastaccesstime(ReplicationState message) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in ReplicationSingleSignOn>>processUpdatelastaccesstime");
        }       
        ReplicationState storedReplica
            = this.getFromReplicationCache((String)message.getId());
        if(storedReplica != null) {
            storedReplica.setLastAccess(message.getLastAccess());
            storedReplica.setVersion(message.getVersion());
        }       
    }
   
    public ReplicationState processBroadcastfindsession(ReplicationState queryState) {
        //complete query and send back response
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindSession:instance: " + ReplicationUtil.getInstanceName());
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindSession:id=" + queryState.getId());                       
        }       
        ReplicationState returnState = null;
        //if queryVersion == -1, then this is version unaware load
        long queryVersion = queryState.getVersion();
        ReplicationState replicaState
            = findReplicatedState(queryState);       
        //first check for none found
        if(replicaState == null) {
            //return nack
            return ReplicationState.createQueryResponseFrom(queryState, true);
        }
        //next check for stale version if version-aware query
        if(queryVersion != -1 && replicaState.getVersion() < queryVersion) {
            this.removeFromReplicationCache(replicaState);
            //return nack
            return ReplicationState.createQueryResponseFrom(replicaState, true);          
        }
        //at this point we know replicaState is not null
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("processBroadcastfindsession:REPLICA_FOUND:replicaStateVersion:" + replicaState.getVersion());
            _logger.fine("processBroadcastfindsession:REPLICA_FOUND:replicaState:" + replicaState.getTrunkState());
            _logger.fine("processBroadcastfindsession:REPLICA_FOUND:replicaAttrState" + replicaState.getState());                    
        }           
        returnState = ReplicationState.createQueryResponseFrom(replicaState);
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("processBroadcastfindsession:replicaStateResponseVersion:" + returnState.getVersion());                 
        }           
        //System.out.println("processBroadcastfindsession:replicaStateResponseVersion:" + returnState.getVersion());
        //FIXME may want to wait for ack before doing this
        //FIXME waiting for Jxta fix to put this next line back in
        //replicatedSessions.remove(replicaState.getId());
        //while here check and remove from manager cache if present
        this.clearFromManagerCache((String)queryState.getId());
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindSession:returnState=" + returnState);
        }       
        return returnState;

    }
   
    public void processBroadcastloadreceived(ReplicationState queryState) {
        //load is acknowledged safe to remove replica now
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastloadreceived:instance: " + ReplicationUtil.getInstanceName());
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastloadreceived:id=" + queryState.getId());                       
        }
        if(queryState == null || queryState.getId() == null) {
            return;
        }

        String ignoreInstance = (String) queryState.getProperty(ReplicationState.IGNORE_REMOVE_INSTANCE_NAME);
        //only safe to remove replica if we are not the replica partner
        //now determined by passed property
        if(ignoreInstance != null && !ignoreInstance.equals(ReplicationUtil.getInstanceName())) {
            removeFromReplicationCache((String)queryState.getId());
        }
       
    }          
   
    protected ReplicationState findReplicatedState(ReplicationState queryState) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("findReplicatedState:id = " + queryState.getId());
        }             
        return this.getFromReplicationCache( (String)queryState.getId() );
    }
   
    protected void clearFromManagerCache(String id) {
        cache.remove(id);
    }
   
    public void processResponse(ReplicationState message) {
        //complete processing response - not sending response to a response
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN" + this.getClass().getName() + ">>processResponse");           
        }       
        ReplicationResponseRepository.putEntry(message);
    }   

    public ReplicationState processBroadcastresponse(ReplicationState queryResponseState) {
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastresponse:queryResponseState=" + queryResponseState);           
        }
        //ReplicationResponseRepository.putEntry(queryResponseState);
        ReplicationResponseRepository.putFederatedEntry(queryResponseState);
        return queryResponseState;
    }
   
    // begin post join reconciliation

    /**
     * do reconciliation processing
     * used for both rolling upgrade and post network partition rejoin
     * @param waitTime the waitTime in seconds
     * @param ctx the RollingUpgradeContext
     */
    public void doPostJoinReconciliation(long waitTime, RollingUpgradeContext ctx) {
        long startTime = 0L;
        if (_logger.isLoggable(Level.FINE)) {
            startTime = System.currentTimeMillis();
        }
        CountDownLatch doneSignal = new CountDownLatch(2);
        //do active cache reconciliation
        ReconcileActive reconcileActive
            = new ReconcileActive((ReplicationManager)this, waitTime, doneSignal, ctx);
        RollingUpgradeHandler.executeTask(reconcileActive);

        //trigger replica cache reconciliation
        TriggerReconcileReplica triggerReconcileReplica
            = new TriggerReconcileReplica((ReplicationManager)this, waitTime, doneSignal, ctx);
        RollingUpgradeHandler.executeTask(triggerReconcileReplica);

        try {
            doneSignal.await(waitTime, TimeUnit.SECONDS);
        } catch(InterruptedException ex) {
            ;
        } finally {
            if(doneSignal.getCount() != 0) {
                String errorMsg = "ReplicationSingleSignOn>>doPostJoinReconciliation timed out after " + waitTime + " seconds";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationSingleSignOn doPostJoinReconciliation successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
        }
    }

    // end post join reconciliation   
   
    //end message processing methods
   
   // begin rolling upgrade related code
   
    public void doRollingUpgradePreShutdownProcessing(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        //do syncpoint save
        doSyncpointSave(waitTime, doneSignal, ctx);
    }
   
    public void doRollingUpgradePostStartupProcessing(String rollingUpgradeType, long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        if(RollingUpgradeHandler.isFileBased(rollingUpgradeType)) {
            //do syncpoint load
            doSyncpointLoad(waitTime, doneSignal, ctx);
        } else {
            doMemoryLoad(waitTime, doneSignal, ctx);
        }
    }

    public void doRollingUpgradePostStartupReconciliationProcessing(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        //do post join reconciliation
        try {
            doPostJoinReconciliation(waitTime, ctx);
        } finally {
            doneSignal.countDown();
        }
    }

    void doSyncpointSave(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        try {
            SSOFileSync syncStore = new SSOFileSync((ReplicationManager)this);
            try {
                syncStore.save(waitTime, ctx);
            } catch (IOException ex) {
                ex.printStackTrace();
                String errorMsg = "ReplicationSingleSignOn>>doSyncPointSave IOException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            }
        } finally {
            doneSignal.countDown();
        }
    }

    void doSyncpointLoad(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        try {
            SSOFileSync syncStore = new SSOFileSync((ReplicationManager)this);
            try {
                syncStore.load(waitTime, ctx);
                markActiveCacheAsSuspected(true);
            } catch (IOException ex) {
                String errorMsg = "ReplicationSingleSignOn>>doSyncPointLoad IOException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } catch (ClassNotFoundException ex2) {
                String errorMsg = "ReplicationSingleSignOn>>doSyncPointLoad ClassNotFoundException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            }
        } finally {
            doneSignal.countDown();
        }
    }

    void doMemoryLoad(long waitTime, CountDownLatch doneSignal, RollingUpgradeContext ctx) {
        try {
            SSOMemorySync syncStore = new SSOMemorySync((ReplicationManager)this);
            try {
                syncStore.load(waitTime, ctx);
                markActiveCacheAsSuspected(true);
            } catch (IOException ex) {
                String errorMsg = "ReplicationSingleSignOn>>doMemoryLoad IOException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } catch (ClassNotFoundException ex2) {
                String errorMsg = "ReplicationSingleSignOn>>doMemoryLoad ClassNotFoundException";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            }
        } finally {
            doneSignal.countDown();
        }
    }

    void restoreActiveCacheViaMemory(long waitTime, RollingUpgradeContext ctx) {
        triggerActiveCacheRestoration(waitTime, ctx);
    }

    void restoreReplicaCacheViaMemory(long waitTime, RollingUpgradeContext ctx) {
        triggerReplicaCacheRestoration(waitTime, ctx);
    }

    public void triggerActiveCacheRestoration(long waitTime, RollingUpgradeContext ctx) {
        long startTime = System.currentTimeMillis();
        int numberOfRespondants = RollingUpgradeUtil.getNumberOfExpectedRespondants();
        restoreActiveCacheDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateTo partner(s) to do
        //active cache restoration
        ownedReplicaListsReceivedForActiveCacheRestoration
            = new ConcurrentHashMap<String, ReplicationState>();
        doTriggerActiveCacheRestoration(ReplicationUtil.checkIsInstanceLoadBalancedByCLB());

        try {
            restoreActiveCacheDoneSignal.await(waitTime, TimeUnit.SECONDS);
            processActiveCacheRestorationResults(ownedReplicaListsReceivedForActiveCacheRestoration);
        } catch(InterruptedException ex) {
            ;
        } finally {
            if(restoreActiveCacheDoneSignal.getCount() != 0) {
                String errorMsg = "ReplicationSingleSignOn>>triggerActiveCacheRestoration timed out after " + waitTime + " seconds";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationSingleSignOn triggerActiveCacheRestoration successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
            ctx.addMessage(getApplicationId(), "Active cache restoration completed for SSO. " +
                    ", Size of active cache = " + getActiveSessionCount() +
                    ", Time taken (ms) = " + (System.currentTimeMillis()-startTime));

        }
    }

    protected void doTriggerActiveCacheRestoration(boolean isInstanceLoadBalancedByCLB) {
        if(!isInstanceLoadBalancedByCLB) {
            //trigger replicateTo partner to do
            //active cache restoration
            ReplicationHealthChecker healthChecker
                = ReplicationHealthChecker.getInstance();
            String replicateToInstanceName = healthChecker.getReshapeReplicateToInstanceName(null, 0L);
            sendRollingUpgradeActiveCacheRestorationAdvisory(replicateToInstanceName);
        } else {
            //trigger all partners to do
            //active cache restoration
            sendRollingUpgradeActiveCacheRestorationAdvisory();
        }
    }

    protected void processActiveCacheRestorationResults(ConcurrentHashMap<String, ReplicationState> ownedReplicaListsReceivedForActiveCacheRestoration) {
        ReplicationSSOStore store = getSingletonSSOStore();
        Collection<ReplicationState> ownedReplicasList = ownedReplicaListsReceivedForActiveCacheRestoration.values();
        Iterator<ReplicationState> it = ownedReplicasList.iterator();
        while(it.hasNext()) {
            ReplicationState nextOwnedListReplicationState = it.next();
            List<ReplicationState>nextOwnedListOfStates
                = (List<ReplicationState>)RollingUpgradeUtil.getObject(nextOwnedListReplicationState.getState());
            for(int i=0; i<nextOwnedListOfStates.size(); i++) {
                ReplicationState nextState = nextOwnedListOfStates.get(i);
                HASingleSignOnEntry nextSSOEntry = null;
                try {
                    nextSSOEntry = store.getSSOEntry(nextState);
                } catch (Exception ex) {}
                if(nextSSOEntry != null) {
                    this.registerInMemory(nextSSOEntry.getId(), nextSSOEntry);
                }
            }
        }
    }

    public void triggerReplicaCacheRestoration(long waitTime, RollingUpgradeContext ctx) {
        long startTime = System.currentTimeMillis();
        int numberOfRespondants = RollingUpgradeUtil.getNumberOfExpectedRespondants();
        restoreReplicaCacheDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateFrom partner(s) to do
        //replica cache restoration
        ownedReplicaListsReceivedForReplicaCacheRestoration
            = new ConcurrentHashMap<String, ReplicationState>();
        doTriggerReplicaCacheRestoration(ReplicationUtil.checkIsInstanceLoadBalancedByCLB());

        try {
            restoreReplicaCacheDoneSignal.await(waitTime, TimeUnit.SECONDS);
            processReplicaCacheRestorationResults(ownedReplicaListsReceivedForReplicaCacheRestoration);
        } catch(InterruptedException ex) {
            ;
        } finally {
            if(restoreReplicaCacheDoneSignal.getCount() != 0) {
                String errorMsg = "ReplicationSingleSignOn>>triggerReplicaCacheRestoration timed out after " + waitTime + " seconds";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationSingleSignOn triggerReplicaCacheRestoration successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
            ctx.addMessage(getApplicationId(), "Replica cache restoration completed for SSO. " +
                    ", Size of replica cache = " + getReplicatedSSOEntries().getEntryCount() +
                    ", Time taken (ms) = " + (System.currentTimeMillis()-startTime));

        }
    }

    protected void doTriggerReplicaCacheRestoration(boolean isInstanceLoadBalancedByCLB) {
        if(!isInstanceLoadBalancedByCLB) {
            //trigger replicateFrom partner to do
            //replica cache restoration
            ReplicationHealthChecker healthChecker
                = ReplicationHealthChecker.getInstance();
            String replicateFromInstanceName = healthChecker.getReshapeReplicateFromInstanceName(null);
            sendRollingUpgradeReplicaCacheRestorationAdvisory(replicateFromInstanceName);
        } else {
            //trigger all partners to do
            //replica cache restoration
            sendRollingUpgradeReplicaCacheRestorationAdvisory();
        }
    }

    protected void processReplicaCacheRestorationResults(ConcurrentHashMap<String, ReplicationState> ownedReplicaListsReceivedForActiveCacheRestoration) {
        Collection<ReplicationState> ownedReplicasList = ownedReplicaListsReceivedForActiveCacheRestoration.values();
        Iterator<ReplicationState> it = ownedReplicasList.iterator();
        while(it.hasNext()) {
            ReplicationState nextOwnedListReplicationState = it.next();
            List<ReplicationState>nextOwnedListOfStates
                = (List<ReplicationState>)RollingUpgradeUtil.getObject(nextOwnedListReplicationState.getState());
            for(int i=0; i<nextOwnedListOfStates.size(); i++) {
                ReplicationState nextState = nextOwnedListOfStates.get(i);
                this.putInReplicationCache(nextState);
            }
        }
    }   
   
    void markActiveCacheAsSuspected(boolean value) {
        Iterator it = cache.values().iterator();
        while(it.hasNext()) {
            ((HASingleSignOnEntry)it.next()).setSuspect(value);
        }
    }

    public void doActiveCacheReconciliation(long waitTime, RollingUpgradeContext ctx) {
        //System.out.println("begin doActiveCacheReconciliation");
        if(!this.isThirdPartyBackingStoreInUse()) {
            stillOwnedSingleSignOnEntryIdsForActiveCacheReconciliation
                = triggerGetIdsForActiveCacheReconciliation(waitTime);
        } else {
            List sessionIdsList = getSSOEntryIdsThirdPartySPI(ReplicationUtil.getInstanceName());
            stillOwnedSingleSignOnEntryIdsForActiveCacheReconciliation
                = RollingUpgradeUtil.mergeQueryListIntoMap(sessionIdsList, stillOwnedSingleSignOnEntryIdsForActiveCacheReconciliation);
        }
        this.setActiveCacheReconciliationOngoing(true);
        try {
            reconcileSSOActiveCache(stillOwnedSingleSignOnEntryIdsForActiveCacheReconciliation);
            //System.out.println("end doActiveCacheReconciliation");
        } finally {
            this.setActiveCacheReconciliationOngoing(false);
        }
    }

    Collection<SSOExtraParams> __findByCriteriaExpiredSSO() {
        return Collections.EMPTY_LIST; //TODO
    }

    public ConcurrentHashMap<String, FederatedQueryListElement> triggerGetIdsForActiveCacheReconciliation(long waitTime) {
        long startTime = 0L;
        if (_logger.isLoggable(Level.FINE)) {
            startTime = System.currentTimeMillis();
        }
        //because sending to all instances
        int numberOfRespondants = ReplicationUtil.getNumberExpectedRespondants();
        getIdsForActiveCacheReconciliationDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateTo partner(s) to
        //provide owned session ids list(s)
        ownedIdsForActiveCacheReconciliation
            = new ConcurrentHashMap<String, ReplicationState>();
        doTriggerGetIdsForActiveCacheReconciliation(ReplicationUtil.checkIsInstanceLoadBalancedByCLB());
        ConcurrentHashMap<String, FederatedQueryListElement> result = null;
        try {
            getIdsForActiveCacheReconciliationDoneSignal.await(waitTime, TimeUnit.SECONDS);
            result = processCacheReconciliationResults(ownedIdsForActiveCacheReconciliation);
        } catch(InterruptedException ex) {
            result = new ConcurrentHashMap<String, FederatedQueryListElement>();
        } finally {
            if(getIdsForActiveCacheReconciliationDoneSignal.getCount() != 0) {
                _logger.log(Level.WARNING, "ReplicationSingleSignOn>>triggerGetIdsForActiveCacheReconciliation timed out after "
                    + waitTime + " seconds");
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationSingleSignOn triggerGetIdsForActiveCacheReconciliation successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
            ownedIdsForActiveCacheReconciliation = null;
        }
        return result;
    }

    protected void doTriggerGetIdsForActiveCacheReconciliation(boolean isInstanceLoadBalancedByCLB) {
        //trigger all partners to do
        //active cache reconciliation
        sendRollingUpgradeGetIdsForActiveCacheReconciliationAdvisory();
    }

    /**
     * broadcast a rolling upgrade getIds
     * for replica cache reconciliation advisory message
     *
     */
    protected void sendRollingUpgradeGetIdsForActiveCacheReconciliationAdvisory() {
        //broadcast rolling upgrade getIds for replica cache reconciliation advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_SSO, MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_ADVISORY);
    }

    public void processRollingupgradegetidsforactivecachereconciliationadvisory(ReplicationState replicationState) {
        String instanceName = ReplicationUtil.getInstanceName();
        String owningInstanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processRollingupgradegetidsforactivecachereconciliationadvisory:instance: " + instanceName);
            _logger.fine("in " + this.getClass().getName() + ">>processRollingupgradegetidsforactivecachereconciliationadvisory:owningInstance=" + owningInstanceName);
        }
        List<FederatedQueryListElement> sessionIds = getSSOEntryIds(owningInstanceName);
        sendReplicasList(sessionIds, owningInstanceName, MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_ACTIVE_CACHE_RECONCILIATION_RESPONSE);
    }

    public void processRollingupgradegetidsforactivecachereconciliationresponse(ReplicationState replicationState) {
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradegetidsforactivecachereconciliationresponse received from " + instanceName);
        }
        ownedIdsForActiveCacheReconciliation.putIfAbsent(instanceName, replicationState);
        getIdsForActiveCacheReconciliationDoneSignal.countDown();
    }

    /**
     * reconcile the active cache with a map containing
     * current set of instance ids owned by this instance
     * @param stillOwnedSingleSignOnEntryIdsForActiveCacheReconciliation
     */
    private void reconcileSSOActiveCache(ConcurrentHashMap stillOwnedSingleSignOnEntryIdsForActiveCacheReconciliation) {
        //first check and reload into active any missing ones from replica partner
        ReplicationHealthChecker healthChecker
            = ReplicationHealthChecker.getInstance();
        String buddyReplicaPartnerInstance
            = healthChecker.getReshapeReplicateToInstanceName(null, 0L);
        Iterator it = stillOwnedSingleSignOnEntryIdsForActiveCacheReconciliation.values().iterator();
        while(it.hasNext()) {
            FederatedQueryListElement nextElement
                = (FederatedQueryListElement)it.next();
            //if replica is from our replica partner and we do not
            //have it (or have lesser version) in active cache then cause it to be loaded
            HASingleSignOnEntry nextSSOEntry = (HASingleSignOnEntry)findSSOEntryFromCacheOnly(nextElement.getId());
            if(RollingUpgradeUtil.isElementFromReplicationPartner(nextElement, buddyReplicaPartnerInstance)
                && (nextSSOEntry == null || nextSSOEntry.getVersion() >= nextElement.getVersion()) ) {
                lookupEntry(nextElement.getId());
            }
        }
        ArrayList idsToBeRemoved = new ArrayList();
        it = cache.values().iterator();
        while(it.hasNext()) {
            HASingleSignOnEntry nextSingleSignOnEntry
                = (HASingleSignOnEntry)it.next();
                //if not suspect - traffic has already cleared it
            synchronized(nextSingleSignOnEntry) {
                if(nextSingleSignOnEntry.isSuspect()) {
                    if(stillOwnedSingleSignOnEntryIdsForActiveCacheReconciliation.get(nextSingleSignOnEntry.getId()) == null) {
                        idsToBeRemoved.add(nextSingleSignOnEntry.getId());
                    } else {
                        nextSingleSignOnEntry.setSuspect(false);
                    }
                }
            }
        }
        for(int i=0; i<idsToBeRemoved.size(); i++) {
            clearFromManagerCache((String)idsToBeRemoved.get(i));
        }
    }

    protected ConcurrentHashMap<String, FederatedQueryListElement> processCacheReconciliationResults(ConcurrentHashMap<String, ReplicationState> ownedIdsForReplicaCacheReconciliation) {
        ConcurrentHashMap<String, FederatedQueryListElement> result =
            new ConcurrentHashMap<String, FederatedQueryListElement>();
        Collection<ReplicationState> ownedReplicasList = ownedIdsForReplicaCacheReconciliation.values();
        Iterator<ReplicationState> it = ownedReplicasList.iterator();
        while(it.hasNext()) {
            ReplicationState nextOwnedListReplicationState = it.next();
            List<FederatedQueryListElement>nextOwnedListOfStates
                = (List<FederatedQueryListElement>)RollingUpgradeUtil.getObject(nextOwnedListReplicationState.getState());
            for(int i=0; i<nextOwnedListOfStates.size(); i++) {
                FederatedQueryListElement nextElement = nextOwnedListOfStates.get(i);
                String nextElementId = nextElement.getId();
                if(nextElementId != null) {
                    FederatedQueryListElement existingElement = result.get(nextElementId);
                    if(existingElement == null || existingElement.getVersion() < nextElement.getVersion()) {
                        result.put(nextElementId, nextElement);
                    }
                }
            }
        }
        return result;
    }

    /**
     * reconcile the replica cache (of your replica partner)
     * query instance1 to get a list of replica id/version data elements
     * then do 2 iterations:
     * iterate over the query result:
     * if an id from this list does not exist in this active cache
     * issue a remove message & load acknowledgment
     * if an id exists and the versions match do nothing
     * if an id exists and the active version is > replica version,
     *   - do a save
     * iterate over the active cache
     * if an id from active cache does not exist in the replica list
     *   - do a save
     * @param waitTime the waitTime
     */
    public void doReplicaCacheReconciliation(long waitTime, RollingUpgradeContext ctx) {
        //System.out.println("begin doReplicaCacheReconciliation");
        //todo - check that zombies are cleaned out during execution of query that follows
        //if so cleanOutZombieReplicas can be removed
        //cleanOutZombieReplicas();
        ConcurrentHashMap<String, FederatedQueryListElement> stillOwnedSessionIdsForReplicaCacheReconciliation
            = null;
        if(!this.isThirdPartyBackingStoreInUse()) {
            stillOwnedSessionIdsForReplicaCacheReconciliation
                = triggerGetIdsForReplicaCacheReconciliation(waitTime);
        } else {
            List ssoEntryIdsList = getSSOEntryIdsThirdPartySPI(ReplicationUtil.getInstanceName());
            stillOwnedSessionIdsForReplicaCacheReconciliation
                = RollingUpgradeUtil.mergeQueryListIntoMap(ssoEntryIdsList, stillOwnedSessionIdsForReplicaCacheReconciliation);
        }
        /*
        Iterator it = stillOwnedSessionIdsForReplicaCacheReconciliation.values().iterator();
        while(it.hasNext()) {
            System.out.println("next id = " + ((FederatedQueryListElement)it.next()).getId());
        }
         */
        reconcileSSOReplicaCache(stillOwnedSessionIdsForReplicaCacheReconciliation);
        //System.out.println("end doReplicaCacheReconciliation");
    }

    public ConcurrentHashMap<String, FederatedQueryListElement> triggerGetIdsForReplicaCacheReconciliation(long waitTime) {
        long startTime = 0L;
        if (_logger.isLoggable(Level.FINE)) {
            startTime = System.currentTimeMillis();
        }
        //because sending to all instances
        int numberOfRespondants = ReplicationUtil.getNumberExpectedRespondants();
        getIdsForReplicaCacheReconciliationDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateTo partner(s) to
        //provide owned sso entry ids list(s)
        ownedIdsForReplicaCacheReconciliation
            = new ConcurrentHashMap<String, ReplicationState>();
        doTriggerGetIdsForReplicaCacheReconciliation(ReplicationUtil.checkIsInstanceLoadBalancedByCLB());
        ConcurrentHashMap<String, FederatedQueryListElement> result = null;
        try {
            getIdsForReplicaCacheReconciliationDoneSignal.await(waitTime, TimeUnit.SECONDS);
            result = processCacheReconciliationResults(ownedIdsForReplicaCacheReconciliation);
        } catch(InterruptedException ex) {
            result = new ConcurrentHashMap<String, FederatedQueryListElement>();
        } finally {
            if(getIdsForReplicaCacheReconciliationDoneSignal.getCount() != 0) {
                _logger.log(Level.WARNING, "ReplicationManagerBase>>triggerGetIdsForReplicaCacheReconciliation timed out after "
                    + waitTime + " seconds");
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationManagerBase triggerGetIdsForReplicaCacheReconciliation successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
            ownedIdsForReplicaCacheReconciliation = null;
        }
        return result;
    }

    protected void doTriggerGetIdsForReplicaCacheReconciliation(boolean isInstanceLoadBalancedByCLB) {
        //trigger all partners to do
        //replica cache reconciliation
        sendRollingUpgradeGetIdsForReplicaCacheReconciliationAdvisory();
    }
   
    void cleanOutZombieReplicas() {
        //FIXME for now issue load acknowledgement for each
        //active cache entry
        //this is functionally correct but not optimal
        //solution will be to improve query for replica
        //ids to track the source instance and then only
        //issue load acks to those ids that are not from
        //our replicate to partner
        HASingleSignOnEntry ssoEntry = null;
        Iterator it = cache.values().iterator();
        while(it.hasNext()) {
            ssoEntry = (HASingleSignOnEntry)it.next();
            String ssoId = ssoEntry.getId();
            long version = ssoEntry.getVersion();
            this.sendLoadAcknowledgement(ssoId, version);
        }       
    }

    public void processRollingupgradegetidsforreplicacachereconciliationadvisory(ReplicationState replicationState) {
        String instanceName = ReplicationUtil.getInstanceName();
        String owningInstanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processRollingupgradegetidsforreplicacachereconciliationadvisory:instance: " + instanceName);
            _logger.fine("in " + this.getClass().getName() + ">>processRollingupgradegetidsforreplicacachereconciliationadvisory:owningInstance=" + owningInstanceName);
        }
        List<FederatedQueryListElement> sessionIds = getSSOEntryIds(owningInstanceName);
        sendReplicasList(sessionIds, owningInstanceName, MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_RESPONSE);
    }

    public void processRollingupgradegetidsforreplicacachereconciliationresponse(ReplicationState replicationState) {
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradegetidsforreplicacachereconciliationresponse received from " + instanceName);
        }
        ownedIdsForReplicaCacheReconciliation.putIfAbsent(instanceName, replicationState);
        getIdsForReplicaCacheReconciliationDoneSignal.countDown();
    }

    protected void doTriggerReplicaCacheReconciliation(long waitTime, boolean isInstanceLoadBalancedByCLB) {
        if(!isInstanceLoadBalancedByCLB) {
            //trigger replicateFrom partner to do
            //replica cache reconciliation
            ReplicationHealthChecker healthChecker
                = ReplicationHealthChecker.getInstance();
            String replicateFromInstanceName = healthChecker.getReshapeReplicateFromInstanceName();
            sendRollingUpgradeAdvisory(waitTime, replicateFromInstanceName);
        } else {
            //trigger all partners to do
            //replica cache reconciliation
            sendRollingUpgradeAdvisory(waitTime);
        }
    }

    public void triggerReplicaCacheReconciliation(long waitTime, RollingUpgradeContext ctx) {
        int numberOfRespondants = 1;
        if(ReplicationUtil.checkIsInstanceLoadBalancedByCLB()) {
            numberOfRespondants = ReplicationUtil.getNumberExpectedRespondants();
        }
        long startTime = 0L;
        if (_logger.isLoggable(Level.FINE)) {
            startTime = System.currentTimeMillis();
        }
        reconcileReplicaCacheDoneSignal = new CountDownLatch(numberOfRespondants);
        //trigger replicateFrom partner(s) to do
        //replica cache reconciliation
        doTriggerReplicaCacheReconciliation(waitTime, ReplicationUtil.checkIsInstanceLoadBalancedByCLB());
        try {
            reconcileReplicaCacheDoneSignal.await(waitTime, TimeUnit.SECONDS);
        } catch(InterruptedException ex) {
            ;
        } finally {
            if(reconcileReplicaCacheDoneSignal.getCount() != 0) {
                String errorMsg = "ReplicationSingleSignOn>>triggerReplicaCacheReconciliation timed out after " + waitTime + " seconds";
                ReplicationUtil.handleErrorMsg(errorMsg, _logger, ctx);
            } else {
                if (_logger.isLoggable(Level.FINE)) {
                    _logger.fine("ReplicationSingleSignOn>>triggerReplicaCacheReconciliation successful after a wait: wait time = " + (System.currentTimeMillis() - startTime));
                }
            }
        }
    }
   
    /**
     * reconcile the replica cache (of your replica partner)
     * with a map containing
     * current set of instance ids owned by this instance
     * @param stillOwnedSessionIdsForReplicaCacheReconciliation
     */   
    private void reconcileSSOReplicaCache(ConcurrentHashMap stillOwnedSessionIdsForReplicaCacheReconciliation) {
        HASingleSignOnEntry ssoEntry = null;
        Iterator it = stillOwnedSessionIdsForReplicaCacheReconciliation.values().iterator();
        while(it.hasNext()) {
            FederatedQueryListElement nextElement
                = (FederatedQueryListElement)it.next();
            ssoEntry = (HASingleSignOnEntry)cache.get(nextElement.getId());
            //if not present in active cache, issue remove
            //and issue load acknowledgement to clean any zombie replicas
            if(ssoEntry == null) {
                String nextId = nextElement.getId();
                try {
                    removeSSOEntry(nextId);
                } catch (IOException ex) {
                    ;
                }
                this.sendLoadAcknowledgement(nextId, nextElement.getVersion());
            } else {
                //if active version > replica version - issue save
                if(ssoEntry.getVersion() > nextElement.getVersion()) {
                    try {
                        saveSSOEntry(ssoEntry);
                    } catch (IOException ex) {
                        ;
                    }
                }
            }
        }
        Iterator it2 = cache.values().iterator();
        while(it2.hasNext()) {
            ssoEntry = (HASingleSignOnEntry)it2.next();
            String activeSSOId = ssoEntry.getId();
            //if session in active cache has no corresponding replica
            //issue a save
            if(stillOwnedSessionIdsForReplicaCacheReconciliation.get(activeSSOId) == null) {
                try {
                    saveSSOEntry(ssoEntry);
                } catch (IOException ex) {
                    ;
                }
            }
        }
    }
   
    protected void sendLoadAcknowledgement(String id, long version) {
        JxtaReplicationSender sender
            = JxtaReplicationSender.createInstance();
        ReplicationState loadReceivedState =
        ReplicationState.createBroadcastLoadReceivedState(MODE_SSO, id, this.getApplicationId(), version, ReplicationUtil.getInstanceName(), MESSAGE_BROADCAST_LOAD_RECEIVED);
        ReplicationHealthChecker healthChecker = ReplicationHealthChecker.getInstance();
        String currentReplicaPartner = healthChecker.getCurrentPartnerInstanceName();
            loadReceivedState.setProperty(ReplicationState.IGNORE_REMOVE_INSTANCE_NAME,
            currentReplicaPartner);              
        sender.sendBroadcastQuery(loadReceivedState);
    }   
   
    protected void readSSOEntries(ObjectInputStream ois)
        throws ClassNotFoundException, IOException {
        int count = ois.readInt();
        for(int i=0; i<count; i++) {
            HASingleSignOnEntry nextEntry = (HASingleSignOnEntry)ois.readObject();
            cache.put(nextEntry.getId(), nextEntry);
        }
    }   
   
    protected void writeSSOEntries(ObjectOutputStream oos)
        throws IOException {
        ReplicationUtil.writeHashMap(cache, oos);
    }
   
    /**
     * process the broadcastfindssoentryids for SSOEntry
     * @param queryState
     */    
    public ReplicationState processBroadcastfindssoentryids(ReplicationState queryState) {
        //complete query and send back response
        String instanceName = ReplicationUtil.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindssoentryids:instance: " + instanceName);
            _logger.fine("in " + this.getClass().getName() + ">>processBroadcastfindssoentryids:owningInstance=" + queryState.getInstanceName());                       
        }        
        String owningInstanceName = queryState.getInstanceName();
        List ssoEntryIds = getSSOEntryIds(owningInstanceName);
        byte[] resultState = null;
        try {
            resultState = ReplicationUtil.getByteArray((ArrayList)ssoEntryIds);
        } catch (IOException ex) {
            //deliberate no-op
            ;
        }
        //first check for none found in either active or replica caches
        if(ssoEntryIds == null || ssoEntryIds.isEmpty()) {
            //return nack
            return ReplicationState.createQueryResponseFrom(queryState, true);
        } else {
            return ReplicationState.createQueryResponseFrom(queryState, resultState);
        }
    }

    List<FederatedQueryListElement> getSSOEntryIds(String owningInstanceName) {
        List ssoEntryIds = new ArrayList();
        //using set to avoid dups
        HashSet ssoEntryIdsSet = new HashSet();
        List replicasToRemove = new ArrayList();
        ReplicationHealthChecker healthChecker = ReplicationHealthChecker.getInstance();
        String correctBuddyReplicaName
            = healthChecker.getReshapeReplicateToInstanceName(null, owningInstanceName, 0L);
        //iterate over sso entry replicas
        Iterator it = getReplicatedSSOEntries().values();
        while(it.hasNext()) {
            ReplicationState nextState
                = (ReplicationState)it.next();
            if(!RollingUpgradeUtil.filterOwnership(owningInstanceName, correctBuddyReplicaName, nextState, ssoEntryIdsSet)) {
                //we remove this entry if it is not on the correct replica partner - i.e. it is a zombie
                replicasToRemove.add(nextState.getId());
            }
        }
        for(int i=0; i<replicasToRemove.size(); i++) {
            String nextId = (String)replicasToRemove.get(i);
            if(nextId != null) {
                removeFromReplicationCache(nextId);
            }
        }       
        ssoEntryIds.addAll(ssoEntryIdsSet);
        return ssoEntryIds;
    }
   
    List<FederatedQueryListElement> getSSOEntryIdsThirdPartySPI(String owningInstanceName) {
        BackingStore backingStore
            = this.getBackingStore();
        SSOExtraParams ssoExtraParamCriteria
            = SSOExtraParams.createSearchCriteria(this, owningInstanceName);
        Collection<SSOExtraParams> ssoColl
            = backingStore.findByCriteria(null, ssoExtraParamCriteria);

        List<FederatedQueryListElement> ssoEntryIds = new ArrayList();
        //using set to avoid dups
        HashSet ssoEntryIdsSet = new HashSet();       
        Iterator<SSOExtraParams> ssoResults =
            (Iterator<SSOExtraParams>) ssoColl.iterator();
        while (ssoResults.hasNext()) {
            SSOExtraParams eParam = ssoResults.next();
            if (owningInstanceName.equals(eParam.getCurrentOwnerInstanceName())) {
                ssoEntryIdsSet.add(new FederatedQueryListElement((String)eParam.getId(), -1L, ReplicationUtil.getInstanceName()));
            }           
        }      
        ssoEntryIds.addAll(ssoEntryIdsSet);       
        return ssoEntryIds;
    }
   
    boolean isThirdPartyBackingStoreInUse() {
        BackingStore backingStore
            = getBackingStore();
        return (!(backingStore instanceof JxtaBackingStoreImpl));
    }

    /**
     * send a rolling upgrade advisory message to instance to trigger
     * it to do rolling upgrade reconciliation for the sending
     * instance
     *
     * @param waitTime the waitTime
     * @param instanceName the instance to be sent the rolling upgrade advisory
     */
    public void sendRollingUpgradeAdvisory(long waitTime, String instanceName) {

        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationSingleSignOn",
                             "sendRollingUpgradeAdvisory",
                             new Object[] {instanceName});
        }
        try {
            RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_SSO, MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY, instanceName, waitTime);
        } catch (IOException ex) {}
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationSingleSignOn",
                            "sendRollingUpgradeAdvisory");
        }
    }

    /**
     * broadcast a rolling upgrade advisory message to trigger
     * rolling upgrade reconciliation for the sending
     * instance
     * @param waitTime the waitTime
     */
    protected void sendRollingUpgradeAdvisory(long waitTime) {
        //broadcast rolling upgrade advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_SSO, MESSAGE_BROADCAST_ROLLING_UPGRADE_ADVISORY, waitTime);
    }

    /**
     * send a rolling upgrade advisory message to instance to trigger
     * it to do rolling upgrade active cache restoration for the sending
     * instance
     *
     * @param instanceName the instance to be sent the rolling upgrade advisory
     */
    public void sendRollingUpgradeActiveCacheRestorationAdvisory(String instanceName) {

        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationSingleSignOn",
                             "sendRollingUpgradeActiveCacheRestorationAdvisory",
                             new Object[] {instanceName});
        }
        try {
            RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_SSO, MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY, instanceName);
        } catch (IOException ex) {}
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationSingleSignOn",
                            "sendRollingUpgradeActiveCacheRestorationAdvisory");
        }
    }

    /**
     * broadcast a rolling upgrade active cache restoration advisory message to trigger
     * rolling upgrade active cache restoration for the sending
     * instance
     */
    protected void sendRollingUpgradeActiveCacheRestorationAdvisory() {
        //broadcast rolling upgrade active cache restoration advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_SSO, MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_ADVISORY);
    }

    /**
     * send a rolling upgrade advisory message to instance to trigger
     * it to do rolling upgrade replica cache restoration for the sending
     * instance
     *
     * @param instanceName the instance to be sent the rolling upgrade advisory
     */
    public void sendRollingUpgradeReplicaCacheRestorationAdvisory(String instanceName) {
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this,
                MODE_SSO, MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_ADVISORY,
                "ReplicationSingleSignOn", "sendRollingUpgradeReplicaCacheRestorationAdvisory", instanceName);
        /*
        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationSingleSignOn",
                             "sendRollingUpgradeReplicaCacheRestorationAdvisory",
                             new Object[] {instanceName});
        }
        try {
            RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_SSO, MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_ADVISORY, instanceName);
        } catch (IOException ex) {}
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationSingleSignOn",
                            "sendRollingUpgradeReplicaCacheRestorationAdvisory");
        }
        */
    }

    /**
     * broadcast a rolling upgrade active cache restoration advisory message to trigger
     * rolling upgrade replica cache restoration for the sending
     * instance
     */
    protected void sendRollingUpgradeReplicaCacheRestorationAdvisory() {
        //broadcast rolling upgrade active cache restoration advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_SSO, MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_ADVISORY);
    }

    /**
     * send a rolling upgrade reconciliation complete advisory message to instance to signal
     * to the original caller instance that replica reconciliation is complete
     *
     * @param instanceName the instance to be sent the rolling upgrade reconciliation complete advisory
     */
    public void sendRollingUpgradeReplicaReconciliationCompleteAdvisory(String instanceName) {

        if(_logger.isLoggable(Level.FINE)) {
            _logger.entering("ReplicationSingleSignOn",
                             "sendRollingUpgradeReplicaReconciliationCompleteAdvisory",
                             new Object[] {instanceName});
        }
        try {
            RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_SSO, MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_RECONCILIATION_COMPLETE_ADVISORY, instanceName);
        } catch (IOException ex) {}
        if(_logger.isLoggable(Level.FINE)) {
            _logger.exiting("ReplicationSingleSignOn",
                            "sendRollingUpgradeReplicaReconciliationCompleteAdvisory");
        }
    }

    /**
     * broadcast a rolling upgrade getIds
     * for replica cache reconciliation advisory message
     *
     */
    protected void sendRollingUpgradeGetIdsForReplicaCacheReconciliationAdvisory() {
        //broadcast rolling upgrade getIds for replica cache reconciliation advisory
        RollingUpgradeUtil.sendRollingUpgradeAdvisory((ReplicationManager)this, MODE_SSO, MESSAGE_BROADCAST_ROLLING_UPGRADE_GET_IDS_FOR_REPLICA_CACHE_RECONCILIATION_ADVISORY);
    }
   
    public void processRollingupgradeadvisory(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradeadvisory received from " + instanceName);
        }
        long waitTime = ((Long)replicationState.getProperty(ReplicationState.WAIT_TIME)).longValue();
        //we have been triggered to do replica
        //cache reconciliation for our replicateTo partner
        doReplicaCacheReconciliation(waitTime, null);// TODO :: check last param, this whole method is unused -- remove it later.
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradeadvisory sending reconciliation completed advisory to: " + instanceName);
        }
        sendRollingUpgradeReplicaReconciliationCompleteAdvisory(instanceName);
    }

    public void processRollingupgradeactivecacherestorationadvisory(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradeactivecacherestorationadvisory received from " + instanceName);
        }
        List<ReplicationState> ownedReplicas = getSSOEntryReplicas(instanceName);
        sendReplicationStateList(ownedReplicas, instanceName, MESSAGE_BROADCAST_ROLLING_UPGRADE_ACTIVE_CACHE_RESTORATION_RESPONSE);

    }

    public void processRollingupgradeactivecacherestorationresponse(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradeactivecacherestorationresponse received from " + instanceName);
        }
        ownedReplicaListsReceivedForActiveCacheRestoration.putIfAbsent(replicationState.getInstanceName(), replicationState);
        restoreActiveCacheDoneSignal.countDown();
    }

    public void processRollingupgradereplicacacherestorationadvisory(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradereplicacacherestorationadvisory received from " + instanceName);
        }
        List<ReplicationState> ownedReplicas = getActiveSessionsToReplicateTo(instanceName);
        sendReplicationStateList(ownedReplicas, instanceName, MESSAGE_BROADCAST_ROLLING_UPGRADE_REPLICA_CACHE_RESTORATION_RESPONSE);
    }

    public void processRollingupgradereplicacacherestorationresponse(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        String instanceName = replicationState.getInstanceName();
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradereplicacacherestorationresponse received from " + instanceName);
        }
        ownedReplicaListsReceivedForReplicaCacheRestoration.putIfAbsent(replicationState.getInstanceName(), replicationState);
        restoreReplicaCacheDoneSignal.countDown();
    }

    public void processRollingupgradereplicareconciliationcompleteadvisory(ReplicationState replicationState) {
        if(replicationState == null) {
            return;
        }
        if(_logger.isLoggable(Level.FINE)) {
            _logger.fine("IN " + this.getClass().getName() + ">>processRollingupgradereplicareconciliationcompleteadvisory received from " + replicationState.getInstanceName());
        }
        reconcileReplicaCacheDoneSignal.countDown();
    }

    List<ReplicationState> getSSOEntryReplicas(String owningInstanceName) {
        List<ReplicationState> ssoEntryIds = new ArrayList();
        //using set to avoid dups
        HashSet ssoEntryIdsSet = new HashSet();
        //iterate over sso entry replicas
        Iterator it = getReplicatedSSOEntries().values();
        while(it.hasNext()) {
            ReplicationState nextState
                = (ReplicationState)it.next();
            RollingUpgradeUtil.filterOwnershipOfReplicas(owningInstanceName, nextState, ssoEntryIdsSet);
        }
        ssoEntryIds.addAll(ssoEntryIdsSet);
        return ssoEntryIds;
    }

    List<ReplicationState> getActiveSessionsToReplicateTo(String owningInstanceName) {
        ReplicationSSOStore store = this.getSingletonSSOStore();
        List<ReplicationState> ssoEntryIds = new ArrayList();
        //using set to avoid dups
        HashSet ssoEntryIdsSet = new HashSet();
        //iterate over active sso entries
        Iterator it = cache.values().iterator();
        while(it.hasNext()) {
        HASingleSignOnEntry nextSsoEntry = (HASingleSignOnEntry)it.next();
            ReplicationState nextState = null;
            try {
                nextState = store.getTransmitState(nextSsoEntry.getId(), nextSsoEntry);
            } catch (IOException ex) {}
            if(nextState != null) {
                ssoEntryIdsSet.add(nextState);
            }
        }
        ssoEntryIds.addAll(ssoEntryIdsSet);
        return ssoEntryIds;
    }

    private void sendReplicationStateList(List<ReplicationState> ownedReplicas,
                               String instanceName, String command) {
        byte[] ownedReplicasState = null;
        try {
            ownedReplicasState = ReplicationUtil.getByteArray((Serializable)ownedReplicas);
        } catch (IOException ex) {}
        //need a unique pseudo-id for this query
        String appId = getApplicationId();
        String id = appId + unique.getAndIncrement();
        ReplicationState ownedReplicasListState = new ReplicationState(MODE_SSO,
                id, appId, 0L, 0L, 0L, null, null, ReplicationUtil.getInstanceName(),
                command, ownedReplicasState, null, null);
        if (ownedReplicas == null || ownedReplicas.isEmpty()) {
            ownedReplicasListState.setNack(true);
        }
        JxtaSocketChannel channel = JxtaSocketChannel.getInstance();
        channel.send(ownedReplicasListState, instanceName);
    }


    private void sendReplicasList(List<FederatedQueryListElement> ownedReplicas,
                               String instanceName, String command) {
        byte[] ownedReplicasState = null;
        try {
            ownedReplicasState = ReplicationUtil.getByteArray((Serializable)ownedReplicas);
        } catch (IOException ex) {}
        //need a unique pseudo-id for this query
        String appId = getApplicationId();
        String id = appId + unique.getAndIncrement();
        ReplicationState ownedReplicasListState = new ReplicationState(MODE_SSO,
                id, appId, 0L, 0L, 0L, null, null, ReplicationUtil.getInstanceName(),
                command, ownedReplicasState, null, null);
        if (ownedReplicas == null || ownedReplicas.isEmpty()) {
            ownedReplicasListState.setNack(true);
        }
        JxtaSocketChannel channel = JxtaSocketChannel.getInstance();
        channel.send(ownedReplicasListState, instanceName);
    }
   
    // end rolling upgrade related code 
   
    /**
     * Sleep for the duration specified by the <code>ssoReapInterval</code>
     * property.
     */
    private void threadSleep() {

        try {
            Thread.sleep(getReapInterval() * 1000L);
        } catch (InterruptedException e) {
        }
    }


   /**
     * Start the background thread that will periodically check for
     * SSO timeouts.
     */
    private void threadStart() {

        if (thread != null)
            return;

        threadDone = false;
        String threadName = "ReplicationSingleSignOnExpiration";
        thread = new Thread(this, threadName);
        thread.setDaemon(true);
        thread.start();
    }


   /**
     * Stop the background thread that is periodically checking for
     * SSO timeouts.
     */
    private void threadStop() {

        if (thread == null)
            return;

        threadDone = true;
        thread.interrupt();
        try {
            thread.join();
        } catch (InterruptedException e) {
            // FIXME evaluate log level
            if (_logger.isLoggable(Level.FINE)) {
                _logger.log(Level.FINE, "Exception in ReplicatorSingleSignOn.threadStop", e);
            }
        }

        thread = null;

    }


    /**
     * The background thread that checks for SSO timeouts and shutdown.
     */
    public void run() {

        // Loop until the termination semaphore is set
        while (!threadDone) {
            threadSleep();
            processExpires();
            processUpdateLat();
            //check and remove stale replica monitors
            replicatedSessionMonitors.processExpired();
        }
    }
   
}
TOP

Related Classes of com.sun.enterprise.ee.web.authenticator.ReplicationSingleSignOn

TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.