package org.jgroups.protocols.pbcast;
import org.apache.commons.logging.Log;
import org.jgroups.*;
import org.jgroups.stack.Protocol;
import org.jgroups.util.*;
import org.jgroups.util.Queue;
import java.io.*;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.jgroups.protocols.pbcast.GmsImpl.Request;
/**
* Group membership protocol. Handles joins/leaves/crashes (suspicions) and emits new views
* accordingly. Use VIEW_ENFORCER on top of this layer to make sure new members don't receive
* any messages until they are members
* @author Bela Ban
* @version $Id: GMS.java,v 1.126.2.24 2009/09/08 12:24:50 belaban Exp $
*/
public class GMS extends Protocol {
private GmsImpl impl=null;
Address local_addr=null;
final Membership members=new Membership(); // real membership
private final Membership tmp_members=new Membership(); // base for computing next view
/** Members joined but for which no view has been received yet */
private final Vector<Address> joining=new Vector<Address>(7);
/** Members excluded from group, but for which no view has been received yet */
private final Vector<Address> leaving=new Vector<Address>(7);
View view=null;
ViewId view_id=null;
private long ltime=0;
long join_timeout=5000;
long leave_timeout=5000;
long merge_timeout=5000; // time to wait for all MERGE_RSPS
private final Object impl_mutex=new Object(); // synchronizes event entry into impl
private final Hashtable<String,GmsImpl> impls=new Hashtable<String,GmsImpl>(3);
private boolean shun=false;
boolean merge_leader=false; // can I initiate a merge ?
private boolean print_local_addr=true;
boolean disable_initial_coord=false; // can the member become a coord on startup or not ?
/** Setting this to false disables concurrent startups. This is only used by unit testing code
* for testing merging. To everybody else: don't change it to false ! */
boolean handle_concurrent_startup=true;
/** Whether view bundling (http://jira.jboss.com/jira/browse/JGRP-144) should be enabled or not. Setting this to
* false forces each JOIN/LEAVE/SUPSECT request to be handled separately. By default these requests are processed
* together if they are queued at approximately the same time */
private boolean view_bundling=true;
private long max_bundling_time=50; // 50ms max to wait for other JOIN, LEAVE or SUSPECT requests
static final String CLIENT="Client";
static final String COORD="Coordinator";
static final String PART="Participant";
TimeScheduler timer=null;
/** Max number of old members to keep in history */
protected int num_prev_mbrs=50;
/** Keeps track of old members (up to num_prev_mbrs) */
BoundedList<Address> prev_members=null;
/** If we receive a JOIN request from P and P is already in the current membership, then we send back a JOIN
* response with an error message when this property is set to true (Channel.connect() will fail). Otherwise,
* we return the current view */
boolean reject_join_from_existing_member=true;
int num_views=0;
/** Stores the last 20 views */
BoundedList<View> prev_views=new BoundedList<View>(20);
/** Class to process JOIN, LEAVE and MERGE requests */
private final ViewHandler view_handler=new ViewHandler();
private Class<Callable<Boolean>> flushInvokerClass;
/** To collect VIEW_ACKs from all members */
final AckCollector ack_collector=new AckCollector();
//[JGRP-700] - FLUSH: flushing should span merge
final AckCollector merge_ack_collector=new AckCollector();
/** Time in ms to wait for all VIEW acks (0 == wait forever) */
long view_ack_collection_timeout=2000;
/** How long should a Resumer wait until resuming the ViewHandler */
long resume_task_timeout=20000;
boolean flushProtocolInStack=false;
public static final String name="GMS";
public GMS() {
initState();
}
public String getName() {
return name;
}
public String getView() {return view_id != null? view_id.toString() : "null";}
public int getNumberOfViews() {return num_views;}
public String getLocalAddress() {return local_addr != null? local_addr.toString() : "null";}
public String getMembers() {return members != null? members.toString() : "[]";}
public int getNumMembers() {return members != null? members.size() : 0;}
public long getJoinTimeout() {return join_timeout;}
public void setJoinTimeout(long t) {join_timeout=t;}
/** @deprecated */
public long getJoinRetryTimeout() {return -1;}
/** @deprecated */
public void setJoinRetryTimeout(long t) {}
public boolean isShun() {return shun;}
public void setShun(boolean s) {shun=s;}
public boolean isPrintLocalAddr() {
return print_local_addr;
}
public void setPrintLocalAddr(boolean print_local_addr) {
this.print_local_addr=print_local_addr;
}
public String printPreviousMembers() {
StringBuilder sb=new StringBuilder();
if(prev_members != null) {
for(Address addr: prev_members) {
sb.append(addr).append("\n");
}
}
return sb.toString();
}
public long getViewAckCollectionTimeout() {
return view_ack_collection_timeout;
}
public void setViewAckCollectionTimeout(long view_ack_collection_timeout) {
this.view_ack_collection_timeout=view_ack_collection_timeout;
}
public boolean isViewBundling() {
return view_bundling;
}
public void setViewBundling(boolean view_bundling) {
this.view_bundling=view_bundling;
}
public long getMaxBundlingTime() {
return max_bundling_time;
}
public void setMaxBundlingTime(long max_bundling_time) {
this.max_bundling_time=max_bundling_time;
}
public int viewHandlerSize() {return view_handler.size();}
public boolean isViewHandlerSuspended() {return view_handler.suspended();}
public String dumpViewHandlerQueue() {
return view_handler.dumpQueue();
}
public String dumpViewHandlerHistory() {
return view_handler.dumpHistory();
}
public void suspendViewHandler() {
view_handler.suspend(null);
}
public void resumeViewHandler() {
view_handler.resumeForce();
}
Log getLog() {return log;}
ViewHandler getViewHandler() {return view_handler;}
public String printPreviousViews() {
StringBuilder sb=new StringBuilder();
for(View view: prev_views) {
sb.append(view).append("\n");
}
return sb.toString();
}
public boolean isCoordinator() {
Address coord=determineCoordinator();
return coord != null && local_addr != null && local_addr.equals(coord);
}
public void resetStats() {
super.resetStats();
num_views=0;
prev_views.clear();
}
public Vector<Integer> requiredDownServices() {
Vector<Integer> retval=new Vector<Integer>(3);
retval.addElement(new Integer(Event.GET_DIGEST));
retval.addElement(new Integer(Event.SET_DIGEST));
retval.addElement(new Integer(Event.FIND_INITIAL_MBRS));
return retval;
}
public void setImpl(GmsImpl new_impl) {
synchronized(impl_mutex) {
if(impl == new_impl) // superfluous
return;
impl=new_impl;
if(log.isDebugEnabled()) {
String msg=(local_addr != null? local_addr.toString()+" " : "") + "changed role to " + new_impl.getClass().getName();
log.debug(msg);
}
}
}
public GmsImpl getImpl() {
return impl;
}
public void init() throws Exception {
prev_members=new BoundedList<Address>(num_prev_mbrs);
timer=getTransport().getTimer();
if(timer == null)
throw new Exception("GMS.init(): timer is null");
if(impl != null)
impl.init();
}
public void start() throws Exception {
if(impl != null) impl.start();
}
public void stop() {
view_handler.stop(true);
if(impl != null) impl.stop();
if(prev_members != null)
prev_members.clear();
}
public void becomeCoordinator() {
CoordGmsImpl tmp=(CoordGmsImpl)impls.get(COORD);
if(tmp == null) {
tmp=new CoordGmsImpl(this);
impls.put(COORD, tmp);
}
try {
tmp.init();
}
catch(Exception e) {
log.error("exception switching to coordinator role", e);
}
setImpl(tmp);
}
public void becomeParticipant() {
ParticipantGmsImpl tmp=(ParticipantGmsImpl)impls.get(PART);
if(tmp == null) {
tmp=new ParticipantGmsImpl(this);
impls.put(PART, tmp);
}
try {
tmp.init();
}
catch(Exception e) {
log.error("exception switching to participant", e);
}
setImpl(tmp);
}
public void becomeClient() {
ClientGmsImpl tmp=(ClientGmsImpl)impls.get(CLIENT);
if(tmp == null) {
tmp=new ClientGmsImpl(this);
impls.put(CLIENT, tmp);
}
try {
tmp.init();
}
catch(Exception e) {
log.error("exception switching to client role", e);
}
setImpl(tmp);
}
boolean haveCoordinatorRole() {
return impl != null && impl instanceof CoordGmsImpl;
}
/**
* Computes the next view. Returns a copy that has <code>old_mbrs</code> and
* <code>suspected_mbrs</code> removed and <code>new_mbrs</code> added.
*/
public View getNextView(Collection<Address> new_mbrs, Collection<Address> old_mbrs, Collection<Address> suspected_mbrs) {
Vector<Address> mbrs;
long vid;
View v;
Membership tmp_mbrs;
synchronized(members) {
if(view_id == null) {
log.error("view_id is null");
return null; // this should *never* happen !
}
vid=Math.max(view_id.getId(), ltime) + 1;
ltime=vid;
tmp_mbrs=tmp_members.copy(); // always operate on the temporary membership
tmp_mbrs.remove(suspected_mbrs);
tmp_mbrs.remove(old_mbrs);
tmp_mbrs.add(new_mbrs);
mbrs=tmp_mbrs.getMembers();
v=new View(local_addr, vid, mbrs);
// Update membership (see DESIGN for explanation):
tmp_members.set(mbrs);
// Update joining list (see DESIGN for explanation)
if(new_mbrs != null) {
for(Iterator<Address> it=new_mbrs.iterator(); it.hasNext();) {
Address tmp_mbr=it.next();
if(!joining.contains(tmp_mbr))
joining.addElement(tmp_mbr);
}
}
// Update leaving list (see DESIGN for explanations)
if(old_mbrs != null) {
for(Iterator<Address> it=old_mbrs.iterator(); it.hasNext();) {
Address addr=it.next();
if(!leaving.contains(addr))
leaving.add(addr);
}
}
if(suspected_mbrs != null) {
for(Iterator<Address> it=suspected_mbrs.iterator(); it.hasNext();) {
Address addr=it.next();
if(!leaving.contains(addr))
leaving.add(addr);
}
}
return v;
}
}
/**
* Broadcasts the new view and digest, and waits for acks from all members in the list given as argument.
* If the list is null, we take the members who are part of new_view
* @param new_view
* @param digest
* @param newMembers
*/
public void castViewChangeWithDest(View new_view, Digest digest, JoinRsp jr, Collection <Address> newMembers) {
if(log.isTraceEnabled())
log.trace("mcasting view {" + new_view + "} (" + new_view.size() + " mbrs)\n");
Message view_change_msg=new Message(); // bcast to all members
GmsHeader hdr=new GmsHeader(GmsHeader.VIEW, new_view);
hdr.my_digest=digest;
view_change_msg.putHeader(name, hdr);
List<Address> ackMembers = new ArrayList<Address>(new_view.getMembers());
if(newMembers != null && !newMembers.isEmpty()) {
ackMembers.removeAll(newMembers);
}
ack_collector.reset(ackMembers);
// Send down a local TMP_VIEW event. This is needed by certain layers (e.g. NAKACK) to compute correct digest
// in case client's next request (e.g. getState()) reaches us *before* our own view change multicast.
// Check NAKACK's TMP_VIEW handling for details
down_prot.up(new Event(Event.TMP_VIEW, new_view));
down_prot.down(new Event(Event.TMP_VIEW, new_view));
down_prot.down(new Event(Event.MSG, view_change_msg));
try {
ack_collector.waitForAllAcks(view_ack_collection_timeout);
if(log.isTraceEnabled())
log.trace("received all ACKs (" + ack_collector.size()
+ ") for "
+ new_view.getVid());
}
catch(TimeoutException e) {
log.warn(local_addr + " failed to collect all ACKs (" + ack_collector.size()
+ ") for mcasted view "
+ new_view
+ " after "
+ view_ack_collection_timeout
+ "ms, missing ACKs from "
+ ack_collector.printMissing()
+ ", local_addr="
+ local_addr);
}
if(jr != null && (newMembers != null && !newMembers.isEmpty())) {
ack_collector.reset(new ArrayList<Address>(newMembers));
for(Address joiner:newMembers) {
sendJoinResponse(jr, joiner);
}
try {
ack_collector.waitForAllAcks(view_ack_collection_timeout);
if(log.isTraceEnabled())
log.trace("received all ACKs (" + ack_collector.size()
+ ") for "
+ new_view.getVid());
}
catch(TimeoutException e) {
log.warn(local_addr + " failed to collect all ACKs (" + ack_collector.size()
+ ") for unicasted view "
+ new_view
+ " after "
+ view_ack_collection_timeout
+ "ms, missing ACKs from "
+ ack_collector.printMissing()
+ ", local_addr="
+ local_addr);
}
}
}
public void sendJoinResponse(JoinRsp rsp, Address dest) {
Message m=new Message(dest, null, null);
GMS.GmsHeader hdr=new GMS.GmsHeader(GMS.GmsHeader.JOIN_RSP, rsp);
m.putHeader(getName(), hdr);
getDownProtocol().down(new Event(Event.MSG, m));
}
public void installView(View new_view) {
installView(new_view, null);
}
/**
* Sets the new view and sends a VIEW_CHANGE event up and down the stack. If the view is a MergeView (subclass
* of View), then digest will be non-null and has to be set before installing the view.
*/
public void installView(View new_view, Digest digest) {
Address coord;
int rc;
ViewId vid=new_view.getVid();
Vector<Address> mbrs=new_view.getMembers();
// Discards view with id lower than our own. Will be installed without check if first view
if(view_id != null) {
rc=vid.compareTo(view_id);
if(rc <= 0) {
if(log.isWarnEnabled() && rc < 0) // only scream if view is smaller, silently discard same views
log.warn("[" + local_addr + "] received view < current view;" +
" discarding it (current vid: " + view_id + ", new vid: " + vid + ')');
return;
}
}
if(digest != null)
mergeDigest(digest);
if(log.isDebugEnabled()) log.debug("[local_addr=" + local_addr + "] view is " + new_view);
if(stats) {
num_views++;
prev_views.add(new_view);
}
ack_collector.handleView(new_view);
merge_ack_collector.handleView(new_view);
ltime=Math.max(vid.getId(), ltime); // compute Lamport logical time
/* Check for self-inclusion: if I'm not part of the new membership, I just discard it.
This ensures that messages sent in view V1 are only received by members of V1 */
if(checkSelfInclusion(mbrs) == false) {
// only shun if this member was previously part of the group. avoids problem where multiple
// members (e.g. X,Y,Z) join {A,B} concurrently, X is joined first, and Y and Z get view
// {A,B,X}, which would cause Y and Z to be shunned as they are not part of the membership
// bela Nov 20 2003
if(shun && local_addr != null && prev_members.contains(local_addr)) {
if(log.isWarnEnabled())
log.warn("I (" + local_addr + ") am not a member of view " + new_view +
", shunning myself and leaving the group (prev_members are " + prev_members +
", current view is " + view + ")");
if(impl != null)
impl.handleExit();
up_prot.up(new Event(Event.EXIT));
}
else {
if(log.isWarnEnabled()) log.warn("I (" + local_addr + ") am not a member of view " + new_view + "; discarding view");
}
return;
}
synchronized(members) { // serialize access to views
// assign new_view to view_id
if(new_view instanceof MergeView)
view=new View(new_view.getVid(), new_view.getMembers());
else
view=new_view;
view_id=vid.copy();
// Set the membership. Take into account joining members
if(mbrs != null && !mbrs.isEmpty()) {
members.set(mbrs);
tmp_members.set(members);
joining.removeAll(mbrs); // remove all members in mbrs from joining
// remove all elements from 'leaving' that are not in 'mbrs'
leaving.retainAll(mbrs);
tmp_members.add(joining); // add members that haven't yet shown up in the membership
tmp_members.remove(leaving); // remove members that haven't yet been removed from the membership
// add to prev_members
for(Iterator<Address> it=mbrs.iterator(); it.hasNext();) {
Address addr=it.next();
if(!prev_members.contains(addr))
prev_members.add(addr);
}
}
// Send VIEW_CHANGE event up and down the stack:
Event view_event=new Event(Event.VIEW_CHANGE, new_view);
// changed order of passing view up and down (http://jira.jboss.com/jira/browse/JGRP-347)
// changed it back (bela Sept 4 2007): http://jira.jboss.com/jira/browse/JGRP-564
down_prot.down(view_event); // needed e.g. by failure detector or UDP
up_prot.up(view_event);
coord=determineCoordinator();
// if(coord != null && coord.equals(local_addr) && !(coord.equals(vid.getCoordAddress()))) {
// changed on suggestion by yaronr and Nicolas Piedeloupe
if(coord != null && coord.equals(local_addr) && !haveCoordinatorRole()) {
becomeCoordinator();
}
else {
if(haveCoordinatorRole() && !local_addr.equals(coord)) {
becomeParticipant();
merge_ack_collector.reset(null);
}
}
}
}
protected Address determineCoordinator() {
synchronized(members) {
return members != null && members.size() > 0? members.elementAt(0) : null;
}
}
/** Checks whether the potential_new_coord would be the new coordinator (2nd in line) */
protected boolean wouldBeNewCoordinator(Address potential_new_coord) {
Address new_coord;
if(potential_new_coord == null) return false;
synchronized(members) {
if(members.size() < 2) return false;
new_coord=members.elementAt(1); // member at 2nd place
return new_coord != null && new_coord.equals(potential_new_coord);
}
}
/** Returns true if local_addr is member of mbrs, else false */
protected boolean checkSelfInclusion(Vector<Address> mbrs) {
Object mbr;
if(mbrs == null)
return false;
for(int i=0; i < mbrs.size(); i++) {
mbr=mbrs.elementAt(i);
if(mbr != null && local_addr.equals(mbr))
return true;
}
return false;
}
public View makeView(Vector<Address> mbrs) {
Address coord=null;
long id=0;
if(view_id != null) {
coord=view_id.getCoordAddress();
id=view_id.getId();
}
return new View(coord, id, mbrs);
}
public static View makeView(Vector<Address> mbrs, ViewId vid) {
Address coord=null;
long id=0;
if(vid != null) {
coord=vid.getCoordAddress();
id=vid.getId();
}
return new View(coord, id, mbrs);
}
/** Send down a SET_DIGEST event */
public void setDigest(Digest d) {
down_prot.down(new Event(Event.SET_DIGEST, d));
}
/** Send down a MERGE_DIGEST event */
public void mergeDigest(Digest d) {
down_prot.down(new Event(Event.MERGE_DIGEST, d));
}
/** Sends down a GET_DIGEST event and waits for the GET_DIGEST_OK response, or
timeout, whichever occurs first */
public Digest getDigest() {
return (Digest)down_prot.down(Event.GET_DIGEST_EVT);
}
boolean startFlush(final View new_view) {
if (flushInvokerClass == null) {
Callable<Boolean> invoker = new Callable<Boolean>() {
public Boolean call() throws Exception {
int maxAttempts = 4;
long randomFloor = 1000L;
long randomCeiling = 5000L;
boolean successfulFlush = true;
boolean validView = new_view != null && new_view.size() > 0;
if (validView && flushProtocolInStack) {
int attemptCount = 0;
while (attemptCount < maxAttempts) {
successfulFlush = (Boolean) up_prot.up(new Event(Event.SUSPEND, new ArrayList<Address>(new_view.getMembers())));
if (successfulFlush)
break;
Util.sleepRandom(randomFloor, randomCeiling);
attemptCount++;
}
if (successfulFlush) {
if (log.isTraceEnabled())
log.trace("Successful GMS flush by coordinator at "
+ getLocalAddress());
} else {
if (log.isWarnEnabled())
log.warn("GMS flush by coordinator at " + getLocalAddress()
+ " failed");
}
}
return successfulFlush;
}
};
try {
return invoker.call();
} catch (Exception e) {
return false;
}
} else {
Callable<Boolean> invoker = null;
try {
invoker = flushInvokerClass.getDeclaredConstructor(View.class).newInstance(new_view);
return invoker.call();
} catch (Exception e) {
return false;
}
}
}
void stopFlush() {
if(flushProtocolInStack) {
if(log.isDebugEnabled()) {
log.debug(getLocalAddress() + " sending RESUME event");
}
up_prot.up(new Event(Event.RESUME));
}
}
void stopFlush(List<Address> members) {
if(log.isDebugEnabled()){
log.debug(getLocalAddress() + " sending RESUME event");
}
up_prot.up(new Event(Event.RESUME,members));
}
public Object up(Event evt) {
switch(evt.getType()) {
case Event.MSG:
Message msg=(Message)evt.getArg();
GmsHeader hdr=(GmsHeader)msg.getHeader(name);
if(hdr == null)
break;
switch(hdr.type) {
case GmsHeader.JOIN_REQ:
view_handler.add(new Request(Request.JOIN, hdr.mbr, false, null));
break;
case GmsHeader.JOIN_REQ_WITH_STATE_TRANSFER:
view_handler.add(new Request(Request.JOIN_WITH_STATE_TRANSFER, hdr.mbr, false, null));
break;
case GmsHeader.JOIN_RSP:
impl.handleJoinResponse(hdr.join_rsp);
break;
case GmsHeader.LEAVE_REQ:
if(log.isDebugEnabled())
log.debug("received LEAVE_REQ for " + hdr.mbr + " from " + msg.getSrc());
if(hdr.mbr == null) {
if(log.isErrorEnabled()) log.error("LEAVE_REQ's mbr field is null");
return null;
}
view_handler.add(new Request(Request.LEAVE, hdr.mbr, false, null));
break;
case GmsHeader.LEAVE_RSP:
impl.handleLeaveResponse();
break;
case GmsHeader.VIEW:
View new_view=hdr.view;
if(new_view == null) {
if(log.isErrorEnabled()) log.error("[VIEW]: view == null");
return null;
}
Address coord=msg.getSrc();
if(!new_view.containsMember(coord)) {
sendViewAck(coord); // we need to send the ack first, otherwise the connection is removed
impl.handleViewChange(new_view, hdr.my_digest);
}
else {
impl.handleViewChange(new_view, hdr.my_digest);
sendViewAck(coord); // send VIEW_ACK to sender of view
}
break;
case GmsHeader.VIEW_ACK:
Address sender=msg.getSrc();
ack_collector.ack(sender);
return null; // don't pass further up
case GmsHeader.MERGE_REQ:
down_prot.down(new Event(Event.SUSPEND_STABLE, 20000));
if(log.isDebugEnabled()){
log.debug("Merge participant " + local_addr + " got merge request from " + msg.getSrc());
}
impl.handleMergeRequest(msg.getSrc(), hdr.merge_id);
break;
case GmsHeader.MERGE_RSP:
MergeData merge_data=new MergeData(msg.getSrc(), hdr.view, hdr.my_digest);
merge_data.merge_rejected=hdr.merge_rejected;
if(log.isDebugEnabled()) {
log.debug("Got merge response at " + local_addr + " from " + msg.getSrc() +
", merge_id=" + hdr.view+ ", merge data is "+ merge_data);
}
impl.handleMergeResponse(merge_data, hdr.merge_id);
break;
case GmsHeader.INSTALL_MERGE_VIEW:
impl.handleMergeView(new MergeData(msg.getSrc(), hdr.view, hdr.my_digest), hdr.merge_id);
down_prot.down(new Event(Event.RESUME_STABLE));
break;
case GmsHeader.INSTALL_MERGE_VIEW_OK:
//[JGRP-700] - FLUSH: flushing should span merge
merge_ack_collector.ack(msg.getSrc());
break;
case GmsHeader.CANCEL_MERGE:
//[JGRP-524] - FLUSH and merge: flush doesn't wrap entire merge process
impl.handleMergeCancelled(hdr.merge_id);
down_prot.down(new Event(Event.RESUME_STABLE));
break;
default:
if(log.isErrorEnabled()) log.error("GmsHeader with type=" + hdr.type + " not known");
}
return null; // don't pass up
case Event.SET_LOCAL_ADDRESS:
local_addr=(Address)evt.getArg();
break; // pass up
case Event.SUSPECT:
Address suspected=(Address)evt.getArg();
view_handler.add(new Request(Request.SUSPECT, suspected, true, null));
ack_collector.suspect(suspected);
merge_ack_collector.suspect(suspected);
break; // pass up
case Event.UNSUSPECT:
impl.unsuspect((Address)evt.getArg());
return null; // discard
case Event.MERGE:
view_handler.add(new Request(Request.MERGE, null, false, (Vector<Address>)evt.getArg()));
return null; // don't pass up
}
if(impl.handleUpEvent(evt))
return up_prot.up(evt);
return null;
}
/**
This method is overridden to avoid hanging on getDigest(): when a JOIN is received, the coordinator needs
to retrieve the digest from the NAKACK layer. It therefore sends down a GET_DIGEST event, to which the NAKACK layer
responds with a GET_DIGEST_OK event.<p>
However, the GET_DIGEST_OK event will not be processed because the thread handling the JOIN request won't process
the GET_DIGEST_OK event until the JOIN event returns. The receiveUpEvent() method is executed by the up-handler
thread of the lower protocol and therefore can handle the event. All we do here is unblock the mutex on which
JOIN is waiting, allowing JOIN to return with a valid digest. The GET_DIGEST_OK event is then discarded, because
it won't be processed twice.
*/
// public void receiveUpEvent(Event evt) {
// switch(evt.getType()) {
// case Event.GET_DIGEST_OK:
// digest_promise.setResult(evt.getArg());
// return; // don't pass further up
// }
// super.receiveUpEvent(evt);
// }
public Object down(Event evt) {
Object arg=null;
switch(evt.getType()) {
case Event.CONNECT:
if(print_local_addr) {
System.out.println("\n---------------------------------------------------------\n" +
"GMS: address is " + local_addr + " (cluster=" + evt.getArg() + ")" +
"\n---------------------------------------------------------");
}
down_prot.down(evt);
if(local_addr == null)
if(log.isFatalEnabled()) log.fatal("[CONNECT] local_addr is null");
try {
impl.join(local_addr);
}
catch(Throwable e) {
arg=e;
}
return arg; // don't pass down: was already passed down
case Event.CONNECT_WITH_STATE_TRANSFER:
if(print_local_addr) {
System.out.println("\n---------------------------------------------------------\n" +
"GMS: address is " + local_addr + " (cluster=" + evt.getArg() + ")" +
"\n---------------------------------------------------------");
}
down_prot.down(evt);
if(local_addr == null)
if(log.isFatalEnabled()) log.fatal("[CONNECT] local_addr is null");
try {
impl.joinWithStateTransfer(local_addr);
}
catch(Throwable e) {
arg=e;
}
return arg; // don't pass down: was already passed down
case Event.DISCONNECT:
impl.leave((Address)evt.getArg());
if(!(impl instanceof CoordGmsImpl)) {
initState(); // in case connect() is called again
}
down_prot.down(evt); // notify the other protocols, but ignore the result
return null;
case Event.CONFIG :
Map<String,Object> config=(Map<String,Object>)evt.getArg();
if((config != null && config.containsKey("flush_supported"))){
flushProtocolInStack=true;
}
break;
}
return down_prot.down(evt);
}
/** Setup the Protocol instance according to the configuration string */
public boolean setProperties(Properties props) {
String str;
super.setProperties(props);
str=props.getProperty("shun");
if(str != null) {
shun=Boolean.valueOf(str).booleanValue();
props.remove("shun");
}
str=props.getProperty("merge_leader");
if(str != null) {
merge_leader=Boolean.valueOf(str).booleanValue();
props.remove("merge_leader");
}
str=props.getProperty("print_local_addr");
if(str != null) {
print_local_addr=Boolean.valueOf(str).booleanValue();
props.remove("print_local_addr");
}
str=props.getProperty("join_timeout"); // time to wait for JOIN
if(str != null) {
join_timeout=Long.parseLong(str);
props.remove("join_timeout");
}
str=props.getProperty("join_retry_timeout"); // time to wait between JOINs
if(str != null) {
props.remove("join_retry_timeout");
if(log.isWarnEnabled())
log.warn("join_retry_timeout has been deprecated and its value will be ignored");
}
str=props.getProperty("leave_timeout"); // time to wait until coord responds to LEAVE req.
if(str != null) {
leave_timeout=Long.parseLong(str);
props.remove("leave_timeout");
}
str=props.getProperty("merge_timeout"); // time to wait for MERGE_RSPS from subgroup coordinators
if(str != null) {
merge_timeout=Long.parseLong(str);
props.remove("merge_timeout");
}
str=props.getProperty("digest_timeout"); // time to wait for GET_DIGEST_OK from PBCAST
if(str != null) {
log.warn("digest_timeout has been deprecated and its value will be ignored");
props.remove("digest_timeout");
}
str=props.getProperty("view_ack_collection_timeout");
if(str != null) {
view_ack_collection_timeout=Long.parseLong(str);
props.remove("view_ack_collection_timeout");
}
str=props.getProperty("resume_task_timeout");
if(str != null) {
resume_task_timeout=Long.parseLong(str);
props.remove("resume_task_timeout");
}
str=props.getProperty("disable_initial_coord");
if(str != null) {
disable_initial_coord=Boolean.valueOf(str).booleanValue();
props.remove("disable_initial_coord");
if(log.isWarnEnabled())
log.warn("disable_initial_coord has been deprecated and will be phased out by 3.0, please don't use it anymore");
}
str=props.getProperty("handle_concurrent_startup");
if(str != null) {
handle_concurrent_startup=Boolean.valueOf(str).booleanValue();
props.remove("handle_concurrent_startup");
}
str=props.getProperty("num_prev_mbrs");
if(str != null) {
num_prev_mbrs=Integer.parseInt(str);
props.remove("num_prev_mbrs");
}
str=props.getProperty("reject_join_from_existing_member");
if(str != null) {
reject_join_from_existing_member=Boolean.parseBoolean(str);
props.remove("reject_join_from_existing_member");
}
str=props.getProperty("use_flush");
if(str != null) {
log.warn("use_flush has been deprecated and its value will be ignored");
props.remove("use_flush");
}
str=props.getProperty("flush_timeout");
if(str != null) {
log.warn("flush_timeout has been deprecated and its value will be ignored");
props.remove("flush_timeout");
}
str=props.getProperty("view_bundling");
if(str != null) {
view_bundling=Boolean.valueOf(str).booleanValue();
props.remove("view_bundling");
}
str=props.getProperty("max_bundling_time");
if(str != null) {
max_bundling_time=Long.parseLong(str);
props.remove("max_bundling_time");
}
str=props.getProperty("flush_invoker_class");
if (str != null) {
try {
flushInvokerClass = (Class<Callable<Boolean>>) Class.forName(str);
flushInvokerClass.getDeclaredConstructor(View.class);
} catch (Exception e) {
log.error("Invalid flush invoker class "
+ str
+ ". Maker sure it implements <Callable<Boolean>> with public constructor using View as a parameter");
}
props.remove("flush_invoker_class");
}
if(!props.isEmpty()) {
log.error("the following properties are not recognized: " + props);
return false;
}
return true;
}
/* ------------------------------- Private Methods --------------------------------- */
final void initState() {
becomeClient();
view_id=null;
view=null;
}
private void sendViewAck(Address dest) {
Message view_ack=new Message(dest, null, null);
view_ack.setFlag(Message.OOB);
GmsHeader tmphdr=new GmsHeader(GmsHeader.VIEW_ACK);
view_ack.putHeader(name, tmphdr);
if(log.isTraceEnabled())
log.trace("sending VIEW_ACK to " + dest);
down_prot.down(new Event(Event.MSG, view_ack));
}
/* --------------------------- End of Private Methods ------------------------------- */
public static class GmsHeader extends Header implements Streamable {
public static final byte JOIN_REQ=1;
public static final byte JOIN_RSP=2;
public static final byte LEAVE_REQ=3;
public static final byte LEAVE_RSP=4;
public static final byte VIEW=5;
public static final byte MERGE_REQ=6;
public static final byte MERGE_RSP=7;
public static final byte INSTALL_MERGE_VIEW=8;
public static final byte CANCEL_MERGE=9;
public static final byte VIEW_ACK=10;
public static final byte JOIN_REQ_WITH_STATE_TRANSFER = 11;
public static final byte INSTALL_MERGE_VIEW_OK=12;
byte type=0;
View view=null; // used when type=VIEW or MERGE_RSP or INSTALL_MERGE_VIEW
Address mbr=null; // used when type=JOIN_REQ or LEAVE_REQ
JoinRsp join_rsp=null; // used when type=JOIN_RSP
Digest my_digest=null; // used when type=MERGE_RSP or INSTALL_MERGE_VIEW
ViewId merge_id=null; // used when type=MERGE_REQ or MERGE_RSP or INSTALL_MERGE_VIEW or CANCEL_MERGE
boolean merge_rejected=false; // used when type=MERGE_RSP
private static final long serialVersionUID=2369798797842183276L;
public GmsHeader() {
} // used for Externalization
public GmsHeader(byte type) {
this.type=type;
}
/** Used for VIEW header */
public GmsHeader(byte type, View view) {
this.type=type;
this.view=view;
}
/** Used for JOIN_REQ or LEAVE_REQ header */
public GmsHeader(byte type, Address mbr) {
this.type=type;
this.mbr=mbr;
}
/** Used for JOIN_RSP header */
public GmsHeader(byte type, JoinRsp join_rsp) {
this.type=type;
this.join_rsp=join_rsp;
}
public byte getType() {
return type;
}
public Address getMember() {
return mbr;
}
public String toString() {
StringBuilder sb=new StringBuilder("GmsHeader");
sb.append('[' + type2String(type) + ']');
switch(type) {
case JOIN_REQ:
sb.append(": mbr=" + mbr);
break;
case JOIN_RSP:
sb.append(": join_rsp=" + join_rsp);
break;
case LEAVE_REQ:
sb.append(": mbr=" + mbr);
break;
case LEAVE_RSP:
break;
case VIEW:
case VIEW_ACK:
sb.append(": view=" + view);
break;
case MERGE_REQ:
sb.append(": merge_id=" + merge_id);
break;
case MERGE_RSP:
sb.append(": view=" + view + ", digest=" + my_digest + ", merge_rejected=" + merge_rejected +
", merge_id=" + merge_id);
break;
case INSTALL_MERGE_VIEW:
sb.append(": view=" + view + ", digest=" + my_digest);
break;
case CANCEL_MERGE:
sb.append(", <merge cancelled>, merge_id=" + merge_id);
break;
}
return sb.toString();
}
public static String type2String(int type) {
switch(type) {
case JOIN_REQ: return "JOIN_REQ";
case JOIN_RSP: return "JOIN_RSP";
case LEAVE_REQ: return "LEAVE_REQ";
case LEAVE_RSP: return "LEAVE_RSP";
case VIEW: return "VIEW";
case MERGE_REQ: return "MERGE_REQ";
case MERGE_RSP: return "MERGE_RSP";
case INSTALL_MERGE_VIEW: return "INSTALL_MERGE_VIEW";
case CANCEL_MERGE: return "CANCEL_MERGE";
case VIEW_ACK: return "VIEW_ACK";
case JOIN_REQ_WITH_STATE_TRANSFER: return "JOIN_REQ_WITH_STATE_TRANSFER";
default: return "<unknown>";
}
}
public void writeExternal(ObjectOutput out) throws IOException {
out.writeByte(type);
out.writeObject(view);
out.writeObject(mbr);
out.writeObject(join_rsp);
out.writeObject(my_digest);
out.writeObject(merge_id);
out.writeBoolean(merge_rejected);
}
public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
type=in.readByte();
view=(View)in.readObject();
mbr=(Address)in.readObject();
join_rsp=(JoinRsp)in.readObject();
my_digest=(Digest)in.readObject();
merge_id=(ViewId)in.readObject();
merge_rejected=in.readBoolean();
}
public void writeTo(DataOutputStream out) throws IOException {
out.writeByte(type);
boolean isMergeView=view != null && view instanceof MergeView;
out.writeBoolean(isMergeView);
Util.writeStreamable(view, out);
Util.writeAddress(mbr, out);
Util.writeStreamable(join_rsp, out);
Util.writeStreamable(my_digest, out);
Util.writeStreamable(merge_id, out); // kludge: we know merge_id is a ViewId
out.writeBoolean(merge_rejected);
}
public void readFrom(DataInputStream in) throws IOException, IllegalAccessException, InstantiationException {
type=in.readByte();
boolean isMergeView=in.readBoolean();
if(isMergeView)
view=(View)Util.readStreamable(MergeView.class, in);
else
view=(View)Util.readStreamable(View.class, in);
mbr=Util.readAddress(in);
join_rsp=(JoinRsp)Util.readStreamable(JoinRsp.class, in);
my_digest=(Digest)Util.readStreamable(Digest.class, in);
merge_id=(ViewId)Util.readStreamable(ViewId.class, in);
merge_rejected=in.readBoolean();
}
public int size() {
int retval=Global.BYTE_SIZE *2; // type + merge_rejected
retval+=Global.BYTE_SIZE; // presence view
retval+=Global.BYTE_SIZE; // MergeView or View
if(view != null)
retval+=view.serializedSize();
retval+=Util.size(mbr);
retval+=Global.BYTE_SIZE; // presence of join_rsp
if(join_rsp != null)
retval+=join_rsp.serializedSize();
retval+=Global.BYTE_SIZE; // presence for my_digest
if(my_digest != null)
retval+=my_digest.serializedSize();
retval+=Global.BYTE_SIZE; // presence for merge_id
if(merge_id != null)
retval+=merge_id.serializedSize();
return retval;
}
}
/**
* Class which processes JOIN, LEAVE and MERGE requests. Requests are queued and processed in FIFO order
* @author Bela Ban
* @version $Id: GMS.java,v 1.126.2.24 2009/09/08 12:24:50 belaban Exp $
*/
class ViewHandler implements Runnable {
volatile Thread thread;
Queue q=new Queue(); // Queue<Request>
boolean suspended=false;
final static long INTERVAL=5000;
private static final long MAX_COMPLETION_TIME=10000;
/** Maintains a list of the last 20 requests */
private final BoundedList<String> history=new BoundedList<String>(20);
/** Map<Object,Future>. Keeps track of Resumer tasks which have not fired yet */
private final Map<Object, Future> resume_tasks=new HashMap<Object,Future>();
private Object merge_id=null;
void add(Request req) {
add(req, false, false);
}
synchronized void add(Request req, boolean at_head, boolean unsuspend) {
if(suspended && !unsuspend) {
log.warn("queue is suspended; request " + req + " is discarded");
return;
}
start(unsuspend);
try {
if(at_head)
q.addAtHead(req);
else
q.add(req);
history.add(new Date() + ": " + req.toString());
}
catch(QueueClosedException e) {
if(log.isTraceEnabled())
log.trace("queue is closed; request " + req + " is discarded");
}
}
void waitUntilCompleted(long timeout) {
waitUntilCompleted(timeout, false);
}
synchronized void waitUntilCompleted(long timeout, boolean resume) {
if(thread != null) {
try {
thread.join(timeout);
}
catch(InterruptedException e) {
Thread.currentThread().interrupt(); // set interrupt flag again
}
//Added after Brian noticed that ViewHandler leaks class loaders
thread = null;
}
if(resume)
resumeForce();
}
/**
* Waits until the current request has been processes, then clears the queue and discards new
* requests from now on
*/
public synchronized void suspend(Object merge_id) {
if(!suspended) {
suspended=true;
this.merge_id=merge_id;
q.clear();
waitUntilCompleted(MAX_COMPLETION_TIME);
q.close(true);
if(log.isDebugEnabled())
log.debug("suspended ViewHandler at " + local_addr);
Resumer resumer=new Resumer(merge_id, resume_tasks, this);
Future<?> future=timer.schedule(resumer, resume_task_timeout, TimeUnit.MILLISECONDS);
Future<?> old_future=resume_tasks.put(merge_id, future);
if(old_future != null)
old_future.cancel(true);
}
else {
if(log.isWarnEnabled()) {
log.warn("attempted suspend on ViewHandler at " + local_addr+ ", however, it is already suspended");
}
}
}
public synchronized void resume(Object merge_id) {
if(suspended) {
boolean same_merge_id=this.merge_id != null && merge_id != null && this.merge_id.equals(merge_id);
same_merge_id=same_merge_id || (this.merge_id == null && merge_id == null);
if(same_merge_id) {
synchronized(resume_tasks) {
Future<?> future=resume_tasks.get(merge_id);
if(future != null) {
future.cancel(false);
resume_tasks.remove(merge_id);
}
}
}
else{
if(log.isWarnEnabled())
log.warn("resume(" + merge_id+ ") does not match "+ this.merge_id);
}
resumeForce();
}
}
public synchronized void resumeForce() {
if(q.closed())
q.reset();
suspended=false;
if(log.isTraceEnabled())
log.trace("resumed ViewHandler");
}
public void run() {
long end_time, wait_time;
List<Request> requests=new LinkedList<Request>();
while(Thread.currentThread().equals(thread) && !suspended) {
try {
boolean keepGoing=false;
end_time=System.currentTimeMillis() + max_bundling_time;
do {
Request firstRequest=(Request)q.remove(INTERVAL); // throws a TimeoutException if it runs into timeout
requests.add(firstRequest);
if(!view_bundling)
break;
if(q.size() > 0) {
Request nextReq=(Request)q.peek();
keepGoing=view_bundling && firstRequest.canBeProcessedTogether(nextReq);
}
else {
wait_time=end_time - System.currentTimeMillis();
if(wait_time > 0)
q.waitUntilClosed(wait_time); // misnomer: waits until element has been added or q closed
keepGoing=q.size() > 0 && firstRequest.canBeProcessedTogether((Request)q.peek());
}
}
while(keepGoing && System.currentTimeMillis() < end_time);
try {
process(requests);
}
finally {
requests.clear();
}
}
catch(QueueClosedException e) {
break;
}
catch(TimeoutException e) {
break;
}
catch(Throwable catchall) {
Util.sleep(50);
}
}
}
public int size() {return q.size();}
public boolean suspended() {return suspended;}
public String dumpQueue() {
StringBuilder sb=new StringBuilder();
List v=q.values();
for(Iterator it=v.iterator(); it.hasNext();) {
sb.append(it.next() + "\n");
}
return sb.toString();
}
public String dumpHistory() {
StringBuilder sb=new StringBuilder();
for(String line: history) {
sb.append(line + "\n");
}
return sb.toString();
}
private void process(List<Request> requests) {
if(requests.isEmpty())
return;
if(log.isTraceEnabled())
log.trace("processing " + requests);
Request firstReq=requests.get(0);
switch(firstReq.type) {
case Request.JOIN:
case Request.JOIN_WITH_STATE_TRANSFER:
case Request.LEAVE:
case Request.SUSPECT:
impl.handleMembershipChange(requests);
break;
case Request.MERGE:
if(requests.size() > 1)
log.error("more than one MERGE request to process, ignoring the others");
impl.merge(firstReq.coordinators);
break;
default:
log.error("request " + firstReq.type + " is unknown; discarded");
}
}
synchronized void start(boolean unsuspend) {
if(q.closed())
q.reset();
if(unsuspend) {
suspended=false;
Future<?> future;
synchronized(resume_tasks) {
future=resume_tasks.remove(merge_id);
}
if(future != null)
future.cancel(true);
}
merge_id=null;
if(thread == null || !thread.isAlive()) {
thread=getThreadFactory().newThread(this, "ViewHandler");
thread.setDaemon(false); // thread cannot terminate if we have tasks left, e.g. when we as coord leave
thread.start();
}
}
synchronized void stop(boolean flush) {
q.close(flush);
synchronized(resume_tasks) {
for(Future<?> future: resume_tasks.values()) {
future.cancel(true);
}
resume_tasks.clear();
}
merge_id=null;
// resumeForce();
// commented as this is already done in waitUnitlCompleted()
// thread = null; // Workaround for potential classloader leak to our thread
}
}
/**
* Resumer is a second line of defense: when the ViewHandler is suspended, it will be resumed when the current
* merge is cancelled, or when the merge completes. However, in a case where this never happens (this
* shouldn't be the case !), the Resumer will nevertheless resume the ViewHandler.
* We chose this strategy because ViewHandler is critical: if it is suspended indefinitely, we would
* not be able to process new JOIN requests ! So, this is for peace of mind, although it most likely
* will never be used...
*/
static class Resumer implements Runnable {
final Object token;
final Map<Object,Future> tasks;
final ViewHandler handler;
public Resumer(final Object token, final Map<Object,Future> t, final ViewHandler handler) {
this.token=token;
this.tasks=t;
this.handler=handler;
}
public void run() {
boolean execute=true;
synchronized(tasks) {
Future future=tasks.get(token);
if(future != null) {
future.cancel(false);
execute=true;
}
else {
execute=false;
}
tasks.remove(token);
}
if(execute) {
handler.resume(token);
}
}
}
}