* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* This abstraction contains the token/identifier of this node

* on the identifier space. This token gets gossiped around.

* This class will also maintain histograms of the load information

* of other nodes in the cluster.

*/

    private static Logger logger_ = LoggerFactory.getLogger(StorageService.class);    

    public static final int RING_DELAY = 30 * 1000; // delay after which we assume ring has stablized

    /* All verb handler identifiers */

    public enum Verb

    {

        MUTATION,

        BINARY,

        READ_REPAIR,

        READ,

        REQUEST_RESPONSE, // client-initiated reads and writes

        STREAM_INITIATE, // Deprecated

        STREAM_INITIATE_DONE, // Deprecated

        STREAM_REPLY,

        STREAM_REQUEST,

        RANGE_SLICE,

        BOOTSTRAP_TOKEN,

        TREE_REQUEST,

        TREE_RESPONSE,

        JOIN, // Deprecated

        GOSSIP_DIGEST_SYN,

        GOSSIP_DIGEST_ACK,

        GOSSIP_DIGEST_ACK2,

        DEFINITIONS_ANNOUNCE,

        DEFINITIONS_UPDATE_RESPONSE,

        TRUNCATE,

        SCHEMA_CHECK,

        INDEX_SCAN,

        REPLICATION_FINISHED,

        INTERNAL_RESPONSE, // responses to internal calls

        // use as padding for backwards compatability where a previous version needs to validate a verb from the future.

        UNUSED_1,

        UNUSED_2,

        UNUSED_3,

        ;

        // remember to add new verbs at the end, since we serialize by ordinal

    }

    public static final Verb[] VERBS = Verb.values();

    public static final EnumMap<StorageService.Verb, Stage> verbStages = new EnumMap<StorageService.Verb, Stage>(StorageService.Verb.class)

    {{

        put(Verb.MUTATION, Stage.MUTATION);

        put(Verb.BINARY, Stage.MUTATION);

        put(Verb.READ_REPAIR, Stage.MUTATION);

        put(Verb.READ, Stage.READ);

        put(Verb.REQUEST_RESPONSE, Stage.REQUEST_RESPONSE);

        put(Verb.STREAM_REPLY, Stage.MISC); // TODO does this really belong on misc? I've just copied old behavior here

        put(Verb.STREAM_REQUEST, Stage.STREAM);

        put(Verb.RANGE_SLICE, Stage.READ);

        put(Verb.BOOTSTRAP_TOKEN, Stage.MISC);

        put(Verb.TREE_REQUEST, Stage.ANTI_ENTROPY);

        put(Verb.TREE_RESPONSE, Stage.ANTI_ENTROPY);

        put(Verb.GOSSIP_DIGEST_ACK, Stage.GOSSIP);

        put(Verb.GOSSIP_DIGEST_ACK2, Stage.GOSSIP);

        put(Verb.GOSSIP_DIGEST_SYN, Stage.GOSSIP);

        put(Verb.DEFINITIONS_ANNOUNCE, Stage.READ);

        put(Verb.DEFINITIONS_UPDATE_RESPONSE, Stage.READ);

        put(Verb.TRUNCATE, Stage.MUTATION);

        put(Verb.SCHEMA_CHECK, Stage.MIGRATION);

        put(Verb.INDEX_SCAN, Stage.READ);

        put(Verb.REPLICATION_FINISHED, Stage.MISC);

        put(Verb.INTERNAL_RESPONSE, Stage.INTERNAL_RESPONSE);

        put(Verb.UNUSED_1, Stage.INTERNAL_RESPONSE);

        put(Verb.UNUSED_2, Stage.INTERNAL_RESPONSE);

        put(Verb.UNUSED_3, Stage.INTERNAL_RESPONSE);

    }};

    /**

     * This pool is used for periodic short (sub-second) tasks.

     */

     public static final RetryingScheduledThreadPoolExecutor scheduledTasks = new RetryingScheduledThreadPoolExecutor("ScheduledTasks");

    /**

     * This pool is used by tasks that can have longer execution times, and usually are non periodic.

     */

    public static final RetryingScheduledThreadPoolExecutor tasks = new RetryingScheduledThreadPoolExecutor("NonPeriodicTasks");

    /* This abstraction maintains the token/endpoint metadata information */

    private TokenMetadata tokenMetadata_ = new TokenMetadata();

    private IPartitioner partitioner = DatabaseDescriptor.getPartitioner();

    public VersionedValue.VersionedValueFactory valueFactory = new VersionedValue.VersionedValueFactory(partitioner);

   

    public static final StorageService instance = new StorageService();

    public static IPartitioner getPartitioner()

    {

        return instance.partitioner;

    }

    public Collection<Range> getLocalRanges(String table)

    {

        return getRangesForEndpoint(table, FBUtilities.getLocalAddress());

    }

    public Range getLocalPrimaryRange()

    {

        return getPrimaryRangeForEndpoint(FBUtilities.getLocalAddress());

    }

    private Set<InetAddress> replicatingNodes = Collections.synchronizedSet(new HashSet<InetAddress>());

    private CassandraDaemon daemon;

    private InetAddress removingNode;

    /* Are we starting this node in bootstrap mode? */

    private boolean isBootstrapMode;

    /* when intialized as a client, we shouldn't write to the system table. */

    private boolean isClientMode;

    private boolean initialized;

    private volatile boolean joined = false;

    private String operationMode;

    private volatile boolean efficientCrossDCWrites;

    private MigrationManager migrationManager = new MigrationManager();

    /* Used for tracking drain progress */

    private volatile int totalCFs, remainingCFs;

    public void finishBootstrapping()

    {

        isBootstrapMode = false;

        SystemTable.setBootstrapped(true);

        setToken(getLocalToken());

        logger_.info("Bootstrap/move completed! Now serving reads.");

    }

    /** This method updates the local token on disk  */

    public void setToken(Token token)

    {

        if (logger_.isDebugEnabled())

            logger_.debug("Setting token to {}", token);

        SystemTable.updateToken(token);

        tokenMetadata_.updateNormalToken(token, FBUtilities.getLocalAddress());

        Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS, valueFactory.normal(getLocalToken()));

        setMode("Normal", false);

    }

    public StorageService()

    {

        MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();

        try

        {

            mbs.registerMBean(this, new ObjectName("org.apache.cassandra.db:type=StorageService"));

        }

        catch (Exception e)

        {

            throw new RuntimeException(e);

        }

        /* register the verb handlers */

        MessagingService.instance().registerVerbHandlers(Verb.BINARY, new BinaryVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.MUTATION, new RowMutationVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.READ_REPAIR, new ReadRepairVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.READ, new ReadVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.RANGE_SLICE, new RangeSliceVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.INDEX_SCAN, new IndexScanVerbHandler());

        // see BootStrapper for a summary of how the bootstrap verbs interact

        MessagingService.instance().registerVerbHandlers(Verb.BOOTSTRAP_TOKEN, new BootStrapper.BootstrapTokenVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.STREAM_REQUEST, new StreamRequestVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.STREAM_REPLY, new StreamReplyVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.REPLICATION_FINISHED, new ReplicationFinishedVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.REQUEST_RESPONSE, new ResponseVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.INTERNAL_RESPONSE, new ResponseVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.TREE_REQUEST, new TreeRequestVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.TREE_RESPONSE, new AntiEntropyService.TreeResponseVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.GOSSIP_DIGEST_SYN, new GossipDigestSynVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.GOSSIP_DIGEST_ACK, new GossipDigestAckVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.GOSSIP_DIGEST_ACK2, new GossipDigestAck2VerbHandler());

       

        MessagingService.instance().registerVerbHandlers(Verb.DEFINITIONS_ANNOUNCE, new DefinitionsAnnounceVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.DEFINITIONS_UPDATE_RESPONSE, new DefinitionsUpdateResponseVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.TRUNCATE, new TruncateVerbHandler());

        MessagingService.instance().registerVerbHandlers(Verb.SCHEMA_CHECK, new SchemaCheckVerbHandler());

        // spin up the streaming serivice so it is available for jmx tools.

        if (StreamingService.instance == null)

            throw new RuntimeException("Streaming service is unavailable.");

    }

    public void registerDaemon(CassandraDaemon daemon)

    {

        this.daemon = daemon;

    }

    // should only be called via JMX

    public void stopGossiping()

    {

        if (initialized)

        {

            logger_.warn("Stopping gossip by operator request");

            Gossiper.instance.stop();

            initialized = false;

        }

    }

    // should only be called via JMX

    public void startGossiping()

    {

        if (!initialized)

        {

            logger_.warn("Starting gossip by operator request");

            Gossiper.instance.start((int)(System.currentTimeMillis() / 1000));

            initialized = true;

        }

    }

    // should only be called via JMX

    public void startRPCServer()

    {

        if (daemon == null)

        {

            throw new IllegalStateException("No configured RPC daemon");

        }

        daemon.startRPCServer();

    }

    // should only be called via JMX

    public void stopRPCServer()

    {

        if (daemon == null)

        {

            throw new IllegalStateException("No configured RPC daemon");

        }

        daemon.stopRPCServer();

    }

    public boolean isRPCServerRunning()

    {

        if (daemon == null)

        {

            throw new IllegalStateException("No configured RPC daemon");

        }

        return daemon.isRPCServerRunning();

    }

    public void stopClient()

    {

        Gossiper.instance.unregister(migrationManager);

        Gossiper.instance.unregister(this);

        Gossiper.instance.stop();

        MessagingService.instance().shutdown();

        StageManager.shutdownNow();

    }

   

    public boolean isInitialized()

    {

        return initialized;

    }

    public synchronized void initClient() throws IOException, ConfigurationException

    {

        if (initialized)

        {

            if (!isClientMode)

                throw new UnsupportedOperationException("StorageService does not support switching modes.");

            return;

        }

        initialized = true;

        isClientMode = true;

        logger_.info("Starting up client gossip");

        setMode("Client", false);

        Gossiper.instance.register(this);

        Gossiper.instance.start((int)(System.currentTimeMillis() / 1000)); // needed for node-ring gathering.

        MessagingService.instance().listen(FBUtilities.getLocalAddress());

       

        // sleep a while to allow gossip to warm up (the other nodes need to know about this one before they can reply).

        try

        {

            Thread.sleep(5000L);

        }

        catch (Exception ex)

        {

            throw new IOError(ex);

        }

        MigrationManager.announce(DatabaseDescriptor.getDefsVersion(), DatabaseDescriptor.getSeeds());

    }

    public synchronized void initServer() throws IOException, org.apache.cassandra.config.ConfigurationException

    {

        logger_.info("Cassandra version: " + FBUtilities.getReleaseVersionString());

        logger_.info("Thrift API version: " + Constants.VERSION);

        if (initialized)

        {

            if (isClientMode)

                throw new UnsupportedOperationException("StorageService does not support switching modes.");

            return;

        }

        initialized = true;

        isClientMode = false;

        if (Boolean.parseBoolean(System.getProperty("cassandra.load_ring_state", "true")))

        {

            logger_.info("Loading persisted ring state");

            for (Map.Entry<Token, InetAddress> entry : SystemTable.loadTokens().entrySet())

            {

                tokenMetadata_.updateNormalToken(entry.getKey(), entry.getValue());

                Gossiper.instance.addSavedEndpoint(entry.getValue());

            }

        }

        // daemon threads, like our executors', continue to run while shutdown hooks are invoked

        Thread drainOnShutdown = new Thread(new WrappedRunnable()

        {

            public void runMayThrow() throws ExecutionException, InterruptedException, IOException

            {

                ThreadPoolExecutor mutationStage = StageManager.getStage(Stage.MUTATION);

                if (!mutationStage.isShutdown())

                {

                    mutationStage.shutdown();

                    mutationStage.awaitTermination(1, TimeUnit.SECONDS);

                    CommitLog.instance.shutdownBlocking();

                }

            }

        });

        Runtime.getRuntime().addShutdownHook(drainOnShutdown);

        if (Boolean.parseBoolean(System.getProperty("cassandra.join_ring", "true")))

        {

            joinTokenRing();

        }

        else

        {

            logger_.info("Not joining ring as requested. Use JMX (StorageService->joinRing()) to initiate ring joining");

        }

    }

    private void joinTokenRing() throws IOException, org.apache.cassandra.config.ConfigurationException

    {

        logger_.info("Starting up server gossip");

        joined = true;

        // have to start the gossip service before we can see any info on other nodes.  this is necessary

        // for bootstrap to get the load info it needs.

        // (we won't be part of the storage ring though until we add a nodeId to our state, below.)

        Gossiper.instance.register(this);

        Gossiper.instance.register(migrationManager);

        Gossiper.instance.start(SystemTable.incrementAndGetGeneration()); // needed for node-ring gathering.

        MessagingService.instance().listen(FBUtilities.getLocalAddress());

        StorageLoadBalancer.instance.startBroadcasting();

        MigrationManager.announce(DatabaseDescriptor.getDefsVersion(), DatabaseDescriptor.getSeeds());

        Gossiper.instance.addLocalApplicationState(ApplicationState.RELEASE_VERSION, valueFactory.releaseVersion());

        HintedHandOffManager.instance.registerMBean();

        if (DatabaseDescriptor.isAutoBootstrap()

                && DatabaseDescriptor.getSeeds().contains(FBUtilities.getLocalAddress())

                && !SystemTable.isBootstrapped())

            logger_.info("This node will not auto bootstrap because it is configured to be a seed node.");

        Token token;

        if (DatabaseDescriptor.isAutoBootstrap()

            && !(DatabaseDescriptor.getSeeds().contains(FBUtilities.getLocalAddress()) || SystemTable.isBootstrapped()))

        {

            setMode("Joining: getting load information", true);

            StorageLoadBalancer.instance.waitForLoadInfo();

            if (logger_.isDebugEnabled())

                logger_.debug("... got load info");

            if (tokenMetadata_.isMember(FBUtilities.getLocalAddress()))

            {

                String s = "This node is already a member of the token ring; bootstrap aborted. (If replacing a dead node, remove the old one from the ring first.)";

                throw new UnsupportedOperationException(s);

            }

            setMode("Joining: getting bootstrap token", true);

            token = BootStrapper.getBootstrapToken(tokenMetadata_, StorageLoadBalancer.instance.getLoadInfo());

            // don't bootstrap if there are no tables defined.

            if (DatabaseDescriptor.getNonSystemTables().size() > 0)

            {

                bootstrap(token);

                assert !isBootstrapMode; // bootstrap will block until finished

            }

            else

            {

                // nothing to bootstrap, go directly to participating in ring

                SystemTable.setBootstrapped(true);

                setToken(token);

            }

        }

        else

        {

            token = SystemTable.getSavedToken();

            if (token == null)

            {

                String initialToken = DatabaseDescriptor.getInitialToken();

                if (initialToken == null)

                {

                    token = partitioner.getRandomToken();

                    logger_.warn("Generated random token " + token + ". Random tokens will result in an unbalanced ring; see http://wiki.apache.org/cassandra/Operations");

                }

                else

                {

                    token = partitioner.getTokenFactory().fromString(initialToken);

                    logger_.info("Saved token not found. Using " + token + " from configuration");

                }

            }

            else

            {

                logger_.info("Using saved token " + token);

            }

            SystemTable.setBootstrapped(true); // first startup is only chance to bootstrap

            setToken(token);

        }

        assert tokenMetadata_.sortedTokens().size() > 0;

    }

    public synchronized void joinRing() throws IOException, org.apache.cassandra.config.ConfigurationException

    {

        if (!joined)

        {

            logger_.info("Joining ring by operator request");

            joinTokenRing();

        }

    }

    public boolean isJoined()

    {

        return joined;

    }

    private void setMode(String m, boolean log)

    {

        operationMode = m;

        if (log)

            logger_.info(m);

        else

            logger_.debug(m);

    }

    private void bootstrap(Token token) throws IOException

    {

        isBootstrapMode = true;

        SystemTable.updateToken(token); // DON'T use setToken, that makes us part of the ring locally which is incorrect until we are done bootstrapping

        Gossiper.instance.addLocalApplicationState(ApplicationState.STATUS, valueFactory.bootstrapping(token));

        setMode("Joining: sleeping " + RING_DELAY + " ms for pending range setup", true);

        try

        {

            Thread.sleep(RING_DELAY);

        }

        catch (InterruptedException e)

        {

            throw new AssertionError(e);

        }

        setMode("Bootstrapping", true);

        new BootStrapper(FBUtilities.getLocalAddress(), token, tokenMetadata_).bootstrap(); // handles token update

    }

    public boolean isBootstrapMode()

    {

        return isBootstrapMode;

    }

    public TokenMetadata getTokenMetadata()

    {

        return tokenMetadata_;

    }

    /**

     * for a keyspace, return the ranges and corresponding hosts for a given keyspace.

     * @param keyspace

     * @return

     */

    public Map<Range, List<String>> getRangeToEndpointMap(String keyspace)

    {

        // some people just want to get a visual representation of things. Allow null and set it to the first

        // non-system table.

        if (keyspace == null)

            keyspace = DatabaseDescriptor.getNonSystemTables().get(0);

        /* All the ranges for the tokens */

        Map<Range, List<String>> map = new HashMap<Range, List<String>>();

        for (Map.Entry<Range,List<InetAddress>> entry : getRangeToAddressMap(keyspace).entrySet())

        {

            map.put(entry.getKey(), stringify(entry.getValue()));

        }

        return map;

    }

    public Map<Range, List<String>> getPendingRangeToEndpointMap(String keyspace)

    {

        // some people just want to get a visual representation of things. Allow null and set it to the first

        // non-system table.

        if (keyspace == null)

            keyspace = DatabaseDescriptor.getNonSystemTables().get(0);

        Map<Range, List<String>> map = new HashMap<Range, List<String>>();

        for (Map.Entry<Range, Collection<InetAddress>> entry : tokenMetadata_.getPendingRanges(keyspace).entrySet())

        {

            List<InetAddress> l = new ArrayList<InetAddress>(entry.getValue());

            map.put(entry.getKey(), stringify(l));

        }

        return map;

    }

    public Map<Range, List<InetAddress>> getRangeToAddressMap(String keyspace)

    {

        List<Range> ranges = getAllRanges(tokenMetadata_.sortedTokens());

        return constructRangeToEndpointMap(keyspace, ranges);

    }

    public Map<Token, String> getTokenToEndpointMap()

    {

        Map<Token, InetAddress> mapInetAddress = tokenMetadata_.getTokenToEndpointMap();

        Map<Token, String> mapString = new HashMap<Token, String>(mapInetAddress.size());

        for (Map.Entry<Token, InetAddress> entry : mapInetAddress.entrySet())

        {

            mapString.put(entry.getKey(), entry.getValue().getHostAddress());

        }

        return mapString;

    }

    /**

     * Construct the range to endpoint mapping based on the true view

     * of the world.

     * @param ranges

     * @return mapping of ranges to the replicas responsible for them.

    */

    private Map<Range, List<InetAddress>> constructRangeToEndpointMap(String keyspace, List<Range> ranges)

    {

        Map<Range, List<InetAddress>> rangeToEndpointMap = new HashMap<Range, List<InetAddress>>();

        for (Range range : ranges)

        {

            rangeToEndpointMap.put(range, Table.open(keyspace).getReplicationStrategy().getNaturalEndpoints(range.right));

        }

        return rangeToEndpointMap;

    }

    /*

     * onChange only ever sees one ApplicationState piece change at a time, so we perform a kind of state machine here.

     * We are concerned with two events: knowing the token associated with an endpoint, and knowing its operation mode.

     * Nodes can start in either bootstrap or normal mode, and from bootstrap mode can change mode to normal.

     * A node in bootstrap mode needs to have pendingranges set in TokenMetadata; a node in normal mode

     * should instead be part of the token ring.

     *

     * Normal MOVE_STATE progression of a node should be like this:

     * STATE_BOOTSTRAPPING,token

     *   if bootstrapping. stays this way until all files are received.

     * STATE_NORMAL,token

     *   ready to serve reads and writes.

     * STATE_NORMAL,token,REMOVE_TOKEN,token

     *   specialized normal state in which this node acts as a proxy to tell the cluster about a dead node whose

     *   token is being removed. this value becomes the permanent state of this node (unless it coordinates another

     *   removetoken in the future).

     * STATE_LEAVING,token

     *   get ready to leave the cluster as part of a decommission or move

     * STATE_LEFT,token

     *   set after decommission or move is completed.

     *

     * Note: Any time a node state changes from STATE_NORMAL, it will not be visible to new nodes. So it follows that

     * you should never bootstrap a new node during a removetoken, decommission or move.

     */

    public void onChange(InetAddress endpoint, ApplicationState state, VersionedValue value)

    {

        switch (state)

        {

            case RELEASE_VERSION:

                updateEfficientCrossDCWriteMode();

                break;

            case STATUS:

                String apStateValue = value.value;

                String[] pieces = apStateValue.split(VersionedValue.DELIMITER_STR, -1);

                assert (pieces.length > 0);

                String moveName = pieces[0];

                if (moveName.equals(VersionedValue.STATUS_BOOTSTRAPPING))

                    handleStateBootstrap(endpoint, pieces);

                else if (moveName.equals(VersionedValue.STATUS_NORMAL))

                    handleStateNormal(endpoint, pieces);

                else if (moveName.equals(VersionedValue.STATUS_LEAVING))

                    handleStateLeaving(endpoint, pieces);

                else if (moveName.equals(VersionedValue.STATUS_LEFT))

                    handleStateLeft(endpoint, pieces);

        }

    }

    /**

     * We can remove this in 0.8, since mixing 0.7.0 with 0.8 is not supported (0.7.1 is required)

     */

    private void updateEfficientCrossDCWriteMode()

    {

        for (Map.Entry<InetAddress, EndpointState> entry : Gossiper.instance.getEndpointStates())

        {

            VersionedValue version = entry.getValue().getApplicationState(ApplicationState.RELEASE_VERSION);

            // no version means it's old code that doesn't gossip version, < 0.7.1.

            if (version == null)

            {

                efficientCrossDCWrites = false;

                return;

            }

        }

        efficientCrossDCWrites = true;

    }

    /**

     * Handle node bootstrap

     *

     * @param endpoint bootstrapping node

     * @param pieces STATE_BOOTSTRAPPING,bootstrap token as string

     */

    private void handleStateBootstrap(InetAddress endpoint, String[] pieces)

    {

        assert pieces.length == 2;

        Token token = getPartitioner().getTokenFactory().fromString(pieces[1]);

        if (logger_.isDebugEnabled())

            logger_.debug("Node " + endpoint + " state bootstrapping, token " + token);

        // if this node is present in token metadata, either we have missed intermediate states

        // or the node had crashed. Print warning if needed, clear obsolete stuff and

        // continue.

        if (tokenMetadata_.isMember(endpoint))

        {

            // If isLeaving is false, we have missed both LEAVING and LEFT. However, if

            // isLeaving is true, we have only missed LEFT. Waiting time between completing

            // leave operation and rebootstrapping is relatively short, so the latter is quite

            // common (not enough time for gossip to spread). Therefore we report only the

            // former in the log.

            if (!tokenMetadata_.isLeaving(endpoint))

                logger_.info("Node " + endpoint + " state jump to bootstrap");

            tokenMetadata_.removeEndpoint(endpoint);

        }

        tokenMetadata_.addBootstrapToken(token, endpoint);

        calculatePendingRanges();

    }

    /**

     * Handle node move to normal state. That is, node is entering token ring and participating

     * in reads.

     *

     * @param endpoint node

     * @param pieces STATE_NORMAL,token[,other_state,token]

     */

    private void handleStateNormal(InetAddress endpoint, String[] pieces)

    {

        assert pieces.length >= 2;

        Token token = getPartitioner().getTokenFactory().fromString(pieces[1]);

        if (logger_.isDebugEnabled())

            logger_.debug("Node " + endpoint + " state normal, token " + token);

        if (tokenMetadata_.isMember(endpoint))

            logger_.info("Node " + endpoint + " state jump to normal");

        // we don't want to update if this node is responsible for the token and it has a later startup time than endpoint.

        InetAddress currentOwner = tokenMetadata_.getEndpoint(token);

        if (currentOwner == null)

        {

            logger_.debug("New node " + endpoint + " at token " + token);

            tokenMetadata_.updateNormalToken(token, endpoint);

            if (!isClientMode)

                SystemTable.updateToken(endpoint, token);

        }

        else if (endpoint.equals(currentOwner))

        {

            // set state back to normal, since the node may have tried to leave, but failed and is now back up

            // no need to persist, token/ip did not change

            tokenMetadata_.updateNormalToken(token, endpoint);

        }

        else if (Gossiper.instance.compareEndpointStartup(endpoint, currentOwner) > 0)

        {

            logger_.info(String.format("Nodes %s and %s have the same token %s.  %s is the new owner",

                                       endpoint, currentOwner, token, endpoint));

            tokenMetadata_.updateNormalToken(token, endpoint);

            if (!isClientMode)

                SystemTable.updateToken(endpoint, token);

        }

        else

        {

            logger_.info(String.format("Nodes %s and %s have the same token %s.  Ignoring %s",

                                       endpoint, currentOwner, token, endpoint));

        }

        if (pieces.length > 2)

        {

            assert pieces.length == 4;

            handleStateRemoving(endpoint, getPartitioner().getTokenFactory().fromString(pieces[3]), pieces[2]);

        }

        calculatePendingRanges();

    }

    /**

     * Handle node preparing to leave the ring

     *

     * @param endpoint node

     * @param pieces STATE_LEAVING,token

     */

    private void handleStateLeaving(InetAddress endpoint, String[] pieces)

    {

        assert pieces.length == 2;

        String moveValue = pieces[1];

        Token token = getPartitioner().getTokenFactory().fromString(moveValue);

        if (logger_.isDebugEnabled())

            logger_.debug("Node " + endpoint + " state leaving, token " + token);

        // If the node is previously unknown or tokens do not match, update tokenmetadata to

        // have this node as 'normal' (it must have been using this token before the

        // leave). This way we'll get pending ranges right.

        if (!tokenMetadata_.isMember(endpoint))

        {

            logger_.info("Node " + endpoint + " state jump to leaving");

            tokenMetadata_.updateNormalToken(token, endpoint);

        }

        else if (!tokenMetadata_.getToken(endpoint).equals(token))

        {

            logger_.warn("Node " + endpoint + " 'leaving' token mismatch. Long network partition?");

            tokenMetadata_.updateNormalToken(token, endpoint);

        }

        // at this point the endpoint is certainly a member with this token, so let's proceed

        // normally

        tokenMetadata_.addLeavingEndpoint(endpoint);

        calculatePendingRanges();

    }

    /**

     * Handle node leaving the ring. This can be either because of decommission or loadbalance

     *

     * @param endpoint If reason for leaving is decommission or loadbalance

     * endpoint is the leaving node.

     * @param pieces STATE_LEFT,token

     */

    private void handleStateLeft(InetAddress endpoint, String[] pieces)

    {

        assert pieces.length == 2;

        Token token = getPartitioner().getTokenFactory().fromString(pieces[1]);

        if (logger_.isDebugEnabled())

            logger_.debug("Node " + endpoint + " state left, token " + token);

        excise(token, endpoint);

    }

    /**

     * Handle notification that a node being actively removed from the ring via 'removetoken'

     *

     * @param endpoint node

     * @param state either REMOVED_TOKEN (node is gone) or REMOVING_TOKEN (replicas need to be restored)

     */

    private void handleStateRemoving(InetAddress endpoint, Token removeToken, String state)

    {

        InetAddress removeEndpoint = tokenMetadata_.getEndpoint(removeToken);

       

        if (removeEndpoint == null)

            return;

       

        if (removeEndpoint.equals(FBUtilities.getLocalAddress()))

        {

            logger_.info("Received removeToken gossip about myself. Is this node a replacement for a removed one?");

            return;

        }

        if (VersionedValue.REMOVED_TOKEN.equals(state))

        {

            excise(removeToken, removeEndpoint);

        }

        else if (VersionedValue.REMOVING_TOKEN.equals(state))

        {

            if (logger_.isDebugEnabled())

                logger_.debug("Token " + removeToken + " removed manually (endpoint was " + removeEndpoint + ")");

            // Note that the endpoint is being removed

            tokenMetadata_.addLeavingEndpoint(removeEndpoint);

            calculatePendingRanges();

            // grab any data we are now responsible for and notify responsible node

            restoreReplicaCount(removeEndpoint, endpoint);

        }

    }

    private void excise(Token token, InetAddress endpoint)

    {

        Gossiper.instance.removeEndpoint(endpoint);

        tokenMetadata_.removeEndpoint(endpoint);

        HintedHandOffManager.instance.deleteHintsForEndpoint(endpoint);

        tokenMetadata_.removeBootstrapToken(token);

        calculatePendingRanges();

        if (!isClientMode)

        {

            logger_.info("Removing token " + token + " for " + endpoint);

            SystemTable.removeToken(token);

        }

    }

    /**

     * Calculate pending ranges according to bootsrapping and leaving nodes. Reasoning is:

     *

     * (1) When in doubt, it is better to write too much to a node than too little. That is, if

     * there are multiple nodes moving, calculate the biggest ranges a node could have. Cleaning

     * up unneeded data afterwards is better than missing writes during movement.

     * (2) When a node leaves, ranges for other nodes can only grow (a node might get additional

     * ranges, but it will not lose any of its current ranges as a result of a leave). Therefore

     * we will first remove _all_ leaving tokens for the sake of calculation and then check what

     * ranges would go where if all nodes are to leave. This way we get the biggest possible

     * ranges with regard current leave operations, covering all subsets of possible final range

     * values.

     * (3) When a node bootstraps, ranges of other nodes can only get smaller. Without doing

     * complex calculations to see if multiple bootstraps overlap, we simply base calculations

     * on the same token ring used before (reflecting situation after all leave operations have

     * completed). Bootstrapping nodes will be added and removed one by one to that metadata and

     * checked what their ranges would be. This will give us the biggest possible ranges the

     * node could have. It might be that other bootstraps make our actual final ranges smaller,

     * but it does not matter as we can clean up the data afterwards.

     *

     * NOTE: This is heavy and ineffective operation. This will be done only once when a node

     * changes state in the cluster, so it should be manageable.

     */

    private void calculatePendingRanges()

    {

        for (String table : DatabaseDescriptor.getNonSystemTables())

            calculatePendingRanges(Table.open(table).getReplicationStrategy(), table);

    }

    // public & static for testing purposes

    public static void calculatePendingRanges(AbstractReplicationStrategy strategy, String table)

    {

        TokenMetadata tm = StorageService.instance.getTokenMetadata();

        Multimap<Range, InetAddress> pendingRanges = HashMultimap.create();

        Map<Token, InetAddress> bootstrapTokens = tm.getBootstrapTokens();

        Set<InetAddress> leavingEndpoints = tm.getLeavingEndpoints();

        if (bootstrapTokens.isEmpty() && leavingEndpoints.isEmpty())

        {

            if (logger_.isDebugEnabled())

                logger_.debug("No bootstrapping or leaving nodes -> empty pending ranges for {}", table);

            tm.setPendingRanges(table, pendingRanges);

            return;

        }

        Multimap<InetAddress, Range> addressRanges = strategy.getAddressRanges();

        // Copy of metadata reflecting the situation after all leave operations are finished.

        TokenMetadata allLeftMetadata = tm.cloneAfterAllLeft();

        // get all ranges that will be affected by leaving nodes

        Set<Range> affectedRanges = new HashSet<Range>();

        for (InetAddress endpoint : leavingEndpoints)

            affectedRanges.addAll(addressRanges.get(endpoint));

        // for each of those ranges, find what new nodes will be responsible for the range when

        // all leaving nodes are gone.

        for (Range range : affectedRanges)

        {

            Collection<InetAddress> currentEndpoints = strategy.calculateNaturalEndpoints(range.right, tm);

            Collection<InetAddress> newEndpoints = strategy.calculateNaturalEndpoints(range.right, allLeftMetadata);

            newEndpoints.removeAll(currentEndpoints);

            pendingRanges.putAll(range, newEndpoints);

        }

        // At this stage pendingRanges has been updated according to leave operations. We can

        // now finish the calculation by checking bootstrapping nodes.

        // For each of the bootstrapping nodes, simply add and remove them one by one to

        // allLeftMetadata and check in between what their ranges would be.

        for (Map.Entry<Token, InetAddress> entry : bootstrapTokens.entrySet())

        {

            InetAddress endpoint = entry.getValue();

            allLeftMetadata.updateNormalToken(entry.getKey(), endpoint);

            for (Range range : strategy.getAddressRanges(allLeftMetadata).get(endpoint))

                pendingRanges.put(range, endpoint);

            allLeftMetadata.removeEndpoint(endpoint);

        }

        tm.setPendingRanges(table, pendingRanges);

        if (logger_.isDebugEnabled())

            logger_.debug("Pending ranges:\n" + (pendingRanges.isEmpty() ? "<empty>" : tm.printPendingRanges()));

    }

    /**

     * Finds living endpoints responsible for the given ranges

     *

     * @param table the table ranges belong to

     * @param ranges the ranges to find sources for

     * @return multimap of addresses to ranges the address is responsible for

     */

    private Multimap<InetAddress, Range> getNewSourceRanges(String table, Set<Range> ranges)

    {

        InetAddress myAddress = FBUtilities.getLocalAddress();

        Multimap<Range, InetAddress> rangeAddresses = Table.open(table).getReplicationStrategy().getRangeAddresses(tokenMetadata_);

        Multimap<InetAddress, Range> sourceRanges = HashMultimap.create();

        IFailureDetector failureDetector = FailureDetector.instance;

        // find alive sources for our new ranges

        for (Range range : ranges)

        {

            Collection<InetAddress> possibleRanges = rangeAddresses.get(range);

            IEndpointSnitch snitch = DatabaseDescriptor.getEndpointSnitch();

            List<InetAddress> sources = snitch.getSortedListByProximity(myAddress, possibleRanges);

            assert (!sources.contains(myAddress));

            for (InetAddress source : sources)

            {

                if (failureDetector.isAlive(source))

                {

                    sourceRanges.put(source, range);