* Licensed to the Apache Software Foundation (ASF) under one or more

* contributor license agreements. See the NOTICE file distributed with

* this work for additional information regarding copyright ownership.

* The ASF licenses this file to You under the Apache License, Version 2.0

* (the "License"); you may not use this file except in compliance with

* the License. You may obtain a copy of the License at

*

* http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* rate in two ways: first, controlling the overall bandwidth used per server, and second, limiting the number

* of simultaneous open connections per server.

* An instance of this class would very probably need to have a lifetime consistent with the long-term nature

* of these values, and be static.

*/

  public static final String _rcsid = "@(#)$Id: ThrottledFetcher.java 989847 2010-08-26 17:52:30Z kwright $";

  /** This flag determines whether we record everything to the disk, as a means of doing a web snapshot */

  protected static final boolean recordEverything = false;

  protected static final long TIME_2HRS = 7200000L;

  protected static final long TIME_5MIN = 300000L;

  protected static final long TIME_15MIN = 1500000L;

  protected static final long TIME_6HRS = 6L * 60L * 60000L;

  protected static final long TIME_1DAY = 24L * 60L * 60000L;

  /** This is the static pool of ConnectionBin's, keyed by bin name. */

  protected static HashMap connectionBins = new HashMap();

  /** This is the static pool of ThrottleBin's, keyed by bin name. */

  protected static HashMap throttleBins = new HashMap();

  /** This global lock protects the "distributed pool" resource, and insures that a connection

  * can get pulled out of all the right pools and wind up in only the hands of one thread. */

  protected static Integer poolLock = new Integer(0);

  /** Current host name */

  private static String currentHost = null;

  static

  {

    // Find the current host name

    try

    {

      java.net.InetAddress addr = java.net.InetAddress.getLocalHost();

      // Get hostname

      currentHost = addr.getHostName();

    }

    catch (java.net.UnknownHostException e)

    {

    }

  }

  /** The read chunk length */

  protected static final int READ_CHUNK_LENGTH = 4096;

  /** Constructor.

  */

  public ThrottledFetcher()

  {

  }

  /** Obtain a connection to specified protocol, server, and port.  We use the protocol because the

  * setup for some protocols is extensive (e.g. https) and hopefully would not need to be repeated if

  * we distinguish connections based on that.

  *@param protocol is the protocol, e.g. "http"

  *@param server is the server IP address, e.g. "10.32.65.1"

  *@param port is the port to connect to, e.g. 80.  Pass -1 if the default port for the protocol is desired.

  *@param authentication is the page credentials object to use for the fetch.  If null, no credentials are available.

  *@param trustStore is the current trust store in effect for the fetch.

  *@param binNames is the set of bins, in order, that should be used for throttling this connection.

  *       Note that the bin names for a given IP address and port MUST be the same for every connection!

  *       This must be enforced by whatever it is that builds the bins - it must do so given an IP and port.

  *@param throttleDescription is the description of all the throttling that should take place.

  *@param connectionLimit isthe maximum number of connections permitted.

  *@return an IThrottledConnection object that can be used to fetch from the port.

  */

  public static IThrottledConnection getConnection(String protocol, String server, int port,

    PageCredentials authentication,

    IKeystoreManager trustStore,

    ThrottleDescription throttleDescription, String[] binNames,

    int connectionLimit,

    String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)

    throws ManifoldCFException

  {

    // Create the https scheme for this connection

    javax.net.ssl.SSLSocketFactory baseFactory;

    String trustStoreString;

    if (trustStore != null)

    {

      baseFactory = trustStore.getSecureSocketFactory();

      trustStoreString = trustStore.getString();

    }

    else

    {

      baseFactory = KeystoreManagerFactory.getTrustingSecureSocketFactory();

      trustStoreString = null;

    }

    ConnectionBin[] bins = new ConnectionBin[binNames.length];

    // Now, start looking for a connection

    int i = 0;

    while (i < binNames.length)

    {

      String binName = binNames[i];

      // Find or create the bin object

      ConnectionBin cb;

      synchronized (connectionBins)

      {

        cb = (ConnectionBin)connectionBins.get(binName);

        if (cb == null)

        {

          cb = new ConnectionBin(binName);

          connectionBins.put(binName,cb);

        }

        //cb.sanityCheck();

      }

      bins[i] = cb;

      i++;

    }

    ThrottledConnection connectionToReuse;

    long startTime = 0L;

    if (Logging.connectors.isDebugEnabled())

    {

      startTime = System.currentTimeMillis();

      Logging.connectors.debug("WEB: Waiting to start getting a connection to "+protocol+"://"+server+":"+port);

    }

    synchronized (poolLock)

    {

      // If the number of outstanding connections is greater than the global limit, close pooled connections until we are under the limit

      long idleTimeout = 64000L;

      while (true)

      {

        int openCount = 0;

        // Lock up everything for a moment

        synchronized (connectionBins)

        {

          // Time out connections that have been idle too long.  To do this, we need to go through

          // all connection bins and look at the pool

          Iterator binIter = connectionBins.keySet().iterator();

          while (binIter.hasNext())

          {

            String binName = (String)binIter.next();

            ConnectionBin cb = (ConnectionBin)connectionBins.get(binName);

            openCount += cb.countConnections();

          }

        }

        if (openCount < connectionLimit)

          break;

        if (idleTimeout == 0L)

        {

          // Can't actually conclude anything here unfortunately

          // Logging.connectors.warn("Web: Exceeding connection limit!  Open count = "+Integer.toString(openCount)+"; limit = "+Integer.toString(connectionLimit));

          break;

        }

        idleTimeout = idleTimeout/4L;

        // Lock up everything for a moment, since otherwise we could delete something people

        // expect to stick around.

        synchronized (connectionBins)

        {

          // Time out connections that have been idle too long.  To do this, we need to go through

          // all connection bins and look at the pool

          Iterator binIter = connectionBins.keySet().iterator();

          while (binIter.hasNext())

          {

            String binName = (String)binIter.next();

            ConnectionBin cb = (ConnectionBin)connectionBins.get(binName);

            cb.flushIdleConnections(idleTimeout);

          }

        }

      }

      try

      {

        // Retry until we get the connection.

        while (true)

        {

          if (Logging.connectors.isDebugEnabled())

            Logging.connectors.debug("WEB: Attempting to get connection to "+protocol+"://"+server+":"+port+" ("+new Long(System.currentTimeMillis()-startTime).toString()+" ms)");

          i = 0;

          connectionToReuse = null;

          try

          {

            // Now, start looking for a connection

            while (i < binNames.length)

            {

              String binName = binNames[i];

              ConnectionBin cb = bins[i];

              // Figure out the connection limit for this bin, based on the throttle description

              int maxConnections = throttleDescription.getMaxOpenConnections(binName);

              // If no restriction, use a very large value.

              if (maxConnections == -1)

                maxConnections = Integer.MAX_VALUE;

              else if (maxConnections == 0)

                maxConnections = 1;

              // Now, do what we need to do to reserve our connection for this bin.

              // If we can't reserve it now, we plan on undoing everything we did, so

              // whatever we do must be reversible.  Furthermore, nothing we call here

              // should actually wait(); that will occur if we can't get what we need out

              // here at this level.

              if (connectionToReuse != null)

              {

                // We have a reuse candidate already, so just make sure each remaining bin is within

                // its limits.

                cb.insureWithinLimits(maxConnections,connectionToReuse);

              }

              else

              {

                connectionToReuse = cb.findConnection(maxConnections,bins,protocol,server,port,authentication,trustStoreString,

                  proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);

              }

              // Increment after we successfully handled this bin

              i++;

            }

            // That loop completed, meaning that we think we got a connection.  Now, go through all the bins and make sure there's enough time since the last

            // fetch.  If not, we have to clean everything up and try again.

            long currentTime = System.currentTimeMillis();

            // Global lock needed to insure that fetch time is updated across all bins simultaneously

            synchronized (connectionBins)

            {

              i = 0;

              while (i < binNames.length)

              {

                String binName = binNames[i];

                ConnectionBin cb = bins[i];

                //cb.sanityCheck();

                // Get the minimum time between fetches for this bin, based on the throttle description

                long minMillisecondsPerFetch = throttleDescription.getMinimumMillisecondsPerFetch(binName);

                if (cb.getLastFetchTime() + minMillisecondsPerFetch > currentTime)

                  throw new WaitException(cb.getLastFetchTime() + minMillisecondsPerFetch - currentTime);

                i++;

              }

              i = 0;

              while (i < binNames.length)

              {

                ConnectionBin cb = bins[i++];

                cb.setLastFetchTime(currentTime);

              }

            }

          }

          catch (Throwable e)

          {

            // We have to free everything and retry, because otherwise we are subject to deadlock.

            // The only thing we have reserved is the connection, which we must free if there's a

            // problem.

            if (connectionToReuse != null)

            {

              // Return this connection to the pool.  That is, the pools for all the bins.

              int k = 0;

              while (k < binNames.length)

              {

                String binName = binNames[k++];

                ConnectionBin cb;

                synchronized (connectionBins)

                {

                  cb = (ConnectionBin)connectionBins.get(binName);

                  if (cb == null)

                  {

                    cb = new ConnectionBin(binName);

                    connectionBins.put(binName,cb);

                  }

                }

                //cb.sanityCheck();

                cb.addToPool(connectionToReuse);

                //cb.sanityCheck();

              }

              connectionToReuse = null;

              // We should not need to notify here because nothing has really changed from

              // when the attempt started to get the connection.  We just undid what we did.

            }

            if (e instanceof Error)

              throw (Error)e;

            if (e instanceof ManifoldCFException)

              throw (ManifoldCFException)e;

            if (e instanceof WaitException)

            {

              // Wait because we need a certain amount of time after a previous fetch.

              WaitException we = (WaitException)e;

              long waitAmount = we.getWaitAmount();

              if (Logging.connectors.isDebugEnabled())

                Logging.connectors.debug("WEB: Waiting "+new Long(waitAmount).toString()+" ms before starting fetch on "+protocol+"://"+server+":"+port);

              // Really don't want to sleep inside the pool lock!

              // The easiest thing to do instead is to use a timed wait.  There is no reason why we need

              // to wake before the wait time is exceeded - but it's harmless, and the alternative is to

              // do more reorganization than probably is wise.

              poolLock.wait(waitAmount);

              continue;

            }

            if (e instanceof PoolException)

            {

              if (Logging.connectors.isDebugEnabled())

                Logging.connectors.debug("WEB: Going into wait for connection to "+protocol+"://"+server+":"+port+" ("+new Long(System.currentTimeMillis()-startTime).toString()+" ms)");

              // Now, wait for something external to change.  The only thing that can help us is if

              // some other thread frees a connection.

              poolLock.wait();

              // Go back around and try again.

              continue;

            }

            throw new ManifoldCFException("Unexpected exception encountered: "+e.getMessage(),e);

          }

          if (Logging.connectors.isDebugEnabled())

            Logging.connectors.debug("WEB: Successfully got connection to "+protocol+"://"+server+":"+port+" ("+new Long(System.currentTimeMillis()-startTime).toString()+" ms)");

          // If we have a connection located, activate it.

          if (connectionToReuse == null)

            connectionToReuse = new ThrottledConnection(protocol,server,port,authentication,baseFactory,trustStoreString,bins,

              proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword);

          connectionToReuse.setup(throttleDescription);

          return connectionToReuse;

        }

      }

      catch (InterruptedException e)

      {

        throw new ManifoldCFException(e.getMessage(),ManifoldCFException.INTERRUPTED);

      }

    }

  }

  /** Flush connections that have timed out from inactivity. */

  public static void flushIdleConnections()

    throws ManifoldCFException

  {

    synchronized (poolLock)

    {

      // Lock up everything for a moment, since otherwise we could delete something people

      // expect to stick around.

      synchronized (connectionBins)

      {

        // Time out connections that have been idle too long.  To do this, we need to go through

        // all connection bins and look at the pool

        Iterator binIter = connectionBins.keySet().iterator();

        while (binIter.hasNext())

        {

          String binName = (String)binIter.next();

          ConnectionBin cb = (ConnectionBin)connectionBins.get(binName);

          if (cb.flushIdleConnections(60000L))

          {

            // Bin is no longer doing anything; get rid of it.

            // I've determined this is safe - inUseConnections is designed to prevent any active connection from getting

            // whacked.

            // Oops.  Hang results again when I enabled this, so out it goes again.

            //connectionBins.remove(binName);

            //binIter = connectionBins.keySet().iterator();

          }

        }

      }

    }

  }

  /** Connection pool for a bin.

  * An instance of this class tracks the connections that are pooled and that are in use for a specific bin.

  */

  protected static class ConnectionBin

  {

    /** This is the bin name which this connection pool belongs to */

    protected String binName;

    /** This is the number of connections in this bin that are signed out and presumably in use */

    protected int inUseConnections = 0;

    /** This is the last time a fetch was done on this bin */

    protected long lastFetchTime = 0L;

    /** This object is what we synchronize on when we are waiting on a connection to free up for this

    * bin.  This is a separate object, because we also want to protect the integrity of the

    * ConnectionBin object itself, for which we'll use the ConnectionBin's synchronizer. */

    protected Integer connectionWait = new Integer(0);

    /** This map contains ThrottledConnection objects that are in the pool, and are not in use. */

    protected HashMap freePool = new HashMap();

    /** Constructor. */

    public ConnectionBin(String binName)

    {

      this.binName = binName;

    }

    /** Get the bin name. */

    public String getBinName()

    {

      return binName;

    }

    /** Note the creation of an active connection that belongs to this bin.  The slots all must

    * have been reserved prior to the connection being created.

    */

    public synchronized void noteConnectionCreation()

    {

      inUseConnections++;

    }

    /** Note the destruction of an active connection that belongs to this bin.

    */

    public synchronized void noteConnectionDestruction()

    {

      inUseConnections--;

    }

    /** Activate a connection that should be in the pool.

    * Removes the connection from the pool.

    */

    public synchronized void takeFromPool(ThrottledConnection tc)

    {

      // Remove this connection from the pool list

      freePool.remove(tc);

      inUseConnections++;

    }

    /** Put a connection into the pool.

    */

    public synchronized void addToPool(ThrottledConnection tc)

    {

      // Add this connection to the pool list

      freePool.put(tc,tc);

      inUseConnections--;

    }

    /** Verify that this bin is within limits.

    */

    public synchronized void insureWithinLimits(int maxConnections, ThrottledConnection existingConnection)

      throws PoolException

    {

      //sanityCheck();

      // See if the connection is in fact within the pool; if so, we just presume the limits are fine as they are.

      // This is necessary because if the connection that's being checked for is freed, then we wreck the data structures.

      if (existsInPool(existingConnection))

        return;

      while (maxConnections > 0 && inUseConnections + freePool.size() > maxConnections)

      {

        //sanityCheck();

        // If there are any pool connections, free them one at a time

        ThrottledConnection freeMe = getPoolConnection();

        if (freeMe != null)

        {

          // It's okay to call activate since we guarantee that only one thread is trying to grab

          // a connection at a time.

          freeMe.activate();

          freeMe.destroy();

          continue;

        }

        // Instead of waiting, throw a pool exception, so that we can wait and retry at the next level up.

        throw new PoolException("Waiting for a connection");

      }

    }

    /** This method is called only when there is no existing connection yet identified that can be used

    * for contacting the server and port specified.  This method returns a connection if a matching one can be found;

    * otherwise it returns null.

    * If a matching connection is found, it is activated before it is returned.  That removes the connection from all

    * pools in which it lives.

    */

    public synchronized ThrottledConnection findConnection(int maxConnections,

      ConnectionBin[] binNames, String protocol, String server, int port,

      PageCredentials authentication, String trustStoreString,

      String proxyHost, int proxyPort, String proxyAuthDomain, String proxyAuthUsername, String proxyAuthPassword)

      throws PoolException

    {

      //sanityCheck();

      // First, wait until there's no excess.

      while (maxConnections > 0 && inUseConnections + freePool.size() > maxConnections)

      {

        //sanityCheck();

        // If there are any pool connections, free them one at a time

        ThrottledConnection freeMe = getPoolConnection();

        if (freeMe != null)

        {

          // It's okay to call activate since we guarantee that only one thread is trying to grab

          // a connection at a time.

          freeMe.activate();

          freeMe.destroy();

          continue;

        }

        // Instead of waiting, throw a pool exception, so that we can wait and retry at the next level up.

        throw new PoolException("Waiting for a connection");

      }

      // Wait until there's a free one

      if (maxConnections > 0 && inUseConnections > maxConnections-1)

      {

        // Instead of waiting, throw a pool exception, so that we can wait and retry at the next level up.

        throw new PoolException("Waiting for a connection");

      }

      // A null return means that there is no existing pooled connection that matches, and the  caller is free to create a new connection

      ThrottledConnection rval = getPoolConnection();

      if (rval == null)

        return null;

      // It's okay to call activate since we guarantee that only one thread is trying to grab

      // a connection at a time.

      rval.activate();

      //sanityCheck();

      if (!rval.matches(binNames,protocol,server,port,authentication,trustStoreString,

        proxyHost,proxyPort,proxyAuthDomain,proxyAuthUsername,proxyAuthPassword))

      {

        // Destroy old connection.  That should free up space for a new creation.

        rval.destroy();

        // Return null to indicate that we can create a new connection now

        return null;

      }

      // Existing entry matched.  Activate and return it.

      return rval;

    }

    /** Note a new time for connection fetch for this pool.

    *@param currentTime is the time the fetch was started.

    */

    public synchronized void setLastFetchTime(long currentTime)

    {

      if (currentTime > lastFetchTime)

        lastFetchTime = currentTime;

    }

    /** Get the last fetch time.

    *@return the time.

    */

    public synchronized long getLastFetchTime()

    {

      return lastFetchTime;

    }

    /** Count connections that are in use.

    *@return connections that are in use.

    */

    public synchronized int countConnections()

    {

      return freePool.size() + inUseConnections;

    }

    /** Flush any idle connections.

    *@return true if the connection bin is now, in fact, empty.

    */

    public synchronized boolean flushIdleConnections(long idleTimeout)

    {

      //sanityCheck();

      // We have to time out the pool connections.  When there are no pool connections

      // left, AND the in-use counts are zero, we can delete the whole thing.

      Iterator iter = freePool.keySet().iterator();

      while (iter.hasNext())

      {

        ThrottledConnection tc = (ThrottledConnection)iter.next();

        if (tc.flushIdleConnections(idleTimeout))

        {

          // Can delete this connection, since it timed out.

          tc.activate();

          tc.destroy();

          iter = freePool.keySet().iterator();

        }

      }

      //sanityCheck();

      return (freePool.size() == 0 && inUseConnections == 0);

    }

    /** Grab a connection from the current pool.  This does not remove the connection from the pool;

    * it just sets it up so that later methods can do that.

    */

    protected ThrottledConnection getPoolConnection()

    {

      if (freePool.size() == 0)

        return null;

      Iterator iter = freePool.keySet().iterator();

      ThrottledConnection rval = (ThrottledConnection)iter.next();

      return rval;

    }

    /** Check if a connection exists in the pool already.

    */

    protected boolean existsInPool(ThrottledConnection tc)

    {

      return freePool.get(tc) != null;

    }

    public synchronized void sanityCheck()

    {

      // Make sure all the connections in the current pool in fact have a reference to this bin.

      Iterator iter = freePool.keySet().iterator();

      while (iter.hasNext())

      {

        ThrottledConnection tc = (ThrottledConnection)iter.next();

        tc.mustHaveReference(this);

      }

    }

  }

  /** Throttles for a bin.

  * An instance of this class keeps track of the information needed to bandwidth throttle access

  * to a url belonging to a specific bin.

  *

  * In order to calculate

  * the effective "burst" fetches per second and bytes per second, we need to have some idea what the window is.

  * For example, a long hiatus from fetching could cause overuse of the server when fetching resumes, if the

  * window length is too long.

  *

  * One solution to this problem would be to keep a list of the individual fetches as records.  Then, we could

  * "expire" a fetch by discarding the old record.  However, this is quite memory consumptive for all but the

  * smallest intervals.

  *

  * Another, better, solution is to hook into the start and end of individual fetches.  These will, presumably, occur

  * at the fastest possible rate without long pauses spent doing something else.  The only complication is that

  * fetches may well overlap, so we need to "reference count" the fetches to know when to reset the counters.

  * For "fetches per second", we can simply make sure we "schedule" the next fetch at an appropriate time, rather

  * than keep records around.  The overall rate may therefore be somewhat less than the specified rate, but that's perfectly

  * acceptable.

  *

  * Some notes on the algorithms used to limit server bandwidth impact

  * ==================================================================

  *

  * In a single connection case, the algorithm we'd want to use works like this.  On the first chunk of a series,

  * the total length of time and the number of bytes are recorded.  Then, prior to each subsequent chunk, a calculation

  * is done which attempts to hit the bandwidth target by the end of the chunk read, using the rate of the first chunk

  * access as a way of estimating how long it will take to fetch those next n bytes.

  *

  * For a multi-connection case, which this is, it's harder to either come up with a good maximum bandwidth estimate,

  * and harder still to "hit the target", because simultaneous fetches will intrude.  The strategy is therefore:

  *

  * 1) The first chunk of any series should proceed without interference from other connections to the same server.