/*
* Created on 12-Jan-2005
* Created by Paul Gardner
* Copyright (C) 2004, 2005, 2006 Aelitis, All Rights Reserved.
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*
* AELITIS, SAS au capital de 46,603.30 euros
* 8 Allee Lenotre, La Grille Royale, 78600 Le Mesnil le Roi, France.
*
*/
package com.aelitis.azureus.core.dht.control.impl;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
import java.math.BigInteger;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.*;
import javax.crypto.Cipher;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import org.bouncycastle.crypto.CipherParameters;
import org.bouncycastle.crypto.engines.RC4Engine;
import org.bouncycastle.crypto.params.KeyParameter;
import org.gudy.azureus2.core3.util.*;
import com.aelitis.azureus.core.dht.*;
import com.aelitis.azureus.core.dht.control.*;
import com.aelitis.azureus.core.dht.db.DHTDB;
import com.aelitis.azureus.core.dht.db.DHTDBFactory;
import com.aelitis.azureus.core.dht.db.DHTDBLookupResult;
import com.aelitis.azureus.core.dht.db.DHTDBValue;
import com.aelitis.azureus.core.dht.impl.DHTLog;
import com.aelitis.azureus.core.dht.netcoords.DHTNetworkPosition;
import com.aelitis.azureus.core.dht.netcoords.DHTNetworkPositionManager;
import com.aelitis.azureus.core.dht.router.DHTRouter;
import com.aelitis.azureus.core.dht.router.DHTRouterAdapter;
import com.aelitis.azureus.core.dht.router.DHTRouterContact;
import com.aelitis.azureus.core.dht.router.DHTRouterFactory;
import com.aelitis.azureus.core.dht.transport.*;
import com.aelitis.azureus.core.dht.transport.udp.DHTTransportUDP;
/**
* @author parg
*
*/
public class
DHTControlImpl
implements DHTControl, DHTTransportRequestHandler
{
private static final boolean DISABLE_REPLICATE_ON_JOIN = true;
public static int EXTERNAL_LOOKUP_CONCURRENCY = 16;
private static final int EXTERNAL_PUT_CONCURRENCY = 8;
private static final int RANDOM_QUERY_PERIOD = 5*60*1000;
private static final int INTEGRATION_TIME_MAX = 15*1000;
private DHTControlAdapter adapter;
private DHTTransport transport;
private DHTTransportContact local_contact;
private DHTRouter router;
private DHTDB database;
private DHTControlStatsImpl stats;
private DHTLogger logger;
private int node_id_byte_count;
private int search_concurrency;
private int lookup_concurrency;
private int cache_at_closest_n;
private int K;
private int B;
private int max_rep_per_node;
private long router_start_time;
private int router_count;
private ThreadPool internal_lookup_pool;
private ThreadPool external_lookup_pool;
private ThreadPool internal_put_pool;
private ThreadPool external_put_pool;
private Map imported_state = new HashMap();
private volatile boolean seeded;
private long last_lookup;
private ListenerManager listeners = ListenerManager.createAsyncManager(
"DHTControl:listenDispatcher",
new ListenerManagerDispatcher()
{
public void
dispatch(
Object _listener,
int type,
Object value )
{
DHTControlListener target = (DHTControlListener)_listener;
target.activityChanged((DHTControlActivity)value, type );
}
});
private List activities = new ArrayList();
private AEMonitor activity_mon = new AEMonitor( "DHTControl:activities" );
protected AEMonitor estimate_mon = new AEMonitor( "DHTControl:estimate" );
private long last_dht_estimate_time;
private long local_dht_estimate;
private long combined_dht_estimate;
private int combined_dht_estimate_mag;
private static final int LOCAL_ESTIMATE_HISTORY = 32;
private Map local_estimate_values =
new LinkedHashMap(LOCAL_ESTIMATE_HISTORY,0.75f,true)
{
protected boolean
removeEldestEntry(
Map.Entry eldest)
{
return( size() > LOCAL_ESTIMATE_HISTORY );
}
};
private static final int REMOTE_ESTIMATE_HISTORY = 128;
private List remote_estimate_values = new LinkedList();
protected AEMonitor spoof_mon = new AEMonitor( "DHTControl:spoof" );
private Cipher spoof_cipher;
private SecretKey spoof_key;
private DHTTransportContact spoof_last_verify_contact;
private int spoof_last_verify_result;
private long last_node_add_check;
private byte[] node_add_check_uninteresting_limit;
private long rbs_time;
private byte[] rbs_id = {};
public
DHTControlImpl(
DHTControlAdapter _adapter,
DHTTransport _transport,
int _K,
int _B,
int _max_rep_per_node,
int _search_concurrency,
int _lookup_concurrency,
int _original_republish_interval,
int _cache_republish_interval,
int _cache_at_closest_n,
DHTLogger _logger )
{
adapter = _adapter;
transport = _transport;
logger = _logger;
K = _K;
B = _B;
max_rep_per_node = _max_rep_per_node;
search_concurrency = _search_concurrency;
lookup_concurrency = _lookup_concurrency;
cache_at_closest_n = _cache_at_closest_n;
// set this so we don't do initial calculation until reasonably populated
last_dht_estimate_time = SystemTime.getCurrentTime();
database = DHTDBFactory.create(
adapter.getStorageAdapter(),
_original_republish_interval,
_cache_republish_interval,
transport.getProtocolVersion(),
logger );
internal_lookup_pool = new ThreadPool("DHTControl:internallookups", lookup_concurrency );
internal_put_pool = new ThreadPool("DHTControl:internalputs", lookup_concurrency );
// external pools queue when full ( as opposed to blocking )
external_lookup_pool = new ThreadPool("DHTControl:externallookups", EXTERNAL_LOOKUP_CONCURRENCY, true );
external_put_pool = new ThreadPool("DHTControl:puts", EXTERNAL_PUT_CONCURRENCY, true );
createRouter( transport.getLocalContact());
node_id_byte_count = router.getID().length;
stats = new DHTControlStatsImpl( this );
// don't bother computing anti-spoof stuff if we don't support value storage
if ( transport.supportsStorage()){
try{
spoof_cipher = Cipher.getInstance("DESede/ECB/PKCS5Padding");
KeyGenerator keyGen = KeyGenerator.getInstance("DESede");
spoof_key = keyGen.generateKey();
}catch( Throwable e ){
Debug.printStackTrace( e );
logger.log( e );
}
}
transport.setRequestHandler( this );
transport.addListener(
new DHTTransportListener()
{
public void
localContactChanged(
DHTTransportContact new_local_contact )
{
logger.log( "Transport ID changed, recreating router" );
List old_contacts = router.findBestContacts( 0 );
byte[] old_router_id = router.getID();
createRouter( new_local_contact );
// sort for closeness to new router id
Set sorted_contacts = new sortedTransportContactSet( router.getID(), true ).getSet();
for (int i=0;i<old_contacts.size();i++){
DHTRouterContact contact = (DHTRouterContact)old_contacts.get(i);
if ( !Arrays.equals( old_router_id, contact.getID())){
if ( contact.isAlive()){
DHTTransportContact t_contact = ((DHTControlContactImpl)contact.getAttachment()).getTransportContact();
sorted_contacts.add( t_contact );
}
}
}
// fill up with non-alive ones to lower limit in case this is a start-of-day
// router change and we only have imported contacts in limbo state
for (int i=0;sorted_contacts.size() < 32 && i<old_contacts.size();i++){
DHTRouterContact contact = (DHTRouterContact)old_contacts.get(i);
if ( !Arrays.equals( old_router_id, contact.getID())){
if ( !contact.isAlive()){
DHTTransportContact t_contact = ((DHTControlContactImpl)contact.getAttachment()).getTransportContact();
sorted_contacts.add( t_contact );
}
}
}
Iterator it = sorted_contacts.iterator();
int added = 0;
// don't add them all otherwise we can skew the smallest-subtree. better
// to seed with some close ones and then let the normal seeding process
// populate it correctly
while( it.hasNext() && added < 128 ){
DHTTransportContact contact = (DHTTransportContact)it.next();
router.contactAlive( contact.getID(), new DHTControlContactImpl( contact ));
added++;
}
seed( false );
}
public void
resetNetworkPositions()
{
List<DHTRouterContact> contacts = router.getAllContacts();
for (int i=0;i<contacts.size();i++){
DHTRouterContact rc = contacts.get(i);
if ( !router.isID( rc.getID())){
((DHTControlContactImpl)rc.getAttachment()).getTransportContact().createNetworkPositions( false );
}
}
}
public void
currentAddress(
String address )
{
}
public void
reachabilityChanged(
boolean reacheable )
{
}
});
}
protected void
createRouter(
DHTTransportContact _local_contact)
{
router_start_time = SystemTime.getCurrentTime();
router_count++;
local_contact = _local_contact;
if ( router != null ){
router.destroy();
}
router = DHTRouterFactory.create(
K, B, max_rep_per_node,
local_contact.getID(),
new DHTControlContactImpl( local_contact ),
logger);
router.setAdapter(
new DHTRouterAdapter()
{
public void
requestPing(
DHTRouterContact contact )
{
DHTControlImpl.this.requestPing( contact );
}
public void
requestLookup(
byte[] id,
String description )
{
lookup( internal_lookup_pool, false,
id,
description,
(byte)0,
false,
0,
search_concurrency,
1,
router.getK(), // (parg - removed this) decrease search accuracy for refreshes
new lookupResultHandler(new DHTOperationAdapter())
{
public void
diversify(
DHTTransportContact cause,
byte diversification_type )
{
}
public void
closest(
List res )
{
}
});
}
public void
requestAdd(
DHTRouterContact contact )
{
nodeAddedToRouter( contact );
}
});
database.setControl( this );
}
public long
getRouterUptime()
{
long now = SystemTime.getCurrentTime();
if ( now < router_start_time ){
router_start_time = now;
}
return( now - router_start_time );
}
public int
getRouterCount()
{
return( router_count );
}
public DHTControlStats
getStats()
{
return( stats );
}
public DHTTransport
getTransport()
{
return( transport );
}
public DHTRouter
getRouter()
{
return( router );
}
public DHTDB
getDataBase()
{
return( database );
}
public void
contactImported(
DHTTransportContact contact )
{
router.contactKnown( contact.getID(), new DHTControlContactImpl(contact));
}
public void
contactRemoved(
DHTTransportContact contact )
{
// obviously we don't want to remove ourselves
if ( !router.isID( contact.getID())){
router.contactDead( contact.getID(), true );
}
}
public void
exportState(
DataOutputStream daos,
int max )
throws IOException
{
/*
* We need to be a bit smart about exporting state to deal with the situation where a
* DHT is started (with good import state) and then stopped before the goodness of the
* state can be re-established. So we remember what we imported and take account of this
* on a re-export
*/
// get all the contacts
List contacts = router.findBestContacts( 0 );
// give priority to any that were alive before and are alive now
List to_save = new ArrayList();
List reserves = new ArrayList();
//System.out.println( "Exporting" );
for (int i=0;i<contacts.size();i++){
DHTRouterContact contact = (DHTRouterContact)contacts.get(i);
Object[] imported = (Object[])imported_state.get( new HashWrapper( contact.getID()));
if ( imported != null ){
if ( contact.isAlive()){
// definitely want to keep this one
to_save.add( contact );
}else if ( !contact.isFailing()){
// dunno if its still good or not, however its got to be better than any
// new ones that we didn't import who aren't known to be alive
reserves.add( contact );
}
}
}
//System.out.println( " initial to_save = " + to_save.size() + ", reserves = " + reserves.size());
// now pull out any live ones
for (int i=0;i<contacts.size();i++){
DHTRouterContact contact = (DHTRouterContact)contacts.get(i);
if ( contact.isAlive() && !to_save.contains( contact )){
to_save.add( contact );
}
}
//System.out.println( " after adding live ones = " + to_save.size());
// now add any reserve ones
for (int i=0;i<reserves.size();i++){
DHTRouterContact contact = (DHTRouterContact)reserves.get(i);
if ( !to_save.contains( contact )){
to_save.add( contact );
}
}
//System.out.println( " after adding reserves = " + to_save.size());
// now add in the rest!
for (int i=0;i<contacts.size();i++){
DHTRouterContact contact = (DHTRouterContact)contacts.get(i);
if (!to_save.contains( contact )){
to_save.add( contact );
}
}
// and finally remove the invalid ones
Iterator it = to_save.iterator();
while( it.hasNext()){
DHTRouterContact contact = (DHTRouterContact)it.next();
DHTTransportContact t_contact = ((DHTControlContactImpl)contact.getAttachment()).getTransportContact();
if ( !t_contact.isValid()){
it.remove();
}
}
//System.out.println( " finally = " + to_save.size());
int num_to_write = Math.min( max, to_save.size());
daos.writeInt( num_to_write );
for (int i=0;i<num_to_write;i++){
DHTRouterContact contact = (DHTRouterContact)to_save.get(i);
//System.out.println( "export:" + contact.getString());
daos.writeLong( contact.getTimeAlive());
DHTTransportContact t_contact = ((DHTControlContactImpl)contact.getAttachment()).getTransportContact();
try{
t_contact.exportContact( daos );
}catch( DHTTransportException e ){
// shouldn't fail as for a contact to make it to the router
// it should be valid...
Debug.printStackTrace( e );
throw( new IOException( e.getMessage()));
}
}
daos.flush();
}
public void
importState(
DataInputStream dais )
throws IOException
{
int num = dais.readInt();
for (int i=0;i<num;i++){
try{
long time_alive = dais.readLong();
DHTTransportContact contact = transport.importContact( dais );
imported_state.put( new HashWrapper( contact.getID()), new Object[]{ new Long( time_alive ), contact });
}catch( DHTTransportException e ){
Debug.printStackTrace( e );
}
}
}
public void
seed(
final boolean full_wait )
{
final AESemaphore sem = new AESemaphore( "DHTControl:seed" );
lookup( internal_lookup_pool, false,
router.getID(),
"Seeding DHT",
(byte)0,
false,
0,
search_concurrency*4,
1,
router.getK(),
new lookupResultHandler(new DHTOperationAdapter())
{
public void
diversify(
DHTTransportContact cause,
byte diversification_type )
{
}
public void
closest(
List res )
{
if ( !full_wait ){
sem.release();
}
seeded = true;
try{
router.seed();
}finally{
if ( full_wait ){
sem.release();
}
}
}
});
// we always wait at least a minimum amount of time before returning
long start = SystemTime.getCurrentTime();
sem.reserve( INTEGRATION_TIME_MAX );
long now = SystemTime.getCurrentTime();
if ( now < start ){
start = now;
}
long remaining = INTEGRATION_TIME_MAX - ( now - start );
if ( remaining > 500 && !full_wait ){
logger.log( "Initial integration completed, waiting " + remaining + " ms for second phase to start" );
try{
Thread.sleep( remaining );
}catch( Throwable e ){
Debug.out(e);
}
}
}
public boolean
isSeeded()
{
return( seeded );
}
protected void
poke()
{
long now = SystemTime.getCurrentTime();
if ( now < last_lookup ||
now - last_lookup > RANDOM_QUERY_PERIOD ){
last_lookup = now;
// we don't want this to be blocking as it'll stuff the stats
external_lookup_pool.run(
new DhtTask(external_lookup_pool)
{
private byte[] target = {};
public void
runSupport()
{
target = router.refreshRandom();
}
protected void
cancel()
{
}
public byte[]
getTarget()
{
return( target );
}
public String
getDescription()
{
return( "Random Query" );
}
});
}
}
public void
put(
byte[] _unencoded_key,
String _description,
byte[] _value,
byte _flags,
byte _life_hours,
byte _replication_control,
boolean _high_priority,
DHTOperationListener _listener )
{
// public entry point for explicit publishes
if ( _value.length == 0 ){
// zero length denotes value removal
throw( new RuntimeException( "zero length values not supported"));
}
byte[] encoded_key = encodeKey( _unencoded_key );
if ( DHTLog.isOn()){
DHTLog.log( "put for " + DHTLog.getString( encoded_key ));
}
DHTDBValue value = database.store( new HashWrapper( encoded_key ), _value, _flags, _life_hours, _replication_control );
put( external_put_pool,
_high_priority,
encoded_key,
_description,
value,
_flags,
0,
true,
new HashSet(),
1,
_listener instanceof DHTOperationListenerDemuxer?
(DHTOperationListenerDemuxer)_listener:
new DHTOperationListenerDemuxer(_listener));
}
public void
putEncodedKey(
byte[] encoded_key,
String description,
DHTTransportValue value,
long timeout,
boolean original_mappings )
{
put( internal_put_pool,
false,
encoded_key,
description,
value,
(byte)0,
timeout,
original_mappings,
new HashSet(),
1,
new DHTOperationListenerDemuxer( new DHTOperationAdapter()));
}
protected void
put(
ThreadPool thread_pool,
boolean high_priority,
byte[] initial_encoded_key,
String description,
DHTTransportValue value,
byte flags,
long timeout,
boolean original_mappings,
Set things_written,
int put_level,
DHTOperationListenerDemuxer listener )
{
put( thread_pool,
high_priority,
initial_encoded_key,
description,
new DHTTransportValue[]{ value },
flags,
timeout,
original_mappings,
things_written,
put_level,
listener );
}
protected void
put(
final ThreadPool thread_pool,
final boolean high_priority,
final byte[] initial_encoded_key,
final String description,
final DHTTransportValue[] values,
final byte flags,
final long timeout,
final boolean original_mappings,
final Set things_written,
final int put_level,
final DHTOperationListenerDemuxer listener )
{
// get the initial starting point for the put - may have previously been diversified
byte[][] encoded_keys =
adapter.diversify(
description,
null,
true,
true,
initial_encoded_key,
DHT.DT_NONE,
original_mappings,
getMaxDivDepth());
if ( encoded_keys.length == 0 ){
// over-diversified
listener.diversified( "Over-diversification of [" + description + "]" );
listener.complete( false );
return;
}
// may be > 1 if diversification is replicating (for load balancing)
for (int i=0;i<encoded_keys.length;i++){
final byte[] encoded_key = encoded_keys[i];
HashWrapper hw = new HashWrapper( encoded_key );
synchronized( things_written ){
if ( things_written.contains( hw )){
// System.out.println( "put: skipping key as already written" );
continue;
}
things_written.add( hw );
}
final String this_description =
Arrays.equals( encoded_key, initial_encoded_key )?
description:
("Diversification of [" + description + "]" );
lookup( thread_pool,
high_priority,
encoded_key,
this_description,
flags,
false,
timeout,
search_concurrency,
1,
router.getK(),
new lookupResultHandler(listener)
{
public void
diversify(
DHTTransportContact cause,
byte diversification_type )
{
Debug.out( "Shouldn't get a diversify on a lookup-node" );
}
public void
closest(
List _closest )
{
put( thread_pool,
high_priority,
new byte[][]{ encoded_key },
"Store of [" + this_description + "]",
new DHTTransportValue[][]{ values },
flags,
_closest,
timeout,
listener,
true,
things_written,
put_level,
false );
}
});
}
}
public void
putDirectEncodedKeys(
byte[][] encoded_keys,
String description,
DHTTransportValue[][] value_sets,
List contacts )
{
// we don't consider diversification for direct puts (these are for republishing
// of cached mappings and we maintain these as normal - its up to the original
// publisher to diversify as required)
put( internal_put_pool,
false,
encoded_keys,
description,
value_sets,
(byte)0,
contacts,
0,
new DHTOperationListenerDemuxer( new DHTOperationAdapter()),
false,
new HashSet(),
1,
false );
}
public void
putDirectEncodedKeys(
byte[][] encoded_keys,
String description,
DHTTransportValue[][] value_sets,
DHTTransportContact contact,
DHTOperationListener listener )
{
// we don't consider diversification for direct puts (these are for republishing
// of cached mappings and we maintain these as normal - its up to the original
// publisher to diversify as required)
List<DHTTransportContact> contacts = new ArrayList<DHTTransportContact>(1);
contacts.add( contact );
put( internal_put_pool,
false,
encoded_keys,
description,
value_sets,
(byte)0,
contacts,
0,
new DHTOperationListenerDemuxer( listener ),
false,
new HashSet(),
1,
false );
}
public byte[]
getObfuscatedKey(
byte[] plain_key )
{
int length = plain_key.length;
byte[] obs_key = new byte[ length ];
System.arraycopy( plain_key, 0, obs_key, 0, 5 );
// ensure plain key and obfuscated one differ at subsequent bytes to prevent potential
// clashes with code that uses 'n' byte prefix (e.g. DB survey code)
for (int i=6;i<length;i++){
if ( plain_key[i] == 0 ){
obs_key[i] = 1;
}
}
// finally copy over last two bytes for code that uses challenge-response on this
// (survey code)
obs_key[length-2] = plain_key[length-2];
obs_key[length-1] = plain_key[length-1];
return( obs_key );
}
protected byte[]
getObfuscatedValue(
byte[] plain_key )
{
RC4Engine engine = new RC4Engine();
CipherParameters params = new KeyParameter( new SHA1Simple().calculateHash( plain_key ));
engine.init( true, params );
byte[] temp = new byte[1024];
engine.processBytes( temp, 0, 1024, temp, 0 );
final byte[] obs_value = new byte[ plain_key.length ];
engine.processBytes( plain_key, 0, plain_key.length, obs_value, 0 );
return( obs_value );
}
protected DHTTransportValue
getObfuscatedValue(
final DHTTransportValue basis,
byte[] plain_key )
{
final byte[] obs_value = getObfuscatedValue( plain_key );
return(
new DHTTransportValue()
{
public boolean
isLocal()
{
return( basis.isLocal());
}
public long
getCreationTime()
{
return( basis.getCreationTime());
}
public byte[]
getValue()
{
return( obs_value );
}
public int
getVersion()
{
return( basis.getVersion());
}
public DHTTransportContact
getOriginator()
{
return( basis.getOriginator());
}
public int
getFlags()
{
return( basis.getFlags());
}
public int
getLifeTimeHours()
{
return( basis.getLifeTimeHours());
}
public byte
getReplicationControl()
{
return( basis.getReplicationControl());
}
public byte
getReplicationFactor()
{
return( basis.getReplicationFactor());
}
public byte
getReplicationFrequencyHours()
{
return( basis.getReplicationFrequencyHours());
}
public String
getString()
{
return( "obs: " + basis.getString());
}
});
}
protected void
put(
final ThreadPool thread_pool,
final boolean high_priority,
byte[][] initial_encoded_keys,
final String description,
final DHTTransportValue[][] initial_value_sets,
final byte flags,
final List contacts,
final long timeout,
final DHTOperationListenerDemuxer listener,
final boolean consider_diversification,
final Set things_written,
final int put_level,
final boolean immediate )
{
int max_depth = getMaxDivDepth();
if ( put_level > max_depth ){
Debug.out( "Put level exceeded, terminating diversification (level=" + put_level + ",max=" + max_depth + ")" );
listener.incrementCompletes();
listener.complete( false );
return;
}
boolean[] ok = new boolean[initial_encoded_keys.length];
int failed = 0;
for (int i=0;i<initial_encoded_keys.length;i++){
if ( ! (ok[i] = !database.isKeyBlocked( initial_encoded_keys[i]))){
failed++;
}
}
// if all failed then nothing to do
if ( failed == ok.length ){
listener.incrementCompletes();
listener.complete( false );
return;
}
final byte[][] encoded_keys = failed==0?initial_encoded_keys:new byte[ok.length-failed][];
final DHTTransportValue[][] value_sets = failed==0?initial_value_sets:new DHTTransportValue[ok.length-failed][];
if ( failed > 0 ){
int pos = 0;
for (int i=0;i<ok.length;i++){
if ( ok[i] ){
encoded_keys[ pos ] = initial_encoded_keys[i];
value_sets[ pos ] = initial_value_sets[i];
pos++;
}
}
}
final byte[][] obs_keys;
final DHTTransportValue[][] obs_vals;
if ( ( flags & DHT.FLAG_OBFUSCATE_LOOKUP ) != 0 ){
if ( encoded_keys.length != 1 ){
Debug.out( "inconsistent - expected one key" );
}
if ( value_sets[0].length != 1 ){
Debug.out( "inconsistent - expected one value" );
}
obs_keys = new byte[1][];
obs_vals = new DHTTransportValue[1][1];
obs_keys[0] = getObfuscatedKey( encoded_keys[0] );
obs_vals[0][0] = getObfuscatedValue( value_sets[0][0], encoded_keys[0] );
}else{
obs_keys = null;
obs_vals = null;
}
// only diversify on one hit as we're storing at closest 'n' so we only need to
// do it once for each key
final boolean[] diversified = new boolean[encoded_keys.length];
int skipped = 0;
for (int i=0;i<contacts.size();i++){
final DHTTransportContact contact = (DHTTransportContact)contacts.get(i);
if ( router.isID( contact.getID())){
// don't send to ourselves!
skipped++;
}else{
boolean skip_this = false;
synchronized( things_written ){
if ( things_written.contains( contact )){
// if we've come back to an already hit contact due to a diversification loop
// then ignore it
Debug.out( "Put: contact encountered for a second time, ignoring" );
skipped++;
skip_this = true;
}else{
things_written.add( contact );
}
}
if ( !skip_this ){
try{
for (int j=0;j<value_sets.length;j++){
for (int k=0;k<value_sets[j].length;k++){
listener.wrote( contact, value_sets[j][k] );
}
}
// each store is going to report its complete event
listener.incrementCompletes();
contact.sendStore(
new DHTTransportReplyHandlerAdapter()
{
public void
storeReply(
DHTTransportContact _contact,
byte[] _diversifications )
{
boolean complete_is_async = false;
try{
if ( DHTLog.isOn()){
DHTLog.log( "Store OK " + DHTLog.getString( _contact ));
}
router.contactAlive( _contact.getID(), new DHTControlContactImpl(_contact));
// can be null for old protocol versions
boolean div_done = false;
if ( consider_diversification && _diversifications != null ){
for (int j=0;j<_diversifications.length;j++){
if ( _diversifications[j] != DHT.DT_NONE && !diversified[j] ){
div_done = true;
diversified[j] = true;
byte[][] diversified_keys =
adapter.diversify( description, _contact, true, false, encoded_keys[j], _diversifications[j], false, getMaxDivDepth());
logDiversification( _contact, encoded_keys, diversified_keys );
for (int k=0;k<diversified_keys.length;k++){
put( thread_pool,
high_priority,
diversified_keys[k],
"Diversification of [" + description + "]",
value_sets[j],
flags,
timeout,
false,
things_written,
put_level + 1,
listener );
}
}
}
}
if ( !div_done ){
if ( obs_keys != null ){
contact.sendStore(
new DHTTransportReplyHandlerAdapter()
{
public void
storeReply(
DHTTransportContact _contact,
byte[] _diversifications )
{
if ( DHTLog.isOn()){
DHTLog.log( "Obs store OK " + DHTLog.getString( _contact ));
}
listener.complete( false );
}
public void
failed(
DHTTransportContact _contact,
Throwable _error )
{
if ( DHTLog.isOn()){
DHTLog.log( "Obs store failed " + DHTLog.getString( _contact ) + " -> failed: " + _error.getMessage());
}
listener.complete( true );
}
},
obs_keys,
obs_vals,
immediate );
complete_is_async = true;
}
}
}finally{
if ( !complete_is_async ){
listener.complete( false );
}
}
}
public void
failed(
DHTTransportContact _contact,
Throwable _error )
{
try{
if ( DHTLog.isOn()){
DHTLog.log( "Store failed " + DHTLog.getString( _contact ) + " -> failed: " + _error.getMessage());
}
router.contactDead( _contact.getID(), false );
}finally{
listener.complete( true );
}
}
public void
keyBlockRequest(
DHTTransportContact contact,
byte[] request,
byte[] key_signature )
{
DHTStorageBlock key_block = database.keyBlockRequest( null, request, key_signature );
if ( key_block != null ){
// remove this key for any subsequent publishes. Quickest hack
// is to change it into a random key value - this will be rejected
// by the recipient as not being close enough anyway
for (int i=0;i<encoded_keys.length;i++){
if ( Arrays.equals( encoded_keys[i], key_block.getKey())){
byte[] dummy = new byte[encoded_keys[i].length];
new Random().nextBytes( dummy );
encoded_keys[i] = dummy;
}
}
}
}
},
encoded_keys,
value_sets,
immediate );
}catch( Throwable e ){
Debug.printStackTrace(e);
}
}
}
}
if ( skipped == contacts.size()){
listener.incrementCompletes();
listener.complete( false );
}
}
protected int
getMaxDivDepth()
{
if ( combined_dht_estimate == 0 ){
getEstimatedDHTSize();
}
int max = Math.max( 2, combined_dht_estimate_mag );
// System.out.println( "net:" + transport.getNetwork() + " - max_div_depth=" + max );
return( max );
}
protected void
logDiversification(
final DHTTransportContact contact,
final byte[][] keys,
final byte[][] div )
{
/*
System.out.println( "Div check starts for " + contact.getString());
String keys_str = "";
for (int i=0;i<keys.length;i++){
keys_str += (i==0?"":",") + ByteFormatter.encodeString( keys[i] );
}
String div_str = "";
for (int i=0;i<div.length;i++){
div_str += (i==0?"":",") + ByteFormatter.encodeString( div[i] );
}
System.out.println( " " + keys_str + " -> " + div_str );
new AEThread2( "sdsd", true )
{
public void
run()
{
DHTTransportFullStats stats = contact.getStats();
System.out.println( contact.getString() + "-> " +(stats==null?"<null>":stats.getString()));
}
}.start();
*/
}
public DHTTransportValue
getLocalValue(
byte[] unencoded_key )
{
final byte[] encoded_key = encodeKey( unencoded_key );
if ( DHTLog.isOn()){
DHTLog.log( "getLocalValue for " + DHTLog.getString( encoded_key ));
}
DHTDBValue res = database.get( new HashWrapper( encoded_key ));
if ( res == null ){
return( null );
}
return( res );
}
public void
get(
byte[] unencoded_key,
String description,
byte flags,
int max_values,
long timeout,
boolean exhaustive,
boolean high_priority,
final DHTOperationListener get_listener )
{
final byte[] encoded_key = encodeKey( unencoded_key );
if ( DHTLog.isOn()){
DHTLog.log( "get for " + DHTLog.getString( encoded_key ));
}
final DhtTaskSet[] task_set = { null };
DHTOperationListenerDemuxer demuxer =
new DHTOperationListenerDemuxer(
new DHTOperationListener()
{
public void
searching(
DHTTransportContact contact,
int level,
int active_searches )
{
get_listener.searching(contact, level, active_searches);
}
public void
diversified(
String desc )
{
get_listener.diversified(desc);
}
public void
found(
DHTTransportContact contact,
boolean is_closest )
{
get_listener.found(contact,is_closest);
}
public void
read(
DHTTransportContact contact,
DHTTransportValue value )
{
get_listener.read(contact, value);
}
public void
wrote(
DHTTransportContact contact,
DHTTransportValue value )
{
get_listener.wrote(contact, value);
}
public void
complete(
boolean timeout )
{
get_listener.complete(timeout);
if ( task_set[0] != null ){
task_set[0].cancel();
}
}
});
task_set[0] = getSupport( encoded_key, description, flags, max_values, timeout, exhaustive, high_priority, demuxer );
}
public boolean
isDiversified(
byte[] unencoded_key )
{
final byte[] encoded_key = encodeKey( unencoded_key );
return( adapter.isDiversified( encoded_key ));
}
public boolean
lookup(
byte[] unencoded_key,
String description,
long timeout,
final DHTOperationListener lookup_listener )
{
return( lookupEncoded( encodeKey( unencoded_key ), description, timeout, false, lookup_listener ));
}
public boolean
lookupEncoded(
byte[] encoded_key,
String description,
long timeout,
boolean high_priority,
final DHTOperationListener lookup_listener )
{
if ( DHTLog.isOn()){
DHTLog.log( "lookup for " + DHTLog.getString( encoded_key ));
}
final AESemaphore sem = new AESemaphore( "DHTControl:lookup" );
final boolean[] diversified = { false };
DHTOperationListener delegate =
new DHTOperationListener()
{
public void
searching(
DHTTransportContact contact,
int level,
int active_searches )
{
lookup_listener.searching( contact, level, active_searches );
}
public void
found(
DHTTransportContact contact,
boolean is_closest )
{
}
public void
diversified(
String desc )
{
lookup_listener.diversified( desc );
}
public void
read(
DHTTransportContact contact,
DHTTransportValue value )
{
}
public void
wrote(
DHTTransportContact contact,
DHTTransportValue value )
{
}
public void
complete(
boolean timeout )
{
lookup_listener.complete( timeout );
sem.release();
}
};
lookup( external_lookup_pool,
high_priority,
encoded_key,
description,
(byte)0,
false,
timeout,
search_concurrency,
1,
router.getK(),
new lookupResultHandler( delegate )
{
public void
diversify(
DHTTransportContact cause,
byte diversification_type )
{
diversified( "Diversification of [lookup]" );
diversified[0] = true;
}
public void
closest(
List closest )
{
for (int i=0;i<closest.size();i++){
lookup_listener.found((DHTTransportContact)closest.get(i),true);
}
}
});
sem.reserve();
return( diversified[0] );
}
protected DhtTaskSet
getSupport(
final byte[] initial_encoded_key,
final String description,
final byte flags,
final int max_values,
final long timeout,
final boolean exhaustive,
final boolean high_priority,
final DHTOperationListenerDemuxer get_listener )
{
final DhtTaskSet result = new DhtTaskSet();
// get the initial starting point for the get - may have previously been diversified
byte[][] encoded_keys = adapter.diversify( description, null, false, true, initial_encoded_key, DHT.DT_NONE, exhaustive, getMaxDivDepth());
if ( encoded_keys.length == 0 ){
// over-diversified
get_listener.diversified( "Over-diversification of [" + description + "]" );
get_listener.complete( false );
return( result );
}
for (int i=0;i<encoded_keys.length;i++){
final boolean[] diversified = { false };
final byte[] encoded_key = encoded_keys[i];
boolean div = !Arrays.equals( encoded_key, initial_encoded_key );
final String this_description =
div?("Diversification of [" + description + "]" ):description;
if ( div ){
get_listener.diversified( this_description );
}
boolean is_stats_query = (flags & DHT.FLAG_STATS ) != 0;
result.add(
lookup( external_lookup_pool,
high_priority,
encoded_key,
this_description,
flags,
true,
timeout,
is_stats_query?search_concurrency*2:search_concurrency,
max_values,
router.getK(),
new lookupResultHandler( get_listener )
{
private List found_values = new ArrayList();
public void
diversify(
DHTTransportContact cause,
byte diversification_type )
{
diversified( "Diversification of [" + this_description + "]" );
// we only want to follow one diversification
if ( !diversified[0]){
diversified[0] = true;
int rem = max_values==0?0:( max_values - found_values.size());
if ( max_values == 0 || rem > 0 ){
byte[][] diversified_keys = adapter.diversify( description, cause, false, false, encoded_key, diversification_type, exhaustive, getMaxDivDepth());
if ( diversified_keys.length > 0 ){
// should return a max of 1 (0 if diversification refused)
// however, could change one day to search > 1
for (int j=0;j<diversified_keys.length;j++){
if ( !result.isCancelled()){
result.add(
getSupport( diversified_keys[j], "Diversification of [" + this_description + "]", flags, rem, timeout, exhaustive, high_priority, get_listener ));
}
}
}
}
}
}
public void
read(
DHTTransportContact contact,
DHTTransportValue value )
{
found_values.add( value );
super.read( contact, value );
}
public void
closest(
List closest )
{
/* we don't use teh cache-at-closest kad feature
if ( found_values.size() > 0 ){
DHTTransportValue[] values = new DHTTransportValue[found_values.size()];
found_values.toArray( values );
// cache the values at the 'n' closest seen locations
for (int k=0;k<Math.min(cache_at_closest_n,closest.size());k++){
DHTTransportContact contact = (DHTTransportContact)(DHTTransportContact)closest.get(k);
for (int j=0;j<values.length;j++){
wrote( contact, values[j] );
}
contact.sendStore(
new DHTTransportReplyHandlerAdapter()
{
public void
storeReply(
DHTTransportContact _contact,
byte[] _diversifications )
{
// don't consider diversification for cache stores as we're not that
// bothered
DHTLog.log( "Cache store OK " + DHTLog.getString( _contact ));
router.contactAlive( _contact.getID(), new DHTControlContactImpl(_contact));
}
public void
failed(
DHTTransportContact _contact,
Throwable _error )
{
DHTLog.log( "Cache store failed " + DHTLog.getString( _contact ) + " -> failed: " + _error.getMessage());
router.contactDead( _contact.getID(), false );
}
},
new byte[][]{ encoded_key },
new DHTTransportValue[][]{ values });
}
}
*/
}
}));
}
return( result );
}
public byte[]
remove(
byte[] unencoded_key,
String description,
DHTOperationListener listener )
{
final byte[] encoded_key = encodeKey( unencoded_key );
if ( DHTLog.isOn()){
DHTLog.log( "remove for " + DHTLog.getString( encoded_key ));
}
DHTDBValue res = database.remove( local_contact, new HashWrapper( encoded_key ));
if ( res == null ){
// not found locally, nothing to do
return( null );
}else{
// we remove a key by pushing it back out again with zero length value
put( external_put_pool,
false,
encoded_key,
description,
res,
(byte)res.getFlags(),
0,
true,
new HashSet(),
1,
new DHTOperationListenerDemuxer( listener ));
return( res.getValue());
}
}
public byte[]
remove(
DHTTransportContact[] contacts,
byte[] unencoded_key,
String description,
DHTOperationListener listener )
{
final byte[] encoded_key = encodeKey( unencoded_key );
if ( DHTLog.isOn()){
DHTLog.log( "remove for " + DHTLog.getString( encoded_key ));
}
DHTDBValue res = database.remove( local_contact, new HashWrapper( encoded_key ));
if ( res == null ){
// not found locally, nothing to do
return( null );
}else{
List contacts_l = new ArrayList( contacts.length );
for (int i=0;i<contacts.length;i++ ){
contacts_l.add( contacts[i] );
}
put( external_put_pool,
true,
new byte[][]{ encoded_key },
"Store of [" + description + "]",
new DHTTransportValue[][]{{ res }},
(byte)res.getFlags(),
contacts_l,
0,
new DHTOperationListenerDemuxer( listener ),
true,
new HashSet(),
1,
true );
return( res.getValue());
}
}
/**
* The lookup method returns up to K closest nodes to the target
* @param lookup_id
* @return
*/
protected DhtTask
lookup(
final ThreadPool thread_pool,
boolean high_priority,
final byte[] _lookup_id,
final String description,
final byte flags,
final boolean value_search,
final long timeout,
final int concurrency,
final int max_values,
final int search_accuracy,
final lookupResultHandler handler )
{
final byte[] lookup_id;
final byte[] obs_value;
if (( flags & DHT.FLAG_OBFUSCATE_LOOKUP ) != 0 ){
lookup_id = getObfuscatedKey( _lookup_id );
obs_value = getObfuscatedValue( _lookup_id );
}else{
lookup_id = _lookup_id;
obs_value = null;
}
DhtTask task =
new DhtTask(thread_pool)
{
boolean timeout_occurred = false;
// keep querying successively closer nodes until we have got responses from the K
// closest nodes that we've seen. We might get a bunch of closer nodes that then
// fail to respond, which means we have reconsider further away nodes
// we keep a list of nodes that we have queried to avoid re-querying them
// we keep a list of nodes discovered that we have yet to query
// we have a parallel search limit of A. For each A we effectively loop grabbing
// the currently closest unqueried node, querying it and adding the results to the
// yet-to-query-set (unless already queried)
// we terminate when we have received responses from the K closest nodes we know
// about (excluding failed ones)
// Note that we never widen the root of our search beyond the initial K closest
// that we know about - this could be relaxed
// contacts remaining to query
// closest at front
Set contacts_to_query;
AEMonitor contacts_to_query_mon;
Map level_map;
// record the set of contacts we've queried to avoid re-queries
Map contacts_queried;
// record the set of contacts that we've had a reply from
// furthest away at front
Set ok_contacts;
// this handles the search concurrency
int idle_searches;
int active_searches;
int values_found;
int value_replies;
Set values_found_set;
boolean key_blocked;
long start;
TimerEvent timeoutEvent;
private int runningState = 1; // -1 terminated, 0 waiting, 1 running
private int freeTasksCount = concurrency;
private boolean cancelled;
// start the lookup
public void runSupport()
{
startLookup();
}
private void startLookup()
{
contacts_to_query = getClosestContactsSet(lookup_id, K, false);
contacts_to_query_mon = new AEMonitor("DHTControl:ctq");
level_map = new LightHashMap();
// record the set of contacts we've queried to avoid re-queries
contacts_queried = new LightHashMap();
// record the set of contacts that we've had a reply from
// furthest away at front
ok_contacts = new sortedTransportContactSet(lookup_id, false).getSet();
// this handles the search concurrency
values_found_set = new HashSet();
start = SystemTime.getMonotonousTime();
last_lookup = SystemTime.getCurrentTime();
handler.incrementCompletes();
Iterator it = contacts_to_query.iterator();
while (it.hasNext())
{
DHTTransportContact contact = (DHTTransportContact) it.next();
handler.found(contact,false);
level_map.put(contact, new Integer(0));
}
if ( DHTLog.isOn()){
DHTLog.log("lookup for " + DHTLog.getString(lookup_id));
}
if (value_search && database.isKeyBlocked(lookup_id)){
DHTLog.log("lookup: terminates - key blocked");
// bail out and pretend everything worked with zero results
terminateLookup(false);
return;
}
if (timeout > 0)
{
timeoutEvent = SimpleTimer.addEvent("DHT lookup timeout", SystemTime.getCurrentTime()+timeout, new TimerEventPerformer() {
public void perform(TimerEvent event) {
if ( DHTLog.isOn()){
DHTLog.log("lookup: terminates - timeout");
}
//System.out.println("timeout");
timeout_occurred = true;
terminateLookup(false);
}
});
}
lookupSteps();
}
private void terminateLookup(boolean error)
{
if(timeoutEvent != null)
timeoutEvent.cancel();
synchronized (this)
{
if(runningState == -1)
return;
runningState = -1;
}
if(!error)
{
// maybe unterminated searches still going on so protect ourselves
// against concurrent modification of result set
List closest_res = null;
try
{
contacts_to_query_mon.enter();
if (DHTLog.isOn())
{
DHTLog.log("lookup complete for " + DHTLog.getString(lookup_id));
DHTLog.log(" queried = " + DHTLog.getString(contacts_queried));
DHTLog.log(" to query = " + DHTLog.getString(contacts_to_query));
DHTLog.log(" ok = " + DHTLog.getString(ok_contacts));
}
closest_res = new ArrayList(ok_contacts);
// we need to reverse the list as currently closest is at the end
Collections.reverse(closest_res);
if (timeout <= 0 && !value_search)
// we can use the results of this to estimate the DHT size
estimateDHTSize(lookup_id, contacts_queried, search_accuracy);
} finally
{
contacts_to_query_mon.exit();
}
handler.closest(closest_res);
}
handler.complete(timeout_occurred);
releaseToPool();
}
private synchronized boolean reserve()
{
if(freeTasksCount <= 0 || runningState == -1)
{
//System.out.println("reserve-exit");
if(runningState == 1)
runningState = 0;
return false;
}
freeTasksCount--;
return true;
}
private synchronized void release()
{
freeTasksCount++;
if(runningState == 0)
{
//System.out.println("release-start");
runningState = 1;
new AEThread2("DHT lookup runner",true) {
public void run() {
thread_pool.registerThreadAsChild(worker);
lookupSteps();
thread_pool.deregisterThreadAsChild(worker);
}
}.start();
}
}
protected synchronized void
cancel()
{
if ( runningState != -1 ){
// System.out.println( "Task cancelled" );
}
cancelled = true;
}
// individual lookup steps
private void lookupSteps() {
try
{
boolean terminate = false;
while ( !cancelled )
{
if (timeout > 0)
{
long now = SystemTime.getMonotonousTime();
long remaining = timeout - (now - start);
if (remaining <= 0)
{
if ( DHTLog.isOn()){
DHTLog.log("lookup: terminates - timeout");
}
timeout_occurred = true;
terminate = true;
break;
}
if(!reserve())
break; // temporary stop, will be revived by release() or until a timeout occurs
} else if(!reserve())
break; // temporary stop, will be revived by release()*/
try
{
contacts_to_query_mon.enter();
// for stats queries the values returned are unique to target so don't assume 2 replies sufficient
if (values_found >= max_values || (( flags & DHT.FLAG_STATS ) == 0 && value_replies >= 2 ))
{
// all hits should have the same values anyway...
terminate = true;
break;
}
// if we've received a key block then easiest way to terminate the query is to
// dump any outstanding targets
if (key_blocked)
contacts_to_query.clear();
// if nothing pending then we need to wait for the results of a previous
// search to arrive. Of course, if there are no searches active then
// we've run out of things to do
if (contacts_to_query.size() == 0)
{
if (active_searches == 0)
{
if ( DHTLog.isOn()){
DHTLog.log("lookup: terminates - no contacts left to query");
}
terminate = true;
break;
}
idle_searches++;
continue;
}
// select the next contact to search
DHTTransportContact closest = (DHTTransportContact) contacts_to_query.iterator().next();
// if the next closest is further away than the furthest successful hit so
// far and we have K hits, we're done
if (ok_contacts.size() == search_accuracy)
{
DHTTransportContact furthest_ok = (DHTTransportContact) ok_contacts.iterator().next();
int distance = computeAndCompareDistances(furthest_ok.getID(), closest.getID(), lookup_id);
if (distance <= 0)
{
if ( DHTLog.isOn()){
DHTLog.log("lookup: terminates - we've searched the closest " + search_accuracy + " contacts");
}
terminate = true;
break;
}
}
// we optimise the first few entries based on their Vivaldi distance. Only a few
// however as we don't want to start too far away from the target.
if (contacts_queried.size() < concurrency)
{
DHTNetworkPosition[] loc_nps = local_contact.getNetworkPositions();
DHTTransportContact vp_closest = null;
Iterator vp_it = contacts_to_query.iterator();
int vp_count_limit = (concurrency * 2) - contacts_queried.size();
int vp_count = 0;
float best_dist = Float.MAX_VALUE;
while (vp_it.hasNext() && vp_count < vp_count_limit)
{
vp_count++;
DHTTransportContact entry = (DHTTransportContact) vp_it.next();
DHTNetworkPosition[] rem_nps = entry.getNetworkPositions();
float dist = DHTNetworkPositionManager.estimateRTT(loc_nps, rem_nps);
if ((!Float.isNaN(dist)) && dist < best_dist)
{
best_dist = dist;
vp_closest = entry;
// System.out.println( start + ": lookup for " + DHTLog.getString2( lookup_id ) + ": vp override (dist = " + dist + ")");
}
if (vp_closest != null) // override ID closest with VP closes
closest = vp_closest;
}
}
final DHTTransportContact f_closest = closest;
contacts_to_query.remove(closest);
contacts_queried.put(new HashWrapper(closest.getID()), closest);
// never search ourselves!
if (router.isID(closest.getID()))
{
release();
continue;
}
final int search_level = ((Integer) level_map.get(closest)).intValue();
active_searches++;
handler.searching(closest, search_level, active_searches);
DHTTransportReplyHandlerAdapter replyHandler = new DHTTransportReplyHandlerAdapter() {
private boolean value_reply_received = false;
public void findNodeReply(DHTTransportContact target_contact, DHTTransportContact[] reply_contacts) {
try
{
if ( DHTLog.isOn()){
DHTLog.log("findNodeReply: " + DHTLog.getString(reply_contacts));
}
router.contactAlive(target_contact.getID(), new DHTControlContactImpl(target_contact));
for (int i = 0; i < reply_contacts.length; i++)
{
DHTTransportContact contact = reply_contacts[i];
// ignore responses that are ourselves
if (compareDistances(router.getID(), contact.getID()) == 0)
continue;
// dunno if its alive or not, however record its existance
router.contactKnown(contact.getID(), new DHTControlContactImpl(contact));
}
try
{
contacts_to_query_mon.enter();
ok_contacts.add(target_contact);
if (ok_contacts.size() > search_accuracy)
{
// delete the furthest away
Iterator ok_it = ok_contacts.iterator();
ok_it.next();
ok_it.remove();
}
for (int i = 0; i < reply_contacts.length; i++)
{
DHTTransportContact contact = reply_contacts[i];
// ignore responses that are ourselves
if (compareDistances(router.getID(), contact.getID()) == 0)
continue;
if (contacts_queried.get(new HashWrapper(contact.getID())) == null && (!contacts_to_query.contains(contact)))
{
if ( DHTLog.isOn()){
DHTLog.log(" new contact for query: " + DHTLog.getString(contact));
}
contacts_to_query.add(contact);
handler.found(contact,false);
level_map.put(contact, new Integer(search_level + 1));
if (idle_searches > 0)
{
idle_searches--;
release();
}
} else
{
// DHTLog.log( " already queried: " + DHTLog.getString( contact ));
}
}
} finally
{
contacts_to_query_mon.exit();
}
} finally
{
try
{
contacts_to_query_mon.enter();
active_searches--;
} finally
{
contacts_to_query_mon.exit();
}
release();
}
}
public void
findValueReply(
DHTTransportContact contact,
DHTTransportValue[] values,
byte diversification_type, // hack - this is set to 99 when recursing here during obsfuscated lookup
boolean more_to_come )
{
if ( DHTLog.isOn()){
DHTLog.log("findValueReply: " + DHTLog.getString(values) + ",mtc=" + more_to_come + ", dt=" + diversification_type);
}
boolean obs_recurse = false;
if ( diversification_type == 99 ){
obs_recurse = true;
diversification_type = DHT.DT_NONE;
}
try
{
if (!key_blocked && diversification_type != DHT.DT_NONE){
// diversification instruction
if (( flags & DHT.FLAG_STATS ) == 0 ){
// ignore for stats queries as we're after the
// target key's stats, not the diversification
// thereof
handler.diversify(contact, diversification_type);
}
}
value_reply_received = true;
router.contactAlive(contact.getID(), new DHTControlContactImpl(contact));
int new_values = 0;
if (!key_blocked)
{
for (int i = 0; i < values.length; i++)
{
DHTTransportValue value = values[i];
DHTTransportContact originator = value.getOriginator();
// can't just use originator id as this value can be DOSed (see DB code)
byte[] originator_id = originator.getID();
byte[] value_bytes = value.getValue();
byte[] value_id = new byte[originator_id.length + value_bytes.length];
System.arraycopy(originator_id, 0, value_id, 0, originator_id.length);
System.arraycopy(value_bytes, 0, value_id, originator_id.length, value_bytes.length);
HashWrapper x = new HashWrapper(value_id);
if ( !values_found_set.contains(x)){
if ( obs_value != null && ! obs_recurse ){
// we have read the marker value, now issue a direct read with the
// real key
if ( Arrays.equals( obs_value, value_bytes )){
more_to_come = true;
final DHTTransportReplyHandlerAdapter f_outer = this;
f_closest.sendFindValue(
new DHTTransportReplyHandlerAdapter()
{
public void
findValueReply(
DHTTransportContact contact,
DHTTransportValue[] values,
byte diversification_type,
boolean more_to_come )
{
if ( diversification_type == DHT.DT_NONE ){
f_outer.findValueReply( contact, values, (byte)99, false );
}
}
public void
failed(
DHTTransportContact contact,
Throwable error )
{
f_outer.failed( contact, error );
}
},
_lookup_id, 1, flags );
break;
}
}else{
new_values++;
values_found_set.add(x);
handler.read(contact, values[i]);
}
}
}
}
try
{
contacts_to_query_mon.enter();
if (!more_to_come)
value_replies++;
values_found += new_values;
} finally
{
contacts_to_query_mon.exit();
}
} finally
{
if (!more_to_come)
{
try
{
contacts_to_query_mon.enter();
active_searches--;
} finally
{
contacts_to_query_mon.exit();
}
release();
}
}
}
public void findValueReply(DHTTransportContact contact, DHTTransportContact[] contacts) {
findNodeReply(contact, contacts);
}
public void failed(DHTTransportContact target_contact, Throwable error) {
try
{
// if at least one reply has been received then we
// don't treat subsequent failure as indication of
// a contact failure (just packet loss)
if (!value_reply_received)
{
if ( DHTLog.isOn()){
DHTLog.log("findNode/findValue " + DHTLog.getString(target_contact) + " -> failed: " + error.getMessage());
}
router.contactDead(target_contact.getID(), false);
}
} finally
{
try
{
contacts_to_query_mon.enter();
active_searches--;
} finally
{
contacts_to_query_mon.exit();
}
release();
}
}
public void keyBlockRequest(DHTTransportContact contact, byte[] request, byte[] key_signature) {
// we don't want to kill the contact due to this so indicate that
// it is ok by setting the flag
if (database.keyBlockRequest(null, request, key_signature) != null)
key_blocked = true;
}
};
router.recordLookup(lookup_id);
if (value_search)
{
int rem = max_values - values_found;
if (rem <= 0)
{
Debug.out("eh?");
rem = 1;
}
closest.sendFindValue(replyHandler, lookup_id, rem, flags);
} else
{
closest.sendFindNode(replyHandler, lookup_id);
}
} finally
{
contacts_to_query_mon.exit();
}
}
if(terminate){
terminateLookup(false);
}else if ( cancelled ){
terminateLookup( true );
}
} catch (Throwable e) {
Debug.printStackTrace(e);
terminateLookup(true);
}
}
public byte[] getTarget() {
return (lookup_id);
}
public String getDescription() {
return (description);
}
};
thread_pool.run( task, high_priority, true);
return( task );
}
// Request methods
public void
pingRequest(
DHTTransportContact originating_contact )
{
if ( DHTLog.isOn()){
DHTLog.log( "pingRequest from " + DHTLog.getString( originating_contact.getID()));
}
router.contactAlive( originating_contact.getID(), new DHTControlContactImpl(originating_contact));
}
public void
keyBlockRequest(
DHTTransportContact originating_contact,
byte[] request,
byte[] sig )
{
if ( DHTLog.isOn()){
DHTLog.log( "keyBlockRequest from " + DHTLog.getString( originating_contact.getID()));
}
router.contactAlive( originating_contact.getID(), new DHTControlContactImpl(originating_contact));
database.keyBlockRequest( originating_contact, request, sig );
}
public DHTTransportStoreReply
storeRequest(
DHTTransportContact originating_contact,
byte[][] keys,
DHTTransportValue[][] value_sets )
{
byte[] originator_id = originating_contact.getID();
router.contactAlive( originator_id, new DHTControlContactImpl(originating_contact));
if ( DHTLog.isOn()){
DHTLog.log( "storeRequest from " + DHTLog.getString( originating_contact )+ ", keys = " + keys.length );
}
byte[] diverse_res = new byte[ keys.length];
Arrays.fill( diverse_res, DHT.DT_NONE );
if ( keys.length != value_sets.length ){
Debug.out( "DHTControl:storeRequest - invalid request received from " + originating_contact.getString() + ", keys and values length mismatch");
return( new DHTTransportStoreReplyImpl( diverse_res ));
}
DHTStorageBlock blocked_details = null;
// System.out.println( "storeRequest: received " + originating_contact.getRandomID() + " from " + originating_contact.getAddress());
//System.out.println( "store request: keys=" + keys.length );
if ( keys.length > 0 ){
boolean cache_forward = false;
for ( DHTTransportValue[] values: value_sets ){
for ( DHTTransportValue value: values ){
if ( !Arrays.equals( originator_id, value.getOriginator().getID())){
cache_forward = true;
break;
}
}
if ( cache_forward ){
break;
}
}
// don't start accepting cache forwards until we have a good idea of our
// acceptable key space
if ( cache_forward && !isSeeded()){
//System.out.println( "not seeded" );
if ( DHTLog.isOn()){
DHTLog.log( "Not storing keys as not yet seeded" );
}
}else if ( !verifyContact( originating_contact, !cache_forward )){
//System.out.println( "verification fail" );
logger.log( "Verification of contact '" + originating_contact.getName() + "' failed for store operation" );
}else{
// get the closest contacts to me
byte[] my_id = local_contact.getID();
int c_factor = router.getK();
DHTStorageAdapter sad = adapter.getStorageAdapter();
if ( sad != null && sad.getNetwork() != DHT.NW_CVS ){
c_factor += ( c_factor/2 );
}
boolean store_it = true;
if ( cache_forward ){
long now = SystemTime.getMonotonousTime();
if ( now - rbs_time < 10*1000 && Arrays.equals( originator_id, rbs_id )){
// System.out.println( "contact too far away - repeat" );
store_it = false;
}else{
// make sure the originator is in our group
List<DHTTransportContact>closest_contacts = getClosestContactsList( my_id, c_factor, true );
DHTTransportContact furthest = closest_contacts.get( closest_contacts.size()-1);
if ( computeAndCompareDistances( furthest.getID(), originator_id, my_id ) < 0 ){
rbs_id = originator_id;
rbs_time = now;
// System.out.println( "contact too far away" );
if ( DHTLog.isOn()){
DHTLog.log( "Not storing keys as cache forward and sender too far away" );
}
store_it = false;
}
}
}
if ( store_it ){
for (int i=0;i<keys.length;i++){
byte[] key = keys[i];
HashWrapper hw_key = new HashWrapper( key );
DHTTransportValue[] values = value_sets[i];
if ( DHTLog.isOn()){
DHTLog.log( " key=" + DHTLog.getString(key) + ", value=" + DHTLog.getString(values));
}
// make sure the key isn't too far away from us
if ( !( database.hasKey( hw_key ) ||
isIDInClosestContacts( my_id, key, c_factor, true ))){
// System.out.println( "key too far away" );
if ( DHTLog.isOn()){
DHTLog.log( "Not storing keys as cache forward and sender too far away" );
}
}else{
diverse_res[i] = database.store( originating_contact, hw_key, values );
if ( blocked_details == null ){
blocked_details = database.getKeyBlockDetails( key );
}
}
}
}
}
}
// fortunately we can get away with this as diversifications are only taken note of by initial, single value stores
// and not by the multi-value cache forwards...
if ( blocked_details == null ){
return( new DHTTransportStoreReplyImpl( diverse_res ));
}else{
return( new DHTTransportStoreReplyImpl( blocked_details.getRequest(), blocked_details.getCertificate()));
}
}
public DHTTransportQueryStoreReply
queryStoreRequest(
DHTTransportContact originating_contact,
int header_len,
List<Object[]> keys )
{
router.contactAlive( originating_contact.getID(), new DHTControlContactImpl(originating_contact));
if ( DHTLog.isOn()){
DHTLog.log( "queryStoreRequest from " + DHTLog.getString( originating_contact )+ ", header_len=" + header_len + ", keys=" + keys.size());
}
int rand = generateSpoofID( originating_contact );
originating_contact.setRandomID( rand );
return( database.queryStore( originating_contact, header_len, keys ));
}
public DHTTransportContact[]
findNodeRequest(
DHTTransportContact originating_contact,
byte[] id )
{
if ( DHTLog.isOn()){
DHTLog.log( "findNodeRequest from " + DHTLog.getString( originating_contact.getID()));
}
router.contactAlive( originating_contact.getID(), new DHTControlContactImpl(originating_contact));
List l;
if ( id.length == router.getID().length ){
l = getClosestKContactsList( id, false );
}else{
// this helps both protect against idiot queries and also saved bytes when we use findNode
// to just get a random ID prior to cache-forwards
l = new ArrayList();
}
final DHTTransportContact[] res = new DHTTransportContact[l.size()];
l.toArray( res );
int rand = generateSpoofID( originating_contact );
originating_contact.setRandomID( rand );
return( res );
}
public DHTTransportFindValueReply
findValueRequest(
DHTTransportContact originating_contact,
byte[] key,
int max_values,
byte flags )
{
if ( DHTLog.isOn()){
DHTLog.log( "findValueRequest from " + DHTLog.getString( originating_contact.getID()));
}
DHTDBLookupResult result = database.get( originating_contact, new HashWrapper( key ), max_values, flags, true );
if ( result != null ){
router.contactAlive( originating_contact.getID(), new DHTControlContactImpl(originating_contact));
DHTStorageBlock block_details = database.getKeyBlockDetails( key );
if ( block_details == null ){
return( new DHTTransportFindValueReplyImpl( result.getDiversificationType(), result.getValues()));
}else{
return( new DHTTransportFindValueReplyImpl( block_details.getRequest(), block_details.getCertificate()));
}
}else{
return( new DHTTransportFindValueReplyImpl( findNodeRequest( originating_contact, key )));
}
}
public DHTTransportFullStats
statsRequest(
DHTTransportContact contact )
{
return( stats );
}
protected void
requestPing(
DHTRouterContact contact )
{
((DHTControlContactImpl)contact.getAttachment()).getTransportContact().sendPing(
new DHTTransportReplyHandlerAdapter()
{
public void
pingReply(
DHTTransportContact _contact )
{
if ( DHTLog.isOn()){
DHTLog.log( "ping OK " + DHTLog.getString( _contact ));
}
router.contactAlive( _contact.getID(), new DHTControlContactImpl(_contact));
}
public void
failed(
DHTTransportContact _contact,
Throwable _error )
{
if ( DHTLog.isOn()){
DHTLog.log( "ping " + DHTLog.getString( _contact ) + " -> failed: " + _error.getMessage());
}
router.contactDead( _contact.getID(), false );
}
});
}
protected void
nodeAddedToRouter(
DHTRouterContact new_contact )
{
if ( DISABLE_REPLICATE_ON_JOIN ){
if ( !new_contact.hasBeenAlive()){
requestPing( new_contact );
}
return;
}
// ignore ourselves
if ( router.isID( new_contact.getID())){
return;
}
// when a new node is added we must check to see if we need to transfer
// any of our values to it.
Map keys_to_store = new HashMap();
DHTStorageBlock[] direct_key_blocks = database.getDirectKeyBlocks();
if ( database.isEmpty() && direct_key_blocks.length == 0 ){
// nothing to do, ping it if it isn't known to be alive
if ( !new_contact.hasBeenAlive()){
requestPing( new_contact );
}
return;
}
// see if we're one of the K closest to the new node
// optimise to avoid calculating for things obviously too far away
boolean perform_closeness_check = true;
byte[] router_id = router.getID();
byte[] contact_id = new_contact.getID();
byte[] distance = computeDistance( router_id, contact_id );
long now = SystemTime.getCurrentTime();
byte[] nacul = node_add_check_uninteresting_limit;
// time limit to pick up router changes caused by contacts being deleted
if ( now - last_node_add_check < 30*1000 && nacul != null ){
int res = compareDistances( nacul, distance );
/*
System.out.println(
"r=" + ByteFormatter.encodeString( router_id ) +
",c=" + ByteFormatter.encodeString( contact_id ) +
",d=" + ByteFormatter.encodeString( distance ) +
",l=" + ByteFormatter.encodeString( nacul ) +
",r=" + res );
*/
if ( res < 0 ){
perform_closeness_check = false;
}
}else{
last_node_add_check = now;
node_add_check_uninteresting_limit = nacul = null;
}
boolean close = false;
if ( perform_closeness_check ){
List closest_contacts = getClosestKContactsList( new_contact.getID(), false );
for (int i=0;i<closest_contacts.size();i++){
if ( router.isID(((DHTTransportContact)closest_contacts.get(i)).getID())){
close = true;
break;
}
}
if ( !close ){
if ( nacul == null ){
node_add_check_uninteresting_limit = distance;
}else{
if ( compareDistances( nacul, distance ) > 0 ){
node_add_check_uninteresting_limit = distance;
}
}
}
}
if ( !close ){
if ( !new_contact.hasBeenAlive()){
requestPing( new_contact );
}
return;
}
// System.out.println( "Node added to router: id=" + ByteFormatter.encodeString( contact_id ));
// ok, we're close enough to worry about transferring values
Iterator it = database.getKeys();
while( it.hasNext()){
HashWrapper key = (HashWrapper)it.next();
byte[] encoded_key = key.getHash();
if ( database.isKeyBlocked( encoded_key )){
continue;
}
DHTDBLookupResult result = database.get( null, key, 0, (byte)0, false );
if ( result == null ){
// deleted in the meantime
continue;
}
// even if a result has been diversified we continue to maintain the base value set
// until the original publisher picks up the diversification (next publish period) and
// publishes to the correct place
DHTDBValue[] values = result.getValues();
if ( values.length == 0 ){
continue;
}
// we don't consider any cached further away than the initial location, for transfer
// however, we *do* include ones we originate as, if we're the closest, we have to
// take responsibility for xfer (as others won't)
List sorted_contacts = getClosestKContactsList( encoded_key, false );
// if we're closest to the key, or the new node is closest and
// we're second closest, then we take responsibility for storing
// the value
boolean store_it = false;
if ( sorted_contacts.size() > 0 ){
DHTTransportContact first = (DHTTransportContact)sorted_contacts.get(0);
if ( router.isID( first.getID())){
store_it = true;
}else if ( Arrays.equals( first.getID(), new_contact.getID()) && sorted_contacts.size() > 1 ){
store_it = router.isID(((DHTTransportContact)sorted_contacts.get(1)).getID());
}
}
if ( store_it ){
List values_to_store = new ArrayList(values.length);
for (int i=0;i<values.length;i++){
values_to_store.add( values[i] );
}
keys_to_store.put( key, values_to_store );
}
}
final DHTTransportContact t_contact = ((DHTControlContactImpl)new_contact.getAttachment()).getTransportContact();
final boolean[] anti_spoof_done = { false };
if ( keys_to_store.size() > 0 ){
it = keys_to_store.entrySet().iterator();
final byte[][] keys = new byte[keys_to_store.size()][];
final DHTTransportValue[][] value_sets = new DHTTransportValue[keys.length][];
int index = 0;
while( it.hasNext()){
Map.Entry entry = (Map.Entry)it.next();
HashWrapper key = (HashWrapper)entry.getKey();
List values = (List)entry.getValue();
keys[index] = key.getHash();
value_sets[index] = new DHTTransportValue[values.size()];
for (int i=0;i<values.size();i++){
value_sets[index][i] = ((DHTDBValue)values.get(i)).getValueForRelay( local_contact );
}
index++;
}
// move to anti-spoof for cache forwards. we gotta do a findNode to update the
// contact's latest random id
t_contact.sendFindNode(
new DHTTransportReplyHandlerAdapter()
{
public void
findNodeReply(
DHTTransportContact contact,
DHTTransportContact[] contacts )
{
// System.out.println( "nodeAdded: pre-store findNode OK" );
anti_spoof_done[0] = true;
t_contact.sendStore(
new DHTTransportReplyHandlerAdapter()
{
public void
storeReply(
DHTTransportContact _contact,
byte[] _diversifications )
{
// System.out.println( "nodeAdded: store OK" );
// don't consider diversifications for node additions as they're not interested
// in getting values from us, they need to get them from nodes 'near' to the
// diversification targets or the originator
if ( DHTLog.isOn()){
DHTLog.log( "add store ok" );
}
router.contactAlive( _contact.getID(), new DHTControlContactImpl(_contact));
}
public void
failed(
DHTTransportContact _contact,
Throwable _error )
{
// System.out.println( "nodeAdded: store Failed" );
if ( DHTLog.isOn()){
DHTLog.log( "add store failed " + DHTLog.getString( _contact ) + " -> failed: " + _error.getMessage());
}
router.contactDead( _contact.getID(), false);
}
public void
keyBlockRequest(
DHTTransportContact contact,
byte[] request,
byte[] signature )
{
database.keyBlockRequest( null, request, signature );
}
},
keys,
value_sets,
false );
}
public void
failed(
DHTTransportContact _contact,
Throwable _error )
{
// System.out.println( "nodeAdded: pre-store findNode Failed" );
if ( DHTLog.isOn()){
DHTLog.log( "pre-store findNode failed " + DHTLog.getString( _contact ) + " -> failed: " + _error.getMessage());
}
router.contactDead( _contact.getID(), false);
}
},
t_contact.getProtocolVersion() >= DHTTransportUDP.PROTOCOL_VERSION_ANTI_SPOOF2?new byte[0]:new byte[20] );
}else{
if ( !new_contact.hasBeenAlive()){
requestPing( new_contact );
}
}
// finally transfer any key-blocks
if ( t_contact.getProtocolVersion() >= DHTTransportUDP.PROTOCOL_VERSION_BLOCK_KEYS ){
for (int i=0;i<direct_key_blocks.length;i++){
final DHTStorageBlock key_block = direct_key_blocks[i];
List contacts = getClosestKContactsList( key_block.getKey(), false );
boolean forward_it = false;
// ensure that the key is close enough to us
for (int j=0;j<contacts.size();j++){
final DHTTransportContact contact = (DHTTransportContact)contacts.get(j);
if ( router.isID( contact.getID())){
forward_it = true;
break;
}
}
if ( !forward_it || key_block.hasBeenSentTo( t_contact )){
continue;
}
final Runnable task =
new Runnable()
{
public void
run()
{
t_contact.sendKeyBlock(
new DHTTransportReplyHandlerAdapter()
{
public void
keyBlockReply(
DHTTransportContact _contact )
{
if ( DHTLog.isOn()){
DHTLog.log( "key block forward ok " + DHTLog.getString( _contact ));
}
key_block.sentTo( _contact );
}
public void
failed(
DHTTransportContact _contact,
Throwable _error )
{
if ( DHTLog.isOn()){
DHTLog.log( "key block forward failed " + DHTLog.getString( _contact ) + " -> failed: " + _error.getMessage());
}
}
},
key_block.getRequest(),
key_block.getCertificate());
}
};
if ( anti_spoof_done[0] ){
task.run();
}else{
t_contact.sendFindNode(
new DHTTransportReplyHandlerAdapter()
{
public void
findNodeReply(
DHTTransportContact contact,
DHTTransportContact[] contacts )
{
task.run();
}
public void
failed(
DHTTransportContact _contact,
Throwable _error )
{
// System.out.println( "nodeAdded: pre-store findNode Failed" );
if ( DHTLog.isOn()){
DHTLog.log( "pre-kb findNode failed " + DHTLog.getString( _contact ) + " -> failed: " + _error.getMessage());
}
router.contactDead( _contact.getID(), false);
}
},
t_contact.getProtocolVersion() >= DHTTransportUDP.PROTOCOL_VERSION_ANTI_SPOOF2?new byte[0]:new byte[20] );
}
}
}
}
protected Set<DHTTransportContact>
getClosestContactsSet(
byte[] id,
int num_to_return,
boolean live_only )
{
List<DHTRouterContact> l = router.findClosestContacts( id, num_to_return, live_only );
Set<DHTTransportContact> sorted_set = new sortedTransportContactSet( id, true ).getSet();
// profilers says l.size() is taking CPU (!) so put it into a variable
// this is safe since the list returned is created for us only
long size = l.size();
for (int i=0;i<size;i++){
sorted_set.add(((DHTControlContactImpl)((DHTRouterContact)l.get(i)).getAttachment()).getTransportContact());
}
return( sorted_set );
}
public List<DHTTransportContact>
getClosestKContactsList(
byte[] id,
boolean live_only )
{
return( getClosestContactsList( id, K, live_only ));
}
public List<DHTTransportContact>
getClosestContactsList(
byte[] id,
int num_to_return,
boolean live_only )
{
Set<DHTTransportContact> sorted_set = getClosestContactsSet( id, num_to_return, live_only );
List<DHTTransportContact> res = new ArrayList<DHTTransportContact>(num_to_return);
Iterator<DHTTransportContact> it = sorted_set.iterator();
while( it.hasNext() && res.size() < num_to_return ){
res.add( it.next());
}
return( res );
}
protected boolean
isIDInClosestContacts(
byte[] test_id,
byte[] target_id,
int num_to_consider,
boolean live_only )
{
List<DHTRouterContact> l = router.findClosestContacts( target_id, num_to_consider, live_only );
boolean found = false;
int num_closer = 0;
for ( DHTRouterContact c: l ){
byte[] c_id = c.getID();
if ( Arrays.equals( test_id, c_id )){
found = true;
}else{
if ( computeAndCompareDistances( c_id, test_id, target_id ) < 0 ){
num_closer++;
}
}
}
return( found && num_closer < num_to_consider );
}
protected byte[]
encodeKey(
byte[] key )
{
byte[] temp = new SHA1Simple().calculateHash( key );
byte[] result = new byte[node_id_byte_count];
System.arraycopy( temp, 0, result, 0, node_id_byte_count );
return( result );
}
public int
computeAndCompareDistances(
byte[] t1,
byte[] t2,
byte[] pivot )
{
return( computeAndCompareDistances2( t1, t2, pivot ));
}
protected static int
computeAndCompareDistances2(
byte[] t1,
byte[] t2,
byte[] pivot )
{
for (int i=0;i<t1.length;i++){
byte d1 = (byte)( t1[i] ^ pivot[i] );
byte d2 = (byte)( t2[i] ^ pivot[i] );
int diff = (d1&0xff) - (d2&0xff);
if ( diff != 0 ){
return( diff );
}
}
return( 0 );
}
public byte[]
computeDistance(
byte[] n1,
byte[] n2 )
{
return( computeDistance2( n1, n2 ));
}
protected static byte[]
computeDistance2(
byte[] n1,
byte[] n2 )
{
byte[] res = new byte[n1.length];
for (int i=0;i<res.length;i++){
res[i] = (byte)( n1[i] ^ n2[i] );
}
return( res );
}
/**
* -ve -> n1 < n2
* @param n1
* @param n2
* @return
*/
public int
compareDistances(
byte[] n1,
byte[] n2 )
{
return( compareDistances2( n1,n2 ));
}
protected static int
compareDistances2(
byte[] n1,
byte[] n2 )
{
for (int i=0;i<n1.length;i++){
int diff = (n1[i]&0xff) - (n2[i]&0xff);
if ( diff != 0 ){
return( diff );
}
}
return( 0 );
}
public void
addListener(
DHTControlListener l )
{
try{
activity_mon.enter();
listeners.addListener( l );
for (int i=0;i<activities.size();i++){
listeners.dispatch( DHTControlListener.CT_ADDED, activities.get(i));
}
}finally{
activity_mon.exit();
}
}
public void
removeListener(
DHTControlListener l )
{
listeners.removeListener( l );
}
public DHTControlActivity[]
getActivities()
{
List res;
try{
activity_mon.enter();
res = new ArrayList( activities );
}finally{
activity_mon.exit();
}
DHTControlActivity[] x = new DHTControlActivity[res.size()];
res.toArray( x );
return( x );
}
public void
setTransportEstimatedDHTSize(
int size )
{
if ( size > 0 ){
try{
estimate_mon.enter();
remote_estimate_values.add( new Integer( size ));
if ( remote_estimate_values.size() > REMOTE_ESTIMATE_HISTORY ){
remote_estimate_values.remove(0);
}
}finally{
estimate_mon.exit();
}
}
// System.out.println( "estimated dht size: " + size );
}
public int
getTransportEstimatedDHTSize()
{
return((int)local_dht_estimate );
}
public int
getEstimatedDHTSize()
{
// public method, trigger actual computation periodically
long now = SystemTime.getCurrentTime();
long diff = now - last_dht_estimate_time;
if ( diff < 0 || diff > 60*1000 ){
estimateDHTSize( router.getID(), null, router.getK());
}
// with recent changes we pretty much have a router that only contains routeable contacts
// therefore the apparent size of the DHT is less than real and we need to adjust by the
// routeable percentage to get an accurate figure
int percent = transport.getStats().getRouteablePercentage();
// current assumption is that around 50% are firewalled, so if less (at least during migration) assume unusable
if ( percent < 25 ){
return((int)combined_dht_estimate );
}
double mult = 100.0 / percent;
return((int)( mult * combined_dht_estimate ));
}
protected void
estimateDHTSize(
byte[] id,
Map contacts,
int contacts_to_use )
{
// if called with contacts then this is in internal estimation based on lookup values
long now = SystemTime.getCurrentTime();
long diff = now - last_dht_estimate_time;
// 5 second limiter here
if ( diff < 0 || diff > 5*1000 ){
try{
estimate_mon.enter();
last_dht_estimate_time = now;
List l;
if ( contacts == null ){
l = getClosestKContactsList( id, false );
}else{
Set sorted_set = new sortedTransportContactSet( id, true ).getSet();
sorted_set.addAll( contacts.values());
l = new ArrayList( sorted_set );
if ( l.size() > 0 ){
// algorithm works relative to a starting point in the ID space so we grab
// the first here rather than using the initial lookup target
id = ((DHTTransportContact)l.get(0)).getID();
}
/*
String str = "";
for (int i=0;i<l.size();i++){
str += (i==0?"":",") + DHTLog.getString2( ((DHTTransportContact)l.get(i)).getID());
}
System.out.println( "trace: " + str );
*/
}
// can't estimate with less than 2
if ( l.size() > 2 ){
/*
<Gudy> if you call N0 yourself, N1 the nearest peer, N2 the 2nd nearest peer ... Np the pth nearest peer that you know (for example, N1 .. N20)
<Gudy> and if you call D1 the Kad distance between you and N1, D2 between you and N2 ...
<Gudy> then you have to compute :
<Gudy> Dc = sum(i * Di) / sum( i * i)
<Gudy> and then :
<Gudy> NbPeers = 2^160 / Dc
*/
BigInteger sum1 = new BigInteger("0");
BigInteger sum2 = new BigInteger("0");
// first entry should be us
for (int i=1;i<Math.min( l.size(), contacts_to_use );i++){
DHTTransportContact node = (DHTTransportContact)l.get(i);
byte[] dist = computeDistance( id, node.getID());
BigInteger b_dist = IDToBigInteger( dist );
BigInteger b_i = new BigInteger(""+i);
sum1 = sum1.add( b_i.multiply(b_dist));
sum2 = sum2.add( b_i.multiply( b_i ));
}
byte[] max = new byte[id.length+1];
max[0] = 0x01;
long this_estimate;
if ( sum1.compareTo( new BigInteger("0")) == 0 ){
this_estimate = 0;
}else{
this_estimate = IDToBigInteger(max).multiply( sum2 ).divide( sum1 ).longValue();
}
// there's always us!!!!
if ( this_estimate < 1 ){
this_estimate = 1;
}
local_estimate_values.put( new HashWrapper( id ), new Long( this_estimate ));
long new_estimate = 0;
Iterator it = local_estimate_values.values().iterator();
String sizes = "";
while( it.hasNext()){
long estimate = ((Long)it.next()).longValue();
sizes += (sizes.length()==0?"":",") + estimate;
new_estimate += estimate;
}
local_dht_estimate = new_estimate/local_estimate_values.size();
// System.out.println( "getEstimatedDHTSize: " + sizes + "->" + dht_estimate + " (id=" + DHTLog.getString2(id) + ",cont=" + (contacts==null?"null":(""+contacts.size())) + ",use=" + contacts_to_use );
}
List rems = new ArrayList(new TreeSet( remote_estimate_values ));
// ignore largest and smallest few values
long rem_average = local_dht_estimate;
int rem_vals = 1;
for (int i=3;i<rems.size()-3;i++){
rem_average += ((Integer)rems.get(i)).intValue();
rem_vals++;
}
combined_dht_estimate = rem_average / rem_vals;
long test_val = 10;
int test_mag = 1;
while( test_val < combined_dht_estimate ){
test_val *= 10;
test_mag++;
}
combined_dht_estimate_mag = test_mag+1;
// System.out.println( "estimateDHTSize: loc =" + local_dht_estimate + ", comb = " + combined_dht_estimate + " [" + remote_estimate_values.size() + "]");
}finally{
estimate_mon.exit();
}
}
}
protected BigInteger
IDToBigInteger(
byte[] data )
{
StringBuilder str_key = new StringBuilder( data.length*2 );
for (int i=0;i<data.length;i++){
String hex = Integer.toHexString( data[i]&0xff );
if ( hex.length() < 2 ){
str_key.append( "0" );
}
str_key.append( hex );
}
BigInteger res = new BigInteger( str_key.toString(), 16 );
return( res );
}
protected int
generateSpoofID(
DHTTransportContact contact )
{
if ( spoof_cipher == null ){
return( 0 );
}
try{
spoof_mon.enter();
// during cache forwarding we get a lot of consecutive requests from the
// same contact so we can save CPU by caching the latest result and optimising for this
if ( contact == spoof_last_verify_contact ){
return( spoof_last_verify_result );
}
spoof_cipher.init(Cipher.ENCRYPT_MODE, spoof_key );
byte[] address = contact.getAddress().getAddress().getAddress();
byte[] data_out = spoof_cipher.doFinal( address );
int res = (data_out[0]<<24)&0xff000000 |
(data_out[1] << 16)&0x00ff0000 |
(data_out[2] << 8)&0x0000ff00 |
data_out[3]&0x000000ff;
// System.out.println( "anti-spoof: generating " + res + " for " + contact.getAddress());
spoof_last_verify_contact = contact;
spoof_last_verify_result = res;
return( res );
}catch( Throwable e ){
logger.log(e);
}finally{
spoof_mon.exit();
}
return( 0 );
}
public boolean
verifyContact(
DHTTransportContact c,
boolean direct )
{
boolean ok = c.getRandomID() == generateSpoofID( c );
if ( DHTLog.CONTACT_VERIFY_TRACE ){
System.out.println( " net " + transport.getNetwork() +"," + (direct?"direct":"indirect") + " verify for " + c.getName() + " -> " + ok + ", version = " + c.getProtocolVersion());
}
return( ok );
}
public List
getContacts()
{
List contacts = router.getAllContacts();
List res = new ArrayList( contacts.size());
for (int i=0;i<contacts.size();i++){
DHTRouterContact rc = (DHTRouterContact)contacts.get(i);
res.add( rc.getAttachment());
}
return( res );
}
public void
pingAll()
{
List contacts = router.getAllContacts();
final AESemaphore sem = new AESemaphore( "pingAll", 32 );
final int[] results = {0,0};
for (int i=0;i<contacts.size();i++){
sem.reserve();
DHTRouterContact rc = (DHTRouterContact)contacts.get(i);
((DHTControlContactImpl)rc.getAttachment()).getTransportContact().sendPing(
new DHTTransportReplyHandlerAdapter()
{
public void
pingReply(
DHTTransportContact _contact )
{
results[0]++;
print();
sem.release();
}
public void
failed(
DHTTransportContact _contact,
Throwable _error )
{
results[1]++;
print();
sem.release();
}
protected void
print()
{
System.out.println( "ok=" + results[0] + ",bad=" + results[1] );
}
});
}
}
public void
print(
boolean full )
{
DHTNetworkPosition[] nps = transport.getLocalContact().getNetworkPositions();
String np_str = "";
for (int j=0;j<nps.length;j++){
np_str += (j==0?"":",") + nps[j];
}
logger.log( "DHT Details: external IP = " + transport.getLocalContact().getAddress() +
", network = " + transport.getNetwork() +
", protocol = V" + transport.getProtocolVersion() +
", nps = " + np_str );
router.print();
database.print( full );
/*
List c = getContacts();
for (int i=0;i<c.size();i++){
DHTControlContact cc = (DHTControlContact)c.get(i);
System.out.println( " " + cc.getTransportContact().getVivaldiPosition());
}
*/
}
protected static class
sortedTransportContactSet
{
private TreeSet<DHTTransportContact> tree_set;
private byte[] pivot;
private boolean ascending;
protected
sortedTransportContactSet(
byte[] _pivot,
boolean _ascending )
{
pivot = _pivot;
ascending = _ascending;
tree_set = new TreeSet<DHTTransportContact>(
new Comparator<DHTTransportContact>()
{
public int
compare(
DHTTransportContact t1,
DHTTransportContact t2 )
{
// this comparator ensures that the closest to the key
// is first in the iterator traversal
int distance = computeAndCompareDistances2( t1.getID(), t2.getID(), pivot );
if ( ascending ){
return( distance );
}else{
return( -distance );
}
}
});
}
public Set<DHTTransportContact>
getSet()
{
return( tree_set );
}
}
protected static class
DHTOperationListenerDemuxer
implements DHTOperationListener
{
private AEMonitor this_mon = new AEMonitor( "DHTOperationListenerDemuxer" );
private DHTOperationListener delegate;
private boolean complete_fired;
private boolean complete_included_ok;
private int complete_count = 0;
protected
DHTOperationListenerDemuxer(
DHTOperationListener _delegate )
{
delegate = _delegate;
if ( delegate == null ){
Debug.out( "invalid: null delegate" );
}
}
public void
incrementCompletes()
{
try{
this_mon.enter();
complete_count++;
}finally{
this_mon.exit();
}
}
public void
searching(
DHTTransportContact contact,
int level,
int active_searches )
{
delegate.searching( contact, level, active_searches );
}
public void
diversified(
String desc )
{
delegate.diversified( desc );
}
public void
found(
DHTTransportContact contact,
boolean is_closest )
{
delegate.found( contact, is_closest );
}
public void
read(
DHTTransportContact contact,
DHTTransportValue value )
{
delegate.read( contact, value );
}
public void
wrote(
DHTTransportContact contact,
DHTTransportValue value )
{
delegate.wrote( contact, value );
}
public void
complete(
boolean timeout )
{
boolean fire = false;
try{
this_mon.enter();
if ( !timeout ){
complete_included_ok = true;
}
complete_count--;
if (complete_count <= 0 && !complete_fired ){
complete_fired = true;
fire = true;
}
}finally{
this_mon.exit();
}
if ( fire ){
delegate.complete( !complete_included_ok );
}
}
}
abstract static class
lookupResultHandler
extends DHTOperationListenerDemuxer
{
protected
lookupResultHandler(
DHTOperationListener delegate )
{
super( delegate );
}
public abstract void
closest(
List res );
public abstract void
diversify(
DHTTransportContact cause,
byte diversification_type );
}
protected static class
DHTTransportFindValueReplyImpl
implements DHTTransportFindValueReply
{
private byte dt = DHT.DT_NONE;
private DHTTransportValue[] values;
private DHTTransportContact[] contacts;
private byte[] blocked_key;
private byte[] blocked_sig;
protected
DHTTransportFindValueReplyImpl(
byte _dt,
DHTTransportValue[] _values )
{
dt = _dt;
values = _values;
boolean copied = false;
for (int i=0;i<values.length;i++){
DHTTransportValue value = values[i];
if ( ( value.getFlags() & DHT.FLAG_ANON ) != 0 ){
if ( !copied ){
values = new DHTTransportValue[ _values.length ];
System.arraycopy( _values, 0, values, 0, values.length );
copied = true;
}
values[i] = new anonValue( value );
}
}
}
protected
DHTTransportFindValueReplyImpl(
DHTTransportContact[] _contacts )
{
contacts = _contacts;
}
protected
DHTTransportFindValueReplyImpl(
byte[] _blocked_key,
byte[] _blocked_sig )
{
blocked_key = _blocked_key;
blocked_sig = _blocked_sig;
}
public byte
getDiversificationType()
{
return( dt );
}
public boolean
hit()
{
return( values != null );
}
public boolean
blocked()
{
return( blocked_key != null );
}
public DHTTransportValue[]
getValues()
{
return( values );
}
public DHTTransportContact[]
getContacts()
{
return( contacts );
}
public byte[]
getBlockedKey()
{
return( blocked_key );
}
public byte[]
getBlockedSignature()
{
return( blocked_sig );
}
}
protected static class
anonValue
implements DHTTransportValue
{
private DHTTransportValue delegate;
protected
anonValue(
DHTTransportValue v )
{
delegate = v;
}
public boolean
isLocal()
{
return( delegate.isLocal());
}
public long
getCreationTime()
{
return( delegate.getCreationTime());
}
public byte[]
getValue()
{
return( delegate.getValue());
}
public int
getVersion()
{
return( delegate.getVersion());
}
public DHTTransportContact
getOriginator()
{
return( new anonContact( delegate.getOriginator()));
}
public int
getFlags()
{
return( delegate.getFlags());
}
public int
getLifeTimeHours()
{
return( delegate.getLifeTimeHours());
}
public byte
getReplicationControl()
{
return( delegate.getReplicationControl());
}
public byte
getReplicationFactor()
{
return( delegate.getReplicationFactor());
}
public byte
getReplicationFrequencyHours()
{
return( delegate.getReplicationFrequencyHours());
}
public String
getString()
{
return( delegate.getString());
}
}
protected static class
anonContact
implements DHTTransportContact
{
private static InetSocketAddress anon_address;
static{
try{
anon_address = new InetSocketAddress( InetAddress.getByName( "0.0.0.0" ), 0);
}catch( Throwable e ){
Debug.printStackTrace(e);
}
}
private DHTTransportContact delegate;
protected
anonContact(
DHTTransportContact c )
{
delegate = c;
}
public int
getMaxFailForLiveCount()
{
return( delegate.getMaxFailForLiveCount());
}
public int
getMaxFailForUnknownCount()
{
return( delegate.getMaxFailForUnknownCount());
}
public int
getInstanceID()
{
return( delegate.getInstanceID());
}
public byte[]
getID()
{
Debug.out( "hmm" );
return( delegate.getID());
}
public byte
getProtocolVersion()
{
return( delegate.getProtocolVersion());
}
public long
getClockSkew()
{
return( delegate.getClockSkew());
}
public void
setRandomID(
int id )
{
delegate.setRandomID( id );
}
public int
getRandomID()
{
return( delegate.getRandomID());
}
public String
getName()
{
return( delegate.getName());
}
public InetSocketAddress
getAddress()
{
return( anon_address );
}
public InetSocketAddress
getExternalAddress()
{
return( getAddress());
}
public boolean
isAlive(
long timeout )
{
return( delegate.isAlive( timeout ));
}
public void
isAlive(
DHTTransportReplyHandler handler,
long timeout )
{
delegate.isAlive( handler, timeout );
}
public boolean
isValid()
{
return( delegate.isValid());
}
public void
sendPing(
DHTTransportReplyHandler handler )
{
delegate.sendPing(handler);
}
public void
sendImmediatePing(
DHTTransportReplyHandler handler,
long timeout )
{
delegate.sendImmediatePing(handler, timeout);
}
public void
sendStats(
DHTTransportReplyHandler handler )
{
delegate.sendStats(handler);
}
public void
sendStore(
DHTTransportReplyHandler handler,
byte[][] keys,
DHTTransportValue[][] value_sets,
boolean immediate )
{
delegate.sendStore(handler, keys, value_sets, immediate);
}
public void
sendQueryStore(
DHTTransportReplyHandler handler,
int header_length,
List<Object[]> key_details )
{
delegate.sendQueryStore( handler, header_length, key_details );
}
public void
sendFindNode(
DHTTransportReplyHandler handler,
byte[] id )
{
delegate.sendFindNode(handler, id);
}
public void
sendFindValue(
DHTTransportReplyHandler handler,
byte[] key,
int max_values,
byte flags )
{
delegate.sendFindValue(handler, key, max_values, flags);
}
public void
sendKeyBlock(
DHTTransportReplyHandler handler,
byte[] key_block_request,
byte[] key_block_signature )
{
delegate.sendKeyBlock(handler, key_block_request, key_block_signature);
}
public DHTTransportFullStats
getStats()
{
return( delegate.getStats());
}
public void
exportContact(
DataOutputStream os )
throws IOException, DHTTransportException
{
delegate.exportContact( os );
}
public void
remove()
{
delegate.remove();
}
public void
createNetworkPositions(
boolean is_local)
{
delegate.createNetworkPositions(is_local);
}
public DHTNetworkPosition[]
getNetworkPositions()
{
return( delegate.getNetworkPositions());
}
public DHTNetworkPosition
getNetworkPosition(
byte position_type )
{
return( delegate.getNetworkPosition( position_type ));
}
public DHTTransport
getTransport()
{
return( delegate.getTransport());
}
public String
getString()
{
return( delegate.getString());
}
}
protected static class
DHTTransportStoreReplyImpl
implements DHTTransportStoreReply
{
private byte[] divs;
private byte[] block_request;
private byte[] block_sig;
protected
DHTTransportStoreReplyImpl(
byte[] _divs )
{
divs = _divs;
}
protected
DHTTransportStoreReplyImpl(
byte[] _bk,
byte[] _bs )
{
block_request = _bk;
block_sig = _bs;
}
public byte[]
getDiversificationTypes()
{
return( divs );
}
public boolean
blocked()
{
return( block_request != null );
}
public byte[]
getBlockRequest()
{
return( block_request );
}
public byte[]
getBlockSignature()
{
return( block_sig );
}
}
protected abstract class
DhtTask
extends ThreadPoolTask
{
private controlActivity activity;
protected
DhtTask(
ThreadPool thread_pool )
{
activity = new controlActivity( thread_pool, this );
try{
activity_mon.enter();
activities.add( activity );
listeners.dispatch( DHTControlListener.CT_ADDED, activity );
// System.out.println( "activity added:" + activities.size());
}finally{
activity_mon.exit();
}
}
public void
taskStarted()
{
listeners.dispatch( DHTControlListener.CT_CHANGED, activity );
//System.out.println( "activity changed:" + activities.size());
}
public void
taskCompleted()
{
try{
activity_mon.enter();
activities.remove( activity );
listeners.dispatch( DHTControlListener.CT_REMOVED, activity );
// System.out.println( "activity removed:" + activities.size());
}finally{
activity_mon.exit();
}
}
public void
interruptTask()
{
}
protected abstract void
cancel();
public abstract byte[]
getTarget();
public abstract String
getDescription();
}
protected static class
DhtTaskSet
{
private boolean cancelled;
private Object things;
private void
add(
DhtTask task )
{
synchronized( this ){
if ( cancelled ){
task.cancel();
return;
}
addToThings( task );
}
}
private void
add(
DhtTaskSet task_set )
{
synchronized( this ){
if ( cancelled ){
task_set.cancel();
return;
}
addToThings( task_set );
}
}
private void
addToThings(
Object obj )
{
if ( things == null ){
things = obj;
}else{
if ( things instanceof List ){
((List)things).add( obj );
}else{
List l = new ArrayList(2);
l.add( things );
l.add( obj );
things = l;
}
}
}
private void
cancel()
{
Object to_cancel;
synchronized( this ){
if ( cancelled ){
return;
}
cancelled = true;
to_cancel = things;
things = null;
}
if ( to_cancel != null ){
if ( to_cancel instanceof DhtTask ){
((DhtTask)to_cancel).cancel();
}else if ( to_cancel instanceof DhtTaskSet ){
((DhtTaskSet)to_cancel).cancel();
}else{
List l = (List)to_cancel;
for (int i=0;i<l.size();i++){
Object o = l.get(i);
if ( o instanceof DhtTask ){
((DhtTask)o).cancel();
}else{
((DhtTaskSet)o).cancel();
}
}
}
}
}
private boolean
isCancelled()
{
synchronized( this ){
return( cancelled);
}
}
}
protected class
controlActivity
implements DHTControlActivity
{
protected ThreadPool tp;
protected DhtTask task;
protected int type;
protected
controlActivity(
ThreadPool _tp,
DhtTask _task )
{
tp = _tp;
task = _task;
if ( _tp == internal_lookup_pool ){
type = DHTControlActivity.AT_INTERNAL_GET;
}else if ( _tp == external_lookup_pool ){
type = DHTControlActivity.AT_EXTERNAL_GET;
}else if ( _tp == internal_put_pool ){
type = DHTControlActivity.AT_INTERNAL_PUT;
}else{
type = DHTControlActivity.AT_EXTERNAL_PUT;
}
}
public byte[]
getTarget()
{
return( task.getTarget());
}
public String
getDescription()
{
return( task.getDescription());
}
public int
getType()
{
return( type );
}
public boolean
isQueued()
{
return( tp.isQueued( task ));
}
public String
getString()
{
return( type + ":" + DHTLog.getString( getTarget()) + "/" + getDescription() + ", q = " + isQueued());
}
}
}