String seedListFileURL;
DigestURI url;
Iterator<String> enu;
int lc;
final int sc = this.peers.sizeConnected();
ResponseHeader header;
final RequestHeader reqHeader = new RequestHeader();
reqHeader.put(HeaderFramework.PRAGMA, "no-cache");
reqHeader.put(HeaderFramework.CACHE_CONTROL, "no-cache");
reqHeader.put(HeaderFramework.USER_AGENT, ClientIdentification.getUserAgent());
final HTTPClient client = new HTTPClient();
client.setHeader(reqHeader.entrySet());
client.setTimout((int) getConfigLong("bootstrapLoadTimeout", 20000));
yacyCore.log.logInfo("BOOTSTRAP: " + sc + " seeds known from previous run");
// - use the superseed to further fill up the seedDB
int ssc = 0, c = 0;
while (true) {
if (Thread.currentThread().isInterrupted()) {
break;
}
seedListFileURL = sb.getConfig("network.unit.bootstrap.seedlist" + c, "");
if (seedListFileURL.length() == 0) {
break;
}
c++;
if (
seedListFileURL.startsWith("http://") ||
seedListFileURL.startsWith("https://")
) {
// load the seed list
try {
url = new DigestURI(seedListFileURL);
//final long start = System.currentTimeMillis();
client.HEADResponse(url.toString());
header = new ResponseHeader(client.getHttpResponse().getAllHeaders());
//final long loadtime = System.currentTimeMillis() - start;
/*if (header == null) {
if (loadtime > getConfigLong("bootstrapLoadTimeout", 6000)) {
yacyCore.log.logWarning("BOOTSTRAP: seed-list URL " + seedListFileURL + " not available, time-out after " + loadtime + " milliseconds");
} else {
yacyCore.log.logWarning("BOOTSTRAP: seed-list URL " + seedListFileURL + " not available, no content");
}
} else*/ if (header.lastModified() == null) {
yacyCore.log.logWarning("BOOTSTRAP: seed-list URL " + seedListFileURL + " not usable, last-modified is missing");
} else if ((header.age() > 86400000) && (ssc > 0)) {
yacyCore.log.logInfo("BOOTSTRAP: seed-list URL " + seedListFileURL + " too old (" + (header.age() / 86400000) + " days)");
} else {
ssc++;
final byte[] content = client.GETbytes(url);
enu = FileUtils.strings(content);
lc = 0;
while (enu.hasNext()) {
try {
ys = yacySeed.genRemoteSeed(enu.next(), null, false, null);
if ((ys != null) &&
(!this.peers.mySeedIsDefined() || !this.peers.mySeed().hash.equals(ys.hash))) {
final long lastseen = Math.abs((System.currentTimeMillis() - ys.getLastSeenUTC()) / 1000 / 60);
if (lastseen < 240) {
if (this.peers.peerActions.connectPeer(ys, false)) {
lc++;
}
}
}
} catch (final IOException e) {
yacyCore.log.logInfo("BOOTSTRAP: bad seed: " + e.getMessage());
}
}
yacyCore.log.logInfo("BOOTSTRAP: " + lc + " seeds from seed-list URL " + seedListFileURL + ", AGE=" + (header.age() / 3600000) + "h");
}
} catch (final IOException e) {
// this is when wget fails, commonly because of timeout
yacyCore.log.logWarning("BOOTSTRAP: failed (1) to load seeds from seed-list URL " + seedListFileURL + ": " + e.getMessage());