/**
* BlockRankCollector
* Copyright 2011 by Michael Christen
* First released 18.05.2011 at http://yacy.net
*
* $LastChangedDate: 2011-04-26 19:39:16 +0200 (Di, 26 Apr 2011) $
* $LastChangedRevision: 7676 $
* $LastChangedBy: orbiter $
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program in the file lgpl21.txt
* If not, see <http://www.gnu.org/licenses/>.
*/
package de.anomic.search;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import net.yacy.cora.document.ASCII;
import net.yacy.cora.storage.OrderedScoreMap;
import net.yacy.cora.storage.ScoreMap;
import net.yacy.kelondro.index.BinSearch;
import net.yacy.kelondro.index.RowSpaceExceededException;
import net.yacy.kelondro.logging.Log;
import net.yacy.kelondro.order.Base64Order;
import net.yacy.kelondro.order.Digest;
import net.yacy.kelondro.rwi.ReferenceContainer;
import net.yacy.kelondro.rwi.ReferenceContainerCache;
import net.yacy.kelondro.rwi.ReferenceIterator;
import net.yacy.kelondro.util.FileUtils;
import de.anomic.search.MetadataRepository.HostStat;
import de.anomic.yacy.yacyClient;
import de.anomic.yacy.yacySeed;
import de.anomic.yacy.yacySeedDB;
import de.anomic.yacy.graphics.WebStructureGraph;
import de.anomic.yacy.graphics.WebStructureGraph.HostReference;
public class BlockRank {
public static BinSearch[] ybrTables = null; // block-rank tables
/**
* collect host index information from other peers. All peers in the seed database are asked
* this may take some time; please wait up to one minute
* @param seeds
* @return a merged host index from all peers
*/
public static ReferenceContainerCache<HostReference> collect(final yacySeedDB seeds, final WebStructureGraph myGraph) {
ReferenceContainerCache<HostReference> index = new ReferenceContainerCache<HostReference>(WebStructureGraph.hostReferenceFactory, Base64Order.enhancedCoder, 6);
// start all jobs
Iterator<yacySeed> si = seeds.seedsConnected(true, false, null, 0.99f);
ArrayList<IndexRetrieval> jobs = new ArrayList<IndexRetrieval>();
while (si.hasNext()) {
IndexRetrieval loader = new IndexRetrieval(index, si.next());
loader.start();
jobs.add(loader);
}
// get the local index
if (myGraph != null) try {
ReferenceContainerCache<HostReference> myIndex = myGraph.incomingReferences();
Log.logInfo("BlockRank", "loaded " + myIndex.size() + " host indexes from my peer");
index.merge(myIndex);
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
// wait for termination
for (IndexRetrieval job: jobs) try { job.join(); } catch (InterruptedException e) { }
Log.logInfo("BlockRank", "create " + index.size() + " host indexes from all peers");
return index;
}
public static class IndexRetrieval extends Thread {
ReferenceContainerCache<HostReference> index;
yacySeed seed;
public IndexRetrieval(ReferenceContainerCache<HostReference> index, yacySeed seed) {
this.index = index;
this.seed = seed;
}
public void run() {
ReferenceContainerCache<HostReference> partialIndex = yacyClient.loadIDXHosts(this.seed);
if (partialIndex == null || partialIndex.size() == 0) return;
Log.logInfo("BlockRank", "loaded " + partialIndex.size() + " host indexes from peer " + this.seed.getName());
try {
this.index.merge(partialIndex);
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
}
}
/**
* save the index into a BLOB
* @param index
* @param file
*/
public static void saveHostIndex(ReferenceContainerCache<HostReference> index, File file) {
Log.logInfo("BlockRank", "saving " + index.size() + " host indexes to file " + file.toString());
index.dump(file, Segment.writeBufferSize, false);
Log.logInfo("BlockRank", "saved " + index.size() + " host indexes to file " + file.toString());
}
public static ReferenceContainerCache<HostReference> loadHostIndex(File file) {
Log.logInfo("BlockRank", "reading host indexes from file " + file.toString());
ReferenceContainerCache<HostReference> index = new ReferenceContainerCache<HostReference>(WebStructureGraph.hostReferenceFactory, Base64Order.enhancedCoder, 6);
// load from file
try {
ReferenceIterator<HostReference> ri = new ReferenceIterator<HostReference>(file, WebStructureGraph.hostReferenceFactory);
while (ri.hasNext()) {
ReferenceContainer<HostReference> references = ri.next();
index.add(references);
}
} catch (IOException e) {
Log.logException(e);
} catch (RowSpaceExceededException e) {
Log.logException(e);
}
Log.logInfo("BlockRank", "read " + index.size() + " host indexes from file " + file.toString());
return index;
}
public static BinSearch[] evaluate(final ReferenceContainerCache<HostReference> index, Map<String, HostStat> hostHashResolver, BinSearch[] referenceTable, int recusions) {
// first find out the maximum count of the hostHashResolver
int maxHostCount = 1;
for (HostStat stat: hostHashResolver.values()) {
if (stat.count > maxHostCount) maxHostCount = stat.count;
}
// then just count the number of references. all other information from the index is not used because they cannot be trusted
ScoreMap<byte[]> hostScore = new OrderedScoreMap<byte[]>(index.termKeyOrdering());
HostStat hostStat;
int hostCount;
for (ReferenceContainer<HostReference> container: index) {
if (container.size() == 0) continue;
if (referenceTable == null) {
hostStat = hostHashResolver.get(ASCII.String(container.getTermHash()));
hostCount = hostStat == null ? 6 /* high = a penalty for 'i do not know this', this may not be fair*/ : Math.max(1, hostStat.count);
hostScore.set(container.getTermHash(), container.size() * maxHostCount / hostCount);
} else {
int score = 0;
Iterator<HostReference> hri = container.entries();
HostReference hr;
while (hri.hasNext()) {
hr = hri.next();
hostStat = hostHashResolver.get(ASCII.String(hr.urlhash()));
hostCount = hostStat == null ? 6 /* high = a penalty for 'i do not know this', this may not be fair*/ : Math.max(1, hostStat.count);
score += (17 - ranking(hr.urlhash(), referenceTable)) * maxHostCount / hostCount;
}
hostScore.set(container.getTermHash(), score);
}
}
// now divide the scores into two halves until the score map is empty
List<BinSearch> table = new ArrayList<BinSearch>();
while (hostScore.size() > 10) {
List<byte[]> smallest = hostScore.lowerHalf();
if (smallest.size() == 0) break; // should never happen but this ensures termination of the loop
Log.logInfo("BlockRank", "index evaluation: computed partition of size " + smallest.size());
table.add(new BinSearch(smallest, 6));
for (byte[] host: smallest) hostScore.delete(host);
}
if (hostScore.size() > 0) {
ArrayList<byte[]> list = new ArrayList<byte[]>();
for (byte[] entry: hostScore) list.add(entry);
Log.logInfo("BlockRank", "index evaluation: computed last partition of size " + list.size());
table.add(new BinSearch(list, 6));
}
// the last table entry has now a list of host hashes that has the most references
int binTables = Math.min(16, table.size());
BinSearch[] newTables = new BinSearch[binTables];
for (int i = 0; i < binTables; i++) newTables[i] = table.get(table.size() - i - 1);
// re-use the new table for a recursion
if (recusions == 0) return newTables;
return evaluate(index, hostHashResolver, newTables, --recusions); // one recursion step
}
public static void analyse(BinSearch[] tables, final WebStructureGraph myGraph, final Map<String, HostStat> hostHash2hostName) {
byte[] hosth = new byte[6];
String hosths, hostn;
HostStat hs;
for (int ybr = 0; ybr < tables.length; ybr++) {
row: for (int i = 0; i < tables[ybr].size(); i++) {
hosth = tables[ybr].get(i, hosth);
hosths = ASCII.String(hosth);
hostn = myGraph.hostHash2hostName(hosths);
if (hostn == null) {
hs = hostHash2hostName.get(hostn);
if (hs != null) hostn = hs.hostname;
}
if (hostn == null) {
//Log.logInfo("BlockRank", "Ranking for Host: ybr = " + ybr + ", hosthash = " + hosths);
continue row;
}
Log.logInfo("BlockRank", "Ranking for Host: ybr = " + ybr + ", hosthash = " + hosths + ", hostname = " + hostn);
}
}
}
/**
* load YaCy Block Rank tables
* These tables have a very simple structure: every file is a sequence of Domain hashes, ordered by b64.
* Each Domain hash has a length of 6 bytes and there is no separation character between the hashes
* @param rankingPath
* @param count
*/
public static void loadBlockRankTable(final File rankingPath, final int count) {
if (!rankingPath.exists()) return;
ybrTables = new BinSearch[count];
String ybrName;
File f;
try {
for (int i = 0; i < count; i++) {
ybrName = "YBR-4-" + Digest.encodeHex(i, 2) + ".idx";
f = new File(rankingPath, ybrName);
if (f.exists()) {
ybrTables[i] = new BinSearch(FileUtils.read(f), 6);
} else {
ybrTables[i] = null;
}
}
} catch (final IOException e) {
}
}
public static void storeBlockRankTable(final File rankingPath) {
String ybrName;
File f;
try {
// first delete all old files
for (int i = 0; i < 16; i++) {
ybrName = "YBR-4-" + Digest.encodeHex(i, 2) + ".idx";
f = new File(rankingPath, ybrName);
if (!f.exists()) continue;
if (!f.canWrite()) return;
f.delete();
}
// write new files
for (int i = 0; i < Math.min(12, ybrTables.length); i++) {
ybrName = "YBR-4-" + Digest.encodeHex(i, 2) + ".idx";
f = new File(rankingPath, ybrName);
ybrTables[i].write(f);
}
} catch (final IOException e) {
}
}
/**
* returns the YBR ranking value in a range of 0..15, where 0 means best ranking and 15 means worst ranking
* @param hash
* @return
*/
public static int ranking(final byte[] hash) {
return ranking(hash, ybrTables);
}
public static int ranking(final byte[] hash, BinSearch[] rankingTable) {
if (rankingTable == null) return 16;
byte[] hosthash;
if (hash.length == 6) {
hosthash = hash;
} else {
hosthash = new byte[6];
System.arraycopy(hash, 6, hosthash, 0, 6);
}
final int m = Math.min(16, rankingTable.length);
for (int i = 0; i < m; i++) {
if (rankingTable[i] != null && rankingTable[i].contains(hosthash)) {
return i;
}
}
return 16;
}
}