package ivory.regression.cikm2010;
import ivory.core.eval.Qrels;
import ivory.regression.GroundTruth;
import ivory.regression.GroundTruth.Metric;
import ivory.smrf.retrieval.Accumulator;
import ivory.smrf.retrieval.BatchQueryRunner;
import java.util.HashMap;
import java.util.Map;
import junit.framework.JUnit4TestAdapter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.log4j.Logger;
import org.junit.Test;
import edu.umd.cloud9.collection.DocnoMapping;
/* Note: different metrics are optimized separately */
public class Wt10g_Desc_Joint {
private static final Logger sLogger = Logger.getLogger(Wt10g_Desc_Joint.class);
private static String[] x10_rawAP = new String[] {
"501","0.4911","502","0.1483","503","0.1751","504","0.3081","505","0.3241",
"506","0.1699","507","0.0581","508","0.2180","509","0.3713","510","0.5216",
"511","0.1821","512","0.1451","513","0.2027","514","0.0822","515","0.0755",
"516","0.0945","517","0.0676","518","0.0170","519","0.3050","520","0.1654",
"521","0.0099","522","0.1813","523","0.3767","524","0.1031","525","0.0425",
"526","0.0963","527","0.0965","528","0.3805","529","0.3959","530","0.4780",
"531","0.0472","532","0.2495","533","0.2020","534","0.0064","535","0.0058",
"536","0.0505","537","0.0001","538","0.0761","539","0.0434","540","0.0269",
"541","0.0178","542","0.2026","543","0.0007","544","0.5133","545","0.3351",
"546","0.1294","547","0.0000","548","0.3333","549","0.2715","550","0.0966"};
private static String[] x15_rawAP = new String[] {
"501","0.5399","502","0.1072","503","0.1723","504","0.2712","505","0.3827",
"506","0.1277","507","0.0394","508","0.2370","509","0.3163","510","0.5278",
"511","0.2912","512","0.1872","513","0.2336","514","0.1180","515","0.1835",
"516","0.0856","517","0.0595","518","0.0770","519","0.2908","520","0.1787",
"521","0.0124","522","0.3096","523","0.3385","524","0.1858","525","0.1442",
"526","0.0881","527","0.2569","528","0.3880","529","0.3422","530","0.3757",
"531","0.0387","532","0.2610","533","0.1696","534","0.0060","535","0.0034",
"536","0.2426","537","0.0042","538","0.0549","539","0.0730","540","0.0276",
"541","0.0112","542","0.1897","543","0.0010","544","0.4984","545","0.3941",
"546","0.1382","547","0.0203","548","0.2917","549","0.3219","550","0.0701"};
private static String[] x20_rawAP = new String[] {
"501","0.5332","502","0.0766","503","0.1715","504","0.2591","505","0.4361",
"506","0.1027","507","0.0362","508","0.2492","509","0.3095","510","0.5831",
"511","0.3609","512","0.1851","513","0.2269","514","0.1267","515","0.2824",
"516","0.0639","517","0.0603","518","0.0963","519","0.2838","520","0.1590",
"521","0.0248","522","0.3071","523","0.2884","524","0.1059","525","0.1423",
"526","0.0792","527","0.2356","528","0.4535","529","0.3394","530","0.3559",
"531","0.0702","532","0.2620","533","0.1926","534","0.0042","535","0.0037",
"536","0.2425","537","0.0046","538","0.0586","539","0.0733","540","0.0269",
"541","0.2263","542","0.1706","543","0.0010","544","0.4899","545","0.4811",
"546","0.1433","547","0.1605","548","0.2429","549","0.3219","550","0.1308"};
private static String[] x25_rawAP = new String[] {
"501","0.5443","502","0.0766","503","0.1701","504","0.2290","505","0.4366",
"506","0.1027","507","0.1014","508","0.2752","509","0.3037","510","0.6097",
"511","0.3730","512","0.1880","513","0.2475","514","0.1210","515","0.2875",
"516","0.0639","517","0.0608","518","0.0805","519","0.2801","520","0.1590",
"521","0.0157","522","0.3529","523","0.2812","524","0.1378","525","0.1490",
"526","0.0880","527","0.3988","528","0.4368","529","0.2952","530","0.3628",
"531","0.0741","532","0.2880","533","0.2002","534","0.0040","535","0.0190",
"536","0.2828","537","0.0047","538","0.0448","539","0.0622","540","0.0264",
"541","0.2231","542","0.1645","543","0.0010","544","0.4893","545","0.5188",
"546","0.1759","547","0.1605","548","0.2667","549","0.3173","550","0.1264"};
private static String[] x30_rawAP = new String[] {
"501","0.5243","502","0.0950","503","0.1740","504","0.2289","505","0.4585",
"506","0.1027","507","0.1014","508","0.2575","509","0.3491","510","0.6151",
"511","0.3579","512","0.1969","513","0.2405","514","0.1284","515","0.2886",
"516","0.0639","517","0.0612","518","0.0789","519","0.2821","520","0.1720",
"521","0.0162","522","0.3523","523","0.2890","524","0.1358","525","0.1313",
"526","0.0792","527","0.3984","528","0.4368","529","0.2944","530","0.3680",
"531","0.0960","532","0.2823","533","0.2488","534","0.0048","535","0.0207",
"536","0.2501","537","0.0048","538","0.0455","539","0.0822","540","0.0221",
"541","0.0766","542","0.1551","543","0.0010","544","0.4890","545","0.5186",
"546","0.1878","547","0.1605","548","0.2667","549","0.3163","550","0.1240"};
private static String[] x35_rawAP = new String[] {
"501","0.5122","502","0.0950","503","0.1882","504","0.2285","505","0.4802",
"506","0.1027","507","0.0546","508","0.2858","509","0.3562","510","0.6199",
"511","0.4041","512","0.1859","513","0.2566","514","0.1270","515","0.2908",
"516","0.0639","517","0.0625","518","0.0776","519","0.2800","520","0.1713",
"521","0.0163","522","0.3527","523","0.2349","524","0.1416","525","0.1327",
"526","0.0862","527","0.3901","528","0.4367","529","0.2988","530","0.3111",
"531","0.1576","532","0.2654","533","0.2558","534","0.0045","535","0.0273",
"536","0.2448","537","0.0052","538","0.0488","539","0.0974","540","0.0262",
"541","0.1047","542","0.1587","543","0.0010","544","0.4866","545","0.5242",
"546","0.1743","547","0.1605","548","0.2262","549","0.3265","550","0.1233"};
private static String[] x40_rawAP = new String[] {
"501","0.5167","502","0.1154","503","0.1834","504","0.2285","505","0.4704",
"506","0.1027","507","0.0537","508","0.3026","509","0.3516","510","0.6257",
"511","0.4199","512","0.1979","513","0.2755","514","0.1313","515","0.2896",
"516","0.0640","517","0.0628","518","0.0759","519","0.2793","520","0.1650",
"521","0.0164","522","0.4358","523","0.2540","524","0.1351","525","0.1438",
"526","0.0774","527","0.3893","528","0.4784","529","0.2986","530","0.2932",
"531","0.1552","532","0.2809","533","0.2304","534","0.0044","535","0.0270",
"536","0.2481","537","0.0046","538","0.0512","539","0.0998","540","0.0223",
"541","0.2881","542","0.1778","543","0.0010","544","0.4397","545","0.5197",
"546","0.1698","547","0.1573","548","0.2429","549","0.3249","550","0.1244"};
private static String[] x45_rawAP = new String[] {
"501","0.5151","502","0.1154","503","0.1806","504","0.2262","505","0.4704",
"506","0.1027","507","0.0980","508","0.3228","509","0.3374","510","0.6213",
"511","0.4170","512","0.1979","513","0.2733","514","0.0989","515","0.3082",
"516","0.0640","517","0.0631","518","0.0791","519","0.2757","520","0.1658",
"521","0.0266","522","0.4361","523","0.2511","524","0.1375","525","0.1527",
"526","0.0774","527","0.3868","528","0.5216","529","0.2936","530","0.3285",
"531","0.1483","532","0.2679","533","0.2313","534","0.0044","535","0.0607",
"536","0.2151","537","0.0047","538","0.0500","539","0.1110","540","0.0260",
"541","0.3005","542","0.1817","543","0.0010","544","0.4392","545","0.5215",
"546","0.1708","547","0.1573","548","0.2111","549","0.3505","550","0.1309"};
private static String[] x50_rawAP = new String[] {
"501","0.4961","502","0.1154","503","0.1313","504","0.2427","505","0.4704",
"506","0.1027","507","0.0966","508","0.3070","509","0.3346","510","0.6223",
"511","0.4170","512","0.1979","513","0.2733","514","0.0995","515","0.3086",
"516","0.0640","517","0.0656","518","0.0403","519","0.2664","520","0.1667",
"521","0.0263","522","0.4165","523","0.2511","524","0.1097","525","0.1509",
"526","0.0774","527","0.3685","528","0.5216","529","0.2739","530","0.2879",
"531","0.1585","532","0.2788","533","0.2362","534","0.0048","535","0.0436",
"536","0.2536","537","0.0044","538","0.0426","539","0.1110","540","0.0209",
"541","0.3005","542","0.1873","543","0.0009","544","0.3753","545","0.5356",
"546","0.1708","547","0.1573","548","0.2111","549","0.3551","550","0.1237"};
@Test
public void runRegression() throws Exception {
Map<String, GroundTruth> g = new HashMap<String, GroundTruth>();
g.put("joint-x1.0", new GroundTruth("joint-x1.0", Metric.AP, 50, x10_rawAP, 0.1778f));
g.put("joint-x1.5", new GroundTruth("joint-x1.5", Metric.AP, 50, x15_rawAP, 0.1937f));
g.put("joint-x2.0", new GroundTruth("joint-x2.0", Metric.AP, 50, x20_rawAP, 0.2048f));
g.put("joint-x2.5", new GroundTruth("joint-x2.5", Metric.AP, 50, x25_rawAP, 0.2136f));
g.put("joint-x3.0", new GroundTruth("joint-x3.0", Metric.AP, 50, x30_rawAP, 0.2126f));
g.put("joint-x3.5", new GroundTruth("joint-x3.5", Metric.AP, 50, x35_rawAP, 0.2133f));
g.put("joint-x4.0", new GroundTruth("joint-x4.0", Metric.AP, 50, x40_rawAP, 0.2201f));
g.put("joint-x4.5", new GroundTruth("joint-x4.5", Metric.AP, 50, x45_rawAP, 0.2226f));
g.put("joint-x5.0", new GroundTruth("joint-x5.0", Metric.AP, 50, x50_rawAP, 0.2175f));
Qrels qrels = new Qrels("data/wt10g/qrels.wt10g.all");
String[] params = new String[] {
"data/wt10g/run.wt10g.CIKM2010.desc.joint.xml",
"data/wt10g/queries.wt10g.501-550.desc.xml" };
FileSystem fs = FileSystem.getLocal(new Configuration());
BatchQueryRunner qr = new BatchQueryRunner(params, fs);
long start = System.currentTimeMillis();
qr.runQueries();
long end = System.currentTimeMillis();
sLogger.info("Total query time: " + (end - start) + "ms");
DocnoMapping mapping = qr.getDocnoMapping();
for (String model : qr.getModels()) {
sLogger.info("Verifying results of model \"" + model + "\"");
Map<String, Accumulator[]> results = qr.getResults(model);
g.get(model).verify(results, mapping, qrels);
sLogger.info("Done!");
}
}
public static junit.framework.Test suite() {
return new JUnit4TestAdapter(Wt10g_Desc_Joint.class);
}
}