Examples of readCollectionName()

Examples of ivory.core.RetrievalEnvironment.readCollectionName()

240
241
242
243
244
245
246
247
248
249
250
    FileSystem fs = FileSystem.get(conf);

    String indexPath = conf.get(Constants.IndexPath);
    RetrievalEnvironment env = new RetrievalEnvironment(indexPath, fs);

    String collectionName = env.readCollectionName();

    int reduceTasks = conf.getInt(Constants.NumReduceTasks, 0);
    int minSplitSize = conf.getInt(Constants.MinSplitSize, 0);
    int collectionDocCnt = env.readCollectionDocumentCount();
View Full Code Here

Examples of ivory.core.RetrievalEnvironment.readCollectionName()

130
131
132
133
134
135
136
137
138
139
140
      throw new RuntimeException("Parameters not read properly! Quitting...");
    }
    JobConf job = new JobConf(conf, ComputeSignaturesSimhash.class);
    FileSystem fs = FileSystem.get(job);
    RetrievalEnvironment env = new RetrievalEnvironment(dir, fs);
    job.setJobName("ComputeSignatures_simhash"+"_D=64_"+env.readCollectionName());

    String inputPath = PwsimEnvironment.getFileNameWithPars(dir, "TermDocs");
    String outputPath = PwsimEnvironment.getFileNameWithPars(dir, "SignaturesSimhash");
   
    int numMappers = 300;
View Full Code Here

Examples of ivory.core.RetrievalEnvironment.readCollectionName()

64
65
66
67
68
69
70
71
72
73
74
    config.setInt("Ivory.NumMapTasks", 100);

    RetrievalEnvironment targetEnv = new RetrievalEnvironment(targetLangDir, hdfs);
    RetrievalEnvironment srcEnv = new RetrievalEnvironment(srcLangDir, hdfs);

    config.set("Ivory.CollectionName", targetEnv.readCollectionName()+"_"+srcEnv.readCollectionName());
    config.set("Ivory.IndexPath", targetLangDir);

    // collection size is the sum of the two collections' sizes
    int collSize = targetEnv.readCollectionDocumentCount()+srcEnv.readCollectionDocumentCount();
    config.setInt("Ivory.CollectionDocumentCount", collSize);
View Full Code Here

Examples of ivory.core.RetrievalEnvironment.readCollectionName()

46
47
48
49
50
51
52
53
54
55
56
    config.set("Ivory.IndexPath", dir);
    config.setInt("Ivory.NumOfBits", numOfBits);
   
    String type = (signatureType.charAt(0)+"").toUpperCase()+signatureType.substring(1, signatureType.length());    //capitalize first character
    RetrievalEnvironment env = new RetrievalEnvironment(dir, fs);
    String collName = env.readCollectionName();
    config.set("Ivory.CollectionName", collName);
   
    PwsimEnvironment.setClassTypes(config);
    int batchSize = -1;
    try {
View Full Code Here

Examples of ivory.core.RetrievalEnvironment.readCollectionName()

153
154
155
156
157
158
159
160
161
162
163
    }
    JobConf job = new JobConf(getConf(), ComputeSignaturesRandom.class);
    FileSystem fs = FileSystem.get(job);

    RetrievalEnvironment re = new RetrievalEnvironment(dir, fs);
    job.setJobName("ComputeSignatures_random_D=" + D + ":" + re.readCollectionName());

    String inputPath = PwsimEnvironment.getIntDocvectorsFile(dir, fs);
    String outputPath = PwsimEnvironment.getSignaturesDir(dir, D, "random");
    String randomVectorFile = PwsimEnvironment.getRandomVectorsDir(dir, D) + "/part-00000";
View Full Code Here

Examples of ivory.core.RetrievalEnvironment.readCollectionName()

56
57
58
59
60
61
62
63
64
65
66
    config.set("Ivory.IndexPath", dir);
    config.setInt("Ivory.NumOfBits", numOfBits);

    String type = (signatureType.charAt(0)+"").toUpperCase() + signatureType.substring(1, signatureType.length());    //capitalize first character
    RetrievalEnvironment env = new RetrievalEnvironment(dir, fs);
    String collName = env.readCollectionName();
    config.set("Ivory.CollectionName", collName);

    PwsimEnvironment.setClassTypes(signatureType, config);
    int batchSize = -1;
    if (cmdline.hasOption(BATCH_OPTION)) {
View Full Code Here

Examples of ivory.core.RetrievalEnvironment.readCollectionName()

117
118
119
120
121
122
123
124
125
126
127
      throw new RuntimeException("Parameters not read properly! Quitting...");
    }
    JobConf job = new JobConf(conf, ComputeSignaturesSimhash.class);
    FileSystem fs = FileSystem.get(job);
    RetrievalEnvironment env = new RetrievalEnvironment(dir, fs);
    job.setJobName("ComputeSignatures_simhash" + "_D=64_" + env.readCollectionName());

    String inputPath = PwsimEnvironment.getTermDocvectorsFile(dir, fs);
    String outputPath = PwsimEnvironment.getSignaturesDir(dir, 64, "simhash");
   
    int numMappers = 300;
View Full Code Here

Examples of ivory.core.RetrievalEnvironment.readCollectionName()

240
241
242
243
244
245
246
247
248
249
250
    FileSystem fs = FileSystem.get(conf);

    String indexPath = conf.get(Constants.IndexPath);
    RetrievalEnvironment env = new RetrievalEnvironment(indexPath, fs);

    String collectionName = env.readCollectionName();

    int reduceTasks = conf.getInt(Constants.NumReduceTasks, 0);
    int minSplitSize = conf.getInt(Constants.MinSplitSize, 0);
    int collectionDocCnt = env.readCollectionDocumentCount();
    //int maxHeap = conf.getInt(Constants.MaxHeap, 2048);
View Full Code Here

Examples of ivory.core.RetrievalEnvironment.readCollectionName()

68
69
70
71
72
73
74
75
76
77
78
    config.setInt("Ivory.NumMapTasks", 100);

    RetrievalEnvironment targetEnv = new RetrievalEnvironment(targetLangDir, hdfs);
    RetrievalEnvironment srcEnv = new RetrievalEnvironment(srcLangDir, hdfs);

    config.set("Ivory.CollectionName", targetEnv.readCollectionName()+"_"+srcEnv.readCollectionName());
    config.set("Ivory.IndexPath", targetLangDir);

    // collection size is the sum of the two collections' sizes
    int srcCollSize = srcEnv.readCollectionDocumentCount();
    int collSize = targetEnv.readCollectionDocumentCount()+srcCollSize;
View Full Code Here

Examples of ivory.core.RetrievalEnvironment.readCollectionName()

56
57
58
59
60
61
62
63
64
65
66
    config.set("Ivory.IndexPath", dir);
    config.setInt("Ivory.NumOfBits", numOfBits);

    String type = (signatureType.charAt(0)+"").toUpperCase() + signatureType.substring(1, signatureType.length());    //capitalize first character
    RetrievalEnvironment env = new RetrievalEnvironment(dir, fs);
    String collName = env.readCollectionName();
    config.set("Ivory.CollectionName", collName);

    PwsimEnvironment.setClassTypes(config);
    int batchSize = -1;
    if (cmdline.hasOption(BATCH_OPTION)) {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.