Examples of SolrIndexer

Examples of com.hmsonline.virgil.index.SolrIndexer

33
34
35
36
37
38
39
40
41
42
43
    return options;
  }

  private CassandraStorage createCassandraStorage(CommandLine params, VirgilConfiguration config)
      throws Exception {
    SolrIndexer indexer = new SolrIndexer(config);

    if (params.hasOption("embedded")) {
      System.out.println("Starting virgil with embedded cassandra server.");
      String yamlFile = config.getCassandraYaml();
      if (yamlFile == null)
View Full Code Here

Examples of com.hmsonline.virgil.index.SolrIndexer

11
12
13
14
15
16
17
18
19
20
21
22
23
24
    private static final String COLUMN_FAMILY = "TEST_CF";
    private static final String KEY = "TEST_ROW";

  @Test
  public void testIndexing() throws Exception {
    Indexer indexer = new SolrIndexer(null);
    String json = "{\"ADDR1\":\"1234 Collin St.\",\"CITY\":\"Souderton\"}";
    indexer.index(COLUMN_FAMILY, KEY, json);

    json = "{\"ADDR1\":\"1234 Owen St.\",\"CITY\":\"Pottstown\"}";
    indexer.index(COLUMN_FAMILY, KEY + "2", json);
    // TODO: Need to figure out how to test w/o a live SOLR
    indexer.delete(COLUMN_FAMILY, KEY);
  }
View Full Code Here

Examples of org.apache.nutch.indexer.solr.SolrIndexer

140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
      linkDbTool.invert(linkDb, segments, true, true, false); // invert links

      // index, dedup & merge
      FileStatus[] fstats = fs.listStatus(segments, HadoopFSUtil.getPassDirectoriesFilter(fs));
      if (isSolrIndex) {
        SolrIndexer indexer = new SolrIndexer(conf);
        indexer.indexSolr(solrUrl, crawlDb, linkDb,
            Arrays.asList(HadoopFSUtil.getPaths(fstats)));
      }
      else {
       
        DeleteDuplicates dedup = new DeleteDuplicates(conf);       
        if(indexes != null) {
          // Delete old indexes
          if (fs.exists(indexes)) {
            LOG.info("Deleting old indexes: " + indexes);
            fs.delete(indexes, true);
          }

          // Delete old index
          if (fs.exists(index)) {
            LOG.info("Deleting old merged index: " + index);
            fs.delete(index, true);
          }
        }
       
        Indexer indexer = new Indexer(conf);
        indexer.index(indexes, crawlDb, linkDb,
            Arrays.asList(HadoopFSUtil.getPaths(fstats)));
       
        IndexMerger merger = new IndexMerger(conf);
        if(indexes != null) {
          dedup.dedup(new Path[] { indexes });
View Full Code Here

Examples of org.apache.nutch.indexer.solr.SolrIndexer

142
143
144
145
146
147
148
149
150
151
152
153
      linkDbTool.invert(linkDb, segments, true, true, false); // invert links

      if (solrUrl != null) {
        // index, dedup & merge
        FileStatus[] fstats = fs.listStatus(segments, HadoopFSUtil.getPassDirectoriesFilter(fs));
        SolrIndexer indexer = new SolrIndexer(getConf());
        indexer.indexSolr(solrUrl, crawlDb, linkDb,
          Arrays.asList(HadoopFSUtil.getPaths(fstats)));
        SolrDeleteDuplicates dedup = new SolrDeleteDuplicates();
        dedup.setConf(getConf());
        dedup.dedup(solrUrl);
      }
View Full Code Here

Examples of org.apache.nutch.indexer.solr.SolrIndexer

143
144
145
146
147
148
149
150
151
152
153
154
      linkDbTool.invert(linkDb, segments, true, true, false); // invert links

      if (solrUrl != null) {
        // index, dedup & merge
        FileStatus[] fstats = fs.listStatus(segments, HadoopFSUtil.getPassDirectoriesFilter(fs));
        SolrIndexer indexer = new SolrIndexer(getConf());
        indexer.indexSolr(solrUrl, crawlDb, linkDb,
          Arrays.asList(HadoopFSUtil.getPaths(fstats)));
        SolrDeleteDuplicates dedup = new SolrDeleteDuplicates();
        dedup.setConf(getConf());
        dedup.dedup(solrUrl);
      }
View Full Code Here

Examples of org.apache.nutch.indexer.solr.SolrIndexer

143
144
145
146
147
148
149
150
151
152
153
154
      linkDbTool.invert(linkDb, segments, true, true, false); // invert links

      if (solrUrl != null) {
        // index, dedup & merge
        FileStatus[] fstats = fs.listStatus(segments, HadoopFSUtil.getPassDirectoriesFilter(fs));
        SolrIndexer indexer = new SolrIndexer(getConf());
        indexer.indexSolr(solrUrl, crawlDb, linkDb,
          Arrays.asList(HadoopFSUtil.getPaths(fstats)));
        SolrDeleteDuplicates dedup = new SolrDeleteDuplicates();
        dedup.setConf(getConf());
        dedup.dedup(solrUrl);
      }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.