Examples of EnwikiContentSource

org.apache.lucene.benchmark.byTask.feeds.EnwikiContentSource
A {@link ContentSource} which reads the English Wikipedia dump. You can readthe .bz2 file directly (it will be decompressed on the fly). Config properties:
- keep.image.only.docs=false|true (default true).
- docs.file=<path to the file>

Examples of org.apache.lucene.benchmark.byTask.feeds.EnwikiContentSource

  }


  public int index(File wikipediaXML, int numDocs, int batchSize) throws Exception {
    int result = 0;
    if (wikipediaXML != null && wikipediaXML.exists()) {
      EnwikiContentSource contentSource = new EnwikiContentSource();
      Properties properties = new Properties();
      //fileName = config.get("docs.file", null);
      String filePath = wikipediaXML.getAbsolutePath();
      properties.setProperty("docs.file", filePath);
      properties.setProperty("doc.maker.forever", "false");
      contentSource.setConfig(new Config(properties));
      contentSource.resetInputs();
      //docMaker.openFile();
      List<SolrInputDocument> docs = new ArrayList<SolrInputDocument>(1000);
      int i = 0;
      SolrInputDocument sDoc = null;
      long start = System.currentTimeMillis();
      try {
        DocData docData = new DocData();


        while ((docData = contentSource.getNextDocData(docData)) != null && i < numDocs) {
          int mod = i % batchSize;


          sDoc = new SolrInputDocument();
          docs.add(sDoc);
          sDoc.addField("file", filePath + "_" + i);

View Full Code Here

Examples of org.apache.lucene.benchmark.byTask.feeds.EnwikiContentSource

    properties.setProperty("docs.file", wikipedia.getAbsolutePath());
    properties.setProperty("content.source.forever", "false");
    properties.setProperty("keep.image.only.docs", String.valueOf(keepImageOnlyDocs));
    Config config = new Config(properties);


    ContentSource source = new EnwikiContentSource();
    source.setConfig(config);
    
    DocMaker docMaker = new DocMaker();
    docMaker.setConfig(config, source);
    docMaker.resetInputs();
    if (wikipedia.exists()) {

View Full Code Here

Examples of org.apache.lucene.benchmark.byTask.feeds.EnwikiContentSource

    properties.setProperty("docs.file", wikipedia.getAbsolutePath());
    properties.setProperty("content.source.forever", "false");
    properties.setProperty("keep.image.only.docs", String.valueOf(keepImageOnlyDocs));
    Config config = new Config(properties);


    ContentSource source = new EnwikiContentSource();
    source.setConfig(config);
    
    DocMaker docMaker = new DocMaker();
    docMaker.setConfig(config, source);
    docMaker.resetInputs();
    if (wikipedia.exists()) {

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.