if (!file.exists()) {
System.err.println("Error: " + file + " does not exist!");
System.exit(-1);
}
StatusStream stream = new JsonStatusCorpusReader(file);
Directory dir = FSDirectory.open(new File(indexPath));
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, IndexStatuses.ANALYZER);
config.setOpenMode(OpenMode.CREATE);
IndexWriter writer = new IndexWriter(dir, config);
int cnt = 0;
Status status;
try {
while ((status = stream.next()) != null) {
if (status.getText() == null) {
continue;
}
// Skip deletes tweetids.
if (deletes != null && deletes.contains(status.getId())) {
continue;
}
if (status.getId() > maxId) {
continue;
}
cnt++;
Document doc = new Document();
doc.add(new LongField(StatusField.ID.name, status.getId(), Field.Store.YES));
doc.add(new LongField(StatusField.EPOCH.name, status.getEpoch(), Field.Store.YES));
doc.add(new TextField(StatusField.SCREEN_NAME.name, status.getScreenname(), Store.YES));
doc.add(new Field(StatusField.TEXT.name, status.getText(), textOptions));
doc.add(new IntField(StatusField.FRIENDS_COUNT.name, status.getFollowersCount(), Store.YES));
doc.add(new IntField(StatusField.FOLLOWERS_COUNT.name, status.getFriendsCount(), Store.YES));
doc.add(new IntField(StatusField.STATUSES_COUNT.name, status.getStatusesCount(), Store.YES));
long inReplyToStatusId = status.getInReplyToStatusId();
if (inReplyToStatusId > 0) {
doc.add(new LongField(StatusField.IN_REPLY_TO_STATUS_ID.name, inReplyToStatusId, Field.Store.YES));
doc.add(new LongField(StatusField.IN_REPLY_TO_USER_ID.name, status.getInReplyToUserId(), Field.Store.YES));
}
String lang = status.getLang();
if (!lang.equals("unknown")) {
doc.add(new TextField(StatusField.LANG.name, status.getLang(), Store.YES));
}
long retweetStatusId = status.getRetweetedStatusId();
if (retweetStatusId > 0) {
doc.add(new LongField(StatusField.RETWEETED_STATUS_ID.name, retweetStatusId, Field.Store.YES));
doc.add(new LongField(StatusField.RETWEETED_USER_ID.name, status.getRetweetedUserId(), Field.Store.YES));
doc.add(new IntField(StatusField.RETWEET_COUNT.name, status.getRetweetCount(), Store.YES));
if ( status.getRetweetCount() < 0 || status.getRetweetedStatusId() < 0) {
LOG.warn("Error parsing retweet fields of " + status.getId());
}
}
writer.addDocument(doc);
if (cnt % 100000 == 0) {
LOG.info(cnt + " statuses indexed");
}
}
LOG.info(String.format("Total of %s statuses added", cnt));
if (cmdline.hasOption(OPTIMIZE_OPTION)) {
LOG.info("Merging segments...");
writer.forceMerge(1);
LOG.info("Done!");
}
LOG.info("Total elapsed time: " + (System.currentTimeMillis() - startTime) + "ms");
} catch (Exception e) {
e.printStackTrace();
} finally {
writer.close();
dir.close();
stream.close();
}
}