return ingestFiles(Arrays.asList(files));
}
public HashMap<String, Change> ingestFiles(Collection<File> files)
{
Timer timer = new Timer();
Storage storage = new Storage(getStorageDirectory());
BillProcessor billProcessor = new BillProcessor();
CalendarProcessor calendarProcessor = new CalendarProcessor();
AgendaProcessor agendaProcessor = new AgendaProcessor();
TranscriptProcessor transcriptProcessor = new TranscriptProcessor();
Collections.sort((List<File>)files, new FileNameComparator());
// Process each file individually, flushing changes to storage as necessary
// Each file processor should produce a change log indicating what happened
timer.start();
for(File file : files) {
try {
logger.debug("Ingesting: "+file);
String type = file.getParentFile().getName();
if (type.equals("bills")) {
billProcessor.process(file, storage);
}
else if (type.equals("calendars")) {
calendarProcessor.process(file, storage);
}
else if (type.equals("agendas")) {
agendaProcessor.process(file, storage);
}
else if (type.equals("annotations")) {
continue;
}
else if (type.equals("transcripts")) {
transcriptProcessor.process(file, storage);
}
// To avoid memory issues, occasionally flush changes to file-system and truncate memory
if (storage.memory.size() > 4000) {
storage.flush();
storage.clear();
}
}
catch (IOException e) {
logger.error("Issue with "+file.getName(), e);
}
catch (UnmarshalException e) {
logger.error("Issue with "+file.getName(), e);
}
catch (JAXBException e) {
logger.error("Unable to parse xml "+file.getName(), e);
}
}
storage.flush();
logger.info(timer.stop()+" seconds to injest "+files.size()+" files.");
return ChangeLogger.getChangeLog();
}