{
BufferedReader bufReader = new BufferedReader(new InputStreamReader(
new FileInputStream(filepath)), 16 * 1024 * 1024);
String line = null;
String delimiter_ = new String(",");
RowMutation rm = null;
Map<String, RowMutation> rms = new HashMap<String, RowMutation>();
if(importer_.columnFamily.delimiter != null)
{
delimiter_ = importer_.columnFamily.delimiter;
}
while ((line = bufReader.readLine()) != null)
{
StringTokenizer st = new StringTokenizer(line, delimiter_);
List<String> tokenList = new ArrayList<String>();
String key = null;
while (st.hasMoreElements())
{
tokenList.add((String)st.nextElement());
}
/* Construct the Key */
List<String> keyFields = new ArrayList<String> ();
for(int fieldId: importer_.key.fields.field)
{
keyFields.add(tokenList.get(fieldId));
}
key = merge(keyFields, importer_.key.combiner);
if(importer_.key.optimizeIt != null && !importer_.key.optimizeIt)
{
if(!checkIfProcessKey(key))
{
continue;
}
}
rm = rms.get(key);
if( rm == null)
{
rm = new RowMutation(importer_.table, key);
rms.put(key, rm);
}
if(importer_.columnFamily.superColumn != null)
{
List<String> superColumnList = new ArrayList<String>();
for(int fieldId : importer_.columnFamily.superColumn.fields.field)
{
superColumnList.add(tokenList.get(fieldId));
}
String superColumnName = merge(superColumnList, " ");
superColumnList.clear();
if(importer_.columnFamily.superColumn.tokenize)
{
Analyzer analyzer = new StandardAnalyzer();
TokenStream ts = analyzer.tokenStream("superColumn", new StringReader(superColumnName));
Token token = null;
token = ts.next();
while(token != null)
{
superColumnList.add(token.termText());
token = ts.next();
}
}
else
{
superColumnList.add(superColumnName);
}
for(String sName : superColumnList)
{
String cfName = importer_.columnFamily.name + ":" + sName;
if(importer_.columnFamily.column != null)
{
for(ColumnType column : importer_.columnFamily.column )
{
String cfColumn = cfName +":" + (column.name == null ? tokenList.get(column.field):column.name);
rm.add(cfColumn, tokenList.get(column.value.field).getBytes(), Integer.parseInt(tokenList.get(column.timestamp.field)));
}
}
}
}
else
{
if(importer_.columnFamily.column != null)
{
for(ColumnType column : importer_.columnFamily.column )
{
String cfColumn = importer_.columnFamily.name +":" + (column.name == null ? tokenList.get(column.field):column.name);
rm.add(cfColumn, tokenList.get(column.value.field).getBytes(), Integer.parseInt(tokenList.get(column.timestamp.field)));
}
}
}
}
// Now apply the data for all keys
// TODO : Add checks for large data
// size maybe we want to check the
// data size and then apply.
Set<String> keys = rms.keySet();
for(String pKey : keys)
{
rm = rms.get(pKey);
if( rm != null)
{
rm.apply();
}
}
}