private static final ExecutorService EXECUTE = ExecutorSerives.EXECUTE;
public static String addTxt(final String tableName,final String store,final String callback) throws JSONException, SQLException, IOException
{
JSONObject jsonObj = new JSONObject();
final HashMap<String,String> tableInfo=getTableInfo(tableName);
if(tableInfo==null||tableInfo.isEmpty())
{
jsonObj.put("code", "0");
jsonObj.put("message", "该表不存在");
if (callback != null && callback.length() > 0) {
return callback + "(" + jsonObj.toString() + ")";
} else {
return jsonObj.toString();
}
}
if(tableInfo.get("status").equals("INDEXING"))
{
jsonObj.put("code", "0");
jsonObj.put("message", "正在创建索引中请稍后");
if (callback != null && callback.length() > 0) {
return callback + "(" + jsonObj.toString() + ")";
} else {
return jsonObj.toString();
}
}
final Map stormconf = Utils.readStormConfig();
final Configuration conf=getConf(stormconf);
final FileSystem fs=FileSystem.get(conf);
long size=HadoopUtil.duSize(fs, new Path(store));
if(size>500l*1024*1024)
{
jsonObj.put("code", "0");
jsonObj.put("message", "上传的文件不能超过500m");
if (callback != null && callback.length() > 0) {
return callback + "(" + jsonObj.toString() + ")";
} else {
return jsonObj.toString();
}
}
TableJoin.updatePercent(tableName, "Stage-1 map = 0%, reduce = 0%", "INDEXING") ;
jsonObj.put("code", "1");
EXECUTE.submit(new Runnable() {
@Override
public void run() {
try{
SimpleDateFormat fmt = new SimpleDateFormat("yyyyMMdd");
String day = fmt.format(new Date());
if(!fs.exists(new Path(store)))
{
fs.mkdirs(new Path(store));
}
Path txtpath=new Path(tableInfo.get("txtStorePath"));
if(!fs.exists(txtpath))
{
fs.mkdirs(txtpath);
}
for( FileStatus outpath:fs.listStatus(new Path(store)))
{
if(!outpath.isDir())
{
fs.rename(outpath.getPath(), new Path(txtpath,outpath.getPath().getName()+"_"+System.currentTimeMillis()));
}
}
fs.delete(new Path(store),true);
Path basepath=new Path(MdrillService.getBasePath());
FileStatus[] tablelist=fs.listStatus(basepath);
String solrHome=tablelist[0].getPath().toString();
for(FileStatus tbl:tablelist){
if(tbl.isDir()&&fs.exists(new Path(tbl.getPath(),"solr")))
{
solrHome=tbl.getPath().toString();
}
}
HashSet<String> inputs=new HashSet<String>();
inputs.add(txtpath.getName());
TableJoin.updateKb(tableName, HadoopUtil.duSize(fs, txtpath));
String strjoins="";
if(tableInfo!=null&&tableInfo.get("joins")!=null)
{
for(String s:tableInfo.get("joins").split(","))
{
String[] cols=s.split(":");
if(cols.length>=3)
{
strjoins=cols[2];
}
}
}
String index = (String) stormconf.get("higo.download.offline.store")+ "/" + day + "/tmp_" + java.util.UUID.randomUUID().toString();
MakeIndex.make(fs, solrHome, conf, "txt", txtpath.getParent().toString(), inputs, "*", tableInfo.get("indexStorePath"), new Path(index), 1, tableInfo.get("splitString"), false, tableInfo.get("colsName"),new updateStatus() {
@Override
public void update(int statge, Job job) {
try {
TableJoin.LOG.info("update "+tableName+ ",stage:"+statge+",map:"+job.mapProgress()+",reduce:"+job.reduceProgress()+ ":INDEXING");
String percent="Stage-"+statge+" map = "+(job.mapProgress()*100)+"%, reduce = "+(job.reduceProgress()*100)+"%";
TableJoin.updatePercent(tableName, percent, "INDEXING") ;
} catch (Exception e) {
TableJoin.LOG.error("updatePercent",e);
}
}
@Override
public void finish() {
try {
TableJoin.LOG.info("update "+tableName+ ",INDEX");
TableJoin.updatePercent(tableName, "Stage-2 map = 100%, reduce = 100%", "INDEX") ;
} catch (Exception e) {
TableJoin.LOG.error("updatePercent",e);
}
}
@Override
public boolean dump(Job job) {
Counters counters;
try {
counters = job.getCounters();
long wsize=counters.findCounter("higo", "dumpcount").getValue();
if(wsize>0)
{
try {
TableJoin.LOG.info("update "+tableName+ ",dump");
TableJoin.updatePercent(tableName, "Stage-2 map = 100%, reduce = 100%", "DUMP") ;
} catch (Exception e) {
TableJoin.LOG.error("updatePercent",e);
}
return true;
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return false;
}
},strjoins,40);
}catch(Exception e){
TableJoin.LOG.error("make index",e);
try {
TableJoin.updatePercent(tableName, "Stage-2 map = 100%, reduce = 100%", "FAIL") ;
} catch (Exception e2) {
TableJoin.LOG.error("updatePercent",e2);
}
}
}
});
if (callback != null && callback.length() > 0) {
return callback + "(" + jsonObj.toString() + ")";
} else {
return jsonObj.toString();
}
}