Package com.alimama.mdrill.solr.hbaserealtime

Examples of com.alimama.mdrill.solr.hbaserealtime.DirectoryInfo

577
578
579
580
581
582
583
584
585
586
587
          File f = new File(s1);
          if (!f.exists()) {
            continue;
          }
          FSDirectory d = LinkFSDirectory.open(f,true);
          DirectoryInfo info = new DirectoryInfo();
          info.d = d;
          info.tp = DirectoryInfo.DirTpe.file;
          this.diskDirector.put(s1, info);
          SolrCore.log.info(">>>>> add links "+ s1);
        }
View Full Code Here
213
214
215
216
217
218
219
220
221
222
223
                    if(!f.isDir())
                    {
                      continue;
                    }
                    try{
                    DirectoryInfo d=new DirectoryInfo();
                    File findex=new File(f.getPath().toString());
                    d.d=FSDirectory.open(findex);
                    d.tp=DirectoryInfo.DirTpe.file;
                    data.AddIndex(findex.getAbsolutePath(), d); 
                    needPurge=true;
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
  public List<Directory> copyForSearch() {
    List<Directory> rtn = new ArrayList<Directory>();
    for (Entry<String, DirectoryInfo> e : diskDirector.entrySet()) {
      rtn.add(e.getValue().d);
    }
    DirectoryInfo ram = this.RamDirector.get(RAM_KEY);
    if (ram != null) {
      rtn.add(ram.d);
    }
    LOG.info("####remakeSearch####" + rtn.toString());
    return rtn;
View Full Code Here
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
  }
 
  public void mergerBuffer(DirectoryInfo expire)
      throws CorruptIndexException, IOException {
    LOG.info("####mergerBuffer start####");
    DirectoryInfo m1 = expire;
    if (m1 == null) {
      m1 = this.bufferDirector.remove(RAM_KEY);
    }
   
    if (m1 == null) {
      return;
    }
   
    try {
      DirectoryInfo d = null;

      DirectoryInfo m2 = RamDirector.get(RAM_KEY);
      if (m2 == null) {
        d = m1;
        d.UpTxid(m1.readTxid());
        d.tp = DirectoryInfo.DirTpe.ram;
      } else {
        d = new DirectoryInfo();
        d.createtime = m2.createtime;
        d.d = new RAMDirectory();
        d.tp = DirectoryInfo.DirTpe.ram;
        ArrayList<DirectoryInfo> merger=new ArrayList<DirectoryInfo>();
        merger.add(m1);
View Full Code Here
269
270
271
272
273
274
275
276
277
278
279
          Pair p=list.get(i);
          pairlist.add(p);
          dinfolist.add(p.value);
        }
       
        DirectoryInfo d=new DirectoryInfo();
        String path=this.mallocPath();
        d.d=LinkFSDirectory.open(new File(path),true);
        d.tp=DirectoryInfo.DirTpe.file;
        this.merger(dinfolist, d);
       
View Full Code Here
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
  }

  public void mergerRam(DirectoryInfo expire) throws CorruptIndexException, IOException
  {
    LOG.info("####mergerRam start####");
    DirectoryInfo m1 = expire;
    if (m1 == null) {
      m1 = RamDirector.remove(RAM_KEY);
    }
    if (m1 == null) {
      return;
    }
    try{
     
      ArrayList<Pair> pairlist=new ArrayList<Pair>();
      ArrayList<DirectoryInfo> dinfolist=new ArrayList<DirectoryInfo>();
   
      if(this.diskDirector.size()>UniqConfig.RealTimeMaxIndexCount())
      {
        ArrayList<Pair> list=this.getSortList();
        for(int i=0;i<list.size()&&i<UniqConfig.RealTimeMergeFactor();i++)
        {
          Pair p=list.get(i);
          pairlist.add(p);
          dinfolist.add(p.value);
        }
      }
     
      DirectoryInfo d=new DirectoryInfo();
      String path=this.mallocPath();
      d.d=LinkFSDirectory.open(new File(path),true);
      d.tp=DirectoryInfo.DirTpe.file;
      dinfolist.add(m1);
      this.merger(dinfolist, d);
View Full Code Here
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
  }
 

  public void maybeMerger()
  {
    DirectoryInfo buffer=bufferDirector.get(RAM_KEY);
    if(buffer!=null)
    {
      if(buffer.filelength()>=UniqConfig.RealTimeBufferSize()||(buffer.createtime+bufferdely)<System.currentTimeMillis())
      {
        try {
          this.mergerBuffer(null);
        } catch (Throwable e) {
          LOG.error("mayBeMerger_buffer",e);
        }
      }
    }
   
   
    DirectoryInfo ram=this.RamDirector.get(RAM_KEY);
    if(ram!=null)
    {
      if((debugIndex++)%10==1)
      {
        LOG.info("####ramsize####"+ram.filelength()+","+UniqConfig.RealTimeRamSize()+","+debugIndex);
        if(debugIndex>100000000)
        {
          debugIndex=0;
        }
      }

      if(ram.filelength()>=UniqConfig.RealTimeRamSize()||(ram.createtime+ramdelay)<System.currentTimeMillis())
      {
        try {
          this.mergerRam(null);
        } catch (Throwable e) {
          LOG.error("mayBeMerger_ram",e);
View Full Code Here
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
  /**
   * 将内存中的数据flush到硬盘上
   */
  public void flushToDisk()
  {
    DirectoryInfo buffer=bufferDirector.get(RAM_KEY);
    if(buffer!=null)
    {
      try {
        this.mergerBuffer(null);
      } catch (Throwable e) {
        LOG.error("mayBeMerger_buffer",e);
      }
    }
   
   
   
    DirectoryInfo ram=this.RamDirector.get(RAM_KEY);
    if(ram!=null)
    {
      try {
        this.mergerRam(null);
      } catch (Throwable e) {
View Full Code Here
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
 
  public void flushDocList(ArrayList<SolrInputDocument> flush)
      throws CorruptIndexException, LockObtainFailedException,
      IOException {
    LOG.info("flushDocList start");
    DirectoryInfo buffer = bufferDirector.get(RAM_KEY);
    if (buffer == null) {
      buffer = new DirectoryInfo();
      buffer.d = new RAMDirectory();
      buffer.tp = DirectoryInfo.DirTpe.buffer;
      LOG.info("####create buffer####");
      bufferDirector.put(RAM_KEY, buffer);
    }

    buffer.d.setSchema(params.core.getSchema());
    IndexWriter writer = new IndexWriter(buffer.d, null,
        new KeepOnlyLastCommitDeletionPolicy(),
        MaxFieldLength.UNLIMITED);
    writer.setMergeFactor(10);
    writer.setUseCompoundFile(false);

    ArrayList<Document> doclist = new ArrayList<Document>(flush.size());
    long txid = 0;
    for (SolrInputDocument sdoc : flush) {
      txid = Math.max(txid, sdoc.getTxid());
      sdoc.remove("mdrillPartion");
      sdoc.remove("mdrillCmd");
      sdoc.remove("mdrill_uuid");
     
     
      Document lucenedoc = DocumentBuilder.toDocument(sdoc,
          params.core.getSchema());
      doclist.add(lucenedoc);
    }
    writer.addDocuments(doclist);
    writer.close();
    buffer.UpTxid(txid);
    LOG.info("flushDocList end");
    this.maybeMerger();
  }
View Full Code Here
TOP

Related Classes of com.alimama.mdrill.solr.hbaserealtime.DirectoryInfo

  • com.alimama.mdrill.solr.hbaserealtime.realtime.RealTimeDirectoryData
  • com.alimama.mdrill.solr.hbaserealtime.realtime.RealTimeDirectoryThread
  • org.apache.lucene.store.IndexInput
  • org.apache.lucene.store.IndexOutput
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.