Package org.apache.lucene.store

Examples of org.apache.lucene.store.MMapDirectory


  boolean unmapHack;
  private int maxChunk;

  @Override
  public Directory open(String path) throws IOException {
    MMapDirectory mapDirectory = new MMapDirectory(new File(path));
    try {
      mapDirectory.setUseUnmap(unmapHack);
    } catch (Exception e) {
      log.warn("Unmap not supported on this JVM, continuing on without setting unmap", e);
    }
    mapDirectory.setMaxChunkSize(maxChunk);
    return mapDirectory;
  }
View Full Code Here


      int shardCount = _tableContext.getDescriptor().getShardCount();
      for (int i = 0; i < shardCount; i++) {
        String shardName = BlurUtil.getShardName(BlurConstants.SHARD_PREFIX, i);
        File file = new File(tableFile, shardName);
        file.mkdirs();
        MMapDirectory directory = new MMapDirectory(file);
        if (!DirectoryReader.indexExists(directory)) {
          new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer())).close();
        }
        shards.put(shardName, openIndex(table, shardName, directory));
      }
View Full Code Here

@TimeoutSuite(millis = 4 * TimeUnits.HOUR)
public class Test4GBStoredFields extends LuceneTestCase {

  @Nightly
  public void test() throws Exception {
    MockDirectoryWrapper dir = new MockDirectoryWrapper(random(), new MMapDirectory(_TestUtil.getTempDir("4GBStoredFields")));
    dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER);

    IndexWriter w = new IndexWriter(dir,
        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
        .setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH)
View Full Code Here

  public void testBigDocuments() throws IOException {
    // "big" as "much bigger than the chunk size"
    // for this test we force a FS dir
    // we can't just use newFSDirectory, because this test doesn't really index anything.
    // so if we get NRTCachingDir+SimpleText, we make massive stored fields and OOM (LUCENE-4484)
    Directory dir = new MockDirectoryWrapper(random(), new MMapDirectory(_TestUtil.getTempDir("testBigDocuments")));
    IndexWriterConfig iwConf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
    iwConf.setMaxBufferedDocs(RandomInts.randomIntBetween(random(), 2, 30));
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConf);

    if (dir instanceof MockDirectoryWrapper) {
View Full Code Here

  public void test() throws Exception {
    int[] ints = new int[7];
    IntsRef input = new IntsRef(ints, 0, ints.length);
    long seed = random().nextLong();

    Directory dir = new MMapDirectory(_TestUtil.getTempDir("2BFST"));

    for(int doPackIter=0;doPackIter<2;doPackIter++) {
      boolean doPack = doPackIter == 1;

      // Build FST w/ NoOutputs and stop when nodeCount > 2.2B
      if (!doPack) {
        System.out.println("\nTEST: 3B nodes; doPack=false output=NO_OUTPUTS");
        Outputs<Object> outputs = NoOutputs.getSingleton();
        Object NO_OUTPUT = outputs.getNoOutput();
        final Builder<Object> b = new Builder<Object>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs,
                                                      null, doPack, PackedInts.COMPACT, true, 15);

        int count = 0;
        Random r = new Random(seed);
        int[] ints2 = new int[200];
        IntsRef input2 = new IntsRef(ints2, 0, ints2.length);
        while(true) {
          //System.out.println("add: " + input + " -> " + output);
          for(int i=10;i<ints2.length;i++) {
            ints2[i] = r.nextInt(256);
          }
          b.add(input2, NO_OUTPUT);
          count++;
          if (count % 100000 == 0) {
            System.out.println(count + ": " + b.fstSizeInBytes() + " bytes; " + b.getTotStateCount() + " nodes");
          }
          if (b.getTotStateCount() > Integer.MAX_VALUE + 100L * 1024 * 1024) {
            break;
          }
          nextInput(r, ints2);
        }

        FST<Object> fst = b.finish();

        for(int verify=0;verify<2;verify++) {
          System.out.println("\nTEST: now verify [fst size=" + fst.sizeInBytes() + "; nodeCount=" + fst.getNodeCount() + "; arcCount=" + fst.getArcCount() + "]");

          Arrays.fill(ints2, 0);
          r = new Random(seed);

          for(int i=0;i<count;i++) {
            if (i % 1000000 == 0) {
              System.out.println(i + "...: ");
            }
            for(int j=10;j<ints2.length;j++) {
              ints2[j] = r.nextInt(256);
            }
            assertEquals(NO_OUTPUT, Util.get(fst, input2));
            nextInput(r, ints2);
          }

          System.out.println("\nTEST: enum all input/outputs");
          IntsRefFSTEnum<Object> fstEnum = new IntsRefFSTEnum<Object>(fst);

          Arrays.fill(ints2, 0);
          r = new Random(seed);
          int upto = 0;
          while(true) {
            IntsRefFSTEnum.InputOutput<Object> pair = fstEnum.next();
            if (pair == null) {
              break;
            }
            for(int j=10;j<ints2.length;j++) {
              ints2[j] = r.nextInt(256);
            }
            assertEquals(input2, pair.input);
            assertEquals(NO_OUTPUT, pair.output);
            upto++;
            nextInput(r, ints2);
          }
          assertEquals(count, upto);

          if (verify == 0) {
            System.out.println("\nTEST: save/load FST and re-verify");
            IndexOutput out = dir.createOutput("fst", IOContext.DEFAULT);
            fst.save(out);
            out.close();
            IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
            fst = new FST<Object>(in, outputs);
            in.close();
          } else {
            dir.deleteFile("fst");
          }
        }
      }

      // Build FST w/ ByteSequenceOutputs and stop when FST
      // size = 3GB
      {
        System.out.println("\nTEST: 3 GB size; doPack=" + doPack + " outputs=bytes");
        Outputs<BytesRef> outputs = ByteSequenceOutputs.getSingleton();
        final Builder<BytesRef> b = new Builder<BytesRef>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs,
                                                          null, doPack, PackedInts.COMPACT, true, 15);

        byte[] outputBytes = new byte[20];
        BytesRef output = new BytesRef(outputBytes);
        Arrays.fill(ints, 0);
        int count = 0;
        Random r = new Random(seed);
        while(true) {
          r.nextBytes(outputBytes);
          //System.out.println("add: " + input + " -> " + output);
          b.add(input, BytesRef.deepCopyOf(output));
          count++;
          if (count % 1000000 == 0) {
            System.out.println(count + "...: " + b.fstSizeInBytes() + " bytes");
          }
          if (b.fstSizeInBytes() > LIMIT) {
            break;
          }
          nextInput(r, ints);
        }

        FST<BytesRef> fst = b.finish();
        for(int verify=0;verify<2;verify++) {

          System.out.println("\nTEST: now verify [fst size=" + fst.sizeInBytes() + "; nodeCount=" + fst.getNodeCount() + "; arcCount=" + fst.getArcCount() + "]");

          r = new Random(seed);
          Arrays.fill(ints, 0);

          for(int i=0;i<count;i++) {
            if (i % 1000000 == 0) {
              System.out.println(i + "...: ");
            }
            r.nextBytes(outputBytes);
            assertEquals(output, Util.get(fst, input));
            nextInput(r, ints);
          }

          System.out.println("\nTEST: enum all input/outputs");
          IntsRefFSTEnum<BytesRef> fstEnum = new IntsRefFSTEnum<BytesRef>(fst);

          Arrays.fill(ints, 0);
          r = new Random(seed);
          int upto = 0;
          while(true) {
            IntsRefFSTEnum.InputOutput<BytesRef> pair = fstEnum.next();
            if (pair == null) {
              break;
            }
            assertEquals(input, pair.input);
            r.nextBytes(outputBytes);
            assertEquals(output, pair.output);
            upto++;
            nextInput(r, ints);
          }
          assertEquals(count, upto);

          if (verify == 0) {
            System.out.println("\nTEST: save/load FST and re-verify");
            IndexOutput out = dir.createOutput("fst", IOContext.DEFAULT);
            fst.save(out);
            out.close();
            IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
            fst = new FST<BytesRef>(in, outputs);
            in.close();
          } else {
            dir.deleteFile("fst");
          }
        }
      }

      // Build FST w/ PositiveIntOutputs and stop when FST
      // size = 3GB
      {
        System.out.println("\nTEST: 3 GB size; doPack=" + doPack + " outputs=long");
        Outputs<Long> outputs = PositiveIntOutputs.getSingleton();
        final Builder<Long> b = new Builder<Long>(FST.INPUT_TYPE.BYTE1, 0, 0, true, true, Integer.MAX_VALUE, outputs,
                                                  null, doPack, PackedInts.COMPACT, true, 15);

        long output = 1;

        Arrays.fill(ints, 0);
        int count = 0;
        Random r = new Random(seed);
        while(true) {
          //System.out.println("add: " + input + " -> " + output);
          b.add(input, output);
          output += 1+r.nextInt(10);
          count++;
          if (count % 1000000 == 0) {
            System.out.println(count + "...: " + b.fstSizeInBytes() + " bytes");
          }
          if (b.fstSizeInBytes() > LIMIT) {
            break;
          }
          nextInput(r, ints);
        }

        FST<Long> fst = b.finish();

        for(int verify=0;verify<2;verify++) {

          System.out.println("\nTEST: now verify [fst size=" + fst.sizeInBytes() + "; nodeCount=" + fst.getNodeCount() + "; arcCount=" + fst.getArcCount() + "]");

          Arrays.fill(ints, 0);

          output = 1;
          r = new Random(seed);
          for(int i=0;i<count;i++) {
            if (i % 1000000 == 0) {
              System.out.println(i + "...: ");
            }

            // forward lookup:
            assertEquals(output, Util.get(fst, input).longValue());
            // reverse lookup:
            assertEquals(input, Util.getByOutput(fst, output));
            output += 1 + r.nextInt(10);
            nextInput(r, ints);
          }

          System.out.println("\nTEST: enum all input/outputs");
          IntsRefFSTEnum<Long> fstEnum = new IntsRefFSTEnum<Long>(fst);

          Arrays.fill(ints, 0);
          r = new Random(seed);
          int upto = 0;
          output = 1;
          while(true) {
            IntsRefFSTEnum.InputOutput<Long> pair = fstEnum.next();
            if (pair == null) {
              break;
            }
            assertEquals(input, pair.input);
            assertEquals(output, pair.output.longValue());
            output += 1 + r.nextInt(10);
            upto++;
            nextInput(r, ints);
          }
          assertEquals(count, upto);

          if (verify == 0) {
            System.out.println("\nTEST: save/load FST and re-verify");
            IndexOutput out = dir.createOutput("fst", IOContext.DEFAULT);
            fst.save(out);
            out.close();
            IndexInput in = dir.openInput("fst", IOContext.DEFAULT);
            fst = new FST<Long>(in, outputs);
            in.close();
          } else {
            dir.deleteFile("fst");
          }
        }
      }
    }
    dir.close();
  }
View Full Code Here

    @Inject public MmapFsStore(ShardId shardId, @IndexSettings Settings indexSettings, IndexStore indexStore, ByteBufferCache byteBufferCache) throws IOException {
        super(shardId, indexSettings, indexStore);
        LockFactory lockFactory = buildLockFactory();
        File location = ((FsIndexStore) indexStore).shardIndexLocation(shardId);
        FileSystemUtils.mkdirs(location);
        this.fsDirectory = new MMapDirectory(location, lockFactory);

        boolean suggestUseCompoundFile;
        Tuple<SwitchDirectory, Boolean> switchDirectory = buildSwitchDirectoryIfNeeded(fsDirectory, byteBufferCache);
        if (switchDirectory != null) {
            suggestUseCompoundFile = DEFAULT_SUGGEST_USE_COMPOUND_FILE;
View Full Code Here

      break;
    case NIO:
      dir = new NIOFSDirectory(_location);
      break;
    case MMAP:
      dir = new MMapDirectory(_location);
      break;
    }
    log.info("created Directory: " + dir);
    return dir;
  }
View Full Code Here

      break;
    case NIO:
      dir = new NIOFSDirectory(f);
      break;
    case MMAP:
      dir = new MMapDirectory(f);
      break;
    }
    return dir;
  }
View Full Code Here

        if (_ramDir) {
          directory = new RAMDirectory();
        } else {
          File file = new File(tableFile, shardName);
          file.mkdirs();
          directory = new MMapDirectory(file);
        }
        if (!DirectoryReader.indexExists(directory)) {
          new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer())).close();
        }
        shards.put(shardName, openIndex(table, shardName, directory));
View Full Code Here

      System.exit(-1);
    }

    PrintStream out = new PrintStream(System.out, true, "UTF-8");

    Directory dir = new MMapDirectory(indexLocation);
    IndexReader reader = IndexReader.open(dir);
    IndexSearcher searcher = new IndexSearcher(reader);

    QueryParser qparser = new QueryParser(Version.LUCENE_31, IndexStatuses.StatusField.TEXT.name,
        IndexStatuses.ANALYZER);
View Full Code Here

TOP

Related Classes of org.apache.lucene.store.MMapDirectory

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.