Examples of org.apache.lucene.store.IndexOutput

org.apache.lucene.store.IndexOutput
Abstract base class for output to a file in a Directory. A random-access output stream. Used for all Lucene index output operations. @see Directory @see IndexInput

    return s;
  }


  public void copyFile(Directory dir, String src, String dest) throws IOException {
    IndexInput in = dir.openInput(src);
    IndexOutput out = dir.createOutput(dest);
    byte[] b = new byte[1024];
    long remainder = in.length();
    while(remainder > 0) {
      int len = (int) Math.min(b.length, remainder);
      in.readBytes(b, 0, len);
      out.writeBytes(b, len);
      remainder -= len;
    }
    in.close();
    out.close();
  }

View Full Code Here

    }
    return hasVectors;
  }


  public void write(Directory d, String name) throws IOException {
    IndexOutput output = d.createOutput(name);
    try {
      write(output);
    } finally {
      output.close();
    }
  }

View Full Code Here

      return;


    if (numPostings > maxNumPostings)
      maxNumPostings = numPostings;


    final IndexOutput tvf = perThread.doc.perDocTvf;


    // This is called once, after inverting all occurrences
    // of a given field in the doc.  At this point we flush
    // our hash into the DocWriter.


    assert fieldInfo.storeTermVector;
    assert perThread.vectorFieldsInOrder(fieldInfo);


    perThread.doc.addField(termsHashPerField.fieldInfo.number);


    final RawPostingList[] postings = termsHashPerField.sortPostings();


    tvf.writeVInt(numPostings);
    byte bits = 0x0;
    if (doVectorPositions)
      bits |= TermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR;
    if (doVectorOffsets) 
      bits |= TermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR;
    tvf.writeByte(bits);


    int encoderUpto = 0;
    int lastTermBytesCount = 0;


    final ByteSliceReader reader = perThread.vectorSliceReader;
    final char[][] charBuffers = perThread.termsHashPerThread.charPool.buffers;
    for(int j=0;j<numPostings;j++) {
      final TermVectorsTermsWriter.PostingList posting = (TermVectorsTermsWriter.PostingList) postings[j];
      final int freq = posting.freq;
          
      final char[] text2 = charBuffers[posting.textStart >> DocumentsWriter.CHAR_BLOCK_SHIFT];
      final int start2 = posting.textStart & DocumentsWriter.CHAR_BLOCK_MASK;


      // We swap between two encoders to save copying
      // last Term's byte array
      final UnicodeUtil.UTF8Result utf8Result = perThread.utf8Results[encoderUpto];


      // TODO: we could do this incrementally
      UnicodeUtil.UTF16toUTF8(text2, start2, utf8Result);
      final int termBytesCount = utf8Result.length;


      // TODO: UTF16toUTF8 could tell us this prefix
      // Compute common prefix between last term and
      // this term
      int prefix = 0;
      if (j > 0) {
        final byte[] lastTermBytes = perThread.utf8Results[1-encoderUpto].result;
        final byte[] termBytes = perThread.utf8Results[encoderUpto].result;
        while(prefix < lastTermBytesCount && prefix < termBytesCount) {
          if (lastTermBytes[prefix] != termBytes[prefix])
            break;
          prefix++;
        }
      }
      encoderUpto = 1-encoderUpto;
      lastTermBytesCount = termBytesCount;


      final int suffix = termBytesCount - prefix;
      tvf.writeVInt(prefix);
      tvf.writeVInt(suffix);
      tvf.writeBytes(utf8Result.result, prefix, suffix);
      tvf.writeVInt(freq);


      if (doVectorPositions) {
        termsHashPerField.initReader(reader, posting, 0);
        reader.writeTo(tvf);
      }

View Full Code Here

            store = new JEDirectory(txn, index, blocks);


            for (int i = 0; i < count; i++) {
                String name = i + ".dat";
                int length = gen.nextInt() & LENGTH_MASK;
                IndexOutput file = store.createOutput(name);


                totalLength += length;


                for (int j = 0; j < length; j++) {
                    byte b = (byte) (gen.nextInt() & 0x7F);
                    file.writeByte(b);
                }


                file.close();
            }
        } catch (IOException e) {
            txn.abort();
            txn = null;
            throw e;
        } catch (DatabaseException e) {
            if (txn != null) {
                txn.abort();
                txn = null;
            }
            throw e;
        } finally {
            if (txn != null)
                txn.commit();


            store.close();
        }


        end = new Date();


        duration = (int) (end.getTime() - start.getTime());
        System.out.print(duration);
        System.out.print(" total milliseconds to create, ");
        System.out.print(totalLength / duration);
        System.out.println(" kb/s");


        try {
            txn = env.beginTransaction(null, null);
            store = new JEDirectory(txn, index, blocks);


            gen = new Random(seed);
            start = new Date();


            for (int i = 0; i < count; i++) {
                String name = i + ".dat";
                int length = gen.nextInt() & LENGTH_MASK;
                IndexInput file = store.openInput(name);


                if (file.length() != length)
                    throw new Exception("length incorrect");


                for (int j = 0; j < length; j++) {
                    byte b = (byte) (gen.nextInt() & 0x7F);


                    if (file.readByte() != b)
                        throw new Exception("contents incorrect");
                }


                file.close();
            }
        } catch (IOException e) {
            txn.abort();
            txn = null;
            throw e;

View Full Code Here

            store = new JEDirectory(txn, index, blocks);


            for (int i = 0; i < count; i++) {
                String name = i + ".dat";
                int length = gen.nextInt() & LENGTH_MASK;
                IndexOutput file = store.createOutput(name);


                totalLength += length;


                for (int j = 0; j < length; j++) {
                    byte b = (byte) (gen.nextInt() & 0x7F);
                    file.writeByte(b);
                }


                file.close();
            }
        } catch (IOException e) {
            txn.abort();
            txn = null;
            throw e;
        } catch (DatabaseException e) {
            if (txn != null) {
                txn.abort();
                txn = null;
            }
            throw e;
        } finally {
            if (txn != null)
                txn.commit();


            store.close();
        }


        end = new Date();


        duration = (int) (end.getTime() - start.getTime());
        System.out.print(duration);
        System.out.print(" total milliseconds to read, ");
        System.out.print(totalLength / duration);
        System.out.println(" kb/s");


        try {
            txn = env.beginTransaction(null, null);
            store = new JEDirectory(txn, index, blocks);


            gen = new Random(seed);
            start = new Date();


            for (int i = 0; i < count; i++) {
                if (i % 2 == 0) {
                    String name = i + ".dat";
                    store.deleteFile(name);
                }
            }
        } catch (IOException e) {
            txn.abort();
            txn = null;
            throw e;
        } catch (DatabaseException e) {
            if (txn != null) {
                txn.abort();
                txn = null;
            }
            throw e;
        } finally {
            if (txn != null)
                txn.commit();


            store.close();
        }


        end = new Date();


        System.out.print(end.getTime() - start.getTime());
        System.out.println(" total milliseconds to delete even files");


        duration = (int) (end.getTime() - start.getTime());
        System.out.print(duration);
        System.out.print(" total milliseconds to create, ");
        System.out.print(totalLength / duration);
        System.out.println(" kb/s");


        try {
            txn = env.beginTransaction(null, null);
            store = new JEDirectory(txn, index, blocks);


            gen = new Random(seed);
            start = new Date();


            for (int i = 0; i < count; i++) {
                int length = gen.nextInt() & LENGTH_MASK;


                if (i % 2 != 0) {
                    String name = i + ".dat";
                    IndexInput file = store.openInput(name);
                    if (file.length() != length)
                        throw new Exception("length incorrect");


                    for (int j = 0; j < length; j++) {
                        byte b = (byte) (gen.nextInt() & 0x7F);


                        if (file.readByte() != b)
                            throw new Exception("contents incorrect");
                    }


                    file.close();
                } else {
                    for (int j = 0; j < length; j++) {
                        gen.nextInt();
                    }
                }

View Full Code Here

            store = new JEDirectory(txn, index, blocks);


            for (int i = 0; i < count; i++) {
                String name = i + ".dat";
                int length = gen.nextInt() & LENGTH_MASK;
                IndexOutput file = store.createOutput(name);
                byte[] data = new byte[length];


                totalLength += length;
                gen.nextBytes(data);
                file.writeBytes(data, length);


                file.close();
            }
        } catch (IOException e) {
            txn.abort();
            txn = null;
            throw e;
        } catch (DatabaseException e) {
            if (txn != null) {
                txn.abort();
                txn = null;
            }
            throw e;
        } finally {
            if (txn != null)
                txn.commit();


            store.close();
        }


        end = new Date();


        duration = (int) (end.getTime() - start.getTime());
        System.out.print(duration);
        System.out.print(" total milliseconds to create, ");
        System.out.print(totalLength / duration);
        System.out.println(" kb/s");


        try {
            txn = env.beginTransaction(null, null);
            store = new JEDirectory(txn, index, blocks);


            gen = new Random(seed);
            start = new Date();


            for (int i = 0; i < count; i++) {
                String name = i + ".dat";
                int length = gen.nextInt() & LENGTH_MASK;
                IndexInput file = store.openInput(name);


                if (file.length() != length)
                    throw new Exception("length incorrect");


                byte[] data = new byte[length];
                byte[] read = new byte[length];
                gen.nextBytes(data);
                file.readBytes(read, 0, length);


                if (!Arrays.equals(data, read))
                    throw new Exception("contents incorrect");


                file.close();
            }
        } catch (IOException e) {
            txn.abort();
            txn = null;
            throw e;

View Full Code Here


  /** Writes this vector to the file <code>name</code> in Directory
    <code>d</code>, in a format that can be read by the constructor {@link
    #BitVector(Directory, String)}.  */
  public final void write(Directory d, String name) throws IOException {
    IndexOutput output = d.createOutput(name);
    try {
      if (isSparse()) { 
        writeDgaps(output); // sparse bit-set more efficiently saved as d-gaps.
      } else {
        writeBits(output);
      }
    } finally {
      output.close();
    }
  }

View Full Code Here

      }
    }


    final String normsFileName = state.segmentName + "." + IndexFileNames.NORMS_EXTENSION;
    state.flushedFiles.add(normsFileName);
    IndexOutput normsOut = state.directory.createOutput(normsFileName);


    try {
      normsOut.writeBytes(SegmentMerger.NORMS_HEADER, 0, SegmentMerger.NORMS_HEADER.length);


      final int numField = fieldInfos.size();


      int normCount = 0;


      for(int fieldNumber=0;fieldNumber<numField;fieldNumber++) {


        final FieldInfo fieldInfo = fieldInfos.fieldInfo(fieldNumber);


        List<NormsWriterPerField> toMerge = byField.get(fieldInfo);
        int upto = 0;
        if (toMerge != null) {


          final int numFields = toMerge.size();


          normCount++;


          final NormsWriterPerField[] fields = new NormsWriterPerField[numFields];
          int[] uptos = new int[numFields];


          for(int j=0;j<numFields;j++)
            fields[j] = toMerge.get(j);


          int numLeft = numFields;
              
          while(numLeft > 0) {


            assert uptos[0] < fields[0].docIDs.length : " uptos[0]=" + uptos[0] + " len=" + (fields[0].docIDs.length);


            int minLoc = 0;
            int minDocID = fields[0].docIDs[uptos[0]];


            for(int j=1;j<numLeft;j++) {
              final int docID = fields[j].docIDs[uptos[j]];
              if (docID < minDocID) {
                minDocID = docID;
                minLoc = j;
              }
            }


            assert minDocID < state.numDocs;


            // Fill hole
            for(;upto<minDocID;upto++)
              normsOut.writeByte(defaultNorm);


            normsOut.writeByte(fields[minLoc].norms[uptos[minLoc]]);
            (uptos[minLoc])++;
            upto++;


            if (uptos[minLoc] == fields[minLoc].upto) {
              fields[minLoc].reset();
              if (minLoc != numLeft-1) {
                fields[minLoc] = fields[numLeft-1];
                uptos[minLoc] = uptos[numLeft-1];
              }
              numLeft--;
            }
          }
          
          // Fill final hole with defaultNorm
          for(;upto<state.numDocs;upto++)
            normsOut.writeByte(defaultNorm);
        } else if (fieldInfo.isIndexed && !fieldInfo.omitNorms) {
          normCount++;
          // Fill entire field with default norm:
          for(;upto<state.numDocs;upto++)
            normsOut.writeByte(defaultNorm);
        }


        assert 4+normCount*state.numDocs == normsOut.getFilePointer() : ".nrm file size mismatch: expected=" + (4+normCount*state.numDocs) + " actual=" + normsOut.getFilePointer();
      }


    } finally {
      normsOut.close();
    }
  }

View Full Code Here

    return s;
  }


  public void copyFile(Directory dir, String src, String dest) throws IOException {
    IndexInput in = dir.openInput(src);
    IndexOutput out = dir.createOutput(dest);
    byte[] b = new byte[1024];
    long remainder = in.length();
    while(remainder > 0) {
      int len = (int) Math.min(b.length, remainder);
      in.readBytes(b, 0, len);
      out.writeBytes(b, len);
      remainder -= len;
    }
    in.close();
    out.close();
  }

View Full Code Here

    public void reWrite(SegmentInfo si) throws IOException {
      assert refCount > 0 && (origNorm == null || origNorm.refCount > 0): "refCount=" + refCount + " origNorm=" + origNorm;


      // NOTE: norms are re-written in regular directory, not cfs
      si.advanceNormGen(this.number);
      IndexOutput out = directory().createOutput(si.getNormFileName(this.number));
      try {
        out.writeBytes(bytes, maxDoc());
      } finally {
        out.close();
      }
      this.dirty = false;
    }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.lucene.store.IndexOutput

com.bah.lucene.blockcache.BlockDirectory

com.bah.lucene.blockcache_v2.CacheDirectory

org.apache.blur.lucene.warmup.IndexWarmup

org.apache.blur.mapreduce.BlurReducer

org.apache.blur.store.BaseDirectoryTestSuite

org.apache.blur.store.blockcache.BlockDirectory

org.apache.blur.store.blockcache.BlockDirectoryTest

org.apache.blur.store.blockcache_v2.CacheDirectoryTest

org.apache.blur.store.blockcache_v2.CacheIndexInputTest

org.apache.blur.store.blockcache_v2.CacheIndexOutputTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.