Package it.unimi.dsi.io

Examples of it.unimi.dsi.io.OutputBitStream

358
359
360
361
362
363
364
365
366
367
368
        }
      }
     
      // We now run through the indices used by this term and copy from the temporary buffer.

      OutputBitStream obs;
     
      for( int k = 0; k < usedIndices; k++ ) {
        final int i = usedIndex[ k ];

        localFrequencies[ i ].writeGamma( localFrequency[ i ] );
View Full Code Here
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
      invNonTermPerm = new int[ nonTermFrequency.length ];
      for( int i = invNonTermPerm.length; i-- != 0; ) invNonTermPerm[ nonTermPerm[ i ] ] = i;
    }

    File newDocumentsFile = File.createTempFile( SimpleCompressedDocumentCollection.class.getSimpleName(), "temp", new File( basename.toString() ).getParentFile() );
    OutputBitStream newDocumentsObs = new OutputBitStream( newDocumentsFile );
    documentsIbs.position( 0 );
    for( int i = (int)collection.documents; i-- != 0; ) {
      readSelfDelimitedUtf8String( documentsIbs, s ); // Skip URI
      SimpleCompressedDocumentCollectionBuilder.writeSelfDelimitedUtf8String( newDocumentsObs, s );
      readSelfDelimitedUtf8String( documentsIbs, s ); // Skip title
      SimpleCompressedDocumentCollectionBuilder.writeSelfDelimitedUtf8String( newDocumentsObs, s );
      for( int f = factory.numberOfFields() - 1; f-- !=0; ) {
        int len = documentsIbs.readDelta();
        newDocumentsObs.writeDelta( len );
        while( len-- != 0 ) {
          newDocumentsObs.writeDelta( invTermPerm[ documentsIbs.readDelta() ] );
          if ( exact ) newDocumentsObs.writeDelta( invNonTermPerm[ documentsIbs.readDelta() ] );
        }
      }
    }
    newDocumentsObs.close();
    new File( basename + DOCUMENTS_EXTENSION ).delete();
    newDocumentsFile.renameTo( new File( basename + DOCUMENTS_EXTENSION ) );
    newDocumentsObs = null;
    invTermPerm = invNonTermPerm = null;
   
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
    // Just to check that the index is of the right type
    final BitStreamHPIndex index = (BitStreamHPIndex)Index.getInstance( basename, false, false );
   
    final InputBitStream ibs = new InputBitStream( basename + DiskBasedIndex.INDEX_EXTENSION );
    final InputBitStream offsets = new InputBitStream( basename + DiskBasedIndex.OFFSETS_EXTENSION );
    final OutputBitStream numBitsPos = new OutputBitStream( basename + DiskBasedIndex.POSITIONS_NUMBER_OF_BITS_EXTENSION );
    long pos = 0, positionOffset = 0, o;
    for( int i = 0; i < index.numberOfTerms; i++ ) {
      ibs.position( pos += offsets.readLongGamma() );
      // Read offset into position file
      o = ibs.readLongDelta();

      if ( i > 0 ) numBitsPos.writeLongGamma( o - positionOffset );
      positionOffset = o;
    }

    // This is necessarily imprecise
    numBitsPos.writeLongGamma( new File( basename + DiskBasedIndex.POSITIONS_EXTENSION ).length() * 8 - positionOffset );

    numBitsPos.close();
    offsets.close();
    ibs.close(); 
  }
View Full Code Here
121
122
123
124
125
126
127
128
129
130
131
132
   * @param writeOffsets if <code>true</code>, the offset file will also be produced.
   * @param flags a flag map setting the coding techniques to be used (see {@link CompressionFlags}).
   */
  public BitStreamIndexWriter( final CharSequence basename, final int numberOfDocuments, final boolean writeOffsets, final Map<Component,Coding> flags ) throws IOException {
    this(
      new OutputBitStream( basename + DiskBasedIndex.INDEX_EXTENSION ),
      writeOffsets ? new OutputBitStream( basename + DiskBasedIndex.OFFSETS_EXTENSION ) : null,
      writeOffsets && flags.get( Component.POSITIONS ) != null ? new OutputBitStream( basename + DiskBasedIndex.POSITIONS_NUMBER_OF_BITS_EXTENSION ) : null,
      numberOfDocuments,
      flags
     );
  }
View Full Code Here
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
*/
public class SemiExternalOffsetListTest extends TestCase {

  private static InputBitStream buildInputStream( LongList offsets ) throws IOException {
    byte[] array = new byte[ offsets.size() * 4 ];
    OutputBitStream streamer = new OutputBitStream( array );
    long previous = 0;
    for ( int i = 0; i < offsets.size(); i++ ) {
      final long value = offsets.getLong( i );
      streamer.writeLongGamma( value - previous );
      previous = value;
    }
    int size = (int)( streamer.writtenBits() / 8 ) + ( ( streamer.writtenBits() % 8 ) == 0 ? 0 : 1 );
    byte[] smaller = new byte[ size ];
    System.arraycopy( array, 0, smaller, 0, size );

    return new InputBitStream( smaller );

View Full Code Here
14
15
16
17
18
19
20
21
22
23
24
25
26
   *
   * @param payload a payload containing a current value.
   */
  public static void testWriteAndRead( Payload payload ) throws IOException {
    final FastByteArrayOutputStream fbos = new FastByteArrayOutputStream();
    final OutputBitStream obs = new OutputBitStream( fbos );
    Object o = payload.get();
    payload.write( obs );
    obs.flush();
    final InputBitStream ibs = new InputBitStream( fbos.array );
    payload.read( ibs );
    assertEquals( o, payload.get() );
  }
View Full Code Here
TOP

Related Classes of it.unimi.dsi.io.OutputBitStream

  • it.unimi.dsi.mg4j.document.SimpleCompressedDocumentCollection
  • it.unimi.dsi.mg4j.document.SimpleCompressedDocumentCollectionBuilder
  • it.unimi.dsi.mg4j.index.BitStreamHPIndexWriter
  • it.unimi.dsi.mg4j.index.BitStreamIndexWriter
  • it.unimi.dsi.mg4j.index.payload.PayloadTestCase
  • it.unimi.dsi.mg4j.index.remote.RemoteIndexReader$ServerThread
  • it.unimi.dsi.mg4j.index.SkipBitStreamIndexWriter
  • it.unimi.dsi.mg4j.tool.Combine
  • it.unimi.dsi.mg4j.tool.ComputeNumBitsPositions
  • it.unimi.dsi.mg4j.tool.Concatenate
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.