Package it.unimi.dsi.logging

Examples of it.unimi.dsi.logging.ProgressLogger


    LongArrayList p = new LongArrayList();
    pointers = new ObjectArrayList<EliasFanoMonotoneLongBigList>( file.length );
    firstDocument = new int[ file.length + 1 ];
    int count = 0;
   
    final ProgressLogger pl = new ProgressLogger( LOGGER );
    pl.expectedUpdates = file.length;
    pl.itemsName = "files";
    pl.start( "Scanning files..." );
   
    // Scan files and retrieve page pointers
    for( String f : file ) {
      p.clear();
      final FastBufferedInputStream fbis = gzipped ? new FastBufferedInputStream( new GZIPInputStream( new FileInputStream( f ) ) ) : new FastBufferedInputStream( new FileInputStream( f ) );
      long position;
      for(;;) {
        position = fbis.position();
        if ( readLine( fbis ) == -1 ) break;
        if ( startsWith( lineBuffer, DOC_MARKER ) ) p.add( position );
        if ( phrase && startsWith( lineBuffer, SENTENCE_MARKER ) ) p.add( position );
      }
     
      count += p.size();
      p.add( fbis.position() );
      fbis.close();
     
      pointers.add( new EliasFanoMonotoneLongBigList( p ) );
      firstDocument[ pointers.size() ] = count;
     
      pl.update();
    }
   
    pl.done();
   
    size = count;
  }
View Full Code Here


    );
    JSAPResult jsapResult = jsap.parse( arg );
    if ( jsap.messagePrinted() ) return;

    DocumentSequence documentSequence = Scan.getSequence( jsapResult.getString( "sequence" ), jsapResult.getClass( "factory" ), jsapResult.getStringArray( "property" ), jsapResult.getInt( "delimiter" ), LOGGER );
    final ProgressLogger progressLogger = new ProgressLogger( LOGGER, "documents" );
    if ( documentSequence instanceof DocumentCollection ) progressLogger.expectedUpdates = ((DocumentCollection)documentSequence).size();
    final ZipDocumentCollectionBuilder zipDocumentCollectionBuilder = new ZipDocumentCollectionBuilder( jsapResult.getString( "basename" ), documentSequence.factory(), !jsapResult.getBoolean( "approximated") );
    zipDocumentCollectionBuilder.open( "" );
    zipDocumentCollectionBuilder.build( documentSequence );
  }
View Full Code Here

     * @throws Exception
     */
    @SuppressWarnings("unchecked")
    public WebgraphWebServer(String pathToWebgraphBase, int port) throws Exception {
        _port = port;
        ProgressLogger pl = new ProgressLogger();

        String fcl = pathToWebgraphBase + ".fcl";
        _node2url = (List<CharSequence>) BinIO.loadObject(fcl);

        String mph = pathToWebgraphBase + ".mph";
View Full Code Here

        this.bufferSize = bufferSize;
        this.descriptors = new ObjectBigArrayBigList<>();
        this.compression = compression;
        this.metadataFile = metadataFile;

        final ProgressLogger progressLogger = new ProgressLogger(LOGGER);
        progressLogger.expectedUpdates = files.length;
        progressLogger.itemsName = "files";

        progressLogger.start("Parsing files with compression \"" + compression
                + "\"");

        final DataOutputStream metadataStream = new DataOutputStream(new FileOutputStream(metadataFile));
        for (int i = 0; i < files.length; i++) {
            parseContent(i, openFileStream(files[i]), metadataStream);
            progressLogger.update();
        }
        metadataStream.close();

        metadataRandomAccess = new RandomAccessFile(metadataFile, "r");

        progressLogger.done();
    }
View Full Code Here

TOP

Related Classes of it.unimi.dsi.logging.ProgressLogger

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.