Package uk.ac.ucl.panda.utility.io

Examples of uk.ac.ucl.panda.utility.io.IndexInput


        throw new IOException("cannot read directory " + src + ": list() returned null");

      byte[] buf = new byte[BufferedIndexOutput.getBufSize()];
      for (int i = 0; i < files.length; i++) {
        IndexOutput os = null;
        IndexInput is = null;
        try {
          // create file in dest directory
          os = dest.createOutput(files[i]);
          // read current file
          is = src.openInput(files[i]);
          // and copy to dest directory
          long len = is.length();
          long readCount = 0;
          while (readCount < len) {
            int toRead = readCount + BufferedIndexOutput.getBufSize() > len ? (int)(len - readCount) : BufferedIndexOutput.getBufSize();
            is.readBytes(buf, 0, toRead);
            os.writeBytes(buf, toRead);
            readCount += toRead;
          }
        } finally {
          // graceful cleanup
          try {
            if (os != null)
              os.close();
          } finally {
            if (is != null)
              is.close();
          }
        }
      }
      if(closeDirSrc)
        src.close();
View Full Code Here


     *  to reduce memory allocation.
     */
    private void copyFile(FileEntry source, IndexOutput os, byte buffer[])
    throws IOException
    {
        IndexInput is = null;
        try {
            long startPtr = os.getFilePointer();

            is = directory.openInput(source.file);
            long length = is.length();
            long remainder = length;
            int chunk = buffer.length;

            while(remainder > 0) {
                int len = (int) Math.min(chunk, remainder);
                is.readBytes(buffer, 0, len);
                os.writeBytes(buffer, len);
                remainder -= len;
                if (checkAbort != null)
                  // Roughly every 2 MB we will check if
                  // it's time to abort
                  checkAbort.work(80);
            }

            // Verify that remainder is 0
            if (remainder != 0)
                throw new IOException(
                    "Non-zero remainder length after copying: " + remainder
                    + " (id: " + source.file + ", length: " + length
                    + ", buffer size: " + chunk + ")");

            // Verify that the output length diff is equal to original file
            long endPtr = os.getFilePointer();
            long diff = endPtr - startPtr;
            if (diff != length)
                throw new IOException(
                    "Difference in the output file offsets " + diff
                    + " does not match the original file length " + length);

        } finally {
            if (is != null) is.close();
        }
    }
View Full Code Here

  /** Constructs a bit vector from the file <code>name</code> in Directory
    <code>d</code>, as written by the {@link #write} method.
    */
  public BitVector(Directory d, String name) throws IOException {
    IndexInput input = d.openInput(name);
    try {
      size = input.readInt();       // read size
      if (size == -1) {
        readDgaps(input);
      } else {
        readBits(input);
      }
    } finally {
      input.close();
    }
  }
View Full Code Here

    boolean success = false;

    // Clear any previous segments:
    clear();

    IndexInput input = directory.openInput(segmentFileName);

    generation = generationFromSegmentsFileName(segmentFileName);

    lastGeneration = generation;

    try {
      int format = input.readInt();
      if(format < 0){     // file contains explicit format info
        // check that it is a format we can understand
        if (format < CURRENT_FORMAT)
          throw new CorruptIndexException("Unknown format version: " + format);
        version = input.readLong(); // read version
        counter = input.readInt(); // read counter
      }
      else{     // file is in old format without explicit format info
        counter = format;
      }
     
      for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
        addElement(new SegmentInfo(directory, format, input));
      }
     
      if(format >= 0){    // in old format the version number may be at the end of the file
        if (input.getFilePointer() >= input.length())
          version = System.currentTimeMillis(); // old file format without version number
        else
          version = input.readLong(); // read version
      }
      success = true;
    }
    finally {
      input.close();
      if (!success) {
        // Clear any segment infos we had loaded so we
        // have a clean slate on retry:
        clear();
      }
View Full Code Here

    throws CorruptIndexException, IOException {

    return ((Long) new FindSegmentsFile(directory) {
        protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {

          IndexInput input = directory.openInput(segmentFileName);

          int format = 0;
          long version = 0;
          try {
            format = input.readInt();
            if(format < 0){
              if (format < CURRENT_FORMAT)
                throw new CorruptIndexException("Unknown format version: " + format);
              version = input.readLong(); // read version
            }
          }
          finally {
            input.close();
          }
    
          if(format < 0)
            return new Long(version);
View Full Code Here

          // a stale cache (NFS) we have a better chance of
          // getting the right generation.
          long genB = -1;
          if (directory != null) {
            for(int i=0;i<defaultGenFileRetryCount;i++) {
              IndexInput genInput = null;
              try {
                genInput = directory.openInput(IndexFileNames.SEGMENTS_GEN);
              } catch (FileNotFoundException e) {
                message("segments.gen open: FileNotFoundException " + e);
                break;
              } catch (IOException e) {
                message("segments.gen open: IOException " + e);
              }

              if (genInput != null) {
                try {
                  int version = genInput.readInt();
                  if (version == FORMAT_LOCKLESS) {
                    long gen0 = genInput.readLong();
                    long gen1 = genInput.readLong();
                    message("fallback check: " + gen0 + "; " + gen1);
                    if (gen0 == gen1) {
                      // The file is consistent.
                      genB = gen0;
                      break;
                    }
                  }
                } catch (IOException err2) {
                  // will retry
                } finally {
                  genInput.close();
                }
              }
              try {
                Thread.sleep(defaultGenFileRetryPauseMsec);
              } catch (InterruptedException e) {
View Full Code Here

      for (int i = 0; i < files.length; ++i) {
        long len = cfr.fileLength(files[i]);

        if (extract) {
          System.out.println("extract " + files[i] + " with " + len + " bytes to local directory...");
          IndexInput ii = cfr.openInput(files[i]);

          FileOutputStream f = new FileOutputStream(files[i]);

          // read and write with a small buffer, which is more effectiv than reading byte by byte
          byte[] buffer = new byte[1024];
          int chunk = buffer.length;
          while(len > 0) {
            final int bufLen = (int) Math.min(chunk, len);
            ii.readBytes(buffer, 0, bufLen);
            f.write(buffer, 0, bufLen);
            len -= bufLen;
          }

          f.close();
          ii.close();
        }
        else
          System.out.println(files[i] + ": " + len + " bytes");
      }
    } catch (IOException ioe) {
View Full Code Here

   * @param d The directory to open the IndexInput from
   * @param name The name of the file to open the IndexInput from in the Directory
   * @throws IOException
   */
  public FieldInfos(Directory d, String name) throws IOException {
    IndexInput input = d.openInput(name);
    try {
      read(input);
    } finally {
      input.close();
    }
  }
View Full Code Here

        cloneableFieldsStream.close();
      }
      if (indexStream != null) {
        indexStream.close();
      }
      IndexInput localFieldsStream = (IndexInput) fieldsStreamTL.get();
      if (localFieldsStream != null) {
        localFieldsStream.close();
        fieldsStreamTL.set(null);
      }
      closed = true;
    }
  }
View Full Code Here

      this.pointer = pointer;
      lazy = true;
    }

    private IndexInput getFieldStream() {
      IndexInput localFieldsStream = (IndexInput) fieldsStreamTL.get();
      if (localFieldsStream == null) {
        localFieldsStream = (IndexInput) cloneableFieldsStream.clone();
        fieldsStreamTL.set(localFieldsStream);
      }
      return localFieldsStream;
View Full Code Here

TOP

Related Classes of uk.ac.ucl.panda.utility.io.IndexInput

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.