* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

  private static final Log LOG = LogFactory.getLog(RecordReaderImpl.class);

  private final FSDataInputStream file;

  private final long firstRow;

  private final List<StripeInformation> stripes =

    new ArrayList<StripeInformation>();

  private OrcProto.StripeFooter stripeFooter;

  private final long totalRowCount;

  private final CompressionCodec codec;

  private final List<OrcProto.Type> types;

  private final int bufferSize;

  private final boolean[] included;

  private final long rowIndexStride;

  private long rowInStripe = 0;

  private int currentStripe = -1;

  private long rowBaseInStripe = 0;

  private long rowCountInStripe = 0;

  private final Map<StreamName, InStream> streams =

      new HashMap<StreamName, InStream>();

  List<BufferChunk> bufferChunks = new ArrayList<BufferChunk>(0);

  private final TreeReader reader;

  private final OrcProto.RowIndex[] indexes;

  private final SearchArgument sarg;

  // the leaf predicates for the sarg

  private final List<PredicateLeaf> sargLeaves;

  // an array the same length as the sargLeaves that map them to column ids

  private final int[] filterColumns;

  // an array about which row groups aren't skipped

  private boolean[] includedRowGroups = null;

  private final Configuration conf;

  private final ByteBufferAllocatorPool pool = new ByteBufferAllocatorPool();

  private final ZeroCopyReaderShim zcr;

  // this is an implementation copied from ElasticByteBufferPool in hadoop-2,

  // which lacks a clear()/clean() operation

  public final static class ByteBufferAllocatorPool implements ByteBufferPoolShim {

    private static final class Key implements Comparable<Key> {

      private final int capacity;

      private final long insertionGeneration;

      Key(int capacity, long insertionGeneration) {

        this.capacity = capacity;

        this.insertionGeneration = insertionGeneration;

      }

      @Override

      public int compareTo(Key other) {

        return ComparisonChain.start().compare(capacity, other.capacity)

            .compare(insertionGeneration, other.insertionGeneration).result();

      }

      @Override

      public boolean equals(Object rhs) {

        if (rhs == null) {

          return false;

        }

        try {

          Key o = (Key) rhs;

          return (compareTo(o) == 0);

        } catch (ClassCastException e) {

          return false;

        }

      }

      @Override

      public int hashCode() {

        return new HashCodeBuilder().append(capacity).append(insertionGeneration)

            .toHashCode();

      }

    }

    private final TreeMap<Key, ByteBuffer> buffers = new TreeMap<Key, ByteBuffer>();

    private final TreeMap<Key, ByteBuffer> directBuffers = new TreeMap<Key, ByteBuffer>();

    private long currentGeneration = 0;

    private final TreeMap<Key, ByteBuffer> getBufferTree(boolean direct) {

      return direct ? directBuffers : buffers;

    }

    public void clear() {

      buffers.clear();

      directBuffers.clear();

    }

    @Override

    public ByteBuffer getBuffer(boolean direct, int length) {

      TreeMap<Key, ByteBuffer> tree = getBufferTree(direct);

      Map.Entry<Key, ByteBuffer> entry = tree.ceilingEntry(new Key(length, 0));

      if (entry == null) {

        return direct ? ByteBuffer.allocateDirect(length) : ByteBuffer

            .allocate(length);

      }

      tree.remove(entry.getKey());

      return entry.getValue();

    }

    @Override

    public void putBuffer(ByteBuffer buffer) {

      TreeMap<Key, ByteBuffer> tree = getBufferTree(buffer.isDirect());

      while (true) {

        Key key = new Key(buffer.capacity(), currentGeneration++);

        if (!tree.containsKey(key)) {

          tree.put(key, buffer);

          return;

        }

        // Buffers are indexed by (capacity, generation).

        // If our key is not unique on the first try, we try again

      }

    }

  }

  /**

   * Given a list of column names, find the given column and return the index.

   * @param columnNames the list of potential column names

   * @param columnName the column name to look for

   * @param rootColumn offset the result with the rootColumn

   * @return the column number or -1 if the column wasn't found

   */

  static int findColumns(String[] columnNames,

                         String columnName,

                         int rootColumn) {

    for(int i=0; i < columnNames.length; ++i) {

      if (columnName.equals(columnNames[i])) {

        return i + rootColumn;

      }

    }

    return -1;

  }

  /**

   * Find the mapping from predicate leaves to columns.

   * @param sargLeaves the search argument that we need to map

   * @param columnNames the names of the columns

   * @param rootColumn the offset of the top level row, which offsets the

   *                   result

   * @return an array mapping the sarg leaves to concrete column numbers

   */

  static int[] mapSargColumns(List<PredicateLeaf> sargLeaves,

                             String[] columnNames,

                             int rootColumn) {

    int[] result = new int[sargLeaves.size()];

    Arrays.fill(result, -1);

    for(int i=0; i < result.length; ++i) {

      String colName = sargLeaves.get(i).getColumnName();

      result[i] = findColumns(columnNames, colName, rootColumn);

    }

    return result;

  }

  RecordReaderImpl(List<StripeInformation> stripes,

                   FileSystem fileSystem,

                   Path path,

                   Reader.Options options,

                   List<OrcProto.Type> types,

                   CompressionCodec codec,

                   int bufferSize,

                   long strideRate,

                   Configuration conf

                  ) throws IOException {

    this.file = fileSystem.open(path);

    this.codec = codec;

    this.types = types;

    this.bufferSize = bufferSize;

    this.included = options.getInclude();

    this.conf = conf;

    this.sarg = options.getSearchArgument();

    if (sarg != null) {

      sargLeaves = sarg.getLeaves();

      filterColumns = mapSargColumns(sargLeaves, options.getColumnNames(), 0);

    } else {

      sargLeaves = null;

      filterColumns = null;

    }

    long rows = 0;

    long skippedRows = 0;

    long offset = options.getOffset();

    long maxOffset = options.getMaxOffset();

    for(StripeInformation stripe: stripes) {

      long stripeStart = stripe.getOffset();

      if (offset > stripeStart) {

        skippedRows += stripe.getNumberOfRows();

      } else if (stripeStart < maxOffset) {

        this.stripes.add(stripe);

        rows += stripe.getNumberOfRows();

      }

    }

    final boolean zeroCopy = (conf != null)

        && (HiveConf.getBoolVar(conf, HIVE_ORC_ZEROCOPY));

    if (zeroCopy

        && (codec == null || ((codec instanceof DirectDecompressionCodec)

            && ((DirectDecompressionCodec) codec).isAvailable()))) {

      /* codec is null or is available */

      this.zcr = ShimLoader.getHadoopShims().getZeroCopyReader(file, pool);

    } else {

      this.zcr = null;

    }

    firstRow = skippedRows;

    totalRowCount = rows;

    reader = createTreeReader(path, 0, types, included, conf);

    indexes = new OrcProto.RowIndex[types.size()];

    rowIndexStride = strideRate;

    advanceToNextRow(0L);

  }

  private static final class PositionProviderImpl implements PositionProvider {

    private final OrcProto.RowIndexEntry entry;

    private int index = 0;

    PositionProviderImpl(OrcProto.RowIndexEntry entry) {

      this.entry = entry;

    }

    @Override

    public long getNext() {

      return entry.getPositions(index++);

    }

  }

  private abstract static class TreeReader {

    protected final Path path;

    protected final int columnId;

    private BitFieldReader present = null;

    protected boolean valuePresent = false;

    protected final Configuration conf;

    TreeReader(Path path, int columnId, Configuration conf) {

      this.path = path;

      this.columnId = columnId;

      this.conf = conf;

    }

    void checkEncoding(OrcProto.ColumnEncoding encoding) throws IOException {

      if (encoding.getKind() != OrcProto.ColumnEncoding.Kind.DIRECT) {

        throw new IOException("Unknown encoding " + encoding + " in column " +

            columnId + " of " + path);

      }

    }

    IntegerReader createIntegerReader(OrcProto.ColumnEncoding.Kind kind,

        InStream in,

        boolean signed) throws IOException {

      switch (kind) {

      case DIRECT_V2:

      case DICTIONARY_V2:

        return new RunLengthIntegerReaderV2(in, signed, conf);

      case DIRECT:

      case DICTIONARY:

        return new RunLengthIntegerReader(in, signed);

      default:

        throw new IllegalArgumentException("Unknown encoding " + kind);

      }

    }

    void startStripe(Map<StreamName, InStream> streams,

                     List<OrcProto.ColumnEncoding> encoding

                    ) throws IOException {

      checkEncoding(encoding.get(columnId));

      InStream in = streams.get(new StreamName(columnId,

          OrcProto.Stream.Kind.PRESENT));

      if (in == null) {

        present = null;

        valuePresent = true;

      } else {

        present = new BitFieldReader(in, 1);

      }

    }

    /**

     * Seek to the given position.

     * @param index the indexes loaded from the file

     * @throws IOException

     */

    void seek(PositionProvider[] index) throws IOException {

      if (present != null) {

        present.seek(index[columnId]);

      }

    }

    protected long countNonNulls(long rows) throws IOException {

      if (present != null) {

        long result = 0;

        for(long c=0; c < rows; ++c) {

          if (present.next() == 1) {

            result += 1;

          }

        }

        return result;

      } else {

        return rows;

      }

    }

    abstract void skipRows(long rows) throws IOException;

    Object next(Object previous) throws IOException {

      if (present != null) {

        valuePresent = present.next() == 1;

      }

      return previous;

    }

    /**

     * Populates the isNull vector array in the previousVector object based on

     * the present stream values. This function is called from all the child

     * readers, and they all set the values based on isNull field value.

     * @param previousVector The columnVector object whose isNull value is populated

     * @param batchSize Size of the column vector

     * @return

     * @throws IOException

     */

    Object nextVector(Object previousVector, long batchSize) throws IOException {

      ColumnVector result = (ColumnVector) previousVector;

      if (present != null) {

        // Set noNulls and isNull vector of the ColumnVector based on

        // present stream

        result.noNulls = true;

        for (int i = 0; i < batchSize; i++) {

          result.isNull[i] = (present.next() != 1);

          if (result.noNulls && result.isNull[i]) {

            result.noNulls = false;

          }

        }

      } else {

        // There is not present stream, this means that all the values are

        // present.

        result.noNulls = true;

        for (int i = 0; i < batchSize; i++) {

          result.isNull[i] = false;

        }

      }

      return previousVector;

    }

  }

  private static class BooleanTreeReader extends TreeReader{

    private BitFieldReader reader = null;

    BooleanTreeReader(Path path, int columnId, Configuration conf) {

      super(path, columnId, conf);

    }

    @Override

    void startStripe(Map<StreamName, InStream> streams,

                     List<OrcProto.ColumnEncoding> encodings

                     ) throws IOException {

      super.startStripe(streams, encodings);

      reader = new BitFieldReader(streams.get(new StreamName(columnId,

          OrcProto.Stream.Kind.DATA)), 1);

    }

    @Override

    void seek(PositionProvider[] index) throws IOException {

      super.seek(index);

      reader.seek(index[columnId]);

    }

    @Override

    void skipRows(long items) throws IOException {

      reader.skip(countNonNulls(items));

    }

    @Override