Examples of HashMapWrapper

org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper
Simple wrapper for persistent Hashmap implementing only the put/get/remove/clear interface. The main memory hash table acts as a cache and all put/get will operate on it first. If the size of the main memory hash table exceeds a certain threshold, new elements will go into the persistent hash table.

Examples of org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper

      MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerde, false);
      for (Byte pos : order) {
        if (pos == posBigTableAlias) {
          continue;
        }
        mapJoinTables[pos] = new HashMapWrapper(hashTableThreshold, hashTableLoadFactor);        
        TableDesc valueTableDesc = conf.getValueTblFilteredDescs().get(pos);
        SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(), null);
        valueSerDe.initialize(null, valueTableDesc.getProperties());
        mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, new MapJoinObjectSerDeContext(
            valueSerDe, hasFilter(pos)));

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper

      MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerde, false);
      for (Byte pos : order) {
        if (pos == posBigTableAlias) {
          continue;
        }
        mapJoinTables[pos] = new HashMapWrapper(hashTableThreshold, hashTableLoadFactor);
        TableDesc valueTableDesc = conf.getValueTblFilteredDescs().get(pos);
        SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(), null);
        valueSerDe.initialize(null, valueTableDesc.getProperties());
        mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, new MapJoinObjectSerDeContext(
            valueSerDe, hasFilter(pos)));

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper

      LogicalInput input = tezContext.getInput(inputName);


      try {
        KeyValueReader kvReader = (KeyValueReader) input.getReader();


        MapJoinTableContainer tableContainer = new HashMapWrapper(hashTableThreshold,
            hashTableLoadFactor);


        // simply read all the kv pairs into the hashtable.


        while (kvReader.next()) {
          // We pass key in as reference, to find out quickly if optimized keys can be used.
          // However, we do not reuse the object since we are putting them into the hashmap.
          lastKey = MapJoinKey.read(output, lastKey, mapJoinTableSerdes[pos].getKeyContext(),
              (Writable)kvReader.getCurrentKey(), false);


          LazyFlatRowContainer values = (LazyFlatRowContainer)tableContainer.get(lastKey);
          if (values == null) {
            values = new LazyFlatRowContainer();
            tableContainer.put(lastKey, values);
          }
          values.add(mapJoinTableSerdes[pos].getValueContext(),
              (BytesWritable)kvReader.getCurrentValue(), useLazyRows);
        }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper

        isFirstKey = false;
        Long keyCountObj = parentKeyCounts.get(pos);
        long keyCount = (keyCountObj == null) ? -1 : keyCountObj.longValue();
        MapJoinTableContainer tableContainer = useOptimizedTables
            ? new MapJoinBytesTableContainer(hconf, valCtx, keyCount, memUsage)
            : new HashMapWrapper(hconf, keyCount);


        while (kvReader.next()) {
          rowCount++;
          lastKey = tableContainer.putRow(keyCtx, (Writable)kvReader.getCurrentKey(),
              valCtx, (Writable)kvReader.getCurrentValue());

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper

      MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerde, false);
      for (Byte pos : order) {
        if (pos == posBigTableAlias) {
          continue;
        }
        mapJoinTables[pos] = new HashMapWrapper(hconf, -1);
        TableDesc valueTableDesc = conf.getValueTblFilteredDescs().get(pos);
        SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(), null);
        SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null);
        mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, new MapJoinObjectSerDeContext(
            valueSerDe, hasFilter(pos)));

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper

      LogicalInput input = tezContext.getInput(inputName);


      try {
        KeyValueReader kvReader = (KeyValueReader) input.getReader();


        MapJoinTableContainer tableContainer = new HashMapWrapper(hashTableThreshold,
            hashTableLoadFactor);


        // simply read all the kv pairs into the hashtable.


        while (kvReader.next()) {
          // We pass key in as reference, to find out quickly if optimized keys can be used.
          // However, we do not reuse the object since we are putting them into the hashmap.
          lastKey = MapJoinKey.read(output, lastKey, mapJoinTableSerdes[pos].getKeyContext(),
              (Writable)kvReader.getCurrentKey(), false);


          LazyFlatRowContainer values = (LazyFlatRowContainer)tableContainer.get(lastKey);
          if (values == null) {
            values = new LazyFlatRowContainer();
            tableContainer.put(lastKey, values);
          }
          values.add(mapJoinTableSerdes[pos].getValueContext(),
              (BytesWritable)kvReader.getCurrentValue(), useLazyRows);
        }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper

      MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerde, false);
      for (Byte pos : order) {
        if (pos == posBigTableAlias) {
          continue;
        }
        mapJoinTables[pos] = new HashMapWrapper(hashTableThreshold, hashTableLoadFactor);
        TableDesc valueTableDesc = conf.getValueTblFilteredDescs().get(pos);
        SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(), null);
        valueSerDe.initialize(null, valueTableDesc.getProperties());
        mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, new MapJoinObjectSerDeContext(
            valueSerDe, hasFilter(pos)));

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper

      MapJoinObjectSerDeContext keyContext = new MapJoinObjectSerDeContext(keySerde, false);
      for (Byte pos : order) {
        if (pos == posBigTableAlias) {
          continue;
        }
        mapJoinTables[pos] = new HashMapWrapper(hashTableThreshold, hashTableLoadFactor);        
        TableDesc valueTableDesc = conf.getValueTblFilteredDescs().get(pos);
        SerDe valueSerDe = (SerDe) ReflectionUtils.newInstance(valueTableDesc.getDeserializerClass(), null);
        valueSerDe.initialize(null, valueTableDesc.getProperties());
        mapJoinTableSerdes[pos] = new MapJoinTableContainerSerDe(keyContext, new MapJoinObjectSerDeContext(
            valueSerDe, hasFilter(pos)));

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.