Package org.apache.hadoop.hive.serde2

Examples of org.apache.hadoop.hive.serde2.SerDeStats

961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
        if (conf.isGatherStats() && isCollectRWStats) {
          if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) {
            for (int idx = 0; idx < fsp.outWriters.length; idx++) {
              RecordWriter outWriter = fsp.outWriters[idx];
              if (outWriter != null) {
                SerDeStats stats = ((StatsProvidingRecordWriter) outWriter).getStats();
                if (stats != null) {
                  fsp.stat.addToStat(StatsSetupConst.RAW_DATA_SIZE, stats.getRawDataSize());
                  fsp.stat.addToStat(StatsSetupConst.ROW_COUNT, stats.getRowCount());
                }
              }
            }
          } else {
            for (int i = 0; i < fsp.updaters.length; i++) {
              if (fsp.updaters[i] != null) {
                SerDeStats stats = fsp.updaters[i].getStats();
                if (stats != null) {
                  fsp.stat.addToStat(StatsSetupConst.RAW_DATA_SIZE, stats.getRawDataSize());
                  fsp.stat.addToStat(StatsSetupConst.ROW_COUNT, stats.getRowCount());
                }
              }
            }
          }
        }
View Full Code Here
566
567
568
569
570
571
572
573
574
575
576
577
578
        if (current != old.get()) {
          old.set(current);
        }
      } else if (vc.equals(VirtualColumn.RAWDATASIZE)) {
        long current = 0L;
        SerDeStats stats = deserializer.getSerDeStats();
        if(stats != null) {
          current = stats.getRawDataSize();
        }
        LongWritable old = (LongWritable) vcValues[i];
        if (old == null) {
          old = new LongWritable(current);
          vcValues[i] = old;
View Full Code Here
153
154
155
156
157
158
159
      this.file = file;
      numColumns = (types.size() == 0) ? 0 : types.get(0).getSubtypesCount();
      this.offset = split.getStart();
      this.length = split.getLength();
      this.reader = createReaderFromFile(file, conf, offset, length);
      this.stats = new SerDeStats();
    }
View Full Code Here
630
631
632
633
634
635
636
637
638
639
640
          out.close();
        }

        @Override
        public SerDeStats getStats() {
          SerDeStats stats = new SerDeStats();
          stats.setRowCount(numRecordsAdded);
          return stats;
        }
      };
    }
View Full Code Here
708
709
710
711
712
713
714
715
716
717
718
719
720
        if (current != old.get()) {
          old.set(current);
        }
      } else if (vc.equals(VirtualColumn.RAWDATASIZE)) {
        long current = 0L;
        SerDeStats stats = deserializer.getSerDeStats();
        if(stats != null) {
          current = stats.getRawDataSize();
        }
        LongWritable old = (LongWritable) vcValues[i];
        if (old == null) {
          old = new LongWritable(current);
          vcValues[i] = old;
View Full Code Here
618
619
620
621
622
623
624
625
626
627
628
629
630
      }

      rowOutWriters = fpaths.outWriters;
      if (conf.isGatherStats()) {
        if (statsCollectRawDataSize) {
          SerDeStats stats = serializer.getSerDeStats();
          if (stats != null) {
            fpaths.stat.addToStat(StatsSetupConst.RAW_DATA_SIZE, stats.getRawDataSize());
          }
        }
        fpaths.stat.addToStat(StatsSetupConst.ROW_COUNT, 1);
      }
View Full Code Here
198
199
200
201
202
203
204
205
206
207
        + " separator=" + Arrays.asList(serdeParams.separators)
        + " nullstring=" + serdeParams.nullString + " lastColumnTakesRest="
        + serdeParams.lastColumnTakesRest);

    serializedSize = 0;
    stats = new SerDeStats();
    lastOperationSerialize = false;
    lastOperationDeserialize = false;

  }
View Full Code Here
120
121
122
123
124
125
126
127
128
129
    // output debug info
    LOG.debug("LazyBinarySerDe initialized with: columnNames=" + columnNames
        + " columnTypes=" + columnTypes);

    serializedSize = 0;
    stats = new SerDeStats();
    lastOperationSerialize = false;
    lastOperationDeserialize = false;

  }
View Full Code Here
202
203
204
205
206
207
208
209
210
211
        + " separator=" + Arrays.asList(serdeParams.separators)
        + " nullstring=" + serdeParams.nullString + " lastColumnTakesRest="
        + serdeParams.lastColumnTakesRest);

    serializedSize = 0;
    stats = new SerDeStats();
    lastOperationSerialize = false;
    lastOperationDeserialize = false;

  }
View Full Code Here
588
589
590
591
592
593
594
595
596
597
598
599
600
      // doesn't implement stats interface we will fallback to conventional way
      // of gathering stats
      isCollectRWStats = areAllTrue(statsFromRecordWriter);
      if (conf.isGatherStats() && !isCollectRWStats) {
        if (statsCollectRawDataSize) {
          SerDeStats stats = serializer.getSerDeStats();
          if (stats != null) {
            fpaths.stat.addToStat(StatsSetupConst.RAW_DATA_SIZE, stats.getRawDataSize());
          }
        }
        fpaths.stat.addToStat(StatsSetupConst.ROW_COUNT, 1);
      }
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.serde2.SerDeStats

  • com.asakusafw.directio.hive.orc.OrcFileInput
  • com.asakusafw.directio.hive.serde.DataModelDriver
  • com.asakusafw.directio.hive.serde.DataModelDriverTest
  • com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest
  • com.facebook.hive.orc.lazy.LazyDoubleTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDirectTreeReader
  • com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector
  • com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter
  • com.facebook.hive.orc.OrcStruct$OrcStructInspector
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.