Package org.apache.hadoop.hive.serde2

Examples of org.apache.hadoop.hive.serde2.SerDeStats

142
143
144
145
146
147
148
      this.file = file;
      numColumns = (types.size() == 0) ? 0 : types.get(0).getSubtypesCount();
      this.offset = split.getStart();
      this.length = split.getLength();
      this.reader = createReaderFromFile(file, conf, offset, length);
      this.stats = new SerDeStats();
    }
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
157
      }

      rowOutWriters = fpaths.getOutWriters();
      if (conf.isGatherStats()) {
        if (statsCollectRawDataSize) {
          SerDeStats stats = serializer.getSerDeStats();
          if (stats != null) {
            fpaths.getStat().addToStat(StatsSetupConst.RAW_DATA_SIZE, stats.getRawDataSize());
          }
        }
        fpaths.getStat().addToStat(StatsSetupConst.ROW_COUNT, 1);
      }
View Full Code Here
114
115
116
117
118
119
120
121
122
123
124
        } else {
            columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
        }
        assert (columnNames.size() == columnTypes.size());

        stats = new SerDeStats();

        // Create row related objects
        rowTypeInfo = (StructTypeInfo) TypeInfoFactory
                .getStructTypeInfo(columnNames, columnTypes);
       
View Full Code Here
126
127
128
129
130
131
132
133
134
135
    // output debug info
    LOG.debug("LazyBinarySerDe initialized with: columnNames=" + columnNames
        + " columnTypes=" + columnTypes);

    serializedSize = 0;
    stats = new SerDeStats();
    lastOperationSerialize = false;
    lastOperationDeserialize = false;

  }
View Full Code Here
217
218
219
220
221
222
223
224
225
        + " separator=" + Arrays.asList(serdeParams.separators)
        + " nullstring=" + serdeParams.nullString + " lastColumnTakesRest="
        + serdeParams.lastColumnTakesRest);

    serializedSize = 0;
    stats = new SerDeStats();
    lastOperationSerialize = false;
    lastOperationDeserialize = false;
  }
View Full Code Here
139
140
141
142
143
144
145
146
147
148
    // Create row related objects
    rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
    this.objInspector = new ArrayWritableObjectInspector((StructTypeInfo) rowTypeInfo);

    // Stats part
    stats = new SerDeStats();
    serializedSize = 0;
    deserializedSize = 0;
    status = LAST_OPERATION.UNKNOWN;
  }
View Full Code Here
362
363
364
365
366
367
368
369
370
371
372
    writer = null;
  }

  @Override
  public SerDeStats getStats() {
    SerDeStats stats = new SerDeStats();
    stats.setRowCount(rowCountDelta);
    // Don't worry about setting raw data size diff.  I have no idea how to calculate that
    // without finding the row we are updating or deleting, which would be a mess.
    return stats;
  }
View Full Code Here
61
62
63
64
65
66
67
    private final SerDeStats stats;

    OrcRecordWriter(Path path, OrcFile.WriterOptions options) {
      this.path = path;
      this.options = options;
      this.stats = new SerDeStats();
    }
View Full Code Here
665
666
667
668
669
670
671
672
673
674
675
676
677
      // doesn't implement stats interface we will fallback to conventional way
      // of gathering stats
      isCollectRWStats = areAllTrue(statsFromRecordWriter);
      if (conf.isGatherStats() && !isCollectRWStats) {
        if (statsCollectRawDataSize) {
          SerDeStats stats = serializer.getSerDeStats();
          if (stats != null) {
            fpaths.stat.addToStat(StatsSetupConst.RAW_DATA_SIZE, stats.getRawDataSize());
          }
        }
        fpaths.stat.addToStat(StatsSetupConst.ROW_COUNT, 1);
      }
View Full Code Here
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
          prevFsp.closeWriters(false);

          // since we are closing the previous fsp's record writers, we need to see if we can get
          // stats from the record writer and store in the previous fsp that is cached
          if (conf.isGatherStats() && isCollectRWStats) {
            SerDeStats stats = null;
            if (conf.getWriteType() == AcidUtils.Operation.NOT_ACID) {
              RecordWriter outWriter = prevFsp.outWriters[0];
              if (outWriter != null) {
                stats = ((StatsProvidingRecordWriter) outWriter).getStats();
              }
            } else if (prevFsp.updaters[0] != null) {
              stats = prevFsp.updaters[0].getStats();
            }
            if (stats != null) {
                prevFsp.stat.addToStat(StatsSetupConst.RAW_DATA_SIZE, stats.getRawDataSize());
                prevFsp.stat.addToStat(StatsSetupConst.ROW_COUNT, stats.getRowCount());
            }
          }

          // let writers release the memory for garbage collection
          prevFsp.outWriters[0] = null;
View Full Code Here
TOP

Related Classes of org.apache.hadoop.hive.serde2.SerDeStats

  • com.asakusafw.directio.hive.orc.OrcFileInput
  • com.asakusafw.directio.hive.serde.DataModelDriver
  • com.asakusafw.directio.hive.serde.DataModelDriverTest
  • com.asakusafw.directio.hive.serde.ValueSerdeFactoryTest
  • com.facebook.hive.orc.lazy.LazyDoubleTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDictionaryTreeReader
  • com.facebook.hive.orc.lazy.LazyShortDirectTreeReader
  • com.facebook.hive.orc.lazy.OrcLazyStructObjectInspector
  • com.facebook.hive.orc.OrcOutputFormat$OrcRecordWriter
  • com.facebook.hive.orc.OrcStruct$OrcStructInspector
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.