Package org.apache.hadoop.hive.metastore.api

Examples of org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc


    return statsObj;
  }

  public static ColumnStatisticsDesc getTableColumnStatisticsDesc(
      MTableColumnStatistics mStatsObj) {
    ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
    statsDesc.setIsTblLevel(true);
    statsDesc.setDbName(mStatsObj.getDbName());
    statsDesc.setTableName(mStatsObj.getTableName());
    statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed());
    return statsDesc;
  }
View Full Code Here


   return statsObj;
  }

  public static ColumnStatisticsDesc getPartitionColumnStatisticsDesc(
    MPartitionColumnStatistics mStatsObj) {
    ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
    statsDesc.setIsTblLevel(false);
    statsDesc.setDbName(mStatsObj.getDbName());
    statsDesc.setTableName(mStatsObj.getTableName());
    statsDesc.setPartName(mStatsObj.getPartitionName());
    statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed());
    return statsDesc;
  }
View Full Code Here

    boolean committed = false;

    openTransaction();
    try {
      List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
      ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();

      // DataNucleus objects get detached all over the place for no (real) reason.
      // So let's not use them anywhere unless absolutely necessary.
      Table table = ensureGetTable(statsDesc.getDbName(), statsDesc.getTableName());
      for (ColumnStatisticsObj statsObj:statsObjs) {
        // We have to get mtable again because DataNucleus.
        MTableColumnStatistics mStatsObj = StatObjectConverter.convertToMTableColumnStatistics(
            ensureGetMTable(statsDesc.getDbName(), statsDesc.getTableName()), statsDesc, statsObj);
        writeMTableColumnStatistics(table, mStatsObj);
      }
      committed = commitTransaction();
      return committed;
    } finally {
View Full Code Here

    boolean committed = false;

    try {
    openTransaction();
    List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj();
    ColumnStatisticsDesc statsDesc = colStats.getStatsDesc();
    Table table = ensureGetTable(statsDesc.getDbName(), statsDesc.getTableName());
    Partition partition = convertToPart(getMPartition(
        statsDesc.getDbName(), statsDesc.getTableName(), partVals));
    for (ColumnStatisticsObj statsObj:statsObjs) {
      // We have to get partition again because DataNucleus
      MPartition mPartition = getMPartition(
          statsDesc.getDbName(), statsDesc.getTableName(), partVals);
      if (partition == null) {
        throw new NoSuchObjectException("Partition for which stats is gathered doesn't exist.");
      }
      MPartitionColumnStatistics mStatsObj =
          StatObjectConverter.convertToMPartitionColumnStatistics(mPartition, statsDesc, statsObj);
View Full Code Here

          GetHelper<ColumnStatistics> ctx) throws MetaException, NoSuchObjectException {
        List<MTableColumnStatistics> mStats = getMTableColumnStatistics(getTable(), colNames);
        if (mStats.isEmpty()) return null;
        // LastAnalyzed is stored per column, but thrift object has it per multiple columns.
        // Luckily, nobody actually uses it, so we will set to lowest value of all columns for now.
        ColumnStatisticsDesc desc = StatObjectConverter.getTableColumnStatisticsDesc(mStats.get(0));
        List<ColumnStatisticsObj> statObjs = new ArrayList<ColumnStatisticsObj>(mStats.size());
        for (MTableColumnStatistics mStat : mStats) {
          if (desc.getLastAnalyzed() > mStat.getLastAnalyzed()) {
            desc.setLastAnalyzed(mStat.getLastAnalyzed());
          }
          statObjs.add(StatObjectConverter.getTableColumnStatisticsObj(mStat));
        }
        return new ColumnStatistics(desc, statObjs);
      }
View Full Code Here

            getMPartitionColumnStatistics(getTable(), partNames, colNames);
        List<ColumnStatistics> result = new ArrayList<ColumnStatistics>(
            Math.min(mStats.size(), partNames.size()));
        String lastPartName = null;
        List<ColumnStatisticsObj> curList = null;
        ColumnStatisticsDesc csd = null;
        for (int i = 0; i <= mStats.size(); ++i) {
          boolean isLast = i == mStats.size();
          MPartitionColumnStatistics mStatsObj = isLast ? null : mStats.get(i);
          String partName = isLast ? null : (String)mStatsObj.getPartitionName();
          if (isLast || !partName.equals(lastPartName)) {
View Full Code Here

      String[] colType = new String[] {"double", "string"};
      boolean isTblLevel = true;
      String partName = null;
      List<ColumnStatisticsObj> statsObjs = new ArrayList<ColumnStatisticsObj>();

      ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
      statsDesc.setDbName(dbName);
      statsDesc.setTableName(tblName);
      statsDesc.setIsTblLevel(isTblLevel);
      statsDesc.setPartName(partName);

      ColumnStatisticsObj statsObj = new ColumnStatisticsObj();
      statsObj.setColName(colName[0]);
      statsObj.setColType(colType[0]);

      ColumnStatisticsData statsData = new ColumnStatisticsData();
      DoubleColumnStatsData numericStats = new DoubleColumnStatsData();
      statsData.setDoubleStats(numericStats);

      statsData.getDoubleStats().setHighValue(highValue);
      statsData.getDoubleStats().setLowValue(lowValue);
      statsData.getDoubleStats().setNumDVs(numDVs);
      statsData.getDoubleStats().setNumNulls(numNulls);

      statsObj.setStatsData(statsData);
      statsObjs.add(statsObj);

      statsObj = new ColumnStatisticsObj();
      statsObj.setColName(colName[1]);
      statsObj.setColType(colType[1]);

      statsData = new ColumnStatisticsData();
      StringColumnStatsData stringStats = new StringColumnStatsData();
      statsData.setStringStats(stringStats);
      statsData.getStringStats().setAvgColLen(avgColLen);
      statsData.getStringStats().setMaxColLen(maxColLen);
      statsData.getStringStats().setNumDVs(numDVs);
      statsData.getStringStats().setNumNulls(numNulls);

      statsObj.setStatsData(statsData);
      statsObjs.add(statsObj);

      ColumnStatistics colStats = new ColumnStatistics();
      colStats.setStatsDesc(statsDesc);
      colStats.setStatsObj(statsObjs);

      // write stats objs persistently
      client.updateTableColumnStatistics(colStats);

      // retrieve the stats obj that was just written
      ColumnStatisticsObj colStats2 = client.getTableColumnStatistics(
          dbName, tblName, Lists.newArrayList(colName[0])).get(0);

     // compare stats obj to ensure what we get is what we wrote
      assertNotNull(colStats2);
      assertEquals(colStats2.getColName(), colName[0]);
      assertEquals(colStats2.getStatsData().getDoubleStats().getLowValue(), lowValue);
      assertEquals(colStats2.getStatsData().getDoubleStats().getHighValue(), highValue);
      assertEquals(colStats2.getStatsData().getDoubleStats().getNumNulls(), numNulls);
      assertEquals(colStats2.getStatsData().getDoubleStats().getNumDVs(), numDVs);

      // test delete column stats; if no col name is passed all column stats associated with the
      // table is deleted
      boolean status = client.deleteTableColumnStatistics(dbName, tblName, null);
      assertTrue(status);
      // try to query stats for a column for which stats doesn't exist
      assertTrue(client.getTableColumnStatistics(
          dbName, tblName, Lists.newArrayList(colName[1])).isEmpty());

      colStats.setStatsDesc(statsDesc);
      colStats.setStatsObj(statsObjs);

      // update table level column stats
      client.updateTableColumnStatistics(colStats);

      // query column stats for column whose stats were updated in the previous call
      colStats2 = client.getTableColumnStatistics(
          dbName, tblName, Lists.newArrayList(colName[0])).get(0);

      // partition level column statistics test
      // create a table with multiple partitions
      cleanUp(dbName, tblName, typeName);

      List<List<String>> values = new ArrayList<List<String>>();
      values.add(makeVals("2008-07-01 14:13:12", "14"));
      values.add(makeVals("2008-07-01 14:13:12", "15"));
      values.add(makeVals("2008-07-02 14:13:12", "15"));
      values.add(makeVals("2008-07-03 14:13:12", "151"));

      createMultiPartitionTableSchema(dbName, tblName, typeName, values);

      List<String> partitions = client.listPartitionNames(dbName, tblName, (short)-1);

      partName = partitions.get(0);
      isTblLevel = false;

      // create a new columnstatistics desc to represent partition level column stats
      statsDesc = new ColumnStatisticsDesc();
      statsDesc.setDbName(dbName);
      statsDesc.setTableName(tblName);
      statsDesc.setPartName(partName);
      statsDesc.setIsTblLevel(isTblLevel);

      colStats = new ColumnStatistics();
      colStats.setStatsDesc(statsDesc);
      colStats.setStatsObj(statsObjs);
View Full Code Here

    if (!isTblLevel) {
      partName = work.getColStats().getPartName();
    }

    ColumnStatisticsDesc statsDesc = getColumnStatsDesc(dbName, tableName, partName, isTblLevel);

    List<ColumnStatisticsObj> statsObjs = new ArrayList<ColumnStatisticsObj>();
    StructObjectInspector soi = (StructObjectInspector) oi;
    List<? extends StructField> fields = soi.getAllStructFieldRefs();
    List<Object> list = soi.getStructFieldsDataAsList(o);
View Full Code Here

  }

  private ColumnStatisticsDesc getColumnStatsDesc(String dbName, String tableName,
      String partName, boolean isTblLevel)
  {
    ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
    statsDesc.setDbName(dbName);
    statsDesc.setTableName(tableName);
    statsDesc.setIsTblLevel(isTblLevel);

    if (!isTblLevel) {
      statsDesc.setPartName(partName);
    } else {
      statsDesc.setPartName(null);
    }
    return statsDesc;
  }
View Full Code Here

    return statsObj;
  }

  public static ColumnStatisticsDesc getTableColumnStatisticsDesc(
      MTableColumnStatistics mStatsObj) {
    ColumnStatisticsDesc statsDesc = new ColumnStatisticsDesc();
    statsDesc.setIsTblLevel(true);
    statsDesc.setDbName(mStatsObj.getDbName());
    statsDesc.setTableName(mStatsObj.getTableName());
    statsDesc.setLastAnalyzed(mStatsObj.getLastAnalyzed());
    return statsDesc;
  }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.