Package org.apache.hadoop.hive.metastore.api

Examples of org.apache.hadoop.hive.metastore.api.ColumnStatistics$ColumnStatisticsTupleScheme


          if (desc.getLastAnalyzed() > mStat.getLastAnalyzed()) {
            desc.setLastAnalyzed(mStat.getLastAnalyzed());
          }
          statObjs.add(StatObjectConverter.getTableColumnStatisticsObj(mStat));
        }
        return new ColumnStatistics(desc, statObjs);
      }
    }.run(true);
  }
View Full Code Here


          boolean isLast = i == mStats.size();
          MPartitionColumnStatistics mStatsObj = isLast ? null : mStats.get(i);
          String partName = isLast ? null : (String)mStatsObj.getPartitionName();
          if (isLast || !partName.equals(lastPartName)) {
            if (i != 0) {
              result.add(new ColumnStatistics(csd, curList));
            }
            if (isLast) {
              continue;
            }
            csd = StatObjectConverter.getPartitionColumnStatisticsDesc(mStatsObj);
View Full Code Here

      dbName = dbName.toLowerCase();
      tableName = tableName.toLowerCase();
      colName = colName.toLowerCase();
      startFunction("get_column_statistics_by_table: db=" + dbName + " table=" + tableName +
                    " column=" + colName);
      ColumnStatistics statsObj = null;
      try {
        statsObj = getMS().getTableColumnStatistics(
            dbName, tableName, Lists.newArrayList(colName));
        assert statsObj.getStatsObjSize() <= 1;
        return statsObj;
      } finally {
        endFunction("get_column_statistics_by_table: ", statsObj != null, null, tableName);
      }
    }
View Full Code Here

        throws MetaException, NoSuchObjectException, TException {
      String dbName = request.getDbName(), tblName = request.getTblName();
      startFunction("get_table_statistics_req: db=" + dbName + " table=" + tblName);
      TableStatsResult result = null;
      try {
        ColumnStatistics cs = getMS().getTableColumnStatistics(
            dbName, tblName, request.getColNames());
        result = new TableStatsResult(
            cs == null ? Lists.<ColumnStatisticsObj>newArrayList() : cs.getStatsObj());
      } finally {
        endFunction("get_table_statistics_req: ", result == null, null, tblName);
      }
      return result;
    }
View Full Code Here

      tableName = tableName.toLowerCase();
      colName = colName.toLowerCase();
      String convertedPartName = lowerCaseConvertPartName(partName);
      startFunction("get_column_statistics_by_partition: db=" + dbName + " table=" + tableName +
          " partition=" + convertedPartName + " column=" + colName);
      ColumnStatistics statsObj = null;

      try {
        List<ColumnStatistics> list = getMS().getPartitionColumnStatistics(dbName, tableName,
            Lists.newArrayList(convertedPartName), Lists.newArrayList(colName));
        if (list.isEmpty()) return null;
View Full Code Here

      return null;
    }
    List<Object[]> list = ensureList(qResult);
    if (list.isEmpty()) return null;
    ColumnStatisticsDesc csd = new ColumnStatisticsDesc(true, dbName, tableName);
    ColumnStatistics result = makeColumnStats(list, csd, 0);
    timingTrace(doTrace, queryText, start, queryTime);
    query.closeAll();
    return result;
  }
View Full Code Here

    + "\"BIG_DECIMAL_HIGH_VALUE\", \"NUM_NULLS\", \"NUM_DISTINCTS\", \"AVG_COL_LEN\", "
    + "\"MAX_COL_LEN\", \"NUM_TRUES\", \"NUM_FALSES\", \"LAST_ANALYZED\" ";

  private ColumnStatistics makeColumnStats(
      List<Object[]> list, ColumnStatisticsDesc csd, int offset) {
    ColumnStatistics result = new ColumnStatistics();
    result.setStatsDesc(csd);
    List<ColumnStatisticsObj> csos = new ArrayList<ColumnStatisticsObj>(list.size());
    for (Object[] row : list) {
      // LastAnalyzed is stored per column but thrift has it per several;
      // get the lowest for now as nobody actually uses this field.
      Object laObj = row[offset + 14];
      if (laObj != null && (!csd.isSetLastAnalyzed() || csd.getLastAnalyzed() > (Long)laObj)) {
        csd.setLastAnalyzed((Long)laObj);
      }
      ColumnStatisticsData data = new ColumnStatisticsData();
      // see STATS_COLLIST
      int i = offset;
      ColumnStatisticsObj cso = new ColumnStatisticsObj((String)row[i++], (String)row[i++], data);
      Object llow = row[i++], lhigh = row[i++], dlow = row[i++], dhigh = row[i++],
          declow = row[i++], dechigh = row[i++], nulls = row[i++], dist = row[i++],
          avglen = row[i++], maxlen = row[i++], trues = row[i++], falses = row[i++];
      StatObjectConverter.fillColumnStatisticsData(cso.getColType(), data,
          llow, lhigh, dlow, dhigh, declow, dechigh, nulls, dist, avglen, maxlen, trues, falses);
      csos.add(cso);
    }
    result.setStatsObj(csos);
    return result;
  }
View Full Code Here

      statsData.getStringStats().setNumNulls(numNulls);

      statsObj.setStatsData(statsData);
      statsObjs.add(statsObj);

      ColumnStatistics colStats = new ColumnStatistics();
      colStats.setStatsDesc(statsDesc);
      colStats.setStatsObj(statsObjs);

      // write stats objs persistently
      client.updateTableColumnStatistics(colStats);

      // retrieve the stats obj that was just written
      ColumnStatistics colStats2 = client.getTableColumnStatistics(dbName, tblName, colName[0]);

     // compare stats obj to ensure what we get is what we wrote
      assertNotNull(colStats2);
      assertEquals(colStats2.getStatsDesc().getDbName(), dbName);
      assertEquals(colStats2.getStatsDesc().getTableName(), tblName);
      assertEquals(colStats2.getStatsObj().get(0).getColName(), colName[0]);
      assertEquals(colStats2.getStatsObj().get(0).getStatsData().getDoubleStats().getLowValue(),
        lowValue);
      assertEquals(colStats2.getStatsObj().get(0).getStatsData().getDoubleStats().getHighValue(),
        highValue);
      assertEquals(colStats2.getStatsObj().get(0).getStatsData().getDoubleStats().getNumNulls(),
        numNulls);
      assertEquals(colStats2.getStatsObj().get(0).getStatsData().getDoubleStats().getNumDVs(),
        numDVs);
      assertEquals(colStats2.getStatsDesc().isIsTblLevel(), isTblLevel);

      // test delete column stats; if no col name is passed all column stats associated with the
      // table is deleted
      boolean status = client.deleteTableColumnStatistics(dbName, tblName, null);
      assertTrue(status);
      // try to query stats for a column for which stats doesn't exist
      try {
        colStats2 = client.getTableColumnStatistics(dbName, tblName, colName[1]);
        assertTrue(true);
      } catch (NoSuchObjectException e) {
        System.out.println("Statistics for column=" + colName[1] + " not found");
      }

      colStats.setStatsDesc(statsDesc);
      colStats.setStatsObj(statsObjs);

      // update table level column stats
      client.updateTableColumnStatistics(colStats);

      // query column stats for column whose stats were updated in the previous call
      colStats2 = client.getTableColumnStatistics(dbName, tblName, colName[0]);

      // partition level column statistics test
      // create a table with multiple partitions
      cleanUp(dbName, tblName, typeName);

      List<List<String>> values = new ArrayList<List<String>>();
      values.add(makeVals("2008-07-01 14:13:12", "14"));
      values.add(makeVals("2008-07-01 14:13:12", "15"));
      values.add(makeVals("2008-07-02 14:13:12", "15"));
      values.add(makeVals("2008-07-03 14:13:12", "151"));

      createMultiPartitionTableSchema(dbName, tblName, typeName, values);

      List<String> partitions = client.listPartitionNames(dbName, tblName, (short)-1);

      partName = partitions.get(0);
      isTblLevel = false;

      // create a new columnstatistics desc to represent partition level column stats
      statsDesc = new ColumnStatisticsDesc();
      statsDesc.setDbName(dbName);
      statsDesc.setTableName(tblName);
      statsDesc.setPartName(partName);
      statsDesc.setIsTblLevel(isTblLevel);

      colStats = new ColumnStatistics();
      colStats.setStatsDesc(statsDesc);
      colStats.setStatsObj(statsObjs);

     client.updatePartitionColumnStatistics(colStats);

     colStats2 = client.getPartitionColumnStatistics(dbName, tblName, partName, colName[1]);

     // compare stats obj to ensure what we get is what we wrote
     assertNotNull(colStats2);
     assertEquals(colStats2.getStatsDesc().getDbName(), dbName);
     assertEquals(colStats2.getStatsDesc().getTableName(), tblName);
     assertEquals(colStats.getStatsDesc().getPartName(), partName);
     assertEquals(colStats2.getStatsObj().get(0).getColName(), colName[1]);
     assertEquals(colStats2.getStatsObj().get(0).getStatsData().getStringStats().getMaxColLen(),
       maxColLen);
     assertEquals(colStats2.getStatsObj().get(0).getStatsData().getStringStats().getAvgColLen(),
       avgColLen);
     assertEquals(colStats2.getStatsObj().get(0).getStatsData().getStringStats().getNumNulls(),
       numNulls);
     assertEquals(colStats2.getStatsObj().get(0).getStatsData().getStringStats().getNumDVs(),
       numDVs);
     assertEquals(colStats2.getStatsDesc().isIsTblLevel(), isTblLevel);

     // test stats deletion at partition level
     client.deletePartitionColumnStatistics(dbName, tblName, partName, colName[1]);

     colStats2 = client.getPartitionColumnStatistics(dbName, tblName, partName, colName[0]);
View Full Code Here

      statsObj.setColType(colType.get(i));
      unpackStructObject(foi, f, fieldName, statsObj);
      statsObjs.add(statsObj);
    }

    ColumnStatistics colStats = new ColumnStatistics();
    colStats.setStatsDesc(statsDesc);
    colStats.setStatsObj(statsObjs);
    return colStats;
  }
View Full Code Here

    } catch (CommandNeedRetryException e) {
      e.printStackTrace();
    }

    // Construct a column statistics object from the result
    ColumnStatistics colStats = constructColumnStatsFromPackedRow(io.oi, io.o);

    // Persist the column statistics object to the metastore
    try {
      db.updatePartitionColumnStatistics(colStats);
    } catch (Exception e) {
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hive.metastore.api.ColumnStatistics$ColumnStatisticsTupleScheme

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.