Examples of FetchWork

Examples of org.apache.hadoop.hive.ql.plan.FetchWork

    prop.setProperty(serdeConstants.SERIALIZATION_NULL_FORMAT, " ");
    String[] colTypes = schema.split("#");
    prop.setProperty("columns", colTypes[0]);
    prop.setProperty("columns.types", colTypes[1]);
    prop.setProperty(serdeConstants.SERIALIZATION_LIB, LazySimpleSerDe.class.getName());
    FetchWork fetch = new FetchWork(ctx.getResFile(), new TableDesc(
        TextInputFormat.class,IgnoreKeyTextOutputFormat.class, prop), -1);
    fetch.setSerializationNullFormat(" ");
    return (FetchTask) TaskFactory.get(fetch, conf);
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.FetchWork

      for (String path : emptyPath) {
        pathToAliases.remove(path);
      }


      // create fetch work
      FetchWork fetchWork = null;
      List<String> partDir = new ArrayList<String>();
      List<PartitionDesc> partDesc = new ArrayList<PartitionDesc>();


      for (String tablePath : pathSet) {
        PartitionDesc partitionDesc = newWork.getPathToPartitionInfo().get(tablePath);
        // create fetchwork for non partitioned table
        if (partitionDesc.getPartSpec() == null || partitionDesc.getPartSpec().size() == 0) {
          fetchWork = new FetchWork(tablePath, partitionDesc.getTableDesc());
          break;
        }
        // if table is partitioned,add partDir and partitionDesc
        partDir.add(tablePath);
        partDesc.add(partitionDesc);
      }
      // create fetchwork for partitioned table
      if (fetchWork == null) {
        TableDesc table = newWork.getAliasToPartnInfo().get(alias).getTableDesc();
        fetchWork = new FetchWork(partDir, partDesc, table);
      }
      // set alias to fetch work
      newLocalWork.getAliasToFetchWork().put(alias, fetchWork);
    }
    // remove small table ailias from aliasToWork;Avoid concurrent modification

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.FetchWork

    return "COLUMNSTATS TASK";
  }


  @Override
  protected void localizeMRTmpFilesImpl(Context ctx) {
    FetchWork fWork = work.getfWork();
    String s = fWork.getTblDir();
    if ((s != null) && ctx.isMRTmpFileURI(s)) {
      fWork.setTblDir(ctx.localizeMRTmpFileURI(s));
    }
    ArrayList<String> ls = fWork.getPartDir();
    if (ls != null) {
      ctx.localizePaths(ls);
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.FetchWork


    boolean aggressive = "more".equals(mode);
    FetchData fetch = checkTree(aggressive, pctx, alias, source);
    if (fetch != null) {
      int limit = pctx.getQB().getParseInfo().getOuterQueryLimit();
      FetchWork fetchWork = fetch.convertToWork();
      FetchTask fetchTask = (FetchTask) TaskFactory.get(fetchWork, pctx.getConf());
      fetchWork.setSink(fetch.completed(pctx, fetchWork));
      fetchWork.setSource(source);
      fetchWork.setLimit(limit);
      return fetchTask;
    }
    return null;
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.FetchWork

    private FetchWork convertToWork() throws HiveException {
      inputs.clear();
      if (table != null) {
        inputs.add(new ReadEntity(table));
        String path = table.getPath().toString();
        FetchWork work = new FetchWork(path, Utilities.getTableDesc(table));
        PlanUtils.configureInputJobPropertiesForStorageHandler(work.getTblDesc());
        work.setSplitSample(splitSample);
        return work;
      }
      List<String> listP = new ArrayList<String>();
      List<PartitionDesc> partP = new ArrayList<PartitionDesc>();


      for (Partition partition : partsList.getNotDeniedPartns()) {
        inputs.add(new ReadEntity(partition));
        listP.add(partition.getPartitionPath().toString());
        partP.add(Utilities.getPartitionDesc(partition));
      }
      TableDesc table = Utilities.getTableDesc(partsList.getSourceTable());
      FetchWork work = new FetchWork(listP, partP, table);
      if (!work.getPartDesc().isEmpty()) {
        PartitionDesc part0 = work.getPartDesc().get(0);
        PlanUtils.configureInputJobPropertiesForStorageHandler(part0.getTableDesc());
        work.setSplitSample(splitSample);
      }
      return work;
    }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.FetchWork

        conf.addResource("hive-site.xml");
        ArrayList<String> results = new ArrayList<String>();
        ArrayList<String> temp = new ArrayList<String>();
        Hive hive = Hive.get(conf);
        org.apache.hadoop.hive.ql.metadata.Table tbl = hive.getTable(database, table);
        FetchWork work;
        if (!tbl.getPartCols().isEmpty()) {
            List<Partition> partitions = hive.getPartitions(tbl);
            List<PartitionDesc> partDesc = new ArrayList<PartitionDesc>();
            List<String> partLocs = new ArrayList<String>();
            for (Partition part : partitions) {
                partLocs.add(part.getLocation());
                partDesc.add(Utilities.getPartitionDesc(part));
            }
            work = new FetchWork(partLocs, partDesc, Utilities.getTableDesc(tbl));
            work.setLimit(100);
        } else {
            work = new FetchWork(tbl.getDataLocation().toString(), Utilities.getTableDesc(tbl));
        }
        FetchTask task = new FetchTask();
        task.setWork(work);
        task.initialize(conf, null, null);
        task.fetch(temp);

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.FetchWork

    Map<String,FetchWork> aliasToFetchWork = localWork.getAliasToFetchWork();
    Map<String, Operator<? extends OperatorDesc>> aliasToWork = localWork.getAliasToWork();


    for (Map.Entry<String, FetchWork> entry : aliasToFetchWork.entrySet()) {
      String alias = entry.getKey();
      FetchWork fetchWork = entry.getValue();


      Operator<? extends OperatorDesc> forwardOp = aliasToWork.get(alias);
      forwardOp.setExecContext(getExecContext());


      JobConf jobClone = cloneJobConf(hconf, forwardOp);

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.FetchWork

   */
  private void genColumnStatsTask(QB qb) {
    QBParseInfo qbParseInfo = qb.getParseInfo();
    ColumnStatsTask cStatsTask = null;
    ColumnStatsWork cStatsWork = null;
    FetchWork fetch = null;
    String tableName = qbParseInfo.getTableName();
    String partName = qbParseInfo.getPartName();
    List<String> colName = qbParseInfo.getColName();
    List<String> colType = qbParseInfo.getColType();
    boolean isTblLevel = qbParseInfo.isTblLvl();


    String cols = loadFileWork.get(0).getColumns();
    String colTypes = loadFileWork.get(0).getColumnTypes();


    String resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
    TableDesc resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat);


    fetch = new FetchWork(new Path(loadFileWork.get(0).getSourceDir()).toString(),
        resultTab, qb.getParseInfo().getOuterQueryLimit());


    ColumnStatsDesc cStatsDesc = new ColumnStatsDesc(tableName, partName,
                                       colName, colType, isTblLevel);
    cStatsWork = new ColumnStatsWork(fetch, cStatsDesc);

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.FetchWork

      String colTypes = loadFileWork.get(0).getColumnTypes();


      String resFileFormat = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYRESULTFILEFORMAT);
      TableDesc resultTab = PlanUtils.getDefaultQueryOutputTableDesc(cols, colTypes, resFileFormat);


      FetchWork fetch = new FetchWork(new Path(loadFileWork.get(0).getSourceDir()).toString(),
          resultTab, qb.getParseInfo().getOuterQueryLimit());


      FetchTask fetchTask = (FetchTask) TaskFactory.get(fetch, conf);
      setFetchTask(fetchTask);

View Full Code Here

Examples of org.apache.hadoop.hive.ql.plan.FetchWork

      localPlan.getAliasToWork().put(alias_id, topOp);
      if (tblDir == null) {
        tblDesc = Utilities.getTableDesc(partsList.getSourceTable());
        localPlan.getAliasToFetchWork().put(
            alias_id,
            new FetchWork(FetchWork.convertPathToStringArray(partDir), partDesc, tblDesc));
      } else {
        localPlan.getAliasToFetchWork().put(alias_id,
            new FetchWork(tblDir.toString(), tblDesc));
      }
      plan.setMapLocalWork(localPlan);
    }
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.