Examples of FetchTask

bixo.fetcher.FetchTask
Runnable instance for fetching a set of URLs from the same server, using keep-alive.
com.ptaack.seobot.tasks.FetchTask
org.apache.hadoop.hive.ql.exec.FetchTask
FetchTask implementation.

Examples of bixo.fetcher.FetchTask

                } else {
                    List<ScoredUrlDatum> urls = datum.getUrls();
                    String ref = datum.getGroupingRef();
                    trace("Processing %d URLs for %s", urls.size(), ref);


                    Runnable doFetch = new FetchTask(this, _fetcher, urls, ref);
                    if (datum.isLastList()) {
                        makeActive(ref, 0L);
                        trace("Executing fetch of %d URLs from %s (last batch)", urls.size(), ref);
                    } else {
                        Long nextFetchTime = System.currentTimeMillis() + datum.getFetchDelay();

View Full Code Here

Examples of com.ptaack.seobot.tasks.FetchTask

        String[] elemArr = line.split(";");
        if (elemArr[0].toLowerCase().equals("visit")) {
          this.Tasks.add(new VisitTask(elemArr[1], elemArr[2]));
        }
        else if (elemArr[0].toLowerCase().equals("fetchproxy")) {
          this.Tasks.add(new FetchTask(elemArr[1]));
        }
      }
    } catch (FileNotFoundException e) {
      System.err.println("Could not found configuration file");

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.FetchTask


  @SuppressWarnings("nls")
  private void genMapRedTasks(QB qb) throws SemanticException {
    FetchWork fetch = null;
    List<Task<? extends Serializable>> mvTask = new ArrayList<Task<? extends Serializable>>();
    FetchTask fetchTask = null;


    QBParseInfo qbParseInfo = qb.getParseInfo();


    // Does this query need reduce job
    if (qb.isSelectStarQuery() && qbParseInfo.getDestToClusterBy().isEmpty()

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.FetchTask

      // can't get any info without a plan
    } else if (sem.getResultSchema() != null) {
      List<FieldSchema> lst = sem.getResultSchema();
      schema = new Schema(lst, null);
    } else if (sem.getFetchTask() != null) {
      FetchTask ft = sem.getFetchTask();
      TableDesc td = ft.getTblDesc();
      // partitioned tables don't have tableDesc set on the FetchTask. Instead
      // they have a list of PartitionDesc objects, each with a table desc.
      // Let's
      // try to fetch the desc for the first partition and use it's
      // deserializer.
      if (td == null && ft.getWork() != null && ft.getWork().getPartDesc() != null) {
        if (ft.getWork().getPartDesc().size() > 0) {
          td = ft.getWork().getPartDesc().get(0).getTableDesc();
        }
      }


      if (td == null) {
        LOG.info("No returning schema.");

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.FetchTask

    }
  }


  public boolean getResults(ArrayList<String> res) throws IOException {
    if (plan != null && plan.getFetchTask() != null) {
      FetchTask ft = plan.getFetchTask();
      ft.setMaxRows(maxRows);
      return ft.fetch(res);
    }


    if (resStream == null) {
      resStream = ctx.getStream();
    }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.FetchTask

    if (sem != null && sem.getFetchTask() != null) {
      if (!sem.getFetchTaskInit()) {
        sem.setFetchTaskInit(true);
        sem.getFetchTask().initialize(conf);
      }
      FetchTask ft = (FetchTask) sem.getFetchTask();


      tableDesc td = ft.getTblDesc();
      String tableName = "result";
      List<FieldSchema> lst = MetaStoreUtils.getFieldsFromDeserializer(
          tableName, td.getDeserializer());
      String schema = MetaStoreUtils.getDDLFromFieldSchema(tableName, lst);
      return schema;

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.FetchTask

    if (sem != null && sem.getFetchTask() != null) {
      if (!sem.getFetchTaskInit()) {
        sem.setFetchTaskInit(true);
        sem.getFetchTask().initialize(conf);
      }
      FetchTask ft = (FetchTask) sem.getFetchTask();
      ft.setMaxRows(maxRows);
      return ft.fetch(res);
    }


    if (resStream == null)
      resStream = ctx.getStream();
    if (resStream == null)

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.FetchTask

      work = new FetchWork(partLocs, partDesc, Utilities.getTableDesc(tbl));
      work.setLimit(100);
    } else {
      work = new FetchWork(tbl.getDataLocation().toString(), Utilities.getTableDesc(tbl));
    }
    FetchTask task = new FetchTask();
    task.setWork(work);
    task.initialize(conf, null, null);
    task.fetch(temp);
    for (String str : temp) {
      results.add(str.replace("\t", ","));
    }
    return results;
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.FetchTask

      work = new FetchWork(partLocs, partDesc, Utilities.getTableDesc(tbl));
      work.setLimit(100);
    } else {
      work = new FetchWork(tbl.getDataLocation().toString(), Utilities.getTableDesc(tbl));
    }
    FetchTask task = new FetchTask();
    task.setWork(work);
    task.initialize(conf, null, null);
    task.fetch(temp);
    for (String str : temp) {
      results.add(str.replace("\t", ","));
    }
    return results;
  }

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.FetchTask

      // no CTAS or insert, not analyze command, and single sourced.
      String alias = (String) pctx.getTopOps().keySet().toArray()[0];
      Operator topOp = (Operator) pctx.getTopOps().values().toArray()[0];
      if (topOp instanceof TableScanOperator) {
        try {
          FetchTask fetchTask = optimize(pctx, alias, (TableScanOperator) topOp);
          if (fetchTask != null) {
            pctx.setFetchTask(fetchTask);
          }
        } catch (HiveException e) {
          // Has to use full name to make sure it does not conflict with

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.