Examples of DataChannel

de.netseeker.ejoe.io.DataChannel
Utility class handling all socket oriented data IO on nio channels. DataChannels must be implemented as singletons to avoid creation of a new object for each socket IO operation. Otherwise heavy load could result in fast-growing memory consumption. @author netseeker @since 0.3.9.1
org.apache.cayenne.DataChannel
DataChannel is an abstraction used by ObjectContexts to obtain mapping metadata and access a persistent store. There is rarely a need to use it directly. @since 1.2
org.apache.tajo.engine.planner.global.DataChannel
org.imagearchive.lsm.toolbox.info.scaninfo.DataChannel

Examples of org.apache.tajo.engine.planner.global.DataChannel

    Expr context = analyzer.parse(SORT_QUERY[0]);
    LogicalPlan plan = planner.createPlan(context);
    LogicalNode rootNode = optimizer.optimize(plan);


    SortNode sortNode = PlannerUtil.findTopNode(rootNode, NodeType.SORT);
    DataChannel channel = new DataChannel(masterPlan.newExecutionBlockId(), masterPlan.newExecutionBlockId(),
        PartitionType.RANGE_PARTITION);
    channel.setPartitionKey(PlannerUtil.sortSpecsToSchema(sortNode.getSortKeys()).toArray());
    ctx.setDataChannel(channel);


    PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
    PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.DataChannel

    }
  }


  private PhysicalExec buildOutputOperator(TaskAttemptContext context, LogicalNode plan,
                                           PhysicalExec execPlan) throws IOException {
    DataChannel channel = context.getDataChannel();
    ShuffleFileWriteNode shuffleFileWriteNode = LogicalPlan.createNodeWithoutPID(ShuffleFileWriteNode.class);
    shuffleFileWriteNode.setStorageType(context.getDataChannel().getStoreType());
    shuffleFileWriteNode.setInSchema(plan.getOutSchema());
    shuffleFileWriteNode.setOutSchema(plan.getOutSchema());
    shuffleFileWriteNode.setShuffle(channel.getShuffleType(), channel.getShuffleKeys(), channel.getShuffleOutputNum());
    shuffleFileWriteNode.setChild(plan);


    PhysicalExec outExecPlan = createShuffleFileWritePlan(context, shuffleFileWriteNode, execPlan);
    return outExecPlan;
  }

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.DataChannel


    private void finalizeQuery(Query query, QueryCompletedEvent event) {
      MasterPlan masterPlan = query.getPlan();


      ExecutionBlock terminal = query.getPlan().getTerminalBlock();
      DataChannel finalChannel = masterPlan.getChannel(event.getExecutionBlockId(), terminal.getId());
      Path finalOutputDir = commitOutputData(query);


      QueryHookExecutor hookExecutor = new QueryHookExecutor(query.context.getQueryMasterContext());
      try {
        hookExecutor.execute(query.context.getQueryContext(), query, event.getExecutionBlockId(),

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.DataChannel

      statsArray = computeStatFromUnionBlock(this);
    } else {
      statsArray = computeStatFromTasks();
    }


    DataChannel channel = masterPlan.getOutgoingChannels(getId()).get(0);
    // get default or store type
    CatalogProtos.StoreType storeType = CatalogProtos.StoreType.CSV; // default setting


    // if store plan (i.e., CREATE or INSERT OVERWRITE)
    StoreTableNode storeTableNode = PlannerUtil.findTopNode(getBlock().getPlan(), NodeType.STORE);
    if (storeTableNode != null) {
      storeType = storeTableNode.getStorageType();
    }
    schema = channel.getSchema();
    meta = CatalogUtil.newTableMeta(storeType, new Options());
    inputStatistics = statsArray[0];
    resultStatistics = statsArray[1];
  }

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.DataChannel

        if (execBlock.hasUnion()) {
          subQuery.finalizeStats();
          state = SubQueryState.SUCCEEDED;
        } else {
          ExecutionBlock parent = subQuery.getMasterPlan().getParent(subQuery.getBlock());
          DataChannel channel = subQuery.getMasterPlan().getChannel(subQuery.getId(), parent.getId());
          setShuffleIfNecessary(subQuery, channel);
          initTaskScheduler(subQuery);
          schedule(subQuery);
          subQuery.totalScheduledObjectsCount = subQuery.getTaskScheduler().remainingScheduledObjectNum();
          LOG.info(subQuery.totalScheduledObjectsCount + " objects are scheduled");

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.DataChannel

  }


  public static void scheduleFragmentsForNonLeafTasks(TaskSchedulerContext schedulerContext,
                                                      MasterPlan masterPlan, SubQuery subQuery, int maxNum)
      throws IOException {
    DataChannel channel = masterPlan.getIncomingChannels(subQuery.getBlock().getId()).get(0);
    if (channel.getShuffleType() == HASH_SHUFFLE) {
      scheduleHashShuffledFetches(schedulerContext, masterPlan, subQuery, channel, maxNum);
    } else if (channel.getShuffleType() == RANGE_SHUFFLE) {
      scheduleRangeShuffledFetches(schedulerContext, masterPlan, subQuery, channel, maxNum);
    } else {
      throw new InternalException("Cannot support partition type");
    }
  }

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.DataChannel

    LogicalPlan plan = planner.createPlan(session, context);


    int numPartitions = 3;
    Column key1 = new Column("default.score.deptname", Type.TEXT);
    Column key2 = new Column("default.score.class", Type.TEXT);
    DataChannel dataChannel = new DataChannel(masterPlan.newExecutionBlockId(), masterPlan.newExecutionBlockId(),
        ShuffleType.HASH_SHUFFLE, numPartitions);
    dataChannel.setShuffleKeys(new Column[]{key1, key2});
    ctx.setDataChannel(dataChannel);
    LogicalNode rootNode = optimizer.optimize(plan);


    TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getStoreType());


    FileSystem fs = sm.getFileSystem();


    PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
    PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.DataChannel

    ctx.setEnforcer(new Enforcer());
    Expr expr = analyzer.parse(QUERIES[14]);
    LogicalPlan plan = planner.createPlan(session, expr);
    LogicalNode rootNode = plan.getRootBlock().getRoot();
    int numPartitions = 1;
    DataChannel dataChannel = new DataChannel(masterPlan.newExecutionBlockId(), masterPlan.newExecutionBlockId(),
        ShuffleType.HASH_SHUFFLE, numPartitions);
    dataChannel.setShuffleKeys(new Column[]{});
    ctx.setDataChannel(dataChannel);
    optimizer.optimize(plan);


    TableMeta outputMeta = CatalogUtil.newTableMeta(dataChannel.getStoreType());


    PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
    PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);
    exec.init();
    exec.next();

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.DataChannel

    Expr context = analyzer.parse(SORT_QUERY[0]);
    LogicalPlan plan = planner.createPlan(session, context);
    LogicalNode rootNode = optimizer.optimize(plan);


    SortNode sortNode = PlannerUtil.findTopNode(rootNode, NodeType.SORT);
    DataChannel channel = new DataChannel(masterPlan.newExecutionBlockId(), masterPlan.newExecutionBlockId(),
        TajoWorkerProtocol.ShuffleType.RANGE_SHUFFLE);
    channel.setShuffleKeys(PlannerUtil.sortSpecsToSchema(sortNode.getSortKeys()).toArray());
    ctx.setDataChannel(channel);


    PhysicalPlanner phyPlanner = new PhysicalPlannerImpl(conf,sm);
    PhysicalExec exec = phyPlanner.createPlan(ctx, rootNode);


    Tuple tuple;
    exec.init();
    exec.next();
    exec.close();


    Schema keySchema = new Schema();
    keySchema.addColumn("?empId", Type.INT4);
    SortSpec[] sortSpec = new SortSpec[1];
    sortSpec[0] = new SortSpec(keySchema.getColumn(0), true, false);
    TupleComparator comp = new TupleComparator(keySchema, sortSpec);
    BSTIndex bst = new BSTIndex(conf);
    BSTIndex.BSTIndexReader reader = bst.getIndexReader(new Path(workDir, "output/index"),
        keySchema, comp);
    reader.open();
    Path outputPath = StorageUtil.concatPath(workDir, "output", "output");
    TableMeta meta = CatalogUtil.newTableMeta(channel.getStoreType(), new Options());
    SeekableScanner scanner =
        StorageManagerFactory.getSeekableScanner(conf, meta, exec.getSchema(), outputPath);
    scanner.init();


    int cnt = 0;

View Full Code Here

Examples of org.apache.tajo.engine.planner.global.DataChannel

      return dataChannel;
    }
    if (!p.hasDataChannel()) {
      return null;
    }
    this.dataChannel = new DataChannel(p.getDataChannel());
    return this.dataChannel;
  }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.