Examples of TezEdgeDescriptor

org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor
Descriptor for Tez edge. It holds combine plan as well as edge properties.

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor

     * @return EdgeProperty
     * @throws IOException
     */
    private EdgeProperty newEdge(TezOperator from, TezOperator to)
            throws IOException {
        TezEdgeDescriptor edge = to.inEdges.get(from.getOperatorKey());
        PhysicalPlan combinePlan = edge.combinePlan;


        InputDescriptor in = InputDescriptor.create(edge.inputClassName);
        OutputDescriptor out = OutputDescriptor.create(edge.outputClassName);


        Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties(), false);
        if (!combinePlan.isEmpty()) {
            addCombiner(combinePlan, to, conf);
        }


        List<POLocalRearrangeTez> lrs = PlanHelper.getPhysicalOperators(from.plan,
                POLocalRearrangeTez.class);


        for (POLocalRearrangeTez lr : lrs) {
            if (lr.getOutputKey().equals(to.getOperatorKey().toString())) {
                byte keyType = lr.getKeyType();
                setIntermediateOutputKeyValue(keyType, conf, to, lr.isConnectedToPackage());
                // In case of secondary key sort, main key type is the actual key type
                conf.set("pig.reduce.key.type", Byte.toString(lr.getMainKeyType()));
                break;
            }
        }


        conf.setIfUnset(TezRuntimeConfiguration.TEZ_RUNTIME_PARTITIONER_CLASS,
                MRPartitioner.class.getName());


        if (edge.getIntermediateOutputKeyClass() != null) {
            conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_CLASS,
                    edge.getIntermediateOutputKeyClass());
        }


        if (edge.getIntermediateOutputValueClass() != null) {
            conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_VALUE_CLASS,
                    edge.getIntermediateOutputValueClass());
        }


        if (edge.getIntermediateOutputKeyComparatorClass() != null) {
            conf.set(TezRuntimeConfiguration.TEZ_RUNTIME_KEY_COMPARATOR_CLASS,
                    edge.getIntermediateOutputKeyComparatorClass());
        }


        conf.setBoolean(MRConfiguration.MAPPER_NEW_API, true);
        conf.set("pig.pigContext", ObjectSerializer.serialize(pc));
        conf.set("udf.import.list",
                ObjectSerializer.serialize(PigContext.getPackageImportList()));


        if(to.isGlobalSort() || to.isLimitAfterSort()){
            conf.set("pig.sortOrder",
                    ObjectSerializer.serialize(to.getSortOrder()));
        }


        if (edge.isUseSecondaryKey()) {
            conf.set("pig.secondarySortOrder",
                    ObjectSerializer.serialize(edge.getSecondarySortOrder()));
            conf.set(org.apache.hadoop.mapreduce.MRJobConfig.PARTITIONER_CLASS_ATTR,
                    SecondaryKeyPartitioner.class.getName());
            // These needs to be on the vertex as well for POShuffleTezLoad to pick it up.
            // Tez framework also expects this to be per vertex and not edge. IFile.java picks
            // up keyClass and valueClass from vertex config. TODO - check with Tez folks

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor

                POLocalRearrangeTez lr = (POLocalRearrangeTez) leaf;
                lr.setOutputKey(to.getOperatorKey().toString());
            }
        }
        // Add edge descriptors to old and new operators
        TezEdgeDescriptor edge = new TezEdgeDescriptor();
        to.inEdges.put(from.getOperatorKey(), edge);
        from.outEdges.put(to.getOperatorKey(), edge);
        return edge;
    }

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor

                                    .equals(succOp.getOperatorKey().toString())) {
                        succOpVertexGroup = succ;
                        break;
                    }
                }
                TezEdgeDescriptor edge = entry.getValue();
                // Edge cannot be one to one as it will get input from two or
                // more union predecessors. Change it to SCATTER_GATHER
                if (edge.dataMovementType == DataMovementType.ONE_TO_ONE) {
                    edge.dataMovementType = DataMovementType.SCATTER_GATHER;
                    edge.partitionerClass = RoundRobinPartitioner.class;

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor

        if (plan.getSuccessors(splittee)!=null) {
            List<TezOperator> succs = new ArrayList<TezOperator>();
            succs.addAll(plan.getSuccessors(splittee));
            plan.disconnect(splitter, splittee);
            for (TezOperator succTezOperator : succs) {
                TezEdgeDescriptor edge = succTezOperator.inEdges.get(splittee.getOperatorKey());


                splitter.outEdges.remove(splittee.getOperatorKey());
                succTezOperator.inEdges.remove(splittee.getOperatorKey());
                plan.disconnect(splittee, succTezOperator);
                TezCompilerUtil.connect(plan, splitter, succTezOperator, edge);

View Full Code Here

Examples of org.apache.pig.backend.hadoop.executionengine.tez.plan.TezEdgeDescriptor

        if (connectingLR == null) {
            return;
        }


        // Detected the POLocalRearrange -> POPackage pattern
        TezEdgeDescriptor inEdge = to.inEdges.get(from.getOperatorKey());
        // Only optimize for Cogroup case
        if (from.isGlobalSort()) {
            return;
        }


        // If there is a custom partitioner do not do secondary key optimization.
        if (inEdge.partitionerClass != null) {
            return;
        }


        if (from.plan.getOperator(connectingLR.getOperatorKey()) == null) {
            // The POLocalRearrange is sub-plan of a POSplit
            rearrangePlan = PlanHelper.getLocalRearrangePlanFromSplit(from.plan, connectingLR.getOperatorKey());
        }


        SecondaryKeyOptimizerInfo info = SecondaryKeyOptimizerUtil.applySecondaryKeySort(rearrangePlan, to.plan);
        if (info != null) {
            numSortRemoved += info.getNumSortRemoved();
            numDistinctChanged += info.getNumDistinctChanged();
            numUseSecondaryKey += info.getNumUseSecondaryKey();
            if (info.isUseSecondaryKey()) {
                // Set it on the receiving vertex and the connecting edge.
                to.setUseSecondaryKey(true);
                inEdge.setUseSecondaryKey(true);
                inEdge.setSecondarySortOrder(info.getSecondarySortOrder());
                log.info("Using Secondary Key Optimization in the edge between vertex - "
                        + from.getOperatorKey()
                        + " and vertex - "
                        + to.getOperatorKey());
            }

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.