Package org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators

Examples of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc

881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
                }
                // we have stick in the POUserfunc(org.apache.pig.builtin.Distinct)[DataBag]
                // in place of the Project-PODistinct pair
                PhysicalOperator distinctPredecessor = mPlan.getPredecessors(pred).get(0);

                POUserFunc func = null;

                try {
                    String scope = proj.getOperatorKey().scope;
                    List<PhysicalOperator> funcInput = new ArrayList<PhysicalOperator>();
                    FuncSpec fSpec = new FuncSpec(DISTINCT_UDF_CLASSNAME);
                    funcInput.add(distinctPredecessor);
                    // explicitly set distinctPredecessor's result type to
                    // be tuple - this is relevant when distinctPredecessor is
                    // originally a POForeach with return type BAG - we need to
                    // set it to tuple so we get a stream of tuples.
                    distinctPredecessor.setResultType(DataType.TUPLE);
                    func = new POUserFunc(new OperatorKey(scope,
                            NodeIdGenerator.getGenerator().getNextNodeId(scope)),-1, funcInput, fSpec);
                    func.setResultType(DataType.BAG);
                    mPlan.replace(proj, func);
                    mPlan.remove(pred);
                    // connect the the newly added "func" to
                    // the predecessor to the earlier PODistinct
                    mPlan.connect(distinctPredecessor, func);
View Full Code Here
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
                ce1.setValue(ce1val);
                ce1.setResultType(DataType.TUPLE);*/
               
               

                POUserFunc gfc = new POUserFunc(new OperatorKey(scope, nodeGen.getNextNodeId(scope)),cross.getRequestedParallelisam(), Arrays.asList((PhysicalOperator)ce1,(PhysicalOperator)ce2), new FuncSpec(GFCross.class.getName()));
                gfc.setAlias(cross.getAlias());
                gfc.setResultType(DataType.BAG);
                fep1.addAsLeaf(gfc);
                gfc.setInputs(Arrays.asList((PhysicalOperator)ce1,(PhysicalOperator)ce2));
                /*fep1.add(gfc);
                fep1.connect(ce1, gfc);
                fep1.connect(ce2, gfc);*/
               
                PhysicalPlan fep2 = new PhysicalPlan();
View Full Code Here
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
       
        List<PhysicalOperator> ufInps = new ArrayList<PhysicalOperator>();
        ufInps.add(prjStar);
       
      PhysicalPlan ep = new PhysicalPlan();
      POUserFunc uf = new POUserFunc(new OperatorKey(scope,nig.getNextNodeId(scope)), -1, ufInps,
                  new FuncSpec(GetMemNumRows.class.getName(), (String[])null));
      uf.setResultType(DataType.TUPLE);
      ep.add(uf);    
      ep.add(prjStar);
      ep.connect(prjStar, uf);

        transformPlans.add(ep);     
View Full Code Here
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
        ep4.add(prjStar4);
       
        List<PhysicalOperator> ufInps = new ArrayList<PhysicalOperator>();
        ufInps.add(prjStar4);
     
        POUserFunc uf = new POUserFunc(new OperatorKey(scope,nig.getNextNodeId(scope)), -1, ufInps,
            new FuncSpec(udfClassName, udfArgs));
        ep4.add(uf);
        ep4.connect(prjStar4, uf);
       
        List<PhysicalPlan> ep4s = new ArrayList<PhysicalPlan>();
View Full Code Here
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
                }
                // we have stick in the POUserfunc(org.apache.pig.builtin.Distinct)[DataBag]
                // in place of the Project-PODistinct pair
                PhysicalOperator distinctPredecessor = mPlan.getPredecessors(pred).get(0);

                POUserFunc func = null;

                try {
                    String scope = proj.getOperatorKey().scope;
                    List<PhysicalOperator> funcInput = new ArrayList<PhysicalOperator>();
                    FuncSpec fSpec = new FuncSpec(DISTINCT_UDF_CLASSNAME);
                    funcInput.add(distinctPredecessor);
                    // explicitly set distinctPredecessor's result type to
                    // be tuple - this is relevant when distinctPredecessor is
                    // originally a POForeach with return type BAG - we need to
                    // set it to tuple so we get a stream of tuples.
                    distinctPredecessor.setResultType(DataType.TUPLE);
                    func = new POUserFunc(new OperatorKey(scope,
                            NodeIdGenerator.getGenerator().getNextNodeId(scope)),-1, funcInput, fSpec);
                    func.setResultType(DataType.BAG);
                    mPlan.replace(proj, func);
                    mPlan.remove(pred);
                    // connect the the newly added "func" to
                    // the predecessor to the earlier PODistinct
                    mPlan.connect(distinctPredecessor, func);
View Full Code Here
79
80
81
82
83
84
85
86
87
88
89
        // setup udf
        List<PhysicalOperator> udfInps = new ArrayList<PhysicalOperator>();
        udfInps.add(projVal1);
        FuncSpec sumSpec = new FuncSpec(IntSum.Intermediate.class.getName());
        POUserFunc sumUdf = new POUserFunc(GenPhyOp.getOK(), -1, udfInps,
                sumSpec);
        valPlan1.add(sumUdf);
        valPlan1.connect(projVal1, sumUdf);

        List<PhysicalPlan> valuePlans = new ArrayList<PhysicalPlan>();
View Full Code Here
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
       
        List<PhysicalOperator> ufInps = new ArrayList<PhysicalOperator>();
        ufInps.add(prjStar);
       
      PhysicalPlan ep = new PhysicalPlan();
      POUserFunc uf = new POUserFunc(new OperatorKey(scope,nig.getNextNodeId(scope)), -1, ufInps,
                  new FuncSpec(GetMemNumRows.class.getName(), (String[])null));
      uf.setResultType(DataType.TUPLE);
      ep.add(uf);    
      ep.add(prjStar);
      ep.connect(prjStar, uf);

        transformPlans.add(ep);     
View Full Code Here
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
        ep4.add(prjStar4);
       
        List<PhysicalOperator> ufInps = new ArrayList<PhysicalOperator>();
        ufInps.add(prjStar4);
     
        POUserFunc uf = new POUserFunc(new OperatorKey(scope,nig.getNextNodeId(scope)), -1, ufInps,
            new FuncSpec(udfClassName, udfArgs));
        ep4.add(uf);
        ep4.connect(prjStar4, uf);
       
        List<PhysicalPlan> ep4s = new ArrayList<PhysicalPlan>();
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POUserFunc

  • org.apache.hadoop.conf.Configuration
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.CombinerOptimizer$DistinctPatcher
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.util.MonitoredUDFExecutor
  • org.apache.pig.backend.hadoop.executionengine.tez.plan.TezCompiler
  • org.apache.pig.backend.hadoop.executionengine.util.CombinerOptimizerUtil
  • org.apache.pig.backend.hadoop.executionengine.util.CombinerOptimizerUtil$DistinctPatcher
  • org.apache.pig.data.DataByteArray
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.