Examples of org.apache.pig.impl.eval.GenerateSpec

org.apache.pig.impl.eval.GenerateSpec

            // will come out of the combiner in the form (groupkey,
            // {(x, y, z)}).  The second ProjectSpec contains the offset of
            // the projection element we're interested in.
            CompositeEvalSpec cs = new CompositeEvalSpec(new ProjectSpec(1));
            cs.addSpec(new ProjectSpec(position));
            fe.setArgs(new GenerateSpec(cs));




            // Reset the function to call the final instance of itself
            // instead of the general instance.  Have to instantiate the
            // function itself first so we can find out if it's algebraic

View Full Code Here

        //Constructing the query structures by hand, quite ugly.
        
        //group all
        ArrayList<EvalSpec> groupFuncs = new ArrayList<EvalSpec>();
    
        groupFuncs.add(new GenerateSpec(new ConstSpec("all")).getGroupBySpec());
    
        quantileJob.groupFuncs = groupFuncs;
        
        //find the quantiles in the reduce step
        ArrayList<EvalSpec> argsList = new ArrayList<EvalSpec>();
        argsList.add(new ConstSpec(Math.max(loSort.getRequestedParallelism()-1,1)));
        
        //sort the first column of the cogroup output and feed it to the quantiles function
        EvalSpec sortedSampleSpec = new ProjectSpec(1);
        EvalSpec starSpec = new StarSpec();
        starSpec.setComparatorName(loSort.getSortSpec().getComparatorName());
        sortedSampleSpec = sortedSampleSpec.addSpec(new SortDistinctSpec(false, starSpec));
        argsList.add(sortedSampleSpec);
        
        EvalSpec args = new GenerateSpec(argsList);


        EvalSpec reduceSpec = new FuncEvalSpec(pigContext, FindQuantiles.class.getName(), args);
        reduceSpec.setFlatten(true);
        quantileJob.addReduceSpec(new GenerateSpec(reduceSpec));
        
        //a temporary file to hold the quantile data
        String quantileFile = getTempFile(pigContext);
        quantileJob.outputFileSpec = new FileSpec(quantileFile, BinStorage.class.getName());

View Full Code Here

        //same input as the quantile job, but the full BinStorage load function
        sortJob.addInputFile(new FileSpec(quantileJob.getFileSpec(0).getFileName(), BinStorage.class.getName()));
        
        ArrayList<EvalSpec> groupFuncs = new ArrayList<EvalSpec>();
                
        groupFuncs.add(new GenerateSpec(loSort.getSortSpec()).getGroupBySpec());
        
        sortJob.groupFuncs = groupFuncs;
        sortJob.partitionFunction = SortPartitioner.class;
        
        ProjectSpec ps = new ProjectSpec(1);
        ps.setFlatten(true);
        sortJob.addReduceSpec(new GenerateSpec(ps));
    
        sortJob.reduceParallelism = loSort.getRequestedParallelism();
        
        String comparatorFuncName = loSort.getSortSpec().getComparatorName();
        if (comparatorFuncName != null) {

View Full Code Here

        // cogroups can't use the combiner at this point.
        if (mro.groupFuncs.size() > 1) {
            return false;
        }


        GenerateSpec gen = (GenerateSpec)spec;


        // Second, the first immediate child of the generate spec must be
        // a project with a value of 0.
        Iterator<EvalSpec> i = gen.getSpecs().iterator();
        if (!i.hasNext()) return false;
        EvalSpec s = i.next();
        if (!(s instanceof ProjectSpec)) {
            return false;
        } else {
            ProjectSpec p = (ProjectSpec)s;
            if (p.numCols() > 1) return false;
            else if (p.getCol() != 0) return false;
        }


        // Third, all subsequent immediate children of the generate spec
        // must be func eval specs
        while (i.hasNext()) {
            s = i.next();
            if (!(s instanceof FuncEvalSpec)) return false;
        }


        // Third, walk the entire tree of the generate spec and see if we
        // can combine it.
        CombineDeterminer cd = new CombineDeterminer();
        gen.visit(cd);
        return cd.useCombiner();
    }

View Full Code Here

TOP

Related Classes of org.apache.pig.impl.eval.GenerateSpec

org.apache.pig.backend.hadoop.executionengine.MapreducePlanCompiler

org.apache.pig.backend.hadoop.executionengine.MapreducePlanCompiler$ReduceAdjuster

org.apache.pig.data.Tuple

org.apache.pig.impl.eval.collector.DataCollector

org.apache.pig.impl.logicalLayer.schema.Schema

org.apache.pig.impl.logicalLayer.schema.TupleSchema

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.