Package org.apache.pig

Examples of org.apache.pig.FuncSpec


     * @see org.apache.pig.EvalFunc#getArgToFuncMapping()
     */
    @Override
    public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
        List<FuncSpec> funcList = new ArrayList<FuncSpec>();
        funcList.add(new FuncSpec(this.getClass().getName(), Schema.generateNestedSchema(DataType.BAG, DataType.BYTEARRAY)));
        funcList.add(new FuncSpec(DoubleMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.DOUBLE)));
        funcList.add(new FuncSpec(FloatMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.FLOAT)));
        funcList.add(new FuncSpec(IntMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.INTEGER)));
        funcList.add(new FuncSpec(LongMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.LONG)));
        funcList.add(new FuncSpec(StringMax.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.CHARARRAY)));
        return funcList;
    }
View Full Code Here


     * @see org.apache.pig.EvalFunc#getArgToFuncMapping()
     */
    @Override
    public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
        List<FuncSpec> funcList = new ArrayList<FuncSpec>();
        funcList.add(new FuncSpec(this.getClass().getName(), Schema.generateNestedSchema(DataType.BAG, DataType.BYTEARRAY)));
        funcList.add(new FuncSpec(DoubleMin.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.DOUBLE)));
        funcList.add(new FuncSpec(FloatMin.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.FLOAT)));
        funcList.add(new FuncSpec(IntMin.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.INTEGER)));
        funcList.add(new FuncSpec(LongMin.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.LONG)));
        funcList.add(new FuncSpec(StringMin.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.CHARARRAY)));
        return funcList;
    }
View Full Code Here

                String scope = leaf.getOperatorKey().getScope();
                POStore str = new POStore(new OperatorKey(scope,
                    NodeIdGenerator.getGenerator().getNextNodeId(scope)));
                spec = new FileSpec(FileLocalizer.getTemporaryPath(
                    pigContext).toString(),
                    new FuncSpec(Utils.getTmpFileCompressorName(pigContext)));
                str.setSFile(spec);
                plan.addAsLeaf(str);
            } else{
                spec = ((POStore)leaf).getSFile();
            }
View Full Code Here

     * @see org.apache.pig.EvalFunc#getArgToFuncMapping()
     */
    @Override
    public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
        List<FuncSpec> funcList = new ArrayList<FuncSpec>();
        funcList.add(new FuncSpec(this.getClass().getName(), Schema.generateNestedSchema(DataType.BAG, DataType.BYTEARRAY)));
        funcList.add(new FuncSpec(DoubleSum.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.DOUBLE)));
        funcList.add(new FuncSpec(FloatSum.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.FLOAT)));
        funcList.add(new FuncSpec(IntSum.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.INTEGER)));
        funcList.add(new FuncSpec(LongSum.class.getName(), Schema.generateNestedSchema(DataType.BAG, DataType.LONG)));
        return funcList;
    }
View Full Code Here

        LogicalExpression copy = new CastExpression(
                lgExpPlan,
                this.getExpression().deepCopy(lgExpPlan),
                this.getFieldSchema().deepCopy());
        try {
            FuncSpec origFuncSpec = this.getFuncSpec();
            if (origFuncSpec != null ) {
                ((CastExpression)copy).setFuncSpec(origFuncSpec.clone());
            }
        } catch(CloneNotSupportedException e) {
            e.printStackTrace();
        }
        return copy;
View Full Code Here

        List<FuncSpec> funcList = new ArrayList<FuncSpec>();
        Schema s = new Schema();
        s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
        s.add(new Schema.FieldSchema(null, DataType.CHARARRAY));
        s.add(new Schema.FieldSchema(null, DataType.INTEGER));
        funcList.add(new FuncSpec(this.getClass().getName(), s));
        return funcList;
    }
View Full Code Here

          LogicalSchema aschema = new LogicalSchema();
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.BYTEARRAY));
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.BYTEARRAY));
          LOLoad A = new LOLoad(new FileSpec("bla", new FuncSpec("PigStorage", "\t")), aschema, lp, null);
          A.setAlias("A");
          lp.add(A);
         
          // B = load
          LogicalSchema bschema = new LogicalSchema();
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.BYTEARRAY));
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.BYTEARRAY));
          LOLoad B = new LOLoad(new FileSpec("morebla", new FuncSpec("PigStorage", "\t")), bschema, lp, null);
          B.setAlias("B");
          lp.add(B);
         
          // C = join
          LogicalSchema cschema = new LogicalSchema();
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.BYTEARRAY));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.BYTEARRAY));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.BYTEARRAY));
          cschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.BYTEARRAY));
          MultiMap<Integer, LogicalExpressionPlan> mm =
                new MultiMap<Integer, LogicalExpressionPlan>();
          LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
          LOJoin C = new LOJoin(lp, mm, JOINTYPE.HASH, new boolean[] {true, true});
          new ProjectExpression(aprojplan, 0, 0, C);
          LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
          new ProjectExpression(bprojplan, 1, 0, C);
          mm.put(0, aprojplan);
          mm.put(1, bprojplan);
         
          C.setAlias("C");
          lp.add(C);
          lp.connect(A, C);
          lp.connect(B, C);
       
          // D = filter
          LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
          LOFilter D = new LOFilter(lp, filterPlan);
          ProjectExpression fx = new ProjectExpression(filterPlan, 0, 0, D);
          ConstantExpression fc0 = new ConstantExpression(filterPlan, new Integer(0), new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          EqualExpression eq1 = new EqualExpression(filterPlan, fx, fc0);
          ProjectExpression fanotherx = new ProjectExpression(filterPlan, 0, 0, D);
          ProjectExpression fa = new ProjectExpression(filterPlan, 0, 2, D);
          EqualExpression eq2 = new EqualExpression(filterPlan, fanotherx, fa);
          AndExpression and1 = new AndExpression(filterPlan, eq1, eq2);
          ProjectExpression fb = new ProjectExpression(filterPlan, 0, 3, D);
          ConstantExpression fc1 = new ConstantExpression(filterPlan, new Integer(1),new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          EqualExpression eq3 = new EqualExpression(filterPlan, fb, fc1);
          AndExpression and2 = new AndExpression(filterPlan, and1, eq3);
          ProjectExpression fanotherb = new ProjectExpression(filterPlan, 0, 3, D);
          ProjectExpression fy = new ProjectExpression(filterPlan, 0, 1, D);
          EqualExpression eq4 = new EqualExpression(filterPlan, fy, fanotherb);
          new AndExpression(filterPlan, and2, eq4);
         
          D.setAlias("D");
          // Connect D to B, since the transform has happened.
          lp.add(D);
          lp.connect(C, D);
        }
       
        System.out.println(lp);
        LogicalPlanOptimizer optimizer = new LogicalPlanOptimizer(lp, 500, null);
        optimizer.optimize();
       
        LogicalPlan expected = new LogicalPlan();
        {
            // A = load
          LogicalSchema aschema = new LogicalSchema();
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "x", null, DataType.BYTEARRAY));
          aschema.addField(new LogicalSchema.LogicalFieldSchema(
              "y", null, DataType.BYTEARRAY));
          LOLoad A = new LOLoad(new FileSpec("bla", new FuncSpec("PigStorage", "\t")), aschema, expected, null);
          expected.add(A);
         
          // DA = filter
          LogicalExpressionPlan DAfilterPlan = new LogicalExpressionPlan();
          LOFilter DA = new LOFilter(expected, DAfilterPlan);
          ProjectExpression fx = new ProjectExpression(DAfilterPlan, 0, 0, DA);
          fx.neverUseForRealSetFieldSchema(new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          ConstantExpression fc0 = new ConstantExpression(DAfilterPlan, new Integer(0), new LogicalFieldSchema(null, null, DataType.BYTEARRAY));
          new EqualExpression(DAfilterPlan, fx, fc0);
         
          DA.neverUseForRealSetSchema(aschema);
          expected.add(DA);
          expected.connect(A, DA);
         
          // B = load
          LogicalSchema bschema = new LogicalSchema();
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "a", null, DataType.BYTEARRAY));
          bschema.addField(new LogicalSchema.LogicalFieldSchema(
              "b", null, DataType.BYTEARRAY));
          LOLoad B = new LOLoad(new FileSpec("morebla", new FuncSpec("PigStorage", "\t")), bschema, expected, null);
          expected.add(B);
         
          // DB = filter
          LogicalExpressionPlan DBfilterPlan = new LogicalExpressionPlan();
            LOFilter DB = new LOFilter(expected, DBfilterPlan);
View Full Code Here

            }
           
        }
       
        private void setUpHashTable() throws IOException {
            FileSpec replFile = new FileSpec(repl,new FuncSpec(PigStorage.class.getName()+"()"));
            POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L), replFile);
            PigContext pc = new PigContext(ExecType.MAPREDUCE,ConfigurationUtil.toProperties(PigMapReduce.sJobConf));
            try {
                pc.connect();
           
View Full Code Here

                String scope = leaf.getOperatorKey().getScope();
                POStore str = new POStore(new OperatorKey(scope,
                    NodeIdGenerator.getGenerator().getNextNodeId(scope)));
                spec = new FileSpec(FileLocalizer.getTemporaryPath(
                    pigContext).toString(),
                    new FuncSpec(Utils.getTmpFileCompressorName(pigContext)));
                str.setSFile(spec);
                plan.addAsLeaf(str);
            } else{
                spec = ((POStore)leaf).getSFile();
            }
View Full Code Here

                            cast.setFieldSchema(fs);
                            p.add(cast);
                            p.connect(proj, cast);
                           
                            cast.setFieldSchema(fs.clone());
                            FuncSpec loadFuncSpec = null;
                            if(lo instanceof LOLoad) {
                                loadFuncSpec = ((LOLoad)lo).getInputFile().getFuncSpec();
                            } else if (lo instanceof LOStream) {
                                StreamingCommand command = ((LOStream)lo).getStreamingCommand();
                                HandleSpec streamOutputSpec = command.getOutputSpec();
                                loadFuncSpec = new FuncSpec(streamOutputSpec.getSpec());
                            } else {
                                int errCode = 2006;
                                String msg = "TypeCastInserter invoked with an invalid operator class name: " + lo.getClass().getSimpleName();
                                throw new OptimizerException(msg, errCode, PigException.BUG);
                            }
View Full Code Here

TOP

Related Classes of org.apache.pig.FuncSpec

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.