Package org.apache.pig.impl

Examples of org.apache.pig.impl.PigContext$ContextClassLoader


     * Test parallelism for group by constant
     * @throws Throwable
     */
    @Test
    public void testGroupConstWithParallel() throws Throwable {
        PigContext pc = new PigContext(ExecType.MAPREDUCE, cluster.getProperties());
        pc.defaultParallel = 100;
        pc.connect();
       
        LogicalPlanTester planTester = new LogicalPlanTester() ;
        planTester.buildPlan("a = load 'input';");
        LogicalPlan lp = planTester.buildPlan("b = group a by 1;");
       
        PhysicalPlan pp = Util.getNewOptimizedPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        HExecutionEngine exe = pc.getExecutionEngine();
        ConfigurationValidator.validatePigProperties(exe.getConfiguration());
        Configuration conf = ConfigurationUtil.toConfiguration(exe.getConfiguration());
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
       
        JobControl jobControl = jcc.compile(mrPlan, "Test");
View Full Code Here


     *  Test parallelism for group by column
     * @throws Throwable
     */
    @Test
    public void testGroupNonConstWithParallel() throws Throwable {
        PigContext pc = new PigContext(ExecType.MAPREDUCE, cluster.getProperties());
        pc.defaultParallel = 100;
        pc.connect();
       
        LogicalPlanTester planTester = new LogicalPlanTester() ;
        planTester.buildPlan("a = load 'input';");
        LogicalPlan lp = planTester.buildPlan("b = group a by $0;");
       
        PhysicalPlan pp = Util.getNewOptimizedPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        HExecutionEngine exe = pc.getExecutionEngine();
        ConfigurationValidator.validatePigProperties(exe.getConfiguration());
        Configuration conf = ConfigurationUtil.toConfiguration(exe.getConfiguration());
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
       
        JobControl jobControl = jcc.compile(mrPlan, "Test");
View Full Code Here

    private PigServer myPig;

    @Override
    @Before
    public void setUp() throws Exception {
        PigContext context = new PigContext(ExecType.LOCAL, new Properties());
        context.getProperties().setProperty("opt.multiquery", ""+true);
        myPig = new PigServer(context);
        myPig.getPigContext().getProperties().setProperty("pig.usenewlogicalplan", "false");
        deleteOutputFiles();
    }
View Full Code Here

        LogicalPlanTester tester = new LogicalPlanTester();
        tester.buildPlan("A = LOAD '" + INPUT_FILE + "';");
        tester.buildPlan("B = LOAD '" + INPUT_FILE + "';");
        tester.buildPlan("C = join A by $0, B by $0 using \"merge\" parallel 50;");
        LogicalPlan lp = tester.buildPlan("store C into 'out';");
  PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
  MROperPlan mro = Util.buildMRPlan(Util.buildPhysicalPlan(lp, pc),pc);
        Assert.assertEquals(1,mro.getRoots().get(0).getRequestedParallelism());
    }
View Full Code Here

    @Test
    public void testNonCollectableLoader() throws Exception{
        LogicalPlanTester lpt = new LogicalPlanTester();
        lpt.buildPlan("A = LOAD '" + INPUT_FILE + "' as (id, name, grade);");
        LogicalPlan lp = lpt.buildPlan("B = group A by id using 'collected';");
        PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
        try {
            Util.buildMRPlan(Util.buildPhysicalPlan(lp, pc),pc)
            fail("Must throw MRCompiler Exception");
        } catch (Exception e) {
            assertTrue(e instanceof MRCompilerException);
View Full Code Here

    }
   
    @Test
    public void testLocalModeInputPositive() throws Throwable {
       
        PigContext ctx = new PigContext(ExecType.LOCAL, new Properties()) ;
        ctx.connect() ;
       
        String inputfile = generateTempFile().getAbsolutePath() ;
        String outputfile = generateNonExistenceTempFile().getAbsolutePath() ;

        LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getFs()) ;       
       
        CompilationMessageCollector collector = new CompilationMessageCollector() ;       
        boolean isBeforeOptimizer = false; // we are not optimizing in this testcase
        LogicalPlanValidationExecutor executor = new LogicalPlanValidationExecutor(plan, ctx, isBeforeOptimizer) ;
        executor.validate(plan, collector) ;
View Full Code Here

   
      
    @Test
    public void testLocalModeNegative2() throws Throwable {
       
        PigContext ctx = new PigContext(ExecType.LOCAL, new Properties()) ;
        ctx.connect() ;
       
        String inputfile = generateTempFile().getAbsolutePath() ;
        String outputfile = generateTempFile().getAbsolutePath() ;

        LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getDfs()) ;       
       
        CompilationMessageCollector collector = new CompilationMessageCollector() ;       
        boolean isBeforeOptimizer = false; // we are not optimizing in this testcase
        LogicalPlanValidationExecutor executor = new LogicalPlanValidationExecutor(plan, ctx, isBeforeOptimizer) ;
        try {
View Full Code Here

    }
       
    @Test
    public void testMapReduceModeInputPositive() throws Throwable {
       
        PigContext ctx = new PigContext(ExecType.MAPREDUCE, cluster.getProperties()) ;      
        ctx.connect() ;
       
        String inputfile = createHadoopTempFile(ctx) ;
        String outputfile = createHadoopNonExistenceTempFile(ctx) ;

        LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getDfs()) ;                    
       
        CompilationMessageCollector collector = new CompilationMessageCollector() ;       
        boolean isBeforeOptimizer = false; // we are not optimizing in this testcase
        LogicalPlanValidationExecutor executor = new LogicalPlanValidationExecutor(plan, ctx, isBeforeOptimizer) ;
        executor.validate(plan, collector) ;
View Full Code Here

    }
   
    @Test
    public void testMapReduceModeInputNegative2() throws Throwable {
       
        PigContext ctx = new PigContext(ExecType.MAPREDUCE, cluster.getProperties()) ;      
        ctx.connect() ;
       
        String inputfile = createHadoopTempFile(ctx) ;
        String outputfile = createHadoopTempFile(ctx) ;

        LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getDfs()) ;                    
       
        CompilationMessageCollector collector = new CompilationMessageCollector() ;       
        boolean isBeforeOptimizer = false; // we are not optimizing in this testcase
        LogicalPlanValidationExecutor executor = new LogicalPlanValidationExecutor(plan, ctx, isBeforeOptimizer) ;
        try {
View Full Code Here

            lpt.buildPlan("A = LOAD 'data1' using "+ DummyCollectableLoader.class.getName() +"() as (id, name, grade);");
            lpt.buildPlan("B = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
            LogicalPlan lp = lpt.buildPlan("C = join A by id left, B by id using 'merge';");
            assertEquals(LOJoin.JOINTYPE.MERGE, ((LOJoin)lp.getLeaves().get(0)).getJoinType());

            PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
            pc.connect();
            PhysicalPlan phyP = Util.buildPhysicalPlan(lp, pc);
            PhysicalOperator phyOp = phyP.getLeaves().get(0);
            assertTrue(phyOp instanceof POForEach);
            assertEquals(1,phyOp.getInputs().size());
            assertTrue(phyOp.getInputs().get(0) instanceof POMergeCogroup);
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.PigContext$ContextClassLoader

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.