Package org.apache.pig.impl

Examples of org.apache.pig.impl.PigContext

111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
     * Test parallelism for group by constant
     * @throws Throwable
     */
    @Test
    public void testGroupConstWithParallel() throws Throwable {
        PigContext pc = new PigContext(ExecType.MAPREDUCE, cluster.getProperties());
        pc.defaultParallel = 100;
        pc.connect();
       
        LogicalPlanTester planTester = new LogicalPlanTester() ;
        planTester.buildPlan("a = load 'input';");
        LogicalPlan lp = planTester.buildPlan("b = group a by 1;");
       
        PhysicalPlan pp = Util.getNewOptimizedPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        HExecutionEngine exe = pc.getExecutionEngine();
        ConfigurationValidator.validatePigProperties(exe.getConfiguration());
        Configuration conf = ConfigurationUtil.toConfiguration(exe.getConfiguration());
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
       
        JobControl jobControl = jcc.compile(mrPlan, "Test");
View Full Code Here
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
     *  Test parallelism for group by column
     * @throws Throwable
     */
    @Test
    public void testGroupNonConstWithParallel() throws Throwable {
        PigContext pc = new PigContext(ExecType.MAPREDUCE, cluster.getProperties());
        pc.defaultParallel = 100;
        pc.connect();
       
        LogicalPlanTester planTester = new LogicalPlanTester() ;
        planTester.buildPlan("a = load 'input';");
        LogicalPlan lp = planTester.buildPlan("b = group a by $0;");
       
        PhysicalPlan pp = Util.getNewOptimizedPhysicalPlan(lp, pc);
        POStore store = GenPhyOp.dummyPigStorageOp();
        pp.addAsLeaf(store);
        MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

        HExecutionEngine exe = pc.getExecutionEngine();
        ConfigurationValidator.validatePigProperties(exe.getConfiguration());
        Configuration conf = ConfigurationUtil.toConfiguration(exe.getConfiguration());
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
       
        JobControl jobControl = jcc.compile(mrPlan, "Test");
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
    private PigServer myPig;

    @Override
    @Before
    public void setUp() throws Exception {
        PigContext context = new PigContext(ExecType.LOCAL, new Properties());
        context.getProperties().setProperty("opt.multiquery", ""+true);
        myPig = new PigServer(context);
        myPig.getPigContext().getProperties().setProperty("pig.usenewlogicalplan", "false");
        deleteOutputFiles();
    }
View Full Code Here
462
463
464
465
466
467
468
469
470
471
        LogicalPlanTester tester = new LogicalPlanTester();
        tester.buildPlan("A = LOAD '" + INPUT_FILE + "';");
        tester.buildPlan("B = LOAD '" + INPUT_FILE + "';");
        tester.buildPlan("C = join A by $0, B by $0 using \"merge\" parallel 50;");
        LogicalPlan lp = tester.buildPlan("store C into 'out';");
  PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
  MROperPlan mro = Util.buildMRPlan(Util.buildPhysicalPlan(lp, pc),pc);
        Assert.assertEquals(1,mro.getRoots().get(0).getRequestedParallelism());
    }
View Full Code Here
89
90
91
92
93
94
95
96
97
98
99
100
    @Test
    public void testNonCollectableLoader() throws Exception{
        LogicalPlanTester lpt = new LogicalPlanTester();
        lpt.buildPlan("A = LOAD '" + INPUT_FILE + "' as (id, name, grade);");
        LogicalPlan lp = lpt.buildPlan("B = group A by id using 'collected';");
        PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
        pc.connect();
        try {
            Util.buildMRPlan(Util.buildPhysicalPlan(lp, pc),pc); 
            fail("Must throw MRCompiler Exception");
        } catch (Exception e) {
            assertTrue(e instanceof MRCompilerException);
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
    }
   
    @Test
    public void testLocalModeInputPositive() throws Throwable {
       
        PigContext ctx = new PigContext(ExecType.LOCAL, new Properties()) ;
        ctx.connect() ;
       
        String inputfile = generateTempFile().getAbsolutePath() ;
        String outputfile = generateNonExistenceTempFile().getAbsolutePath() ;

        LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getFs()) ;       
       
        CompilationMessageCollector collector = new CompilationMessageCollector() ;       
        boolean isBeforeOptimizer = false; // we are not optimizing in this testcase
        LogicalPlanValidationExecutor executor = new LogicalPlanValidationExecutor(plan, ctx, isBeforeOptimizer) ;
        executor.validate(plan, collector) ;
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
   
      
    @Test
    public void testLocalModeNegative2() throws Throwable {
       
        PigContext ctx = new PigContext(ExecType.LOCAL, new Properties()) ;
        ctx.connect() ;
       
        String inputfile = generateTempFile().getAbsolutePath() ;
        String outputfile = generateTempFile().getAbsolutePath() ;

        LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getDfs()) ;       
       
        CompilationMessageCollector collector = new CompilationMessageCollector() ;       
        boolean isBeforeOptimizer = false; // we are not optimizing in this testcase
        LogicalPlanValidationExecutor executor = new LogicalPlanValidationExecutor(plan, ctx, isBeforeOptimizer) ;
        try {
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
    }
       
    @Test
    public void testMapReduceModeInputPositive() throws Throwable {
       
        PigContext ctx = new PigContext(ExecType.MAPREDUCE, cluster.getProperties()) ;      
        ctx.connect() ;
       
        String inputfile = createHadoopTempFile(ctx) ;
        String outputfile = createHadoopNonExistenceTempFile(ctx) ;

        LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getDfs()) ;                    
       
        CompilationMessageCollector collector = new CompilationMessageCollector() ;       
        boolean isBeforeOptimizer = false; // we are not optimizing in this testcase
        LogicalPlanValidationExecutor executor = new LogicalPlanValidationExecutor(plan, ctx, isBeforeOptimizer) ;
        executor.validate(plan, collector) ;
View Full Code Here
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
    }
   
    @Test
    public void testMapReduceModeInputNegative2() throws Throwable {
       
        PigContext ctx = new PigContext(ExecType.MAPREDUCE, cluster.getProperties()) ;      
        ctx.connect() ;
       
        String inputfile = createHadoopTempFile(ctx) ;
        String outputfile = createHadoopTempFile(ctx) ;

        LogicalPlan plan = genNewLoadStorePlan(inputfile, outputfile, ctx.getDfs()) ;                    
       
        CompilationMessageCollector collector = new CompilationMessageCollector() ;       
        boolean isBeforeOptimizer = false; // we are not optimizing in this testcase
        LogicalPlanValidationExecutor executor = new LogicalPlanValidationExecutor(plan, ctx, isBeforeOptimizer) ;
        try {
View Full Code Here
97
98
99
100
101
102
103
104
105
106
107
108
            lpt.buildPlan("A = LOAD 'data1' using "+ DummyCollectableLoader.class.getName() +"() as (id, name, grade);");
            lpt.buildPlan("B = LOAD 'data2' using "+ DummyIndexableLoader.class.getName() +"() as (id, name, grade);");
            LogicalPlan lp = lpt.buildPlan("C = join A by id left, B by id using 'merge';");
            assertEquals(LOJoin.JOINTYPE.MERGE, ((LOJoin)lp.getLeaves().get(0)).getJoinType());

            PigContext pc = new PigContext(ExecType.MAPREDUCE,cluster.getProperties());
            pc.connect();
            PhysicalPlan phyP = Util.buildPhysicalPlan(lp, pc);
            PhysicalOperator phyOp = phyP.getLeaves().get(0);
            assertTrue(phyOp instanceof POForEach);
            assertEquals(1,phyOp.getInputs().size());
            assertTrue(phyOp.getInputs().get(0) instanceof POMergeCogroup);
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.PigContext

  • com.netflix.lipstick.Main
  • com.netflix.lipstick.pigtolipstick.BasicP2LClientTest
  • com.netflix.lipstick.util.OutputSampler
  • Example1
  • org.apache.cassandra.pig.PigTestBase
  • org.apache.pig.backend.datastorage.DataStorage
  • org.apache.pig.backend.datastorage.ElementDescriptor
  • org.apache.pig.backend.executionengine.PigSlice
  • org.apache.pig.backend.hadoop.datastorage.HDataStorage
  • org.apache.pig.backend.hadoop.executionengine.HExecutionEngine
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.