Package org.apache.pig.backend.hadoop.executionengine

Examples of org.apache.pig.backend.hadoop.executionengine.HExecutionEngine


        switch (execType) {
            case LOCAL:
            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
View Full Code Here


        PhysicalPlan rpep = new PhysicalPlan();
        ConstantExpression rpce = new ConstantExpression(new OperatorKey(scope,nig.getNextNodeId(scope)));
        rpce.setRequestedParallelism(rp);
        int val = rp;
        if(val<=0){
            HExecutionEngine eng = pigContext.getExecutionEngine();
            if(pigContext.getExecType() != ExecType.LOCAL){
                try {
                    if(val<=0)
                        val = pigContext.defaultParallel;
                    if (val<=0)
                        val = eng.getJobConf().getNumReduceTasks();
                    if (val<=0)
                        val = 1;
                } catch (Exception e) {
                    int errCode = 6015;
                    String msg = "Problem getting the default number of reduces from the Job Client.";
View Full Code Here

        MROperPlan mrp = compile(php, pc);
               
        ConfigurationValidator.validatePigProperties(pc.getProperties());
        Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
       
        HExecutionEngine exe = pc.getExecutionEngine();
        JobClient jobClient = new JobClient(exe.getJobConf());
       
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
       
        // start collecting statistics
        PigStatsUtil.startCollection(pc, jobClient, jcc, mrp);
View Full Code Here

            }
            break;

            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
View Full Code Here

            }
            break;

            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
View Full Code Here

        MROperPlan mrp = compile(php, pc);
               
        ConfigurationValidator.validatePigProperties(pc.getProperties());
        Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
       
        HExecutionEngine exe = pc.getExecutionEngine();
        JobClient jobClient = new JobClient(exe.getJobConf());
       
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);

        ScriptState.get().addWorkflowAdjacenciesToConf(mrp, conf);
       
View Full Code Here

            }
            break;

            case MAPREDUCE:
            {
                executionEngine = new HExecutionEngine (this);

                executionEngine.init();
               
                dfs = executionEngine.getDataStorage();
               
View Full Code Here

        MROperPlan mrp = compile(php, pc);
               
        ConfigurationValidator.validatePigProperties(pc.getProperties());
        Configuration conf = ConfigurationUtil.toConfiguration(pc.getProperties());
       
        HExecutionEngine exe = pc.getExecutionEngine();
        JobClient jobClient = new JobClient(exe.getJobConf());
       
        JobControlCompiler jcc = new JobControlCompiler(pc, conf);
       
        // start collecting statistics
        PigStatsUtil.startCollection(pc, jobClient, jcc, mrp);
View Full Code Here

    }

    /* Create a Mock PigContext object with methods stubbed out to
        let the BasicP2LClient.createPlan() method complete */
    public void addMockPigContext(BasicP2LClient client) {
        HExecutionEngine exec_engine = mock(HExecutionEngine.class);
        when(exec_engine.getLogToPhyMap()).thenReturn(new HashMap<Operator, PhysicalOperator>());

        PigContext ctx = mock(PigContext.class);
        when(ctx.getExecutionEngine()).thenReturn(exec_engine);
        when(ctx.getProperties()).thenReturn(new Properties());
        client.setPigContext(ctx);
View Full Code Here

    private P2jPlanGenerator getOpPlanGenerator(LipstickPigServer lps) throws Exception {
        return new P2jPlanGenerator(lps.getLP(null));
    }

    private Map<PhysicalOperator, Operator> getP2lMap(LipstickPigServer lps) throws Exception {
        HExecutionEngine he = new HExecutionEngine(lps.getPigContext());
        he.compile(getLogicalPlan(lps), null);

        Map<Operator, PhysicalOperator> l2pMap = he.getLogToPhyMap();
        Map<PhysicalOperator, Operator> p2lMap = Maps.newHashMap();
        for (Entry<Operator, PhysicalOperator> i : l2pMap.entrySet()) {
            p2lMap.put(i.getValue(), i.getKey());
        }
View Full Code Here

TOP

Related Classes of org.apache.pig.backend.hadoop.executionengine.HExecutionEngine

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.