* Licensed to the Apache Software Foundation (ASF) under one

* or more contributor license agreements.  See the NOTICE file

* distributed with this work for additional information

* regarding copyright ownership.  The ASF licenses this file

* to you under the Apache License, Version 2.0 (the

* "License"); you may not use this file except in compliance

* with the License.  You may obtain a copy of the License at

*

*     http://www.apache.org/licenses/LICENSE-2.0

*

* Unless required by applicable law or agreed to in writing, software

* distributed under the License is distributed on an "AS IS" BASIS,

* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

* See the License for the specific language governing permissions and

* limitations under the License.

*/

* This is compiler class that takes an MROperPlan and converts

* it into a JobControl object with the relevant dependency info

* maintained. The JobControl Object is made up of Jobs each of

* which has a JobConf. The MapReduceOper corresponds to a Job

* and the getJobCong method returns the JobConf that is configured

* as per the MapReduceOper

*

* <h2>Comparator Design</h2>

* <p>

* A few words on how comparators are chosen.  In almost all cases we use raw

* comparators (the one exception being when the user provides a comparison

* function for order by).  For order by queries the PigTYPERawComparator

* functions are used, where TYPE is Int, Long, etc.  These comparators are

* null aware and asc/desc aware.  The first byte of each of the

* NullableTYPEWritable classes contains info on whether the value is null.

* Asc/desc is written as an array into the JobConf with the key pig.sortOrder

* so that it can be read by each of the comparators as part of their

* setConf call.

* <p>

* For non-order by queries, PigTYPEWritableComparator classes are used.

* These are all just type specific instances of WritableComparator.

*

*/

    MROperPlan plan;

    Configuration conf;

    Configuration defaultConf;

    PigContext pigContext;

    private static final Matcher DISTRIBUTED_CACHE_ARCHIVE_MATCHER = Pattern

            .compile("\\.(zip|tgz|tar\\.gz|tar)$").matcher("");

    private static final Log log = LogFactory.getLog(JobControlCompiler.class);

    public static final String LOG_DIR = "_logs";

    public static final String END_OF_INP_IN_MAP = "pig.invoke.close.in.map";

    private static final String REDUCER_ESTIMATOR_KEY = "pig.exec.reducer.estimator";

    private static final String REDUCER_ESTIMATOR_ARG_KEY =  "pig.exec.reducer.estimator.arg";

    public static final String PIG_MAP_COUNTER = "pig.counters.counter_";

    public static final String PIG_MAP_RANK_NAME = "pig.rank_";

    public static final String PIG_MAP_SEPARATOR = "_";

    public HashMap<String, ArrayList<Pair<String,Long>>> globalCounters = new HashMap<String, ArrayList<Pair<String,Long>>>();

    public static final String SMALL_JOB_LOG_MSG = "This job was detected as a small job, will run in-process instead";

    public static final String BIG_JOB_LOG_MSG = "This job cannot be converted run in-process";

    /**

     * We will serialize the POStore(s) present in map and reduce in lists in

     * the Hadoop Conf. In the case of Multi stores, we could deduce these from

     * the map plan and reduce plan but in the case of single store, we remove

     * the POStore from the plan - in either case, we serialize the POStore(s)

     * so that PigOutputFormat and PigOutputCommiter can get the POStore(s) in

     * the same way irrespective of whether it is multi store or single store.

     */

    public static final String PIG_MAP_STORES = "pig.map.stores";

    public static final String PIG_REDUCE_STORES = "pig.reduce.stores";

    // A mapping of job to pair of store locations and tmp locations for that job

    private Map<Job, Pair<List<POStore>, Path>> jobStoreMap;

    private Map<Job, MapReduceOper> jobMroMap;

    private int counterSize;

    public JobControlCompiler(PigContext pigContext, Configuration conf) {

        this(pigContext, conf, null);

    }

    public JobControlCompiler(PigContext pigContext, Configuration conf, Configuration defaultConf) {

        this.pigContext = pigContext;

        this.conf = conf;

        this.defaultConf = defaultConf;

        jobStoreMap = new HashMap<Job, Pair<List<POStore>, Path>>();

        jobMroMap = new HashMap<Job, MapReduceOper>();

    }

    /**

     * Returns all store locations of a previously compiled job

     */

    public List<POStore> getStores(Job job) {

        Pair<List<POStore>, Path> pair = jobStoreMap.get(job);

        if (pair != null && pair.first != null) {

            return pair.first;

        } else {

            return new ArrayList<POStore>();

        }

    }

    /**

     * Resets the state

     */

    public void reset() {

        jobStoreMap = new HashMap<Job, Pair<List<POStore>, Path>>();

        jobMroMap = new HashMap<Job, MapReduceOper>();

        UDFContext.getUDFContext().reset();

    }

    /**

     * Gets the map of Job and the MR Operator

     */

    public Map<Job, MapReduceOper> getJobMroMap() {

        return Collections.unmodifiableMap(jobMroMap);

    }

    /**

     * Moves all the results of a collection of MR jobs to the final

     * output directory. Some of the results may have been put into a

     * temp location to work around restrictions with multiple output

     * from a single map reduce job.

     *

     * This method should always be called after the job execution

     * completes.

     */

    public void moveResults(List<Job> completedJobs) throws IOException {

        for (Job job: completedJobs) {

            Pair<List<POStore>, Path> pair = jobStoreMap.get(job);

            if (pair != null && pair.second != null) {

                Path tmp = pair.second;

                Path abs = new Path(tmp, "abs");

                Path rel = new Path(tmp, "rel");

                FileSystem fs = tmp.getFileSystem(conf);

                if (fs.exists(abs)) {

                    moveResults(abs, abs.toUri().getPath(), fs);

                }

                if (fs.exists(rel)) {

                    moveResults(rel, rel.toUri().getPath()+"/", fs);

                }

            }

        }

    }

    /**

     * Walks the temporary directory structure to move (rename) files

     * to their final location.

     */

    private void moveResults(Path p, String rem, FileSystem fs) throws IOException {

        for (FileStatus fstat: fs.listStatus(p)) {

            Path src = fstat.getPath();

            if (fstat.isDir()) {

                log.info("mkdir: "+src);

                fs.mkdirs(removePart(src, rem));

                moveResults(fstat.getPath(), rem, fs);

            } else {

                Path dst = removePart(src, rem);

                log.info("mv: "+src+" "+dst);

                fs.rename(src,dst);

            }

        }

    }

    private Path removePart(Path src, String part) {

        URI uri = src.toUri();

        String pathStr = uri.getPath().replace(part, "");

        return new Path(pathStr);

    }

    /**

     * Compiles all jobs that have no dependencies removes them from

     * the plan and returns. Should be called with the same plan until

     * exhausted.

     * @param plan - The MROperPlan to be compiled

     * @param grpName - The name given to the JobControl

     * @return JobControl object - null if no more jobs in plan

     * @throws JobCreationException

     */

    public JobControl compile(MROperPlan plan, String grpName) throws JobCreationException{

        // Assert plan.size() != 0

        this.plan = plan;

        int timeToSleep;

        String defaultPigJobControlSleep = pigContext.getExecType().isLocal() ? "100" : "5000";

        String pigJobControlSleep = conf.get("pig.jobcontrol.sleep", defaultPigJobControlSleep);

        if (!pigJobControlSleep.equals(defaultPigJobControlSleep)) {

            log.info("overriding default JobControl sleep (" + defaultPigJobControlSleep + ") to " + pigJobControlSleep);

        }

        try {

            timeToSleep = Integer.parseInt(pigJobControlSleep);

        } catch (NumberFormatException e) {

            throw new RuntimeException("Invalid configuration " +

                    "pig.jobcontrol.sleep=" + pigJobControlSleep +

                    " should be a time in ms. default=" + defaultPigJobControlSleep, e);

        }

        JobControl jobCtrl = HadoopShims.newJobControl(grpName, timeToSleep);

        try {

            List<MapReduceOper> roots = new LinkedList<MapReduceOper>();

            roots.addAll(plan.getRoots());

            for (MapReduceOper mro: roots) {

                if(mro instanceof NativeMapReduceOper) {

                    return null;

                }

                Job job = getJob(plan, mro, conf, pigContext);

                jobMroMap.put(job, mro);

                jobCtrl.addJob(job);

            }

        } catch (JobCreationException jce) {

            throw jce;

        } catch(Exception e) {

            int errCode = 2017;

            String msg = "Internal error creating job configuration.";

            throw new JobCreationException(msg, errCode, PigException.BUG, e);

        }

        return jobCtrl;

    }

    // Update Map-Reduce plan with the execution status of the jobs. If one job

    // completely fail (the job has only one store and that job fail), then we

    // remove all its dependent jobs. This method will return the number of MapReduceOper

    // removed from the Map-Reduce plan

    public int updateMROpPlan(List<Job> completeFailedJobs)

    {

        int sizeBefore = plan.size();

        for (Job job : completeFailedJobs)  // remove all subsequent jobs

        {

            MapReduceOper mrOper = jobMroMap.get(job);

            plan.trimBelow(mrOper);

            plan.remove(mrOper);

        }

        // Remove successful jobs from jobMroMap

        for (Job job : jobMroMap.keySet())

        {

            if (!completeFailedJobs.contains(job))

            {

                MapReduceOper mro = jobMroMap.get(job);

                if (!pigContext.inIllustrator && mro.isCounterOperation())

                    saveCounters(job,mro.getOperationID());

                plan.remove(mro);

            }

        }

        jobMroMap.clear();

        int sizeAfter = plan.size();

        return sizeBefore-sizeAfter;

    }

    /**

     * Reads the global counters produced by a job on the group labeled with PIG_MAP_RANK_NAME.

     * Then, it is calculated the cumulative sum, which consists on the sum of previous cumulative

     * sum plus the previous global counter value.

     * @param job with the global counters collected.

     * @param operationID After being collected on global counters (POCounter),

     * these values are passed via configuration file to PORank, by using the unique

     * operation identifier

     */

    private void saveCounters(Job job, String operationID) {

        Counters counters;

        Group groupCounters;

        Long previousValue = 0L;

        Long previousSum = 0L;

        ArrayList<Pair<String,Long>> counterPairs;

        try {

            counters = HadoopShims.getCounters(job);

            String groupName = getGroupName(counters.getGroupNames());

            // In case that the counter group was not find, we need to find

            // out why. Only acceptable state is that the relation has been

            // empty.

            if (groupName == null) {

                Counter outputRecords =

                    counters.getGroup(MRPigStatsUtil.TASK_COUNTER_GROUP)

                    .getCounterForName(MRPigStatsUtil.MAP_OUTPUT_RECORDS);

                if(outputRecords.getCounter() == 0) {

                    globalCounters.put(operationID, new ArrayList<Pair<String, Long>>());

                    return;

                } else {

                  throw new RuntimeException("Did not found RANK counter group for operationId: " + operationID);

                }

            }

            groupCounters = counters.getGroup(groupName);

            Iterator<Counter> it = groupCounters.iterator();

            HashMap<Integer,Long> counterList = new HashMap<Integer, Long>();

            while(it.hasNext()) {

                try{

                    Counter c = it.next();

                    counterList.put(Integer.valueOf(c.getDisplayName()), c.getValue());

                } catch (Exception ex) {

                    ex.printStackTrace();

                }

            }

            counterSize = counterList.size();

            counterPairs = new ArrayList<Pair<String,Long>>();

            for(int i = 0; i < counterSize; i++){

                previousSum += previousValue;

                previousValue = counterList.get(Integer.valueOf(i));

                counterPairs.add(new Pair<String, Long>(JobControlCompiler.PIG_MAP_COUNTER + operationID + JobControlCompiler.PIG_MAP_SEPARATOR + i, previousSum));

            }

            globalCounters.put(operationID, counterPairs);

        } catch (Exception e) {

            String msg = "Error to read counters into Rank operation counterSize "+counterSize;

            throw new RuntimeException(msg, e);

        }

    }

    private String getGroupName(Collection<String> collection) {

        for (String name : collection) {

            if (name.contains(PIG_MAP_RANK_NAME))

                return name;

        }

        return null;

    }

    private boolean okToRunLocal(org.apache.hadoop.mapreduce.Job job, MapReduceOper mro, List<POLoad> lds) throws IOException {

        Configuration conf = job.getConfiguration();

        if(!conf.getBoolean(PigConfiguration.PIG_AUTO_LOCAL_ENABLED, false)) {

            return false;

        }

        long totalInputFileSize = InputSizeReducerEstimator.getTotalInputFileSize(conf, lds, job);

        long inputByteMax = conf.getLong(PigConfiguration.PIG_AUTO_LOCAL_INPUT_MAXBYTES, 100*1000*1000l);

        log.info("Size of input: " + totalInputFileSize +" bytes. Small job threshold: " + inputByteMax );

        if (totalInputFileSize < 0 || totalInputFileSize > inputByteMax) {

            return false;

        }

        int reducers = conf.getInt("mapred.reduce.tasks", 1);

        log.info("No of reducers: " + reducers);

        if (reducers > 1) {

            return false;

        }

        return true;

    }

    /**

     * The method that creates the Job corresponding to a MapReduceOper.

     * The assumption is that

     * every MapReduceOper will have a load and a store. The JobConf removes

     * the load operator and serializes the input filespec so that PigInputFormat can

     * take over the creation of splits. It also removes the store operator

     * and serializes the output filespec so that PigOutputFormat can take over

     * record writing. The remaining portion of the map plan and reduce plans are

     * serialized and stored for the PigMapReduce or PigMapOnly objects to take over

     * the actual running of the plans.

     * The Mapper &amp; Reducer classes and the required key value formats are set.

     * Checks if this is a map only job and uses PigMapOnly class as the mapper

     * and uses PigMapReduce otherwise.

     * If it is a Map Reduce job, it is bound to have a package operator. Remove it from

     * the reduce plan and serializes it so that the PigMapReduce class can use it to package

     * the indexed tuples received by the reducer.

     * @param mro - The MapReduceOper for which the JobConf is required

     * @param config - the Configuration object from which JobConf is built

     * @param pigContext - The PigContext passed on from execution engine

     * @return Job corresponding to mro

     * @throws JobCreationException

     */

    @SuppressWarnings({ "unchecked" })

    private Job getJob(MROperPlan plan, MapReduceOper mro, Configuration config, PigContext pigContext) throws JobCreationException{

        org.apache.hadoop.mapreduce.Job nwJob = null;

        try{

            nwJob = new org.apache.hadoop.mapreduce.Job(config);

        }catch(Exception e) {

            throw new JobCreationException(e);

        }

        Configuration conf = nwJob.getConfiguration();

        ArrayList<FileSpec> inp = new ArrayList<FileSpec>();

        ArrayList<List<OperatorKey>> inpTargets = new ArrayList<List<OperatorKey>>();

        ArrayList<String> inpSignatureLists = new ArrayList<String>();

        ArrayList<Long> inpLimits = new ArrayList<Long>();

        ArrayList<POStore> storeLocations = new ArrayList<POStore>();

        Path tmpLocation = null;

        // add settings for pig statistics

        String setScriptProp = conf.get(PigConfiguration.INSERT_ENABLED, "true");

        if (setScriptProp.equalsIgnoreCase("true")) {

            MRScriptState ss = MRScriptState.get();

            ss.addSettingsToConf(mro, conf);

        }

        conf.set("mapred.mapper.new-api", "true");

        conf.set("mapred.reducer.new-api", "true");

        String buffPercent = conf.get("mapred.job.reduce.markreset.buffer.percent");

        if (buffPercent == null || Double.parseDouble(buffPercent) <= 0) {

            log.info("mapred.job.reduce.markreset.buffer.percent is not set, set to default 0.3");

            conf.set("mapred.job.reduce.markreset.buffer.percent", "0.3");

        }else{

            log.info("mapred.job.reduce.markreset.buffer.percent is set to " + conf.get("mapred.job.reduce.markreset.buffer.percent"));

        }

        // Convert mapred.output.* to output.compression.*, See PIG-1791

        if( "true".equals( conf.get( "mapred.output.compress" ) ) ) {

            conf.set( "output.compression.enabled",  "true" );

            String codec = conf.get( "mapred.output.compression.codec" );

            if( codec == null ) {

                throw new JobCreationException("'mapred.output.compress' is set but no value is specified for 'mapred.output.compression.codec'." );

            } else {

                conf.set( "output.compression.codec", codec );

            }

        }

        try{

            //Process the POLoads

            List<POLoad> lds = PlanHelper.getPhysicalOperators(mro.mapPlan, POLoad.class);

            if(lds!=null && lds.size()>0){

                for (POLoad ld : lds) {

                    LoadFunc lf = ld.getLoadFunc();

                    lf.setLocation(ld.getLFile().getFileName(), nwJob);

                    //Store the inp filespecs

                    inp.add(ld.getLFile());

                }

            }

            if(!mro.reducePlan.isEmpty()){

                log.info("Reduce phase detected, estimating # of required reducers.");

                adjustNumReducers(plan, mro, nwJob);

            } else {

                nwJob.setNumReduceTasks(0);

            }

            if(lds!=null && lds.size()>0){

                for (POLoad ld : lds) {

                    //Store the target operators for tuples read

                    //from this input

                    List<PhysicalOperator> ldSucs = mro.mapPlan.getSuccessors(ld);

                    List<OperatorKey> ldSucKeys = new ArrayList<OperatorKey>();

                    if(ldSucs!=null){

                        for (PhysicalOperator operator2 : ldSucs) {

                            ldSucKeys.add(operator2.getOperatorKey());

                        }

                    }

                    inpTargets.add(ldSucKeys);

                    inpSignatureLists.add(ld.getSignature());

                    inpLimits.add(ld.getLimit());

                    //Remove the POLoad from the plan

                    if (!pigContext.inIllustrator)

                        mro.mapPlan.remove(ld);

                }

            }

            if (!pigContext.inIllustrator && ! pigContext.getExecType().isLocal())

            {

                if (okToRunLocal(nwJob, mro, lds)) {

                    log.info(SMALL_JOB_LOG_MSG);

                    // override with the default conf to run in local mode

                    for (Entry<String, String> entry : defaultConf) {

                        String key = entry.getKey();

                        if (key.equals("mapred.reduce.tasks") || key.equals("mapreduce.job.reduces")) {

                            // this must not be set back to the default in case it has been set to 0 for example.

                            continue;

                        }

                        if (key.startsWith("fs.")) {

                            // we don't want to change fs settings back

                            continue;

                        }

                        if (key.startsWith("io.")) {

                            // we don't want to change io settings back

                            continue;

                        }

                        String value = entry.getValue();

                        if (conf.get(key) == null || !conf.get(key).equals(value)) {

                            conf.set(key, value);

                        }

                    }

                    conf.setBoolean(PigImplConstants.CONVERTED_TO_LOCAL, true);

                } else {

                    log.info(BIG_JOB_LOG_MSG);

                    // Setup the DistributedCache for this job

                    for (URL extraJar : pigContext.extraJars) {

                        log.debug("Adding jar to DistributedCache: " + extraJar.toString());

                        putJarOnClassPathThroughDistributedCache(pigContext, conf, extraJar);

                    }

                    for (String scriptJar : pigContext.scriptJars) {

                        log.debug("Adding jar to DistributedCache: " + scriptJar.toString());

                        putJarOnClassPathThroughDistributedCache(pigContext, conf, new File(scriptJar).toURI().toURL());

                    }

                    //Create the jar of all functions and classes required

                    File submitJarFile = File.createTempFile("Job", ".jar");

                    log.info("creating jar file "+submitJarFile.getName());

                    // ensure the job jar is deleted on exit

                    submitJarFile.deleteOnExit();

                    FileOutputStream fos = new FileOutputStream(submitJarFile);

                    JarManager.createJar(fos, mro.UDFs, pigContext);

                    log.info("jar file "+submitJarFile.getName()+" created");

                    //Start setting the JobConf properties

                    conf.set("mapred.jar", submitJarFile.getPath());

                }

            }

            if(Utils.isLocal(pigContext, conf)) {

                ConfigurationUtil.replaceConfigForLocalMode(conf);

            }

            conf.set("pig.inputs", ObjectSerializer.serialize(inp));

            conf.set("pig.inpTargets", ObjectSerializer.serialize(inpTargets));

            conf.set("pig.inpSignatures", ObjectSerializer.serialize(inpSignatureLists));

            conf.set("pig.inpLimits", ObjectSerializer.serialize(inpLimits));

            conf.set("pig.pigContext", ObjectSerializer.serialize(pigContext));

            conf.set("udf.import.list", ObjectSerializer.serialize(PigContext.getPackageImportList()));

            // this is for unit tests since some don't create PigServer

            // if user specified the job name using -D switch, Pig won't reset the name then.

            if (System.getProperty("mapred.job.name") == null &&

                    pigContext.getProperties().getProperty(PigContext.JOB_NAME) != null){

                nwJob.setJobName(pigContext.getProperties().getProperty(PigContext.JOB_NAME));

            }

            if (pigContext.getProperties().getProperty(PigContext.JOB_PRIORITY) != null) {

                // If the job priority was set, attempt to get the corresponding enum value

                // and set the hadoop job priority.

                String jobPriority = pigContext.getProperties().getProperty(PigContext.JOB_PRIORITY).toUpperCase();

                try {

                    // Allow arbitrary case; the Hadoop job priorities are all upper case.

                    conf.set("mapred.job.priority", JobPriority.valueOf(jobPriority).toString());

                } catch (IllegalArgumentException e) {

                    StringBuffer sb = new StringBuffer("The job priority must be one of [");

                    JobPriority[] priorities = JobPriority.values();

                    for (int i = 0; i < priorities.length; ++i) {

                        if (i > 0)  sb.append(", ");

                        sb.append(priorities[i]);

                    }

                    sb.append("].  You specified [" + jobPriority + "]");

                    throw new JobCreationException(sb.toString());

                }

            }

            setupDistributedCache(pigContext, conf, pigContext.getProperties(),

                    "pig.streaming.ship.files", true);

            setupDistributedCache(pigContext, conf, pigContext.getProperties(),

                    "pig.streaming.cache.files", false);

            nwJob.setInputFormatClass(PigInputFormat.class);

            // tmp file compression setups

            // PIG-3741 This must be done before setStoreLocation on POStores

            Utils.setTmpFileCompressionOnConf(pigContext, conf);

            //Process POStore and remove it from the plan

            LinkedList<POStore> mapStores = PlanHelper.getPhysicalOperators(mro.mapPlan, POStore.class);

            LinkedList<POStore> reduceStores = PlanHelper.getPhysicalOperators(mro.reducePlan, POStore.class);

            for (POStore st : mapStores) {

                storeLocations.add(st);

                StoreFuncInterface sFunc = st.getStoreFunc();

                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);

                if (sFunc instanceof OverwritableStoreFunc) {

                    OverwritableStoreFunc osf = (OverwritableStoreFunc) sFunc;

                    if (osf.shouldOverwrite()) {

                        osf.cleanupOutput(st, nwJob);

                    }

                }

            }

            for (POStore st : reduceStores) {

                storeLocations.add(st);

                StoreFuncInterface sFunc = st.getStoreFunc();

                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);

                if (sFunc instanceof OverwritableStoreFunc) {

                    OverwritableStoreFunc osf = (OverwritableStoreFunc) sFunc;

                    if (osf.shouldOverwrite()) {

                        osf.cleanupOutput(st, nwJob);

                    }

                }

            }

            // the OutputFormat we report to Hadoop is always PigOutputFormat which

            // can be wrapped with LazyOutputFormat provided if it is supported by

            // the Hadoop version and PigConfiguration.PIG_OUTPUT_LAZY is set

            if ("true".equalsIgnoreCase(conf.get(PigConfiguration.PIG_OUTPUT_LAZY))) {

                try {

                    Class<?> clazz = PigContext.resolveClassName(

                            "org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat");

                    Method method = clazz.getMethod("setOutputFormatClass", nwJob.getClass(),

                            Class.class);

                    method.invoke(null, nwJob, PigOutputFormat.class);

                }

                catch (Exception e) {

                    nwJob.setOutputFormatClass(PigOutputFormat.class);

                    log.warn(PigConfiguration.PIG_OUTPUT_LAZY

                            + " is set but LazyOutputFormat couldn't be loaded. Default PigOutputFormat will be used");

                }

            }

            else {

                nwJob.setOutputFormatClass(PigOutputFormat.class);

            }

            if (mapStores.size() + reduceStores.size() == 1) { // single store case

                log.info("Setting up single store job");

                POStore st;

                if (reduceStores.isEmpty()) {

                    st = mapStores.get(0);

                    if(!pigContext.inIllustrator)

                        mro.mapPlan.remove(st);

                }

                else {

                    st = reduceStores.get(0);

                    if(!pigContext.inIllustrator)

                        mro.reducePlan.remove(st);

                }

               

                MapRedUtil.setupStreamingDirsConfSingle(st, pigContext, conf);

            }

            else if (mapStores.size() + reduceStores.size() > 0) { // multi store case

                log.info("Setting up multi store job");

                MapRedUtil.setupStreamingDirsConfMulti(pigContext, conf);

                boolean disableCounter = conf.getBoolean("pig.disable.counter", false);

                if (disableCounter) {

                    log.info("Disable Pig custom output counters");

                }

                int idx = 0;

                for (POStore sto: storeLocations) {

                    sto.setDisableCounter(disableCounter);

                    sto.setMultiStore(true);

                    sto.setIndex(idx++);

                }

            }

            // store map key type

            // this is needed when the key is null to create

            // an appropriate NullableXXXWritable object

            conf.set("pig.map.keytype", ObjectSerializer.serialize(new byte[] { mro.mapKeyType }));

            // set parent plan in all operators in map and reduce plans

            // currently the parent plan is really used only when POStream is present in the plan

            new PhyPlanSetter(mro.mapPlan).visit();

            new PhyPlanSetter(mro.reducePlan).visit();

            // this call modifies the ReplFiles names of POFRJoin operators

            // within the MR plans, must be called before the plans are

            // serialized

            setupDistributedCacheForJoin(mro, pigContext, conf);

            // Search to see if we have any UDFs that need to pack things into the

            // distributed cache.

            setupDistributedCacheForUdfs(mro, pigContext, conf);

            SchemaTupleFrontend.copyAllGeneratedToDistributedCache(pigContext, conf);

            POPackage pack = null;

            if(mro.reducePlan.isEmpty()){

                //MapOnly Job

                nwJob.setMapperClass(PigMapOnly.Map.class);

                if(!pigContext.inIllustrator)

                    conf.set("pig.mapPlan", ObjectSerializer.serialize(mro.mapPlan));

                if(mro.isEndOfAllInputSetInMap()) {

                    // this is used in Map.close() to decide whether the

                    // pipeline needs to be rerun one more time in the close()

                    // The pipeline is rerun if there either was a stream or POMergeJoin

                    conf.set(END_OF_INP_IN_MAP, "true");

                }

            }

            else{

                //Map Reduce Job

                //Process the POPackage operator and remove it from the reduce plan

                if(!mro.combinePlan.isEmpty()){

                    POPackage combPack = (POPackage)mro.combinePlan.getRoots().get(0);

                    mro.combinePlan.remove(combPack);

                    nwJob.setCombinerClass(PigCombiner.Combine.class);

                    conf.set("pig.combinePlan", ObjectSerializer.serialize(mro.combinePlan));

                    conf.set("pig.combine.package", ObjectSerializer.serialize(combPack));

                } else if (mro.needsDistinctCombiner()) {

                    nwJob.setCombinerClass(DistinctCombiner.Combine.class);

                    log.info("Setting identity combiner class.");

                }

                pack = (POPackage)mro.reducePlan.getRoots().get(0);

                if(!pigContext.inIllustrator)

                    mro.reducePlan.remove(pack);

                nwJob.setMapperClass(PigMapReduce.Map.class);

                nwJob.setReducerClass(PigMapReduce.Reduce.class);

                if (mro.customPartitioner != null)

                    nwJob.setPartitionerClass(PigContext.resolveClassName(mro.customPartitioner));

                if(!pigContext.inIllustrator)

                    conf.set("pig.mapPlan", ObjectSerializer.serialize(mro.mapPlan));

                if(mro.isEndOfAllInputSetInMap()) {

                    // this is used in Map.close() to decide whether the

                    // pipeline needs to be rerun one more time in the close()

                    // The pipeline is rerun only if there was a stream or merge-join.

                    conf.set(END_OF_INP_IN_MAP, "true");

                }

                if(!pigContext.inIllustrator)

                    conf.set("pig.reducePlan", ObjectSerializer.serialize(mro.reducePlan));

                if(mro.isEndOfAllInputSetInReduce()) {

                    // this is used in Map.close() to decide whether the

                    // pipeline needs to be rerun one more time in the close()

                    // The pipeline is rerun only if there was a stream

                    conf.set("pig.stream.in.reduce", "true");

                }

                if (!pigContext.inIllustrator)

                    conf.set("pig.reduce.package", ObjectSerializer.serialize(pack));

                conf.set("pig.reduce.key.type",

                        Byte.toString(pack.getPkgr().getKeyType()));

                if (mro.getUseSecondaryKey()) {

                    nwJob.setGroupingComparatorClass(PigSecondaryKeyGroupComparator.class);

                    nwJob.setPartitionerClass(SecondaryKeyPartitioner.class);

                    nwJob.setSortComparatorClass(PigSecondaryKeyComparator.class);

                    nwJob.setOutputKeyClass(NullableTuple.class);

                    conf.set("pig.secondarySortOrder",

                            ObjectSerializer.serialize(mro.getSecondarySortOrder()));

                }

                else

                {

                    Class<? extends WritableComparable> keyClass = HDataType

                            .getWritableComparableTypes(

                                    pack.getPkgr().getKeyType()).getClass();

                    nwJob.setOutputKeyClass(keyClass);

                    selectComparator(mro, pack.getPkgr().getKeyType(), nwJob);

                }

                nwJob.setOutputValueClass(NullableTuple.class);

            }

            if(mro.isGlobalSort() || mro.isLimitAfterSort()){

                // Only set the quantiles file and sort partitioner if we're a

                // global sort, not for limit after sort.

                if (mro.isGlobalSort()) {

                    String symlink = addSingleFileToDistributedCache(

                            pigContext, conf, mro.getQuantFile(), "pigsample");

                    conf.set("pig.quantilesFile", symlink);

                    nwJob.setPartitionerClass(WeightedRangePartitioner.class);

                }

                if (mro.isUDFComparatorUsed) {

                    boolean usercomparator = false;

                    for (String compFuncSpec : mro.UDFs) {

                        Class comparator = PigContext.resolveClassName(compFuncSpec);

                        if(ComparisonFunc.class.isAssignableFrom(comparator)) {

                            nwJob.setMapperClass(PigMapReduce.MapWithComparator.class);

                            nwJob.setReducerClass(PigMapReduce.ReduceWithComparator.class);

                            conf.set("pig.reduce.package", ObjectSerializer.serialize(pack));

                            conf.set("pig.usercomparator", "true");

                            nwJob.setOutputKeyClass(NullableTuple.class);

                            nwJob.setSortComparatorClass(comparator);

                            usercomparator = true;

                            break;

                        }

                    }

                    if (!usercomparator) {

                        String msg = "Internal error. Can't find the UDF comparator";

                        throw new IOException (msg);

                    }

                } else {

                    conf.set("pig.sortOrder",

                            ObjectSerializer.serialize(mro.getSortOrder()));

                }

            }

            if (mro.isSkewedJoin()) {

                String symlink = addSingleFileToDistributedCache(pigContext,

                        conf, mro.getSkewedJoinPartitionFile(), "pigdistkey");

                conf.set("pig.keyDistFile", symlink);

                nwJob.setPartitionerClass(SkewedPartitioner.class);

                nwJob.setMapperClass(PigMapReduce.MapWithPartitionIndex.class);

                nwJob.setMapOutputKeyClass(NullablePartitionWritable.class);

                nwJob.setGroupingComparatorClass(PigGroupingPartitionWritableComparator.class);

            }

            if (mro.isCounterOperation()) {

                if (mro.isRowNumber()) {

                    nwJob.setMapperClass(PigMapReduceCounter.PigMapCounter.class);

                } else {

                    nwJob.setReducerClass(PigMapReduceCounter.PigReduceCounter.class);

                }

            }

            if(mro.isRankOperation()) {

                Iterator<String> operationIDs = mro.getRankOperationId().iterator();

                while(operationIDs.hasNext()) {

                    String operationID = operationIDs.next();

                    Iterator<Pair<String, Long>> itPairs = globalCounters.get(operationID).iterator();

                    Pair<String,Long> pair = null;

                    while(itPairs.hasNext()) {

                        pair = itPairs.next();

                        conf.setLong(pair.first, pair.second);

                    }

                }

            }

            if (!pigContext.inIllustrator)

            {

                // unset inputs for POStore, otherwise, map/reduce plan will be unnecessarily deserialized

                for (POStore st: mapStores) { st.setInputs(null); st.setParentPlan(null);}

                for (POStore st: reduceStores) { st.setInputs(null); st.setParentPlan(null);}

                conf.set(PIG_MAP_STORES, ObjectSerializer.serialize(mapStores));

                conf.set(PIG_REDUCE_STORES, ObjectSerializer.serialize(reduceStores));

            }

            String tmp;

            long maxCombinedSplitSize = 0;

            if (!mro.combineSmallSplits() || pigContext.getProperties().getProperty("pig.splitCombination", "true").equals("false"))

                conf.setBoolean("pig.noSplitCombination", true);

            else if ((tmp = pigContext.getProperties().getProperty("pig.maxCombinedSplitSize", null)) != null) {

                try {

                    maxCombinedSplitSize = Long.parseLong(tmp);

                } catch (NumberFormatException e) {

                    log.warn("Invalid numeric format for pig.maxCombinedSplitSize; use the default maximum combined split size");

                }

            }

            if (maxCombinedSplitSize > 0)

                conf.setLong("pig.maxCombinedSplitSize", maxCombinedSplitSize);

            // It's a hack to set distributed cache file for hadoop 23. Once MiniMRCluster do not require local

            // jar on fixed location, this can be removed

            if (pigContext.getExecType() == ExecType.MAPREDUCE) {

                String newfiles = conf.get("alternative.mapreduce.job.cache.files");

                if (newfiles!=null) {

                    String files = conf.get("mapreduce.job.cache.files");

                    conf.set("mapreduce.job.cache.files",

                            files == null ? newfiles.toString() : files + "," + newfiles);

                }

            }

            // Serialize the UDF specific context info.

            UDFContext.getUDFContext().serialize(conf);

            Job cjob = new Job(new JobConf(conf), new ArrayList<Job>());

            jobStoreMap.put(cjob,new Pair<List<POStore>, Path>(storeLocations, tmpLocation));

            return cjob;

        } catch (JobCreationException jce) {

            throw jce;

        } catch(Exception e) {

            int errCode = 2017;

            String msg = "Internal error creating job configuration.";

            throw new JobCreationException(msg, errCode, PigException.BUG, e);

        }

    }

    /**

     * Adjust the number of reducers based on the default_parallel, requested parallel and estimated

     * parallel. For sampler jobs, we also adjust the next job in advance to get its runtime parallel as

     * the number of partitions used in the sampler.

     * @param plan the MR plan

     * @param mro the MR operator

     * @param nwJob the current job

     * @throws IOException

     */

    public void adjustNumReducers(MROperPlan plan, MapReduceOper mro,

            org.apache.hadoop.mapreduce.Job nwJob) throws IOException {

        int jobParallelism = calculateRuntimeReducers(mro, nwJob);

        if (mro.isSampler() && plan.getSuccessors(mro) != null) {

            // We need to calculate the final number of reducers of the next job (order-by or skew-join)

            // to generate the quantfile.

            MapReduceOper nextMro = plan.getSuccessors(mro).get(0);

            // Here we use the same conf and Job to calculate the runtime #reducers of the next job

            // which is fine as the statistics comes from the nextMro's POLoads

            int nPartitions = calculateRuntimeReducers(nextMro, nwJob);

            // set the runtime #reducer of the next job as the #partition

            ParallelConstantVisitor visitor =

                    new ParallelConstantVisitor(mro.reducePlan, nPartitions);

            visitor.visit();

        }

        log.info("Setting Parallelism to " + jobParallelism);

        Configuration conf = nwJob.getConfiguration();

        // set various parallelism into the job conf for later analysis, PIG-2779

        conf.setInt("pig.info.reducers.default.parallel", pigContext.defaultParallel);

        conf.setInt("pig.info.reducers.requested.parallel", mro.requestedParallelism);

        conf.setInt("pig.info.reducers.estimated.parallel", mro.estimatedParallelism);

        // this is for backward compatibility, and we encourage to use runtimeParallelism at runtime

        mro.requestedParallelism = jobParallelism;

        // finally set the number of reducers

        conf.setInt("mapred.reduce.tasks", jobParallelism);

    }

    /**

     * Calculate the runtime #reducers based on the default_parallel, requested parallel and estimated

     * parallel, and save it to MapReduceOper's runtimeParallelism.

     * @return the runtimeParallelism

     * @throws IOException

     */

    private int calculateRuntimeReducers(MapReduceOper mro,

            org.apache.hadoop.mapreduce.Job nwJob) throws IOException{

        // we don't recalculate for the same job

        if (mro.runtimeParallelism != -1) {

            return mro.runtimeParallelism;

        }

        int jobParallelism = -1;

        if (mro.requestedParallelism > 0) {

            jobParallelism = mro.requestedParallelism;

        } else if (pigContext.defaultParallel > 0) {

            jobParallelism = pigContext.defaultParallel;

        } else {

            mro.estimatedParallelism = estimateNumberOfReducers(nwJob, mro);

            if (mro.estimatedParallelism > 0) {

                jobParallelism = mro.estimatedParallelism;

            } else {

                // reducer estimation could return -1 if it couldn't estimate

                log.info("Could not estimate number of reducers and no requested or default " +

                        "parallelism set. Defaulting to 1 reducer.");

                jobParallelism = 1;

            }

        }

        // save it

        mro.runtimeParallelism = jobParallelism;

        return jobParallelism;

    }

    /**

     * Looks up the estimator from REDUCER_ESTIMATOR_KEY and invokes it to find the number of

     * reducers to use. If REDUCER_ESTIMATOR_KEY isn't set, defaults to InputSizeReducerEstimator.

     * @param job

     * @param mapReducerOper

     * @throws IOException

     */

    public static int estimateNumberOfReducers(org.apache.hadoop.mapreduce.Job job,

            MapReduceOper mapReducerOper) throws IOException {

        Configuration conf = job.getConfiguration();

        PigReducerEstimator estimator = conf.get(REDUCER_ESTIMATOR_KEY) == null ?

                new InputSizeReducerEstimator() :

                    PigContext.instantiateObjectFromParams(conf,

                            REDUCER_ESTIMATOR_KEY, REDUCER_ESTIMATOR_ARG_KEY, PigReducerEstimator.class);

                log.info("Using reducer estimator: " + estimator.getClass().getName());

                int numberOfReducers = estimator.estimateNumberOfReducers(job, mapReducerOper);

                return numberOfReducers;

    }

    public static class PigSecondaryKeyGroupComparator extends WritableComparator {

        public PigSecondaryKeyGroupComparator() {

            super(NullableTuple.class, true);

        }

        @SuppressWarnings("unchecked")

        @Override

        public int compare(WritableComparable a, WritableComparable b)

        {

            PigNullableWritable wa = (PigNullableWritable)a;

            PigNullableWritable wb = (PigNullableWritable)b;

            if ((wa.getIndex() & PigNullableWritable.mqFlag) != 0) { // this is a multi-query index