Package org.apache.pig.impl.io

Examples of org.apache.pig.impl.io.ValidatingInputFileSpec


        FileSystem fs = FileSystem.get(job);
        List<SliceWrapper> splits = new ArrayList<SliceWrapper>();
        for (int i = 0; i < inputs.size(); i++) {
            DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
            ValidatingInputFileSpec spec;
            if (inputs.get(i) instanceof ValidatingInputFileSpec) {
                spec = (ValidatingInputFileSpec) inputs.get(i);
            } else {
                spec = new ValidatingInputFileSpec(inputs.get(i), store);
            }
            EvalSpec groupBy = groupFuncs == null ? null : groupFuncs.get(i);
            if (isSplittable && (spec.getSlicer() instanceof PigSlicer)) {
                ((PigSlicer)spec.getSlicer()).setSplittable(isSplittable);
            }
            Slice[] pigs = spec.getSlicer().slice(store, spec.getFileName());
            for (Slice split : pigs) {
                splits.add(new SliceWrapper(split, pigContext, groupBy,
                        mapFuncs.get(i), i, fs));
            }
        }
View Full Code Here


        if(pigContext.getExecType() == ExecType.MAPREDUCE) {
            fs.setWorkingDirectory(new Path("/user", job.getUser()));
                }
       
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
        ValidatingInputFileSpec spec;
        if (inputs.get(i).first instanceof ValidatingInputFileSpec) {
            spec = (ValidatingInputFileSpec) inputs.get(i).first;
        } else {
            spec = new ValidatingInputFileSpec(inputs.get(i).first, store);
        }
        boolean isSplittable = inputs.get(i).second;
        if ((spec.getSlicer() instanceof PigSlicer)) {
            ((PigSlicer)spec.getSlicer()).setSplittable(isSplittable);
        }
        Slice[] pigs = spec.getSlicer().slice(store, spec.getFileName());
        for (Slice split : pigs) {
            splits.add(new SliceWrapper(split, pigContext.getExecType(), i, fs, inpTargets.get(i)));
        }
            } catch (ExecException ee) {
              throw ee;
View Full Code Here

        // Pass loader signature to slicer
          List<String> inpSignatureLists = (ArrayList<String>)ObjectSerializer.deserialize(job.get("pig.inpSignatures"));
                if (inpSignatureLists.get(i)!=null)
                    store.getConfiguration().setProperty("pig.loader.signature", inpSignatureLists.get(i));

        ValidatingInputFileSpec spec;
        if (inputs.get(i).first instanceof ValidatingInputFileSpec) {
            spec = (ValidatingInputFileSpec) inputs.get(i).first;
        } else {
            spec = new ValidatingInputFileSpec(inputs.get(i).first, store);
        }
        boolean isSplittable = inputs.get(i).second;
        if ((spec.getSlicer() instanceof PigSlicer)) {
            ((PigSlicer)spec.getSlicer()).setSplittable(isSplittable);
        }
                       
                /* Set the input size in UDF Context if LoadFunc is a sample loader.
                 * This value is used by PoissonSampleLoader to calculate the number of
                 * samplable tuples
                 */
                Object loader = PigContext.instantiateFuncFromSpec(spec.getFuncSpec());
                if (loader instanceof SampleLoader) {
                    Long iSize = FileLocalizer.getSize(spec.getFileName(), pigContext.getProperties());                    
                    UDFContext udfc = UDFContext.getUDFContext();
                    Properties p = udfc.getUDFProperties(SampleLoader.class);
                    p.setProperty("pig.input." + i + ".size", Long.toString(iSize));
                    udfc.serialize(job);
                }
           
        Slice[] pigs = spec.getSlicer().slice(store, spec.getFileName());
        for (Slice split : pigs) {
            splits.add(new SliceWrapper(split, pigContext.getExecType(), i, fs, inpTargets.get(i)));
        }
            } catch (ExecException ee) {
              throw ee;
View Full Code Here

        // working dir to /user/<userid>
        if(pigContext.getExecType() == ExecType.MAPREDUCE)
            fs.setWorkingDirectory(new Path("/user", job.getUser()));
       
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
        ValidatingInputFileSpec spec;
        if (inputs.get(i).first instanceof ValidatingInputFileSpec) {
            spec = (ValidatingInputFileSpec) inputs.get(i).first;
        } else {
            spec = new ValidatingInputFileSpec(inputs.get(i).first, store);
        }
        boolean isSplittable = inputs.get(i).second;
        if (isSplittable && (spec.getSlicer() instanceof PigSlicer)) {
            ((PigSlicer)spec.getSlicer()).setSplittable(isSplittable);
        }
        Slice[] pigs = spec.getSlicer().slice(store, spec.getFileName());
        for (Slice split : pigs) {
            splits.add(new SliceWrapper(split, pigContext, i, fs, inpTargets.get(i)));
        }
            } catch (ExecException ee) {
              throw ee;
View Full Code Here

        if(pigContext.getExecType() == ExecType.MAPREDUCE) {
            fs.setWorkingDirectory(new Path("/user", job.getUser()));
                                }
       
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
        ValidatingInputFileSpec spec;
        if (inputs.get(i).first instanceof ValidatingInputFileSpec) {
            spec = (ValidatingInputFileSpec) inputs.get(i).first;
        } else {
            spec = new ValidatingInputFileSpec(inputs.get(i).first, store);
        }
        boolean isSplittable = inputs.get(i).second;
        if ((spec.getSlicer() instanceof PigSlicer)) {
            ((PigSlicer)spec.getSlicer()).setSplittable(isSplittable);
        }
        Slice[] pigs = spec.getSlicer().slice(store, spec.getFileName());
        for (Slice split : pigs) {
            splits.add(new SliceWrapper(split, pigContext, i, fs, inpTargets.get(i)));
        }
            } catch (ExecException ee) {
              throw ee;
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.io.ValidatingInputFileSpec

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.