Package org.apache.pig

Examples of org.apache.pig.StoreFunc


    @Test
    public void testSFPig() throws Exception {
        byte[] buf = new byte[1024];
        FakeFSOutputStream os = new FakeFSOutputStream(buf);
        StoreFunc sfunc = new PigStorage("\t");
        sfunc.bindTo(os);

        DataByteArray[] input = { new DataByteArray("amy"),
            new DataByteArray("bob"), new DataByteArray("charlene"),
            new DataByteArray("david"), new DataByteArray("erin"),
            new DataByteArray("frank") };
        Tuple f1 = Util.loadTuple(TupleFactory.getInstance().newTuple(input.length), input);

        sfunc.putNext(f1);
        sfunc.finish();
       
        FakeFSInputStream is = new FakeFSInputStream(buf);
        LoadFunc lfunc = new PigStorage();
        lfunc.bindTo(null, new BufferedPositionedInputStream(is), 0, buf.length);
        Tuple f2 = lfunc.getNext();
View Full Code Here


     * @param conf the JobConf object
     * @return the StoreFunc reference
     * @throws ExecException
     */
    public static StoreFunc getStoreFunc(JobConf conf) throws ExecException {
        StoreFunc store;
        try {
            String storeFunc = conf.get("pig.storeFunc", "");
            if (storeFunc.length() == 0) {
                store = new PigStorage();
            } else {
View Full Code Here

                StreamingCommand command = stream.getStreamingCommand();
                HandleSpec streamInputSpec = command.getInputSpec();
                FileSpec loadFileSpec = load.getInputFile();
                // Instantiate both LoadFunc objects to compare them for
                // equality
                StoreFunc streamStorer =
                    (StoreFunc)PigContext.instantiateFuncFromSpec(
                            streamInputSpec.getSpec());
                LoadFunc inputLoader = (LoadFunc)PigContext.instantiateFuncFromSpec(
                                             loadFileSpec.getFuncSpec());
                LogFactory.getLog(this.getClass()).info("streamStorer:" + streamStorer + "," +
                        "inputLoader:" + inputLoader);
                // Check if the streaming command's inputSpec also implements
                // LoadFunc and if it does, are they of the same _reversible_
                // type?
                boolean sameType = false;
                try {
                    // Check if the streamStorer is _reversible_ as
                    // the inputLoader ...
                    if (streamStorer instanceof LoadFunc) {
                        // Cast to check if they are of the same type...
                        streamStorer.getClass().cast(inputLoader);
                        LogFactory.getLog(this.getClass()).info("streamStorer:" + streamStorer + "," +
                                "inputLoader:" + inputLoader);
                        // Now check if they both are reversible...
                        if (streamStorer instanceof ReversibleLoadStoreFunc &&
                            inputLoader instanceof ReversibleLoadStoreFunc) {
                            sameType = true;
                        }
                    }
                } catch (ClassCastException cce) {
                    sameType = false;
                }
                // Check if both LoadFunc objects belong to the same type and
                // are equivalent
                if (sameType && streamStorer.equals(inputLoader)) {
                    // Since they both are the same, we can flip them
                    // for BinaryStorage
                    mOptimizeLoad = true;                   
                }
            }
View Full Code Here

            // Instantiate both to compare them for equality
            LoadFunc streamLoader =
                (LoadFunc)PigContext.instantiateFuncFromSpec(
                        streamOutputSpec.getSpec());
           
            StoreFunc outputStorer = (StoreFunc)PigContext.instantiateFuncFromSpec(
                                         storeFileSpec.getFuncSpec());
           
            // Check if the streaming command's outputSpec also implements
            // StoreFunc and if it does, are they of the same _reversible_
            // type?
View Full Code Here

    }

    public PigRecordWriter getRecordWriter(FileSystem fs, JobConf job,
            Path outputDir, String name, Progressable progress)
            throws IOException {
        StoreFunc store = MapRedUtil.getStoreFunc(job);

        String parentName = FileOutputFormat.getOutputPath(job).getName();
        int suffixStart = parentName.lastIndexOf('.');
        if (suffixStart != -1) {
            String suffix = parentName.substring(suffixStart);
View Full Code Here

     * @throws IOException
     */
    @SuppressWarnings({ "unchecked", "deprecation" })
    private void checkOutputSpecsHelper(FileSystem fs, POStore store, JobConf job)
    throws IOException {
        StoreFunc storeFunc = (StoreFunc)PigContext.instantiateFuncFromSpec(
                store.getSFile().getFuncSpec());
        Class sPrepClass = null;
        try {
            sPrepClass = storeFunc.getStorePreparationClass();
        } catch(AbstractMethodError e) {
            // this is for backward compatibility wherein some old StoreFunc
            // which does not implement getStorePreparationClass() is being
            // used. In this case, we want to just use PigOutputFormat
            sPrepClass = null;
View Full Code Here

     * @param conf the JobConf object
     * @return the StoreFunc reference
     * @throws ExecException
     */
    public static StoreFunc getStoreFunc(JobConf conf) throws ExecException {
        StoreFunc store;
        try {
            String storeFunc = conf.get("pig.storeFunc", "");
            if (storeFunc.length() == 0) {
                store = new PigStorage();
            } else {
View Full Code Here

    @Test
    public void testSFPig() throws Exception {
        byte[] buf = new byte[1024];
        FakeFSOutputStream os = new FakeFSOutputStream(buf);
        StoreFunc sfunc = new PigStorage("\t");
        sfunc.bindTo(os);

        DataByteArray[] input = { new DataByteArray("amy"),
            new DataByteArray("bob"), new DataByteArray("charlene"),
            new DataByteArray("david"), new DataByteArray("erin"),
            new DataByteArray("frank") };
        Tuple f1 = Util.loadTuple(TupleFactory.getInstance().newTuple(input.length), input);

        sfunc.putNext(f1);
        sfunc.finish();
       
        FakeFSInputStream is = new FakeFSInputStream(buf);
        LoadFunc lfunc = new PigStorage();
        lfunc.bindTo(null, new BufferedPositionedInputStream(is), 0, buf.length);
        Tuple f2 = lfunc.getNext();
View Full Code Here

    }

    public PigRecordWriter getRecordWriter(FileSystem fs, JobConf job,
            Path outputDir, String name, Progressable progress)
            throws IOException {
        StoreFunc store;
        String storeFunc = job.get("pig.storeFunc", "");
        if (storeFunc.length() == 0) {
            store = new PigStorage();
        } else {
            try {
View Full Code Here

                StreamingCommand command = stream.getStreamingCommand();
                HandleSpec streamInputSpec = command.getInputSpec();
                FileSpec loadFileSpec = load.getInputFile();
                // Instantiate both LoadFunc objects to compare them for
                // equality
                StoreFunc streamStorer =
                    (StoreFunc)PigContext.instantiateFuncFromSpec(
                            streamInputSpec.getSpec());
                LoadFunc inputLoader = (LoadFunc)PigContext.instantiateFuncFromSpec(
                                             loadFileSpec.getFuncSpec());
                LogFactory.getLog(this.getClass()).info("streamStorer:" + streamStorer + "," +
                        "inputLoader:" + inputLoader);
                // Check if the streaming command's inputSpec also implements
                // LoadFunc and if it does, are they of the same _reversible_
                // type?
                boolean sameType = false;
                try {
                    // Check if the streamStorer is _reversible_ as
                    // the inputLoader ...
                    if (streamStorer instanceof LoadFunc) {
                        // Cast to check if they are of the same type...
                        streamStorer.getClass().cast(inputLoader);
                        LogFactory.getLog(this.getClass()).info("streamStorer:" + streamStorer + "," +
                                "inputLoader:" + inputLoader);
                        // Now check if they both are reversible...
                        if (streamStorer instanceof ReversibleLoadStoreFunc &&
                            inputLoader instanceof ReversibleLoadStoreFunc) {
                            sameType = true;
                        }
                    }
                } catch (ClassCastException cce) {
                    sameType = false;
                }
                // Check if both LoadFunc objects belong to the same type and
                // are equivalent
                if (sameType && streamStorer.equals(inputLoader)) {
                    // Since they both are the same, we can flip them
                    // for BinaryStorage
                    mOptimizeLoad = true;                   
                }
            }
View Full Code Here

TOP

Related Classes of org.apache.pig.StoreFunc

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.