Examples of LOStream

org.apache.pig.impl.logicalLayer.LOStream
{@link LOStream} represents the specification of an externalcommand to be executed in a Pig Query. LOStream encapsulates all relevant details of the command specified by the user either directly via the STREAM operator or indirectly via a DEFINE operator. It includes details such as input/output/error specifications and also files to be shipped to the cluster and files to be cached.
org.apache.pig.newplan.logical.relational.LOStream

Examples of org.apache.pig.impl.logicalLayer.LOStream

            String msg = "Expected " + LOStream.class.getSimpleName()
                    + ", got "
                    + (lo == null ? lo : lo.getClass().getSimpleName());
            throw new OptimizerException(msg, errCode, PigException.BUG);            
        }
        LOStream stream = (LOStream)lo;
        
        // check if either the predecessor of stream is load with
        // the same loader function as the serializer of stream
        // OR if the successor of stream is store with the same
        // storage function as the deserializer of stream.

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOStream

            LogicalOperator lo = nodes.get(0);
            if (lo == null || !(lo instanceof LOStream)) {
                throw new RuntimeException("Expected stream, got " +
                    (lo == null ? lo : lo.getClass().getName()));
            }
            LOStream stream = (LOStream)lo;
            if(mOptimizeLoad) {
                // we have verified in check() that the predecessor was load
                LOLoad load = (LOLoad)mPlan.getPredecessors(lo).get(0);
                FileSpec loadFileSpec = load.getInputFile();
                load.setInputFile(new FileSpec(loadFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.INPUT, BinaryStorage.class.getName());
            }
            if(mOptimizeStore) {
                // we have verified in check() that the predecessor was load
                LOStore store = (LOStore)mPlan.getSuccessors(lo).get(0);
                FileSpec storeFileSpec = store.getOutputFile();
                store.setOutputFile(new FileSpec(storeFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.OUTPUT, BinaryStorage.class.getName());
            }
        } catch (Exception e) {
            int errCode = 2014;
            String msg = "Unable to optimize load-stream-store optimization"; 
            throw new OptimizerException(msg, errCode, PigException.BUG, e);

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOStream

    @Test
    public void testQueryStreamingNoSchema() throws FrontendException {
        String query = "stream (load 'a') through `" + simpleEchoStreamingCommand + "`;";
        LogicalPlan lp = planTester.buildPlan(query);
        
        LOStream stream = (LOStream)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields streamRelevantFields = stream.getRelevantInputs(0, 0).get(0);
        assertTrue(streamRelevantFields.getNeedAllFields() == true);
        assertTrue(streamRelevantFields.getNeedNoFields() == false);
        assertTrue(streamRelevantFields.getFields() == null);
    }

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOStream

    @Test
    public void testQueryStreamingNoSchema1() throws FrontendException {
        String query = "stream (load 'a' as (url, hitCount)) through `" + simpleEchoStreamingCommand + "` ;";
        LogicalPlan lp = planTester.buildPlan(query);
        
        LOStream stream = (LOStream)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields streamRelevantFields = stream.getRelevantInputs(0, 0).get(0);
        assertTrue(streamRelevantFields.getNeedAllFields() == true);
        assertTrue(streamRelevantFields.getNeedNoFields() == false);
        assertTrue(streamRelevantFields.getFields() == null);
    }

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOStream

    @Test
    public void testQueryStreamingWithSchema() throws FrontendException {
        String query = "stream (load 'a') through `" + simpleEchoStreamingCommand + "` as (x, y);";
        LogicalPlan lp = planTester.buildPlan(query);
        
        LOStream stream = (LOStream)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields streamRelevantFields = stream.getRelevantInputs(0, 0).get(0);
        assertTrue(streamRelevantFields.getNeedAllFields() == true);
        assertTrue(streamRelevantFields.getNeedNoFields() == false);
        assertTrue(streamRelevantFields.getFields() == null);
    }

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOStream

    @Test
    public void testQueryStreamingWithSchema1() throws FrontendException {
        String query = "stream (load 'a' as (url, hitCount)) through `" + simpleEchoStreamingCommand + "` as (x, y);";
        LogicalPlan lp = planTester.buildPlan(query);
        
        LOStream stream = (LOStream)lp.getSuccessors(lp.getRoots().get(0)).get(0);
        RequiredFields streamRelevantFields = stream.getRelevantInputs(0, 0).get(0);
        assertTrue(streamRelevantFields.getNeedAllFields() == true);
        assertTrue(streamRelevantFields.getNeedNoFields() == false);
        assertTrue(streamRelevantFields.getFields() == null);
    }

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOStream

        if (lo == null || !(lo instanceof LOStream)) {
            int errCode = 2005;
            String msg = "Expected " + LOStream.class.getSimpleName() + ", got " + lo.getClass().getSimpleName();
            throw new OptimizerException(msg, errCode, PigException.BUG);            
        }
        LOStream stream = (LOStream)lo;
        
        // check if either the predecessor of stream is load with
        // the same loader function as the serializer of stream
        // OR if the successor of stream is store with the same
        // storage function as the deserializer of stream.

View Full Code Here

Examples of org.apache.pig.impl.logicalLayer.LOStream

            LogicalOperator lo = nodes.get(0);
            if (lo == null || !(lo instanceof LOStream)) {
                throw new RuntimeException("Expected stream, got " +
                    lo.getClass().getName());
            }
            LOStream stream = (LOStream)lo;
            if(mOptimizeLoad) {
                // we have verified in check() that the predecessor was load
                LOLoad load = (LOLoad)mPlan.getPredecessors(lo).get(0);
                FileSpec loadFileSpec = load.getInputFile();
                load.setInputFile(new FileSpec(loadFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.INPUT, BinaryStorage.class.getName());
            }
            if(mOptimizeStore) {
                // we have verified in check() that the predecessor was load
                LOStore store = (LOStore)mPlan.getSuccessors(lo).get(0);
                FileSpec storeFileSpec = store.getOutputFile();
                store.setOutputFile(new FileSpec(storeFileSpec.getFileName(), new FuncSpec(BinaryStorage.class.getName())));
                stream.setOptimizedSpec(Handle.OUTPUT, BinaryStorage.class.getName());
            }
        } catch (Exception e) {
            int errCode = 2014;
            String msg = "Unable to optimize load-stream-store optimization"; 
            throw new OptimizerException(msg, errCode, PigException.BUG, e);

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOStream

        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator op;
        if (getOperatorClassName().equals(LOLoad.class.getName()))
            op = new LOLoad(null, null, plan, null);
        else // LOStream
            op = new LOStream(plan, null, null, null);
        plan.add(op);
        return plan;
    }

View Full Code Here

Examples of org.apache.pig.newplan.logical.relational.LOStream

    
    String buildStreamOp(SourceLocation loc, String alias, String inputAlias, StreamingCommand command,
            LogicalSchema schema, IntStream input)
    throws RecognitionException {
        try {
            LOStream op = new LOStream( plan, pigContext.createExecutableManager(), command, schema );
            return buildOp( loc, op, alias, inputAlias, null );
        } catch (ExecException ex) {
            throw new PlanGenerationFailureException( input, loc, ex );
        }
    }

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.