Package org.apache.pig.newplan.logical.relational

Examples of org.apache.pig.newplan.logical.relational.LOLoad


        } catch(Exception ex) {
            throw new ParserValidationException( intStream, loc, ex );
        }

        FileSpec loader = new FileSpec(absolutePath, funcSpec);
        LOLoad op = new LOLoad(
                loader,
                schema,
                plan,
                ConfigurationUtil.toConfiguration(pigContext.getProperties()),
                loFunc,
                alias + "_" + newOperatorKey());
        op.setTmpLoad(false);

        // Check if there's a store in the plan already that this load
        // depends on. If so, add it as an input alias
        List<String> inputAliases = new ArrayList<String>();

        // Get list of stores. The stores are not all sinks in the plan
        // if they've already got successors.
        Iterator<Operator> itr = plan.getOperators();
        List<LOStore> stores = new ArrayList<LOStore>();
        while (itr.hasNext()) {
            Operator lop = itr.next();
            if (lop instanceof LOStore) {
                stores.add((LOStore)lop);
            }
        }

        for (LOStore store : stores) {
            String ifile = op.getFileSpec().getFileName();
            String ofile = store.getFileSpec().getFileName();
            if (ofile.equals(ifile)) {
                inputAliases.add( store.getAlias() );
            }
        }
View Full Code Here


            for( String path : paths )
                pigContext.addJar( path );
            buildOp( loc, op, null, new ArrayList<String>(), null );
            ((LOStore)operators.get( storeAlias )).setTmpStore(true);
            plan.connect( operators.get( storeAlias ), op );
            LOLoad load = (LOLoad)operators.get( loadAlias );
            plan.connect( op, load );
            return load.getAlias();
        } catch (ParserException e) {
            throw new InvalidCommandException( input, loc, cmd );
        } catch (MalformedURLException e) {
            throw new InvalidPathException( input, loc, e);
        }
View Full Code Here

    @Override
    protected OperatorPlan buildPattern() {
        // match each foreach.
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator load = new LOLoad (null, plan);
        plan.add( load );
//        LogicalRelationalOperator filter = new LOFilter( plan );
//        plan.add( filter );
//        plan.connect( load, filter );
        return plan;
View Full Code Here

                // remove the limit
                currentPlan.removeAndReconnect(limit);
            } else if (pred instanceof LOLoad) {
                // Push limit to load
                LOLoad load = (LOLoad) pred;
                if (load.getLimit() == -1)
                    load.setLimit(limit.getLimit());
                else
                    load.setLimit(load.getLimit() < limit.getLimit() ? load
                            .getLimit() : limit.getLimit());
            } else if (pred instanceof LOLimit) {
                // Limit is merged into another LOLimit
                LOLimit beforeLimit = (LOLimit) pred;
                beforeLimit
View Full Code Here

    @Override
    protected OperatorPlan buildPattern() {
        // match each foreach.
        LogicalPlan plan = new LogicalPlan();
        LogicalRelationalOperator load = new LOLoad (null, plan);
        plan.add( load );
//        LogicalRelationalOperator filter = new LOFilter( plan );
//        plan.add( filter );
//        plan.connect( load, filter );
        return plan;
View Full Code Here

    }

    @Override
    protected OperatorPlan buildPattern() {
        LogicalPlan plan = new LogicalPlan();
        plan.add(new LOLoad(null, plan));
        return plan;
    }
View Full Code Here

        } catch(Exception ex) {
            throw new ParserValidationException( intStream, loc, ex );
        }

        FileSpec loader = new FileSpec(absolutePath, funcSpec);
        LOLoad op = new LOLoad(
                loader,
                schema,
                plan,
                ConfigurationUtil.toConfiguration(pigContext.getProperties()),
                loFunc,
                alias + "_" + newOperatorKey());
        op.setTmpLoad(false);

        // Check if there's a store in the plan already that this load
        // depends on. If so, add it as an input alias
        List<String> inputAliases = new ArrayList<String>();

        // Get list of stores. The stores are not all sinks in the plan
        // if they've already got successors.
        Iterator<Operator> itr = plan.getOperators();
        List<LOStore> stores = new ArrayList<LOStore>();
        while (itr.hasNext()) {
            Operator lop = itr.next();
            if (lop instanceof LOStore) {
                stores.add((LOStore)lop);
            }
        }

        for (LOStore store : stores) {
            String ifile = op.getFileSpec().getFileName();
            String ofile = store.getFileSpec().getFileName();
            if (ofile.equals(ifile)) {
                inputAliases.add( store.getAlias() );
            }
        }
View Full Code Here

            for( String path : paths )
                pigContext.addJar( path );
            buildOp( loc, op, null, new ArrayList<String>(), null );
            ((LOStore)operators.get( storeAlias )).setTmpStore(true);
            plan.connect( operators.get( storeAlias ), op );
            LOLoad load = (LOLoad)operators.get( loadAlias );
            plan.connect( op, load );
            return load.getAlias();
        } catch (ParserException e) {
            throw new InvalidCommandException( input, loc, cmd );
        } catch (MalformedURLException e) {
            throw new InvalidPathException( input, loc, e);
        }
View Full Code Here

    private LogicalPlan generateLogicalPlan(String inputFile,
            String outputFile, DataStorage dfs) throws Exception {
        LogicalPlan plan = new LogicalPlan();
        FileSpec filespec1 = new FileSpec(generateTmpFile(inputFile).getAbsolutePath(), new FuncSpec("org.apache.pig.builtin.PigStorage"));
        FileSpec filespec2 = new FileSpec(generateTmpFile(outputFile).getAbsolutePath(), new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = newLOLoad(filespec1, null, plan, ConfigurationUtil.toConfiguration(dfs.getConfiguration()));
        LOStore store = new LOStore(plan, filespec2, (StoreFuncInterface) PigContext.instantiateFuncFromSpec(filespec2.getFuncSpec()), null);

        LOFilter filter = new LOFilter(plan);

        plan.add(load);
View Full Code Here

    public void testNonDfsLocation() throws Exception {
        String nonDfsUrl = "har:///user/foo/f.har";
        String query = "a = load '" + nonDfsUrl + "' using PigStorage('\t','-noschema');" +
                       "store a into 'output';";
        LogicalPlan lp = Util.buildLp(servers[1], query);
        LOLoad load = (LOLoad) lp.getSources().get(0);
        nonDfsUrl = nonDfsUrl.replaceFirst("/$", "");
        assertEquals(nonDfsUrl, load.getFileSpec().getFileName());
    }
View Full Code Here

TOP

Related Classes of org.apache.pig.newplan.logical.relational.LOLoad

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.