Package org.apache.pig.impl.logicalLayer.schema

Examples of org.apache.pig.impl.logicalLayer.schema.Schema


    public ProjectionMap getProjectionMap() {
       
        if(mIsProjectionMapComputed) return mProjectionMap;
        mIsProjectionMapComputed = true;
       
        Schema outputSchema;
       
        try {
            outputSchema = getSchema();
        } catch (FrontendException fee) {
            mProjectionMap = null;
            return mProjectionMap;
        }
       
        if(outputSchema == null) {
            mProjectionMap = null;
            return mProjectionMap;
        }
       
        List<LogicalOperator> predecessors = (ArrayList<LogicalOperator>)mPlan.getPredecessors(this);
        if(predecessors == null) {
            mProjectionMap = null;
            return mProjectionMap;
        }
       
        LogicalOperator predecessor = predecessors.get(0);
       
        Schema inputSchema;
       
        try {
            inputSchema = predecessor.getSchema();
        } catch (FrontendException fee) {
            mProjectionMap = null;
            return mProjectionMap;
        }
       
        List<LogicalPlan> foreachPlans = getForEachPlans();
        List<Boolean> flattenList = getFlatten();
       
        MultiMap<Integer, ProjectionMap.Column> mapFields = new MultiMap<Integer, ProjectionMap.Column>();
        List<Integer> addedFields = new ArrayList<Integer>();
        int outputColumn = 0;
       
        for(int i = 0; i < foreachPlans.size(); ++i) {
            LogicalPlan foreachPlan = foreachPlans.get(i);
            List<LogicalOperator> leaves = foreachPlan.getLeaves();
            if(leaves == null || leaves.size() > 1) {
                mProjectionMap = null;
                return mProjectionMap;
            }
           
            int inputColumn = -1;
            boolean mapped = false;
            LOCast cast = null;
           
           
            if(leaves.get(0) instanceof LOProject || leaves.get(0) instanceof LOCast) {
                //find out if this project is a chain of projects
                Pair<LOProject, LOCast> pair = LogicalPlan.chainOfProjects(foreachPlan);
                if (pair != null) {
                    LOProject topProject = pair.first;
                    cast = pair.second;
                    if (topProject != null) {
                        inputColumn = topProject.getCol();
                        mapped = true;
                    }
                }
            }
           
            Schema.FieldSchema leafFS;
            try {
                leafFS = ((ExpressionOperator)leaves.get(0)).getFieldSchema();
            } catch (FrontendException fee) {
                mProjectionMap = null;
                return mProjectionMap;
            }
           
            if(leafFS == null) {
                mProjectionMap = null;
                return mProjectionMap;
            }
           
            if(flattenList.get(i)) {
                Schema innerSchema = leafFS.schema;
               
                if(innerSchema != null) {                   
                    if(innerSchema.isTwoLevelAccessRequired()) {
                        // this is the case where the schema is that of
                        // a bag which has just one tuple fieldschema which
                        // in turn has a list of fieldschemas. The schema
                        // after flattening would consist of the fieldSchemas
                        // present in the tuple
                       
                        // check that indeed we only have one field schema
                        // which is that of a tuple
                        if(innerSchema.getFields().size() != 1) {
                            mProjectionMap = null;
                            return mProjectionMap;
                        }
                        Schema.FieldSchema tupleFS;
                        try {
                            tupleFS = innerSchema.getField(0);
                        } catch (FrontendException fee) {
                            mProjectionMap = null;
                            return mProjectionMap;
                        }
                       
                        if(tupleFS.type != DataType.TUPLE) {
                            mProjectionMap = null;
                            return mProjectionMap;
                        }
                        innerSchema = tupleFS.schema;
                    }
                   
                    //innerSchema could be modified and hence the second check
                    if(innerSchema != null) {
                        for(int j = 0; j < innerSchema.size(); ++j) {
                            if(mapped) {
                                //map each flattened column to the original column
                                if (cast != null) {
                                    mapFields.put(outputColumn++,
                                            new ProjectionMap.Column(
View Full Code Here


            mSchemaInputMapping = new ArrayList<LogicalOperator>();
            int i=-1;
            boolean seeUnknown = false;
            for (LogicalOperator op : inputs) {
                try {
                    Schema cSchema = op.getSchema();
                    if(cSchema!=null){
                       
                        for (FieldSchema schema : cSchema.getFields()) {
                            ++i;
                            FieldSchema newFS = null;
                            if(schema.alias != null) {
                                if(nonDuplicates.containsKey(schema.alias)) {
                                    if (nonDuplicates.get(schema.alias) != -1) {
                                        nonDuplicates.remove(schema.alias);
                                        nonDuplicates.put(schema.alias, -1);
                                    }
                                } else {
                                    nonDuplicates.put(schema.alias, i);
                                }
                                newFS = new FieldSchema(op.getAlias()+"::"+schema.alias,schema.schema,schema.type);
                            } else {
                                newFS = new Schema.FieldSchema(null, schema.type);
                            }
                            newFS.setParent(schema.canonicalName, op);
                            fss.add(newFS);
                            mSchemaInputMapping.add(op);
                        }
                    } else {
                        seeUnknown = true;
                    }
                } catch (FrontendException ioe) {
                    mIsSchemaComputed = false;
                    mSchema = null;
                    throw ioe;
                }
            }
            mIsSchemaComputed = true;
            mSchema = null;
            if (!seeUnknown)
            {
                mSchema = new Schema(fss);
                for (Entry<String, Integer> ent : nonDuplicates.entrySet()) {
                    int ind = ent.getValue();
                    if(ind==-1) continue;
                    FieldSchema prevSch = fss.get(ind);
                    // this is a non duplicate and hence can be referred to
View Full Code Here

                throw new FrontendException(msg, errCode, PigException.INPUT, false, null);               
            }

        }

        return new Schema(fsList) ;
    }
View Full Code Here

    public ProjectionMap getProjectionMap() {
       
        if(mIsProjectionMapComputed) return mProjectionMap;
        mIsProjectionMapComputed = true;
       
        Schema outputSchema;
       
        try {
            outputSchema = getSchema();
        } catch (FrontendException fee) {
            mProjectionMap = null;
            return mProjectionMap;
        }
       
        if(outputSchema == null) {
            mProjectionMap = null;
            return mProjectionMap;
        }
       
        List<LogicalOperator> predecessors = (ArrayList<LogicalOperator>)mPlan.getPredecessors(this);
        if(predecessors == null) {
            mProjectionMap = null;
            return mProjectionMap;
        }
       
        MultiMap<Integer, ProjectionMap.Column> mapFields = new MultiMap<Integer, ProjectionMap.Column>();
        List<Integer> addedFields = new ArrayList<Integer>();
        boolean[] unknownSchema = new boolean[predecessors.size()];
        boolean anyUnknownInputSchema = false;
        int outputColumnNum = 0;
       
        for(int inputNum = 0; inputNum < predecessors.size(); ++inputNum) {
            LogicalOperator predecessor = predecessors.get(inputNum);
            Schema inputSchema = null;       
           
            try {
                inputSchema = predecessor.getSchema();
            } catch (FrontendException fee) {
                mProjectionMap = null;
                return mProjectionMap;
            }
           
            if(inputSchema == null) {
                unknownSchema[inputNum] = true;
                outputColumnNum++;
                addedFields.add(inputNum);
                anyUnknownInputSchema = true;
            } else {
                unknownSchema[inputNum] = false;
                for(int inputColumn = 0; inputColumn < inputSchema.size(); ++inputColumn) {
                    mapFields.put(outputColumnNum++,
                            new ProjectionMap.Column(new Pair<Integer, Integer>(inputNum, inputColumn)));
                }
            }
        }
View Full Code Here

       
    @Test
    public void testFRJoinSch1() throws IOException{
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        Schema frjSch = null, shjSch = null;
        pigServer.registerQuery("C = join A by $0, B by $0 using \"repl\";");
        frjSch = pigServer.dumpSchema("C");
        pigServer.registerQuery("C = join A by $0, B by $0;");
        shjSch = pigServer.dumpSchema("C");
        Assert.assertEquals(true, shjSch.equals(frjSch));
View Full Code Here

   
    @Test
    public void testFRJoinSch2() throws IOException{
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "';");
        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "';");
        Schema frjSch = null, shjSch = null;
        pigServer.registerQuery("C = join A by $0, B by $0 using \"repl\";");
        frjSch = pigServer.dumpSchema("C");
        pigServer.registerQuery("C = join A by $0, B by $0;");
        shjSch = pigServer.dumpSchema("C");
        Assert.assertTrue(shjSch == null);
View Full Code Here

    @Test
    public void testFRJoinSch3() throws IOException{
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        pigServer.registerQuery("C = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        Schema frjSch = null, shjSch = null;
        pigServer.registerQuery("D = join A by $0, B by $0, C by $0 using \"repl\";");
        frjSch = pigServer.dumpSchema("D");
        pigServer.registerQuery("D = join A by $0, B by $0, C by $0;");
        shjSch = pigServer.dumpSchema("D");
        Assert.assertEquals(true, shjSch.equals(frjSch));
View Full Code Here

        }
    }

    @Override
    public Schema outputSchema(Schema input) {
        return new Schema(new Schema.FieldSchema(null, DataType.INTEGER));
    }
View Full Code Here

    @Test
    public void testFRJoinSch4() throws IOException{
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "';");
        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "';");
        pigServer.registerQuery("C = LOAD '" + INPUT_FILE + "';");
        Schema frjSch = null, shjSch = null;
        pigServer.registerQuery("D = join A by $0, B by $0, C by $0 using \"repl\";");
        frjSch = pigServer.dumpSchema("D");
        pigServer.registerQuery("D = join A by $0, B by $0, C by $0;");
        shjSch = pigServer.dumpSchema("D");
        Assert.assertTrue(shjSch == null);
View Full Code Here

   
    @Test
    public void testFRJoinSch5() throws IOException{
        pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        pigServer.registerQuery("B = LOAD '" + INPUT_FILE + "' as (x:int,y:int);");
        Schema frjSch = null, shjSch = null;
        pigServer.registerQuery("C = join A by ($0,$1), B by ($0,$1) using \"repl\";");
        frjSch = pigServer.dumpSchema("C");
        pigServer.registerQuery("C = join A by ($0,$1), B by ($0,$1);");
        shjSch = pigServer.dumpSchema("C");
        Assert.assertEquals(true, shjSch.equals(frjSch));
View Full Code Here

TOP

Related Classes of org.apache.pig.impl.logicalLayer.schema.Schema

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.