Package org.apache.pig.impl.io

Examples of org.apache.pig.impl.io.FileSpec

1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
        String pigStorage = PigStorage.class.getName() ;

        LOLoad load1 = new LOLoad(plan,
                                  genNewOperatorKey(),
                                  new FileSpec("pi", new FuncSpec(pigStorage)),
                                  null) ;
        LOLoad load2 = new LOLoad(plan,
                                  genNewOperatorKey(),
                                  new FileSpec("pi", new FuncSpec(pigStorage)),
                                  null) ;

        // schema for input#1
        Schema inputSchema1 = null ;
        {
View Full Code Here
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
        String pigStorage = PigStorage.class.getName() ;

        LOLoad load1 = new LOLoad(plan,
                                  genNewOperatorKey(),
                                  new FileSpec("pi", new FuncSpec(pigStorage)),
                                  null) ;
        LOLoad load2 = new LOLoad(plan,
                                  genNewOperatorKey(),
                                  new FileSpec("pi", new FuncSpec(pigStorage)),
                                  null) ;

        // schema for input#1
        Schema inputSchema1 = null ;
        {
View Full Code Here
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
        String pigStorage = PigStorage.class.getName() ;

        LOLoad load1 = new LOLoad(plan,
                                  genNewOperatorKey(),
                                  new FileSpec("pi", new FuncSpec(pigStorage)),
                                  null) ;
        LOLoad load2 = new LOLoad(plan,
                                  genNewOperatorKey(),
                                  new FileSpec("pi", new FuncSpec(pigStorage)),
                                  null) ;

        // schema for input#1
        Schema inputSchema1 = null ;
        {
View Full Code Here
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
        LogicalPlan lp = new LogicalPlan();
        {
            LogicalSchema aschema = new LogicalSchema();
            aschema.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
            LOLoad A = new LOLoad(new FileSpec("/abc",
                new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema, lp, null);
            lp.add(A);
       
            // B = load
            LogicalSchema bschema = new LogicalSchema();
            bschema.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
            LOLoad B = new LOLoad(new FileSpec("/def",
                new FuncSpec("PigStorage", "\t")), bschema, lp, null);
            lp.add(B);
       
            // C = join
            LogicalSchema cschema = new LogicalSchema();
            cschema.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
            cschema.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
            LogicalExpressionPlan aprojplan = new LogicalExpressionPlan();
            new ProjectExpression(aprojplan, 0, 0, null);
            LogicalExpressionPlan bprojplan = new LogicalExpressionPlan();
            new ProjectExpression(bprojplan, 1, 0, null);
            MultiMap<Integer, LogicalExpressionPlan> mm =
                new MultiMap<Integer, LogicalExpressionPlan>();
            mm.put(0, aprojplan);
            mm.put(1, bprojplan);
            LOJoin C = new LOJoin(lp, mm, JOINTYPE.HASH, new boolean[] {true, true});
            C.neverUseForRealSetSchema(cschema);
            lp.add(C);
            lp.connect(A, C);
            lp.connect(B, C);
           
            // D = filter
            LogicalExpressionPlan filterPlan = new LogicalExpressionPlan();
            ProjectExpression fy = new ProjectExpression(filterPlan, 0, 1, null);
            ConstantExpression fc = new ConstantExpression(filterPlan, new Integer(0));
            new EqualExpression(filterPlan, fy, fc);
            LOFilter D = new LOFilter(lp, filterPlan);
            D.neverUseForRealSetSchema(cschema);
            lp.add(D);
            lp.connect(C, D);
        }
       
        // Build a second similar plan to test equality
        // A = load
        LogicalPlan lp1 = new LogicalPlan();
        {
            LogicalSchema aschema = new LogicalSchema();
            aschema.addField(new LogicalSchema.LogicalFieldSchema(
                "x", null, DataType.INTEGER));
            LOLoad A = new LOLoad(new FileSpec("/abc",
                new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema, lp1, null);
            lp1.add(A);
           
            // B = load
            LogicalSchema bschema = new LogicalSchema();
            bschema.addField(new LogicalSchema.LogicalFieldSchema(
                "y", null, DataType.INTEGER));
            LOLoad B = new LOLoad(new FileSpec("/def",
                new FuncSpec("PigStorage", "\t")), bschema, lp1, null);
            lp1.add(B);
           
            // C = join
            LogicalSchema cschema = new LogicalSchema();
View Full Code Here
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
        LogicalPlan lp = new LogicalPlan();
       
        LogicalSchema aschema1 = new LogicalSchema();
        aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        LOLoad load1 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema1, lp, null);
        lp.add(load1);
       
        LOLoad load2 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "z"})), aschema1, lp, null);
        lp.add(load2);
       
        assertFalse(load1.isEqual(load2));
    }
View Full Code Here
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
        LogicalPlan lp = new LogicalPlan();
       
        LogicalSchema aschema1 = new LogicalSchema();
        aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        LOLoad load1 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema1, lp, null);
        lp.add(load1);
       
        LOLoad load3 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), "x")), aschema1, lp, null);
        lp.add(load3);
       
        assertFalse(load1.isEqual(load3));
    }
View Full Code Here
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
        LogicalPlan lp = new LogicalPlan();
       
        LogicalSchema aschema1 = new LogicalSchema();
        aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        LOLoad load1 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema1, lp, null);
        lp.add(load1);
       
         // Different function names in FuncSpec
        LOLoad load4 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "z"})), aschema1, lp, null);
        lp.add(load4);
       
        assertFalse(load1.isEqual(load4));
    }
View Full Code Here
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
    public void testLoadEqualityDifferentFileName() throws FrontendException {
        LogicalPlan lp = new LogicalPlan();
        LogicalSchema aschema1 = new LogicalSchema();
        aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        LOLoad load1 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema1, lp, null);
        lp.add(load1);
   
        // Different file name
        LOLoad load5 = new LOLoad(new FileSpec("/def",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "z"})), aschema1, lp, null);
        lp.add(load5);
       
        assertFalse(load1.isEqual(load5));
    }
View Full Code Here
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
    public void testRelationalEqualityDifferentSchema() throws FrontendException {
        LogicalPlan lp = new LogicalPlan();
        LogicalSchema aschema1 = new LogicalSchema();
        aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        LOLoad load1 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema1, lp, null);
        lp.add(load1);
       
        // Different schema
        LogicalSchema aschema2 = new LogicalSchema();
        aschema2.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.CHARARRAY));
       
        LOLoad load6 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "z"})), aschema2, lp, null);
        lp.add(load6);
           
        assertFalse(load1.isEqual(load6));
    }
View Full Code Here
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
   
    @Test
    public void testRelationalEqualityNullSchemas() throws FrontendException {
        LogicalPlan lp = new LogicalPlan();
        // Test that two loads with no schema are still equal
        LOLoad load7 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), null, lp, null);
        lp.add(load7);
       
        LOLoad load8 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), null, lp, null);
        lp.add(load8);
       
        assertTrue(load7.isEqual(load8));
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.io.FileSpec

  • com.netflix.lipstick.MRPlanCalculator
  • org.apache.pig.backend.executionengine.util.ExecTools
  • org.apache.pig.backend.hadoop.executionengine.HExecutionEngine
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler$JoinDistributedCacheVisitor
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.LimitAdjuster
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler$FindStoreNameVisitor
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MRCompiler$LimitAdjuster
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.NoopStoreRemover$PhysicalRemover
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SampleOptimizer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.