Examples of FileSpec

flex2.compiler.FileSpec
A list of files, which are not required to follow the single public definition rule. This is used by compc's include-sources option and by Flash Builder. It's similar to ASC's include option. @author Clement Wong
org.apache.pig.impl.io.FileSpec
A simple class that specifies a file name and storage function which is used to read/write it
org.pdfclown.documents.fileSpecs.FileSpec
fanochizzolini.it) @since 0.0.7 @version 0.1.0

Examples of org.apache.pig.impl.io.FileSpec

    public void testJoinDifferentNumInputs() throws FrontendException {
        LogicalPlan lp = new LogicalPlan();
               LogicalSchema jaschema1 = new LogicalSchema();
       jaschema1.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
       LOLoad A1 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema1, lp, null);
       lp.add(A1);
        
        // B = load
        LogicalSchema jbschema1 = new LogicalSchema();
        jbschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad B1 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema1, lp, null);
        lp.add(B1);
        
        // C = join
        LogicalSchema jcschema1 = new LogicalSchema();
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan1, 0, 0, null);
        LogicalExpressionPlan bprojplan1 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan1, 1, 0, null);
        MultiMap<Integer, LogicalExpressionPlan> mm1 = 
            new MultiMap<Integer, LogicalExpressionPlan>();
        mm1.put(0, aprojplan1);
        mm1.put(1, bprojplan1);
        LOJoin C1 = new LOJoin(lp, mm1, JOINTYPE.HASH, new boolean[] {true, true});
        C1.neverUseForRealSetSchema(jcschema1);
        lp.add(C1);
        lp.connect(A1, C1);
        lp.connect(B1, C1);
 
        // A = load
        LogicalSchema jaschema5 = new LogicalSchema();
        jaschema5.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A5 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema5, lp, null);
        lp.add(A5);
        
        // B = load
        LogicalSchema jbschema5 = new LogicalSchema();
        jbschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad B5 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema5, lp, null);
        lp.add(B5);
        
        // Beta = load
        LogicalSchema jbetaschema5 = new LogicalSchema();
        jbetaschema5.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LOLoad Beta5 = new LOLoad(new FileSpec("/ghi",
            new FuncSpec("PigStorage", "\t")), jbetaschema5, lp, null);
        lp.add(Beta5);
        
        // C = join
        LogicalSchema jcschema5 = new LogicalSchema();

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

        
        // Test different join keys
        LogicalSchema jaschema6 = new LogicalSchema();
        jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A6 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema6, lp, null);
        lp.add(A6);
        
        // B = load
        LogicalSchema jbschema6 = new LogicalSchema();
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
        LOLoad B6 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema6, lp, null);
        lp.add(B6);
        
        // C = join
        LogicalSchema jcschema6 = new LogicalSchema();
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan6, 0, 0, null);
        LogicalExpressionPlan bprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan6, 1, 0, null);
        LogicalExpressionPlan b2projplan6 = new LogicalExpressionPlan();
        new ProjectExpression(b2projplan6, 1, 1, null);
        MultiMap<Integer, LogicalExpressionPlan> mm6 = 
            new MultiMap<Integer, LogicalExpressionPlan>();
        mm6.put(0, aprojplan6);
        mm6.put(1, bprojplan6);
        mm6.put(1, b2projplan6);
        LOJoin C6 = new LOJoin(lp, mm6, JOINTYPE.HASH, new boolean[] {true, true});
        C6.neverUseForRealSetSchema(jcschema6);
        lp.add(C6);
        lp.connect(A6, C6);
        lp.connect(B6, C6);
        
        
        LogicalSchema jaschema7 = new LogicalSchema();
        jaschema7.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A7 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema7, lp, null);
        lp.add(A7);
        
        // B = load
        LogicalSchema jbschema7 = new LogicalSchema();
        jbschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        jbschema7.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
        LOLoad B7 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema7, lp, null);
        lp.add(B7);
        
        // C = join
        LogicalSchema jcschema7 = new LogicalSchema();

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

        
        // Test different join keys
        LogicalSchema jaschema6 = new LogicalSchema();
        jaschema6.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A6 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema6, lp, null);
        lp.add(A6);
        
        // B = load
        LogicalSchema jbschema6 = new LogicalSchema();
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        jbschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
        LOLoad B6 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema6, lp, null);
        lp.add(B6);
        
        // C = join
        LogicalSchema jcschema6 = new LogicalSchema();
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        jcschema6.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        LogicalExpressionPlan aprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(aprojplan6, 0, 0, null);
        LogicalExpressionPlan bprojplan6 = new LogicalExpressionPlan();
        new ProjectExpression(bprojplan6, 1, 0, null);
        LogicalExpressionPlan b2projplan6 = new LogicalExpressionPlan();
        new ProjectExpression(b2projplan6, 1, 1, null);
        MultiMap<Integer, LogicalExpressionPlan> mm6 = 
            new MultiMap<Integer, LogicalExpressionPlan>();
        mm6.put(0, aprojplan6);
        mm6.put(1, bprojplan6);
        mm6.put(1, b2projplan6);
        LOJoin C6 = new LOJoin(lp, mm6, JOINTYPE.HASH, new boolean[] {true, true});
        C6.neverUseForRealSetSchema(jcschema6);
        lp.add(C6);
        lp.connect(A6, C6);
        lp.connect(B6, C6);
        
        // Test different different number of join keys
        LogicalSchema jaschema8 = new LogicalSchema();
        jaschema8.addField(new LogicalSchema.LogicalFieldSchema(
           "x", null, DataType.INTEGER));
        LOLoad A8 = new LOLoad(new FileSpec("/abc",
           new FuncSpec("/fooload", new String[] {"x", "y"})), jaschema8, lp, null);
        lp.add(A8);
        
        // B = load
        LogicalSchema jbschema8 = new LogicalSchema();
        jbschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "y", null, DataType.INTEGER));
        jbschema8.addField(new LogicalSchema.LogicalFieldSchema(
            "z", null, DataType.LONG));
        LOLoad B8 = new LOLoad(new FileSpec("/def",
            new FuncSpec("PigStorage", "\t")), jbschema8, lp, null);
        lp.add(B8);
        
        // C = join
        LogicalSchema jcschema8 = new LogicalSchema();

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

    public void testRelationalSameOpDifferentPreds() throws FrontendException {
        LogicalPlan lp1 = new LogicalPlan();
        LogicalSchema aschema1 = new LogicalSchema();
        aschema1.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        LOLoad A1 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "y"})), aschema1, lp1, null);
        lp1.add(A1);
        
        LogicalExpressionPlan fp1 = new LogicalExpressionPlan();
        ProjectExpression fy1 = new ProjectExpression(fp1, 0, 0, null);
        ConstantExpression fc1 = new ConstantExpression(fp1, 
            new Integer(0), new LogicalFieldSchema(null, null, DataType.INTEGER));
        new EqualExpression(fp1, fy1, fc1);
        LOFilter D1 = new LOFilter(lp1, fp1);
        LogicalSchema cschema = new LogicalSchema();
        cschema.addField(new LogicalSchema.LogicalFieldSchema(
            "x", null, DataType.INTEGER));
        D1.neverUseForRealSetSchema(cschema);
        lp1.add(D1);
        lp1.connect(A1, D1);
        
        LogicalPlan lp2 = new LogicalPlan();
        LOLoad A2 = new LOLoad(new FileSpec("/abc",
            new FuncSpec(DummyLoad.class.getName(), new String[] {"x", "z"})), null, lp2, null);
        lp2.add(A2);
        
        LogicalExpressionPlan fp2 = new LogicalExpressionPlan();
        ProjectExpression fy2 = new ProjectExpression(fp2, 0, 0, null);

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

                    int errCode = 2025;
                    String msg = "Expected leaf of reduce plan to " +
                        "always be POStore. Found " + mpLeaf.getClass().getSimpleName();
                    throw new MRCompilerException(msg, errCode, PigException.BUG);
                }
                FileSpec oldSpec = ((POStore)mpLeaf).getSFile();
                boolean oldIsTmpStore = ((POStore)mpLeaf).isTmpStore();
                
                FileSpec fSpec = getTempFileSpec();
                ((POStore)mpLeaf).setSFile(fSpec);
                ((POStore)mpLeaf).setIsTmpStore(true);
                mr.setReduceDone(true);
                MapReduceOper limitAdjustMROp = getMROp();
                POLoad ld = getLoad();

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

            this.oldSpec = oldSpec;
        }


        @Override
        public void visitStore(POStore sto) throws VisitorException {
            FileSpec spec = sto.getSFile();
            if (oldSpec.equals(spec)) {
                sto.setSFile(newSpec);
            }           
        }

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

                         + "_" + Long.toString(System.currentTimeMillis()) 
                         + "_" + i;
                     replicatedPath.add(replFiles[i].getFileName() + "#"
                             + symlink);
                 }
                 newReplFiles[i] = new FileSpec(symlink, 
                         (replFiles[i] == null ? null : replFiles[i].getFuncSpec()));               
             }
             
             join.setReplFiles(newReplFiles);

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

        
    private LogicalPlan genNewLoadStorePlan(String inputFile,
                                            String outputFile, DataStorage dfs) 
                                        throws Throwable {
        LogicalPlan plan = new LogicalPlan() ;
        FileSpec filespec1 =
            new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
        FileSpec filespec2 =
            new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
        LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, 
                ConfigurationUtil.toConfiguration(dfs.getConfiguration())) ;       
        LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2, "new") ;
        
        plan.add(load) ;

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

        
        for(MapReduceOper mrOp: mrOpList) {
            for(PhysicalOperator scalar: mrOp.scalars) {                
                MapReduceOper mro = phyToMROpMap.get(scalar);
                if (scalar instanceof POStore) {                                     
                    FileSpec oldSpec = ((POStore)scalar).getSFile();
                    MapReduceOper mro2 = seen.get(oldSpec);
                    boolean hasSeen = false;
                    if (mro2 != null) {
                        hasSeen = true;
                        mro = mro2;
                    }
                    if (!hasSeen
                            && combinable
                            && (mro.reducePlan.isEmpty() ? hasTooManyInputFiles(mro, conf)
                                    : (mro.requestedParallelism >= fileConcatenationThreshold))) {
                        PhysicalPlan pl = mro.reducePlan.isEmpty() ? mro.mapPlan : mro.reducePlan;
                        FileSpec newSpec = getTempFileSpec();
                        
                        // replace oldSpec in mro with newSpec
                        new FindStoreNameVisitor(pl, newSpec, oldSpec).visit();
                        
                        POStore newSto = getStore();

View Full Code Here

Examples of org.apache.pig.impl.io.FileSpec

            if (!mro.isMapDone()) {
                mro.setMapDoneSingle(true);
                curMROp = mro;
            }
            else if(mro.isMapDone() && !mro.isReduceDone()){
                FileSpec fSpec = getTempFileSpec();
                
                POStore st = getStore();
                st.setSFile(fSpec);
                mro.reducePlan.addAsLeaf(st);
                mro.setReduceDone(true);

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.