Package org.apache.pig.impl.io

Examples of org.apache.pig.impl.io.BufferedPositionedInputStream

106
107
108
109
110
111
112
113
114
115
116
    } else {
      is = FileLocalizer.openDFSFile(keyDistFile);
    }
    BinStorage loader = new BinStorage();
    DataBag partitionList;
    loader.bindTo(keyDistFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
    Tuple t = loader.getNext();
    if(t==null) {
      throw new RuntimeException("Empty samples file");
    }
    // The keydist file is structured as (key, min, max)
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
       
        try{
            InputStream is = FileLocalizer.openDFSFile(quantilesFile,ConfigurationUtil.toProperties(job));
            BinStorage loader = new BinStorage();
            DataBag quantilesList;
            loader.bindTo(quantilesFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
            Tuple t = loader.getNext();
            if(t==null) throw new RuntimeException("Empty samples file");
            // the Quantiles file has a tuple as under:
            // (numQuantiles, bag of samples)
            // numQuantiles here is the reduce parallelism
View Full Code Here
100
101
102
103
104
105
106
        String filename = lFile.getFileName();
        loader = (LoadFunc)PigContext.instantiateFuncFromSpec(lFile.getFuncSpec());
       
        is = (this.offset == 0) ? FileLocalizer.open(filename, pc) : FileLocalizer.open(filename, this.offset,pc);
       
        loader.bindTo(filename , new BufferedPositionedInputStream(is), this.offset, Long.MAX_VALUE);
    }
View Full Code Here
269
270
271
272
273
274
275
276
277
278
279
            return null;
        }
       
        InputStream is = FileLocalizer.open(fileName, execType, storage);
      
        bindTo(fileName, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
        // get the first record from the input file
        // and figure out the schema from the data in
        // the first record
        Tuple t = getNext();
        is.close();
View Full Code Here
104
105
106
107
108
109
110
        loader = (LoadFunc)PigContext.instantiateFuncFromSpec(lFile.getFuncSpec());
        if (loader instanceof PigStorage)
            ((PigStorage)loader).setSignature(signature);
        is = (this.offset == 0) ? FileLocalizer.open(filename, pc) : FileLocalizer.open(filename, this.offset,pc);
       
        loader.bindTo(filename , new BufferedPositionedInputStream(is), this.offset, Long.MAX_VALUE);
    }
View Full Code Here
400
401
402
403
404
405
406
407
408
        PigMapReduce.sJobConf.set("pig.loader.signature", signature);
        rightLoader.initialize(PigMapReduce.sJobConf);
        // the main purpose of this bindTo call is supply the input file name
        // to the right loader - in the case of Pig's DefaultIndexableLoader
        // this is really not used since the index has all information required
        rightLoader.bindTo(rightInputFileName, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
        rightLoader.seekNear(
                firstLeftKey instanceof Tuple ? (Tuple)firstLeftKey : mTupleFactory.newTuple(firstLeftKey));
    }
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    public void testPigLineRecordReader() {
        try {
            File testFile = File.createTempFile("testPigLineRecordReader", ".txt");
            LocalSeekableInputStream is = new LocalSeekableInputStream( testFile );
           
            BufferedPositionedInputStream bpis = new BufferedPositionedInputStream( is );
           
            PigLineRecordReader reader = new PigLineRecordReader( bpis, 0, Integer.MAX_VALUE );
           
            Field lineReader = reader.getClass().getDeclaredField("lineReader");
            lineReader.setAccessible(true);
View Full Code Here
115
116
117
118
119
120
121
122
123
124
125
            out.write(buffer, 0, buffer.length);
            out.close();
           
            LocalSeekableInputStream is = new LocalSeekableInputStream( testFile );
            CBZip2InputStream bzis = new CBZip2InputStream( is );
            BufferedPositionedInputStream bpis = new BufferedPositionedInputStream( bzis );
           
            PigLineRecordReader reader = new PigLineRecordReader( bpis, 0, 100 );

            Field lineReader = reader.getClass().getDeclaredField("lineReader");
            lineReader.setAccessible(true);
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
                ps.println( text );
            }
            ps.close();
           
            LocalSeekableInputStream is = new LocalSeekableInputStream( testFile );
            BufferedPositionedInputStream bpis = new BufferedPositionedInputStream( is );
            PigLineRecordReader reader = new PigLineRecordReader( bpis, 0, Integer.MAX_VALUE );
           
            Text value = new Text();
            int counter = 0;
            while( reader.next(value) ) {
View Full Code Here
201
202
203
204
205
206
207
208
209
210
211
            }
            ps.close();
           
            LocalSeekableInputStream is = new LocalSeekableInputStream( testFile );
            CBZip2InputStream bzis = new CBZip2InputStream( is );
            BufferedPositionedInputStream bpis = new BufferedPositionedInputStream( bzis );
            PigLineRecordReader reader = new PigLineRecordReader( bpis, 0, Integer.MAX_VALUE );
           
            Text value = new Text();
            int counter = 0;
            while( reader.next(value) ) {
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.io.BufferedPositionedInputStream

  • org.apache.pig.backend.executionengine.PigSlice
  • org.apache.pig.backend.hadoop.executionengine.HJob
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.SortPartitioner
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.partitioners.WeightedRangePartitioner
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeJoin
  • org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil
  • org.apache.pig.backend.local.executionengine.LocalJob
  • org.apache.pig.backend.local.executionengine.POLoad
  • org.apache.pig.builtin.BinStorage
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.