Package org.apache.pig.impl.io

Examples of org.apache.pig.impl.io.BufferedPositionedInputStream

83
84
85
86
87
88
89
90
91
92
93
       
        try{
            InputStream is = FileLocalizer.openDFSFile(quantilesFile,ConfigurationUtil.toProperties(job));
            BinStorage loader = new BinStorage();
            DataBag quantilesList;
            loader.bindTo(quantilesFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
            Tuple t = loader.getNext();
            if(t!=null)
            {
                // the Quantiles file has a tuple as under:
                // (numQuantiles, bag of samples)
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
    } else {
      is = FileLocalizer.openDFSFile(keyDistFile);
    }
    BinStorage loader = new BinStorage();
    DataBag partitionList;
    loader.bindTo(keyDistFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
    Tuple t = loader.getNext();
    if(t==null) {
      throw new RuntimeException("Empty samples file");
    }
    // The keydist file is structured as (key, min, max)
View Full Code Here
188
189
190
191
192
193
194
        } else if (currentFileName.endsWith(".gz")) {
            is = new GZIPInputStream(is);
        }

       
        loader.bindTo(currentFileName , new BufferedPositionedInputStream(is), offset, Long.MAX_VALUE);
    }
View Full Code Here
135
136
137
138
139
140
141
142
        // We may extend later.
        String signature = ((HDataStorage)base).getConfiguration().getProperty("pig.loader.signature");
        if (loader instanceof PigStorage)
            ((PigStorage)loader).setSignature(signature);
       
        loader.bindTo(file, new BufferedPositionedInputStream(is,
                start), start, end);
    }
View Full Code Here
274
275
276
277
278
279
280
281
282
283
284
    }

    private void setStreams() throws IOException {
        stdout = new DataInputStream(new BufferedInputStream(process
                .getInputStream()));
        outputHandler.bindTo("", new BufferedPositionedInputStream(stdout),
                0, Long.MAX_VALUE);

        stdin = new DataOutputStream(new BufferedOutputStream(process
                .getOutputStream()));
        inputHandler.bindTo(stdin);
View Full Code Here
66
67
68
69
70
71
72
73
74
75
76
        try{
             p = (LoadFunc)PigContext.instantiateFuncFromSpec(outFileSpec.getFuncSpec());

             InputStream is = FileLocalizer.open(outFileSpec.getFileName(), pigContext);

             p.bindTo(outFileSpec.getFileName(), new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);

        }catch (Exception e){
            throw new ExecException("Unable to get results for " + outFileSpec, e);
        }
       
View Full Code Here
77
78
79
80
81
82
83
84
85
86
87
       
        try{
            InputStream is = FileLocalizer.openDFSFile(quantilesFile,ConfigurationUtil.toProperties(job));
            BinStorage loader = new BinStorage();
            ArrayList<PigNullableWritable> quantilesList = new ArrayList<PigNullableWritable>();
            loader.bindTo(quantilesFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
            Tuple t = loader.getNext();
            if(t==null) throw new RuntimeException("Empty samples file");
            // the Quantiles file has a tuple as under:
            // (numQuantiles, bag of samples)
            // numQuantiles here is the reduce parallelism
View Full Code Here
220
221
222
223
224
225
226
227
228
229
230
    }

    private void setStreams() throws IOException {
        stdout = new DataInputStream(new BufferedInputStream(process
                .getInputStream()));
        outputHandler.bindTo("", new BufferedPositionedInputStream(stdout),
                0, Long.MAX_VALUE);
       
        stdin = new DataOutputStream(new BufferedOutputStream(process
                .getOutputStream()));
        inputHandler.bindTo(stdin);
View Full Code Here
212
213
214
215
216
217
218
219
220
221
222
            // Get hold of the stdout of the process
            stdout = new DataInputStream(new BufferedInputStream(process
                    .getInputStream()));

            // Bind the stdout to the OutputHandler
            outputHandler.bindTo("", new BufferedPositionedInputStream(stdout),
                    0, Long.MAX_VALUE);

            // start thread to process output from executable's stdout
            stdoutThread = new ProcessOutputThread(outputHandler, poStream);
            stdoutThread.start();
View Full Code Here
52
53
54
55
56
57
58
59
60
            long offset, long end) throws IOException {
        // This is a trigger to start processing the output from the file ...
        // ... however, we must ignore the input parameters and use ones
        // provided during initialization
        File file = new File(this.fileName);
        BufferedPositionedInputStream fileInStream =
            new BufferedPositionedInputStream(new FileInputStream(file));
        super.bindTo(this.fileName, fileInStream, 0, file.length());
    }
View Full Code Here
TOP

Related Classes of org.apache.pig.impl.io.BufferedPositionedInputStream

  • org.apache.pig.backend.executionengine.PigSlice
  • org.apache.pig.backend.hadoop.executionengine.HJob
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.SortPartitioner
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.partitioners.WeightedRangePartitioner
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad
  • org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POMergeJoin
  • org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil
  • org.apache.pig.backend.local.executionengine.LocalJob
  • org.apache.pig.backend.local.executionengine.POLoad
  • org.apache.pig.builtin.BinStorage
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.