Package org.apache.pig.builtin

Examples of org.apache.pig.builtin.BinStorage$UnImplementedLoadCaster

80
81
82
83
84
85
86
87
88
89
90
91
92
93
        if (quantilesFile.length() == 0)
            throw new RuntimeException(this.getClass().getSimpleName() + " used but no quantiles found");
       
        try{
            InputStream is = FileLocalizer.openDFSFile(quantilesFile,ConfigurationUtil.toProperties(job));
            BinStorage loader = new BinStorage();
            DataBag quantilesList;
            loader.bindTo(quantilesFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
            Tuple t = loader.getNext();
            if(t==null) throw new RuntimeException("Empty samples file");
            // the Quantiles file has a tuple as under:
            // (numQuantiles, bag of samples)
            // numQuantiles here is the reduce parallelism
            Map<String, Object> quantileMap = (Map<String, Object>) t.get(0);
View Full Code Here
160
161
162
163
164
165
166
167
168
169
170
        t.append(weights);
        b.add(t);
       
        String fileName = "file:"+File.createTempFile("tmp", "");
        PigFile f = new PigFile(fileName);
        f.store(b, new BinStorage(), pigServer.getPigContext());
       
       
        pigServer.registerQuery("a = load '" + Util.encodeEscape(fileName) + "' using BinStorage();");
        pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
        Iterator<Tuple> iter = pigServer.openIterator("b");
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
        t.append(weights);
        b.add(t);
       
        String fileName = "file:"+File.createTempFile("tmp", "");
        PigFile f = new PigFile(fileName);
        f.store(b, new BinStorage(), pigServer.getPigContext());
       
       
        pigServer.registerQuery("a = load '" + Util.encodeEscape(fileName) + "' using BinStorage();");
        pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
        Iterator<Tuple> iter = pigServer.openIterator("b");
View Full Code Here
81
82
83
84
85
86
87
88
89
90
91
92
93
94
        if (quantilesFile.length() == 0)
            throw new RuntimeException(this.getClass().getSimpleName() + " used but no quantiles found");
       
        try{
            InputStream is = FileLocalizer.openDFSFile(quantilesFile,ConfigurationUtil.toProperties(job));
            BinStorage loader = new BinStorage();
            DataBag quantilesList;
            loader.bindTo(quantilesFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
            Tuple t = loader.getNext();
            if(t!=null)
            {
                // the Quantiles file has a tuple as under:
                // (numQuantiles, bag of samples)
                // numQuantiles here is the reduce parallelism
View Full Code Here
118
119
120
121
122
123
124
125
126
127
128
129
130
131
    if (job != null) {
      is = FileLocalizer.openDFSFile(keyDistFile,ConfigurationUtil.toProperties(job));
    } else {
      is = FileLocalizer.openDFSFile(keyDistFile);
    }
    BinStorage loader = new BinStorage();
    DataBag partitionList;
    loader.bindTo(keyDistFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
    Tuple t = loader.getNext();
    if(t==null) {
      throw new RuntimeException("Empty samples file");
    }
    // The keydist file is structured as (key, min, max)
    // min, max being the index of the reducers
View Full Code Here
75
76
77
78
79
80
81
82
83
84
85
86
87
88
        if (quantilesFile.length() == 0)
            throw new RuntimeException(this.getClass().getSimpleName() + " used but no quantiles found");
       
        try{
            InputStream is = FileLocalizer.openDFSFile(quantilesFile,ConfigurationUtil.toProperties(job));
            BinStorage loader = new BinStorage();
            ArrayList<PigNullableWritable> quantilesList = new ArrayList<PigNullableWritable>();
            loader.bindTo(quantilesFile, new BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
            Tuple t = loader.getNext();
            if(t==null) throw new RuntimeException("Empty samples file");
            // the Quantiles file has a tuple as under:
            // (numQuantiles, bag of samples)
            // numQuantiles here is the reduce parallelism
            numQuantiles = (Integer) t.get(0);
View Full Code Here
341
342
343
344
345
346
347
348
349
350
351
                    "i:int,d:double);store a into '" + outputFileName + "' using " +
                            "BinStorage();";
            pig.setBatchOn();
            Util.registerMultiLineQuery(pig, query);
            pig.executeBatch();
            ResourceSchema rs = new BinStorage().getSchema(outputFileName,
                    new Job(ConfigurationUtil.toConfiguration(pig.getPigContext().
                            getProperties())));
            Schema expectedSchema = Utils.getSchemaFromString(
                    "c:chararray,i:int,d:double");
            assertTrue("Checking binstorage getSchema output", Schema.equals(
View Full Code Here
371
372
373
374
375
376
377
378
379
380
381
                    "i:int,d:double);store a into '" + outputFileName + "' using " +
                            "BinStorage();";
            pig.setBatchOn();
            Util.registerMultiLineQuery(pig, query);
            pig.executeBatch();
            ResourceSchema rs = new BinStorage().getSchema(outputFileName,
                    new Job(ConfigurationUtil.toConfiguration(pig.getPigContext().
                            getProperties())));
            Schema expectedSchema = Utils.getSchemaFromString(
                    "c:chararray,i:int,d:double");
            assertTrue("Checking binstorage getSchema output", Schema.equals(
View Full Code Here
329
330
331
332
333
334
335
336
337
338
339
                    "i:int,d:double);store a into '" + outputFileName + "' using " +
                            "BinStorage();";
            pig.setBatchOn();
            Util.registerMultiLineQuery(pig, query);
            pig.executeBatch();
            ResourceSchema rs = new BinStorage().getSchema(outputFileName,
                    new Job(ConfigurationUtil.toConfiguration(pig.getPigContext().
                            getProperties())));
            Schema expectedSchema = Utils.getSchemaFromString(
                    "c:chararray,i:int,d:double");
            assertTrue("Checking binstorage getSchema output", Schema.equals(
View Full Code Here
169
170
171
172
173
174
175
176
177
178
179
        t.append(weights);
        b.add(t);
       
        String fileName = "file:"+File.createTempFile("tmp", "");
        PigFile f = new PigFile(fileName);
        f.store(b, new BinStorage(), pigServer.getPigContext());
       
       
        pigServer.registerQuery("a = load '" + Util.encodeEscape(fileName) + "' using BinStorage();");
        pigServer.registerQuery("b = foreach a generate $0#'apple',flatten($1#'orange');");
        Iterator<Tuple> iter = pigServer.openIterator("b");
View Full Code Here
TOP

Related Classes of org.apache.pig.builtin.BinStorage$UnImplementedLoadCaster

  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.Path
  • org.apache.pig.backend.hadoop.datastorage.HDataStorage
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.SortPartitioner
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.partitioners.WeightedRangePartitioner
  • org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil
  • org.apache.pig.data.Tuple
  • org.apache.pig.impl.io.BinStorageInputFormat
  • org.apache.pig.impl.io.BinStorageOutputFormat
  • org.apache.pig.impl.io.BufferedPositionedInputStream
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.