Package org.apache.pig.backend.datastorage

Examples of org.apache.pig.backend.datastorage.DataStorage

220
221
222
223
224
225
226
227
228
229
230
        // working dir to /user/<userid>
        if(pigContext.getExecType() == ExecType.MAPREDUCE) {
            fs.setWorkingDirectory(new Path("/user", job.getUser()));
                }
       
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
        ValidatingInputFileSpec spec;
        if (inputs.get(i).first instanceof ValidatingInputFileSpec) {
            spec = (ValidatingInputFileSpec) inputs.get(i).first;
        } else {
            spec = new ValidatingInputFileSpec(inputs.get(i).first, store);
View Full Code Here
248
249
250
251
252
253
254
255
256
257
258
259
    }

    private void checkStorePath(String orig, String expected, boolean isTmp) throws Exception {
        pc.getProperties().setProperty("opt.multiquery",""+true);

        DataStorage dfs = pc.getDfs();
        dfs.setActiveContainer(dfs.asContainer("/tmp"));
        Map<LogicalOperator, LogicalPlan> aliases = new HashMap<LogicalOperator, LogicalPlan>();
        Map<OperatorKey, LogicalOperator> logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
        Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>();
        Map<String, String> fileNameMap = new HashMap<String, String>();
       
View Full Code Here
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
        return lastConf;
    }

    public RecordReader<Text, Tuple> makeReader(JobConf job) throws IOException {
        lastConf = job;       
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
        // if the execution is against Mapred DFS, set
        // working dir to /user/<userid>
        if(execType == ExecType.MAPREDUCE)
            store.setActiveContainer(store.asContainer("/user/" + job.getUser()));
        PigContext.setPackageImportList((ArrayList<String>)ObjectSerializer.deserialize(job.get("udf.import.list")));
        wrapped.init(store);
       
        job.set("map.target.ops", ObjectSerializer.serialize(targetOps));
        // Mimic org.apache.hadoop.mapred.FileSplit if feasible...
View Full Code Here
40
41
42
43
44
45
46
47
48
49
50
public class TypeCheckingTestUtil {

    public static LOLoad genDummyLOLoad(LogicalPlan plan)  {
        String pigStorage = PigStorage.class.getName() ;
        DataStorage dfs = new HDataStorage(new Properties());
        try {
            LOLoad load = new LOLoad(plan,
                                      genNewOperatorKey(),
                                      new FileSpec("pi", new FuncSpec(pigStorage)),
                                      null, dfs, true) ;
View Full Code Here
198
199
200
201
202
203
204
205
206
207
208
209
    }

    private void checkLoadPath(String orig, String expected, boolean isTmp) throws Exception {
        pc.getProperties().setProperty("opt.multiquery",""+true);
               
        DataStorage dfs = pc.getDfs();
        dfs.setActiveContainer(dfs.asContainer("/tmp"));
        Map<LogicalOperator, LogicalPlan> aliases = new HashMap<LogicalOperator, LogicalPlan>();
        Map<OperatorKey, LogicalOperator> logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
        Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>();
        Map<String, String> fileNameMap = new HashMap<String, String>();
       
View Full Code Here
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
        // working dir to /user/<userid>
        if(pigContext.getExecType() == ExecType.MAPREDUCE) {
            fs.setWorkingDirectory(new Path("/user", job.getUser()));
                }
       
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
       
        // Pass loader signature to slicer
          List<String> inpSignatureLists = (ArrayList<String>)ObjectSerializer.deserialize(job.get("pig.inpSignatures"));
                if (inpSignatureLists.get(i)!=null)
                    store.getConfiguration().setProperty("pig.loader.signature", inpSignatureLists.get(i));

        ValidatingInputFileSpec spec;
        if (inputs.get(i).first instanceof ValidatingInputFileSpec) {
            spec = (ValidatingInputFileSpec) inputs.get(i).first;
        } else {
View Full Code Here
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
        src.rename(dst);

    }

    public void copy(String src, String dst, boolean localDst) throws IOException {
        DataStorage dstStorage = dfs;
       
        if (localDst) {
            dstStorage = lfs;
        }
       
        ElementDescriptor srcElement = null;
        ElementDescriptor dstElement = null;

        try {
            srcElement = dfs.asElement(src);
            dstElement = dstStorage.asElement(dst);
        }
        catch (DataStorageException e) {
            byte errSrc = getErrorSource();           
            int errCode = 0;
            switch(errSrc) {
View Full Code Here
170
171
172
173
174
175
176
177
178
        return openDFSFile(fileName, ConfigurationUtil.toProperties(conf));

    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }
View Full Code Here
194
195
196
197
198
199
200
201
202
203
204
205
        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }
   
    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
      
        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);
       
        long size = 0;
View Full Code Here
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
     * @return an array of Element descriptors for files present (found by traversing all levels of dirs)
     *  in the input element descriptor
     * @throws DataStorageException
     */
    private static ElementDescriptor[] getFileElementDescriptors(ElementDescriptor elem) throws DataStorageException {
        DataStorage store = elem.getDataStorage();
        ElementDescriptor[] elems = store.asCollection(elem.toString());
        // elems could have directories in it, if so
        // get the files out so that it contains only files
        List<ElementDescriptor> paths = new ArrayList<ElementDescriptor>();
        List<ElementDescriptor> filePaths = new ArrayList<ElementDescriptor>();
        for (int m = 0; m < elems.length; m++) {
            paths.add(elems[m]);
        }
        for (int j = 0; j < paths.size(); j++) {
            ElementDescriptor fullPath = store.asElement(store
                    .getActiveContainer(), paths.get(j));
            // Skip hadoop's private/meta files ...
            if (fullPath.systemElement()) {
                continue;
            }
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.datastorage.DataStorage

  • com.packetloop.packetpig.storage.JsonMetadata
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.PigInputFormat
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.SliceWrapper
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SliceWrapper
  • org.apache.pig.builtin.JsonMetadata
  • org.apache.pig.impl.io.FileLocalizer
  • org.apache.pig.impl.PigContext
  • org.apache.pig.parser.QueryParserUtils
  • org.apache.pig.piggybank.storage.JsonMetadata
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.