Package org.apache.pig.backend.datastorage

Examples of org.apache.pig.backend.datastorage.DataStorage

84
85
86
87
88
89
90
91
92
93
        lp.add( store );
        lp.connect( input, store );
    }

    static Path getCurrentDir(PigContext pigContext) throws IOException {
        DataStorage dfs = pigContext.getDfs();
        ContainerDescriptor desc = dfs.getActiveContainer();
        ElementDescriptor el = dfs.asElement(desc);
        return new Path(el.toString());
    }
View Full Code Here
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
        src.rename(dst);

    }

    public void copy(String src, String dst, boolean localDst) throws IOException {
        DataStorage dstStorage = dfs;

        if (localDst) {
            dstStorage = lfs;
        }

        ElementDescriptor srcElement = null;
        ElementDescriptor dstElement = null;

        try {
            srcElement = dfs.asElement(src);
            dstElement = dstStorage.asElement(dst);
        }
        catch (DataStorageException e) {
            byte errSrc = getErrorSource();
            int errCode = 0;
            switch(errSrc) {
View Full Code Here
176
177
178
179
180
181
182
183
184
        return openDFSFile(fileName, ConfigurationUtil.toProperties(conf));

    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }
View Full Code Here
193
194
195
196
197
198
199
200
201
202
203
204
        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }

    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);

        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);

        long size = 0;
View Full Code Here
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
     * @return an array of Element descriptors for files present (found by traversing all levels of dirs)
     *  in the input element descriptor
     * @throws DataStorageException
     */
    private static ElementDescriptor[] getFileElementDescriptors(ElementDescriptor elem) throws DataStorageException {
        DataStorage store = elem.getDataStorage();
        ElementDescriptor[] elems = store.asCollection(elem.toString());
        // elems could have directories in it, if so
        // get the files out so that it contains only files
        List<ElementDescriptor> paths = new ArrayList<ElementDescriptor>();
        List<ElementDescriptor> filePaths = new ArrayList<ElementDescriptor>();
        for (int m = 0; m < elems.length; m++) {
            paths.add(elems[m]);
        }
        for (int j = 0; j < paths.size(); j++) {
            ElementDescriptor fullPath = store.asElement(store
                    .getActiveContainer(), paths.get(j));
            // Skip hadoop's private/meta files ...
            if (fullPath.systemElement()) {
                continue;
            }
View Full Code Here
565
566
567
568
569
570
571
572
573
574
575
576
577
    public long capacity() throws IOException {
        if (pigContext.getExecType() == ExecType.LOCAL) {
            throw new IOException("capacity only supported for non-local execution");
        }
        else {
            DataStorage dds = pigContext.getDfs();
           
            Map<String, Object> stats = dds.getStatistics();

            String rawCapacityStr = (String) stats.get(DataStorage.RAW_CAPACITY_KEY);
            String rawUsedStr = (String) stats.get(DataStorage.RAW_USED_KEY);
           
            if ((rawCapacityStr == null) || (rawUsedStr == null)) {
View Full Code Here
590
591
592
593
594
595
596
597
598
599
600
601
     * @param filename
     * @return length of the file in bytes
     * @throws IOException
     */
    public long fileSize(String filename) throws IOException {
        DataStorage dfs = pigContext.getDfs();
        ElementDescriptor elem = dfs.asElement(filename);
        Map<String, Object> stats = elem.getStatistics();
        long length = (Long) stats.get(ElementDescriptor.LENGTH_KEY);
        int replication = (Short) stats
                .get(ElementDescriptor.BLOCK_REPLICATION_KEY);

View Full Code Here
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
    protected Set<ElementDescriptor> findMetaFile(String path, String metaname, Configuration conf)
        throws IOException {
        Set<ElementDescriptor> metaFileSet = new HashSet<ElementDescriptor>();
        String[] locations = LoadFunc.getPathStrings(path);
        for (String loc : locations) {
            DataStorage storage;

            storage = new HDataStorage(new Path(loc).toUri(), ConfigurationUtil.toProperties(conf));

            String fullPath = FileLocalizer.fullPath(loc, storage);

            if(storage.isContainer(fullPath)) {
                ElementDescriptor metaFilePath = storage.asElement(fullPath, metaname);
                if (exists(metaFilePath)) {
                    metaFileSet.add(metaFilePath);
                }
            } else {
                ElementDescriptor[] descriptors = storage.asCollection(loc);
                for(ElementDescriptor descriptor : descriptors) {
                    ContainerDescriptor container = null;

                    if (descriptor instanceof HFile) {
                        Path descriptorPath = ((HPath) descriptor).getPath();
                        Path parent = descriptorPath.getParent();
                        container = new HDirectory((HDataStorage)storage,parent);
                    } else { // descriptor instanceof HDirectory
                        container = (HDirectory)descriptor;
                    }

                    // if no custom schema, try the parent directory
                    ElementDescriptor metaFilePath = storage.asElement(container, metaname);
                    if (exists(metaFilePath)) {
                        metaFileSet.add(metaFilePath);
                    }
                }
            }
View Full Code Here
271
272
273
274
275
276
277
278
279
280
281
282
283
    // Implementation of StoreMetaData interface

    @Override
    public void storeStatistics(ResourceStatistics stats, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(new Path(location).toUri(),
                ConfigurationUtil.toProperties(conf));
        ElementDescriptor statFilePath = storage.asElement(location, statFileName);
        if(!statFilePath.exists() && stats != null) {
            try {
                new ObjectMapper().writeValue(statFilePath.create(), stats);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
View Full Code Here
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
    }

    @Override
    public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(new Path(location).toUri(),
                ConfigurationUtil.toProperties(conf));
        ElementDescriptor schemaFilePath = storage.asElement(location, schemaFileName);
        if(!schemaFilePath.exists() && schema != null) {
            try {
                new ObjectMapper().writeValue(schemaFilePath.create(), schema);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            } catch (JsonMappingException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            }
        }
        if (printHeaders) {
            ElementDescriptor headerFilePath = storage.asElement(location, headerFileName);
            if (!headerFilePath.exists()) {
                OutputStream os = headerFilePath.create();
                try {
                    String[] names = schema.fieldNames();
                    String fn;
View Full Code Here
TOP

Related Classes of org.apache.pig.backend.datastorage.DataStorage

  • com.packetloop.packetpig.storage.JsonMetadata
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.PigInputFormat
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.SliceWrapper
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SliceWrapper
  • org.apache.pig.builtin.JsonMetadata
  • org.apache.pig.impl.io.FileLocalizer
  • org.apache.pig.impl.PigContext
  • org.apache.pig.parser.QueryParserUtils
  • org.apache.pig.piggybank.storage.JsonMetadata
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.