Examples of org.apache.pig.backend.datastorage.DataStorage

Package org.apache.pig.backend.datastorage

Examples of org.apache.pig.backend.datastorage.DataStorage

org.apache.pig.backend.datastorage.DataStorage
DataStorage provides an abstraction of a generic container. Special instances of it can be a file system.

        return openDFSFile(fileName, ConfigurationUtil.toProperties(conf));


    }


    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }

View Full Code Here


        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }


    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);


        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);


        long size = 0;

View Full Code Here

     * @return an array of Element descriptors for files present (found by traversing all levels of dirs)
     *  in the input element descriptor
     * @throws DataStorageException
     */
    private static ElementDescriptor[] getFileElementDescriptors(ElementDescriptor elem) throws DataStorageException {
        DataStorage store = elem.getDataStorage();
        ElementDescriptor[] elems = store.asCollection(elem.toString());
        // elems could have directories in it, if so
        // get the files out so that it contains only files
        List<ElementDescriptor> paths = new ArrayList<ElementDescriptor>();
        List<ElementDescriptor> filePaths = new ArrayList<ElementDescriptor>();
        for (int m = 0; m < elems.length; m++) {
            paths.add(elems[m]);
        }
        for (int j = 0; j < paths.size(); j++) {
            ElementDescriptor fullPath = store.asElement(store
                    .getActiveContainer(), paths.get(j));
            // Skip hadoop's private/meta files ...
            if (fullPath.systemElement()) {
                continue;
            }

View Full Code Here

        src.rename(dst);


    }


    public void copy(String src, String dst, boolean localDst) throws IOException {
        DataStorage dstStorage = dfs;


        if (localDst) {
            dstStorage = lfs;
        }


        ElementDescriptor srcElement = null;
        ElementDescriptor dstElement = null;


        try {
            srcElement = dfs.asElement(src);
            dstElement = dstStorage.asElement(dst);
        }
        catch (DataStorageException e) {
            byte errSrc = getErrorSource();
            int errCode = 0;
            switch(errSrc) {

View Full Code Here

        lp.add( store );
        lp.connect( input, store );
    }


    static Path getCurrentDir(PigContext pigContext) throws IOException {
        DataStorage dfs = pigContext.getDfs();
        ContainerDescriptor desc = dfs.getActiveContainer();
        ElementDescriptor el = dfs.asElement(desc);
        return new Path(el.toString());
    }

View Full Code Here

    public long capacity() throws IOException {
        if (pigContext.getExecType().isLocal()) {
            throw new IOException("capacity only supported for non-local execution");
        }
        else {
            DataStorage dds = pigContext.getDfs();


            Map<String, Object> stats = dds.getStatistics();


            String rawCapacityStr = (String) stats.get(DataStorage.RAW_CAPACITY_KEY);
            String rawUsedStr = (String) stats.get(DataStorage.RAW_USED_KEY);


            if ((rawCapacityStr == null) || (rawUsedStr == null)) {

View Full Code Here

     * @param filename
     * @return length of the file in bytes
     * @throws IOException
     */
    public long fileSize(String filename) throws IOException {
        DataStorage dfs = pigContext.getDfs();
        ElementDescriptor elem = dfs.asElement(filename);
        Map<String, Object> stats = elem.getStatistics();
        long length = (Long) stats.get(ElementDescriptor.LENGTH_KEY);
        int replication = (Short) stats
                .get(ElementDescriptor.BLOCK_REPLICATION_KEY);

View Full Code Here

        boolean[] multiquery = {true, false};
        
        for (boolean b : multiquery) {
            pc.getProperties().setProperty("opt.multiquery", "" + b);
                    
            DataStorage dfs = pc.getDfs();
            dfs.setActiveContainer(dfs.asContainer("/tmp"));
            Map<String, String> fileNameMap = new HashMap<String, String>();
            
            QueryParserDriver builder = new QueryParserDriver(pc, "Test-Load", fileNameMap);
            
            String query = "a = load '"+orig+"';";

View Full Code Here

    }


    private void checkStorePath(String orig, String expected, boolean isTmp) throws Exception {
        pc.getProperties().setProperty("opt.multiquery",""+true);


        DataStorage dfs = pc.getDfs();
        dfs.setActiveContainer(dfs.asContainer("/tmp"));
        Map<String, String> fileNameMap = new HashMap<String, String>();


        QueryParserDriver builder = new QueryParserDriver(pc, "Test-Store", fileNameMap);


        String query = "a = load 'foo';" + "store a into '"+orig+"';";

View Full Code Here

        // if the execution is against Mapred DFS, set
        // working dir to /user/<userid>
        if(pigContext.getExecType() == ExecType.MAPREDUCE)
            fs.setWorkingDirectory(new Path("/user", job.getUser()));
        
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
        ValidatingInputFileSpec spec;
        if (inputs.get(i).first instanceof ValidatingInputFileSpec) {
            spec = (ValidatingInputFileSpec) inputs.get(i).first;
        } else {
            spec = new ValidatingInputFileSpec(inputs.get(i).first, store);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of org.apache.pig.backend.datastorage.DataStorage

com.packetloop.packetpig.storage.JsonMetadata

org.apache.pig.backend.hadoop.executionengine.mapreduceExec.PigInputFormat

org.apache.pig.backend.hadoop.executionengine.mapreduceExec.SliceWrapper

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat

org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SliceWrapper

org.apache.pig.builtin.JsonMetadata

org.apache.pig.impl.io.FileLocalizer

org.apache.pig.impl.PigContext

org.apache.pig.parser.QueryParserUtils

org.apache.pig.piggybank.storage.JsonMetadata

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.