Package org.apache.pig.backend.datastorage

Examples of org.apache.pig.backend.datastorage.DataStorage


     * @param conf      configuration object
     * @return Set of element descriptors for all metadata files associated with the files on the path.
     */
    protected Set<ElementDescriptor> findMetaFile(String path, String prefix, Configuration conf)
        throws IOException {
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        String fullPath = FileLocalizer.fullPath(path, storage);
        Set<ElementDescriptor> metaFileSet = new HashSet<ElementDescriptor>();
        if(storage.isContainer(fullPath)) {
            ElementDescriptor metaFilePath = storage.asElement(fullPath, prefix);
            if (metaFilePath.exists()) {
                metaFileSet.add(metaFilePath);
            }
        } else {
            ElementDescriptor[] descriptors = storage.asCollection(path);
            for(ElementDescriptor descriptor : descriptors) {
                String fileName = null, parentName = null;
                ContainerDescriptor parentContainer = null;
                if (descriptor instanceof HFile) {
                    Path descriptorPath = ((HFile) descriptor).getPath();
                    fileName = descriptorPath.getName();
                    Path parent = descriptorPath.getParent();
                    parentName = parent.toString();
                    parentContainer = new HDirectory((HDataStorage)storage,parent);
                }
                ElementDescriptor metaFilePath = storage.asElement(parentName, prefix+"."+fileName);

                // if the file has a custom schema, use it
                if (metaFilePath.exists()) {
                    metaFileSet.add(metaFilePath);
                    continue;
                }

                // if no custom schema, try the parent directory
                metaFilePath = storage.asElement(parentContainer, prefix);
                if (metaFilePath.exists()) {
                    metaFileSet.add(metaFilePath);
                }
            }
        }
View Full Code Here


    // Implementation of StoreMetaData interface
   
    @Override
    public void storeStatistics(ResourceStatistics stats, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        ElementDescriptor statFilePath = storage.asElement(location, statFileName);
        if(!statFilePath.exists() && stats != null) {
            try {
                new ObjectMapper().writeValue(statFilePath.create(), stats);                   
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
View Full Code Here

    }

    @Override
    public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        ElementDescriptor schemaFilePath = storage.asElement(location, schemaFileName);
        if(!schemaFilePath.exists() && schema != null) {
            try {
                new ObjectMapper().writeValue(schemaFilePath.create(), schema);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            } catch (JsonMappingException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            }
        }
        if (printHeaders) {
            ElementDescriptor headerFilePath = storage.asElement(location, headerFileName);
            if (!headerFilePath.exists()) {
                OutputStream os = headerFilePath.create();
                try {
                    String[] names = schema.fieldNames();
View Full Code Here

     * @param conf      configuration object
     * @return Set of element descriptors for all metadata files associated with the files on the path.
     */
    protected Set<ElementDescriptor> findMetaFile(String path, String prefix, Configuration conf)
        throws IOException {
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        String fullPath = FileLocalizer.fullPath(path, storage);
        Set<ElementDescriptor> metaFileSet = new HashSet<ElementDescriptor>();
        if(storage.isContainer(fullPath)) {
            ElementDescriptor metaFilePath = storage.asElement(fullPath, prefix);
            if (metaFilePath.exists()) {
                metaFileSet.add(metaFilePath);
            }
        } else {
            ElementDescriptor[] descriptors = storage.asCollection(path);
            for(ElementDescriptor descriptor : descriptors) {
                String fileName = null, parentName = null;
                ContainerDescriptor parentContainer = null;
                if (descriptor instanceof HFile) {
                    Path descriptorPath = ((HFile) descriptor).getPath();
                    fileName = descriptorPath.getName();
                    Path parent = descriptorPath.getParent();
                    parentName = parent.toString();
                    parentContainer = new HDirectory((HDataStorage)storage,parent);
                }
                ElementDescriptor metaFilePath = storage.asElement(parentName, prefix+"."+fileName);

                // if the file has a custom schema, use it
                if (metaFilePath.exists()) {
                    metaFileSet.add(metaFilePath);
                    continue;
                }

                // if no custom schema, try the parent directory
                metaFilePath = storage.asElement(parentContainer, prefix);
                if (metaFilePath.exists()) {
                    metaFileSet.add(metaFilePath);
                }
            }
        }
View Full Code Here

    // Implementation of StoreMetaData interface
   
    @Override
    public void storeStatistics(ResourceStatistics stats, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        ElementDescriptor statFilePath = storage.asElement(location, statFileName);
        if(!statFilePath.exists() && stats != null) {
            try {
                new ObjectMapper().writeValue(statFilePath.create(), stats);                   
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
View Full Code Here

    }

    @Override
    public void storeSchema(ResourceSchema schema, String location, Job job) throws IOException {
        Configuration conf = job.getConfiguration();
        DataStorage storage = new HDataStorage(ConfigurationUtil.toProperties(conf));
        ElementDescriptor schemaFilePath = storage.asElement(location, schemaFileName);
        if(!schemaFilePath.exists() && schema != null) {
            try {
                new ObjectMapper().writeValue(schemaFilePath.create(), schema);
            } catch (JsonGenerationException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            } catch (JsonMappingException e) {
                log.warn("Unable to write Resource Statistics for "+location);
                e.printStackTrace();
            }
        }
        if (printHeaders) {
            ElementDescriptor headerFilePath = storage.asElement(location, headerFileName);
            if (!headerFilePath.exists()) {
                OutputStream os = headerFilePath.create();
                try {
                    String[] names = schema.fieldNames();
View Full Code Here

        src.rename(dst);

    }

    public void copy(String src, String dst, boolean localDst) throws IOException {
        DataStorage dstStorage = dfs;
       
        if (localDst) {
            dstStorage = lfs;
        }
       
        ElementDescriptor srcElement = null;
        ElementDescriptor dstElement = null;

        try {
            srcElement = dfs.asElement(src);
            dstElement = dstStorage.asElement(dst);
        }
        catch (DataStorageException e) {
            byte errSrc = getErrorSource();           
            int errCode = 0;
            switch(errSrc) {
View Full Code Here

    public long capacity() throws IOException {
        if (pigContext.getExecType() == ExecType.LOCAL) {
            throw new IOException("capacity only supported for non-local execution");
        }
        else {
            DataStorage dds = pigContext.getDfs();
           
            Map<String, Object> stats = dds.getStatistics();

            String rawCapacityStr = (String) stats.get(DataStorage.RAW_CAPACITY_KEY);
            String rawUsedStr = (String) stats.get(DataStorage.RAW_USED_KEY);
           
            if ((rawCapacityStr == null) || (rawUsedStr == null)) {
View Full Code Here

     * @param filename
     * @return length of the file in bytes
     * @throws IOException
     */
    public long fileSize(String filename) throws IOException {
        DataStorage dfs = pigContext.getDfs();
        ElementDescriptor elem = dfs.asElement(filename);
        Map<String, Object> stats = elem.getStatistics();
        long length = (Long) stats.get(ElementDescriptor.LENGTH_KEY);
        int replication = (Short) stats
                .get(ElementDescriptor.BLOCK_REPLICATION_KEY);

View Full Code Here

        boolean[] multiquery = {true, false};
       
        for (boolean b : multiquery) {
            pc.getProperties().setProperty("opt.multiquery", "" + b);
                   
            DataStorage dfs = pc.getDfs();
            dfs.setActiveContainer(dfs.asContainer("/tmp"));
            Map<LogicalOperator, LogicalPlan> aliases = new HashMap<LogicalOperator, LogicalPlan>();
            Map<OperatorKey, LogicalOperator> logicalOpTable = new HashMap<OperatorKey, LogicalOperator>();
            Map<String, LogicalOperator> aliasOp = new HashMap<String, LogicalOperator>();
            Map<String, String> fileNameMap = new HashMap<String, String>();
           
View Full Code Here

TOP

Related Classes of org.apache.pig.backend.datastorage.DataStorage

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.