Package org.apache.pig.backend.datastorage

Examples of org.apache.pig.backend.datastorage.DataStorage

121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
    }

    @SuppressWarnings("unchecked")
    public RecordReader<Text, Tuple> makeReader(JobConf job) throws IOException {
        lastConf = job;       
        DataStorage store = new HDataStorage(ConfigurationUtil.toProperties(job));
        // if the execution is against Mapred DFS, set
        // working dir to /user/<userid>
        if(execType == ExecType.MAPREDUCE)
            store.setActiveContainer(store.asContainer("/user/" + job.getUser()));
        PigContext.setPackageImportList((ArrayList<String>)ObjectSerializer.deserialize(job.get("udf.import.list")));
        List<String> inpSignatureLists = (ArrayList<String>)ObjectSerializer.deserialize(job.get("pig.inpSignatures"));
        if (inpSignatureLists.get(index)!=null) {
            // Pass loader signature to slice
            store.getConfiguration().setProperty("pig.loader.signature", inpSignatureLists.get(index));
            job.set("pig.loader.signature", inpSignatureLists.get(index));
        }
       
        // Get the UDF specific context
        UDFContext udfc = UDFContext.getUDFContext();
View Full Code Here
609
610
611
612
613
614
615
616
617
618
619
620
621
    public long capacity() throws IOException {
        if (pigContext.getExecType() == ExecType.LOCAL) {
            throw new IOException("capacity only supported for non-local execution");
        }
        else {
            DataStorage dds = pigContext.getDfs();
           
            Map<String, Object> stats = dds.getStatistics();

            String rawCapacityStr = (String) stats.get(DataStorage.RAW_CAPACITY_KEY);
            String rawUsedStr = (String) stats.get(DataStorage.RAW_USED_KEY);
           
            if ((rawCapacityStr == null) || (rawUsedStr == null)) {
View Full Code Here
634
635
636
637
638
639
640
641
642
643
644
645
     * @param filename
     * @return length of the file in bytes
     * @throws IOException
     */
    public long fileSize(String filename) throws IOException {
        DataStorage dfs = pigContext.getDfs();
        ElementDescriptor elem = dfs.asElement(filename);
        Map<String, Object> stats = elem.getStatistics();
        long length = (Long) stats.get(ElementDescriptor.LENGTH_KEY);
        int replication = (Short) stats
                .get(ElementDescriptor.BLOCK_REPLICATION_KEY);

View Full Code Here
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
    public long capacity() throws IOException {
        if (pigContext.getExecType() == ExecType.LOCAL) {
            throw new IOException("capacity only supported for non-local execution");
        }
        else {
            DataStorage dds = pigContext.getDfs();

            Map<String, Object> stats = dds.getStatistics();

            String rawCapacityStr = (String) stats.get(DataStorage.RAW_CAPACITY_KEY);
            String rawUsedStr = (String) stats.get(DataStorage.RAW_USED_KEY);

            if ((rawCapacityStr == null) || (rawUsedStr == null)) {
View Full Code Here
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
     * @param filename
     * @return length of the file in bytes
     * @throws IOException
     */
    public long fileSize(String filename) throws IOException {
        DataStorage dfs = pigContext.getDfs();
        ElementDescriptor elem = dfs.asElement(filename);
        Map<String, Object> stats = elem.getStatistics();
        long length = (Long) stats.get(ElementDescriptor.LENGTH_KEY);
        int replication = (Short) stats
                .get(ElementDescriptor.BLOCK_REPLICATION_KEY);

View Full Code Here
165
166
167
168
169
170
171
172
173
        return openDFSFile(fileName, ConfigurationUtil.toProperties(conf));

    }

    public static InputStream openDFSFile(String fileName, Properties properties) throws IOException{
        DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
        return openDFSFile(elem);
    }
View Full Code Here
182
183
184
185
186
187
188
189
190
191
192
193
        return getSize(fileName, ConfigurationUtil.toProperties(conf));
    }
   
    public static long getSize(String fileName, Properties properties) throws IOException {
      DataStorage dds = new HDataStorage(properties);
        ElementDescriptor elem = dds.asElement(fileName);
      
        // recursively get all the files under this path
        ElementDescriptor[] allElems = getFileElementDescriptors(elem);
       
        long size = 0;
View Full Code Here
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
     * @return an array of Element descriptors for files present (found by traversing all levels of dirs)
     *  in the input element descriptor
     * @throws DataStorageException
     */
    private static ElementDescriptor[] getFileElementDescriptors(ElementDescriptor elem) throws DataStorageException {
        DataStorage store = elem.getDataStorage();
        ElementDescriptor[] elems = store.asCollection(elem.toString());
        // elems could have directories in it, if so
        // get the files out so that it contains only files
        List<ElementDescriptor> paths = new ArrayList<ElementDescriptor>();
        List<ElementDescriptor> filePaths = new ArrayList<ElementDescriptor>();
        for (int m = 0; m < elems.length; m++) {
            paths.add(elems[m]);
        }
        for (int j = 0; j < paths.size(); j++) {
            ElementDescriptor fullPath = store.asElement(store
                    .getActiveContainer(), paths.get(j));
            // Skip hadoop's private/meta files ...
            if (fullPath.systemElement()) {
                continue;
            }
View Full Code Here
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
    public long capacity() throws IOException {
        if (pigContext.getExecType().isLocal()) {
            throw new IOException("capacity only supported for non-local execution");
        }
        else {
            DataStorage dds = pigContext.getDfs();

            Map<String, Object> stats = dds.getStatistics();

            String rawCapacityStr = (String) stats.get(DataStorage.RAW_CAPACITY_KEY);
            String rawUsedStr = (String) stats.get(DataStorage.RAW_USED_KEY);

            if ((rawCapacityStr == null) || (rawUsedStr == null)) {
View Full Code Here
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
     * @param filename
     * @return length of the file in bytes
     * @throws IOException
     */
    public long fileSize(String filename) throws IOException {
        DataStorage dfs = pigContext.getDfs();
        ElementDescriptor elem = dfs.asElement(filename);
        Map<String, Object> stats = elem.getStatistics();
        long length = (Long) stats.get(ElementDescriptor.LENGTH_KEY);
        int replication = (Short) stats
                .get(ElementDescriptor.BLOCK_REPLICATION_KEY);

View Full Code Here
TOP

Related Classes of org.apache.pig.backend.datastorage.DataStorage

  • com.packetloop.packetpig.storage.JsonMetadata
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.PigInputFormat
  • org.apache.pig.backend.hadoop.executionengine.mapreduceExec.SliceWrapper
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigInputFormat
  • org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.SliceWrapper
  • org.apache.pig.builtin.JsonMetadata
  • org.apache.pig.impl.io.FileLocalizer
  • org.apache.pig.impl.PigContext
  • org.apache.pig.parser.QueryParserUtils
  • org.apache.pig.piggybank.storage.JsonMetadata
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.