Package org.apache.pig

Examples of org.apache.pig.ResourceSchema

58
59
60
61
62
63
64
65
66
67
68
69
    }
   
    public ResourceFieldSchema getBagFieldSchema() throws IOException{
        ResourceFieldSchema tuplefs = getTupleFieldSchema();
       
        ResourceSchema outBagSchema = new ResourceSchema();
        outBagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
        ResourceFieldSchema outBagfs = new ResourceFieldSchema();
        outBagfs.setSchema(outBagSchema);
        outBagfs.setType(DataType.BAG);
       
        return outBagfs;
View Full Code Here
135
136
137
138
139
140
141
142
143
144
145
    }

    private LogicalSchema getSchemaFromMetaData() throws FrontendException {
        if (getLoadFunc()!=null && getLoadFunc() instanceof LoadMetadata) {
            try {
                ResourceSchema resourceSchema = ((LoadMetadata)loadFunc).getSchema(getFileSpec().getFileName(), new Job(conf));
                Schema oldSchema = Schema.getPigSchema(resourceSchema);
                return Util.translateSchema(oldSchema);
            } catch (IOException e) {
                throw new FrontendException("Cannot get schema from loadFunc " + loadFunc.getClass().getName(), 2245, e);
            }
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
            String validationErrStr ="Output Location Validation Failed for: '" + outLoc ;
            Job dummyJob;
           
            try {
                if(store.getSchema() != null){
                    sf.checkSchema(new ResourceSchema(Util.translateSchema(store.getSchema()), store.getSortInfo()));               
                }
                dummyJob = new Job(ConfigurationUtil.toConfiguration(pigCtx.getProperties()));
                sf.setStoreLocation(outLoc, dummyJob);
            } catch (IOException ioe) {
                if(ioe instanceof PigException){
View Full Code Here
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
   
    @Test
    public void testBytesToComplexTypeMisc() throws IOException, ParseException {
        String s = "(a,b";
        Schema schema = Utils.getSchemaFromString("t:tuple(a:chararray, b:chararray)");
        ResourceFieldSchema rfs = new ResourceSchema(schema).getFields()[0];
        Tuple t = ps.getLoadCaster().bytesToTuple(s.getBytes(), rfs);
        assertTrue(t==null);
       
        s = "{(a,b}";
        schema = Utils.getSchemaFromString("b:bag{t:tuple(a:chararray, b:chararray)}");
        rfs = new ResourceSchema(schema).getFields()[0];
        DataBag b = ps.getLoadCaster().bytesToBag(s.getBytes(), rfs);
        assertTrue(b==null);
       
        s = "{(a,b)";
        schema = Utils.getSchemaFromString("b:bag{t:tuple(a:chararray, b:chararray)}");
        rfs = new ResourceSchema(schema).getFields()[0];
        b = ps.getLoadCaster().bytesToBag(s.getBytes(), rfs);
        assertTrue(b==null);
       
        s = "[ab]";
        Map<String, Object> m = ps.getLoadCaster().bytesToMap(s.getBytes());
        assertTrue(m==null);
       
        s = "[a#b";
        m = ps.getLoadCaster().bytesToMap(s.getBytes());
        assertTrue(m==null);
       
        s = "[a#]";
        m = ps.getLoadCaster().bytesToMap(s.getBytes());
        Map.Entry<String, Object> entry = m.entrySet().iterator().next();
        assertTrue(entry.getKey().equals("a"));
        assertTrue(entry.getValue()==null);
       
        s = "[#]";
        m = ps.getLoadCaster().bytesToMap(s.getBytes());
        assertTrue(m==null);
       
        s = "(a,b)";
        schema = Utils.getSchemaFromString("t:tuple()");
        rfs = new ResourceSchema(schema).getFields()[0];
        t = ps.getLoadCaster().bytesToTuple(s.getBytes(), rfs);
        assertTrue(t.size()==1);
        assertTrue(t.get(0) instanceof DataByteArray);
        assertTrue(t.get(0).toString().equals("a,b"));
       
        s = "[a#(1,2,3)]";
        m = ps.getLoadCaster().bytesToMap(s.getBytes());
        entry = m.entrySet().iterator().next();
        assertTrue(entry.getKey().equals("a"));
        assertTrue(entry.getValue() instanceof DataByteArray);
        assertTrue(entry.getValue().toString().equals("(1,2,3)"));
       
        s = "(a,b,(123,456,{(1,2,3)}))";
        schema = Utils.getSchemaFromString("t:tuple()");
        rfs = new ResourceSchema(schema).getFields()[0];
        t = ps.getLoadCaster().bytesToTuple(s.getBytes(), rfs);
        assertTrue(t.size()==1);
        assertTrue(t.get(0) instanceof DataByteArray);
        assertTrue(t.get(0).toString().equals("a,b,(123,456,{(1,2,3)})"));
       
        s = "(a,b,(123,456,{(1,2,3}))";
        schema = Utils.getSchemaFromString("t:tuple()");
        rfs = new ResourceSchema(schema).getFields()[0];
        t = ps.getLoadCaster().bytesToTuple(s.getBytes(), rfs);
        assertTrue(t==null);
    }
View Full Code Here
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
      ResourceFieldSchema rfs;
      Tuple tuple, convertedTuple;
        tuple = TupleFactory.getInstance().newTuple(1);
     
        schema = Utils.getSchemaFromString("t:tuple(a:int)");
        rfs = new ResourceSchema(schema).getFields()[0];

        // long bigger than Integer.MAX_VALUE
      tuple.set(0, Integer.valueOf(Integer.MAX_VALUE).longValue() + 1);
      convertedTuple = ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
        assertNull("Invalid cast to int: " + tuple.get(0) + " -> " + convertedTuple.get(0), convertedTuple.get(0));
       
        // long smaller than Integer.MIN_VALUE
      tuple.set(0, Integer.valueOf(Integer.MIN_VALUE).longValue() - 1);
      convertedTuple = ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
        assertNull("Invalid cast to int: " + tuple.get(0) + " -> " + convertedTuple.get(0), convertedTuple.get(0));
       
        // double bigger than Integer.MAX_VALUE
        tuple.set(0, Integer.valueOf(Integer.MAX_VALUE).doubleValue() + 1);
        convertedTuple = ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
        assertNull("Invalid cast to int: " + tuple.get(0) + " -> " + convertedTuple.get(0), convertedTuple.get(0));
       
        // double smaller than Integer.MIN_VALUE
        tuple.set(0, Integer.valueOf(Integer.MIN_VALUE).doubleValue() - 1);
        convertedTuple = ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
        assertNull("Invalid cast to int: " + tuple.get(0) + " -> " + convertedTuple.get(0), convertedTuple.get(0));
        schema = Utils.getSchemaFromString("t:tuple(a:long)");
        rfs = new ResourceSchema(schema).getFields()[0];
       
        // double bigger than Long.MAX_VALUE
        tuple.set(0, Long.valueOf(Long.MAX_VALUE).doubleValue() + 10000);
        convertedTuple = ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
        assertNull("Invalid cast to long: " + tuple.get(0) + " -> " + convertedTuple.get(0), convertedTuple.get(0));
View Full Code Here
576
577
578
579
580
581
582
        }

        @Override
        public ResourceSchema getSchema(String location, Job job)
                throws IOException {
            return new ResourceSchema(schema);
        }
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
        String validationErrStr ="Output Location Validation Failed for: " + outLoc ;
        Job dummyJob;
       
        try {
            if(store.getSchema() != null){
                sf.checkSchema(new ResourceSchema(store.getSchema(), store.getSortInfo()));               
            }
            dummyJob = new Job(ConfigurationUtil.toConfiguration(pigCtx.getProperties()));
            sf.setStoreLocation(outLoc, dummyJob);
        } catch (IOException ioe) {
             if(ioe instanceof PigException){
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
        } else {
            log.warn("Could not find schema file for "+location);
            return null;
        }
        log.info("Found schema file: "+schemaFile.toString());
        ResourceSchema resourceSchema = null;
        try {
            resourceSchema = new ObjectMapper().readValue(schemaFile.open(), ResourceSchema.class);
        } catch (JsonParseException e) {
            log.warn("Unable to load Resource Schema for "+location);
            e.printStackTrace();
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
   
    private Schema determineSchema() throws IOException {
        if(!mDeterminedSchemaCached) {
            if(LoadMetadata.class.isAssignableFrom(mLoadFunc.getClass())) {
                LoadMetadata loadMetadata = (LoadMetadata)mLoadFunc;
                ResourceSchema rSchema = loadMetadata.getSchema(
                        mInputFileSpec.getFileName(), new Job(conf));
                mDeterminedSchema = Schema.getPigSchema(rSchema);
            }
            // set the flag so that future calls just use mDeterminedSchema
            mDeterminedSchemaCached = true;
View Full Code Here
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
                              "8", "9"});
       
        pig.registerQuery("A = LOAD 'originput2' using org.apache.pig.piggybank.storage.PigStorageSchema() as (f:int);");
        pig.registerQuery("B = group A by f;");
        Schema origSchema = pig.dumpSchema("B");
        ResourceSchema rs1 = new ResourceSchema(origSchema);
        pig.registerQuery("STORE B into 'bout' using org.apache.pig.piggybank.storage.PigStorageSchema();");
       
        pig.registerQuery("C = LOAD 'bout' using org.apache.pig.piggybank.storage.PigStorageSchema();");
        Schema genSchema = pig.dumpSchema("C");
        ResourceSchema rs2 = new ResourceSchema(genSchema);
        assertTrue("generated schema equals original" , ResourceSchema.equals(rs1, rs2));
       
        pig.registerQuery("C1 = LOAD 'bout' as (a0:int, A: {t: (f:int) } );");
        pig.registerQuery("D = foreach C1 generate a0, SUM(A);");
View Full Code Here
TOP

Related Classes of org.apache.pig.ResourceSchema

  • com.acme.io.JsonLoader
  • com.acme.io.JsonStorage
  • com.linkedin.pig.FastAvroStorageCommon
  • com.mongodb.hadoop.pig.BSONLoader
  • com.mongodb.hadoop.pig.BSONStorage
  • com.mongodb.hadoop.pig.BSONStorageTest
  • com.mongodb.hadoop.pig.JSONPigReplace
  • com.mongodb.hadoop.pig.MongoInsertStorage
  • com.mongodb.hadoop.pig.MongoLoader
  • com.mongodb.hadoop.pig.MongoStorage
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.