Examples of PigStorage

org.apache.pig.builtin.PigStorage
A load function that parses a line of input into fields using a delimiter to set the fields. The delimiter is given as a regular expression. See {@link java.lang.String#split(String)} and {@link java.util.regex.Pattern}for more information.

Examples of org.apache.pig.builtin.PigStorage

            for( int i = 0; i < LOOP_COUNT; i++ ) {
                ps.println( text );
            }
            ps.close();
            
            PigStorage storage = new PigStorage(":");
            LocalSeekableInputStream is = 
                new LocalSeekableInputStream( testFile );
            BufferedPositionedInputStream bpis = 
                new BufferedPositionedInputStream( is );
            storage.bindTo(testFile.getName(), bpis, 0, testFile.length());
            
            // Skip till middle of a line
            storage.skip( (text.length() + 1 ) 
                    * (LOOP_COUNT/2) + text.length()/2 );
            
            // Test if we have skipped till end of the line
            assertEquals( "Invalid Bytes Skiped", storage.getPosition(),
                (text.length()+1) * ((LOOP_COUNT/2) +1 ) );           
            
            testFile.deleteOnExit();
            
        } catch (IOException e) {

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

            for( int i = 0; i < LOOP_COUNT; i++ ) {
                ps.println( text );
            }
            ps.close();
            
            PigStorage storage = new PigStorage(":");
            LocalSeekableInputStream is = 
                new LocalSeekableInputStream( testFile );
            CBZip2InputStream bzis = new CBZip2InputStream( is );
            BufferedPositionedInputStream bpis = 
                new BufferedPositionedInputStream( bzis );
            storage.bindTo(testFile.getName(), bpis, 0, testFile.length());
            
            // Skip till middle of a line
            storage.skip( (text.length() + 1 ) 
                    * (LOOP_COUNT/2) + text.length()/2 );
            
            // Test if we have skipped till end of the line
            /*
             * This is what is expected, but this fails.
             * Due to bzip2, data is compressed and hence the bytes
             * reported are different than the one received.
             * The test below is changed to provide a hardcoded value
            assertEquals( "Invalid Bytes Skiiped", storage.getPosition(),
                (text.length()+1) * ((LOOP_COUNT/2) +1 ) );
            */
            assertEquals( "Invalid Bytes Skiped", storage.getPosition(),
                    5 );
            
            testFile.deleteOnExit();
            
        } catch (IOException e) {

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

                String output = counter.concat( text.substring(counter.length() ) );
                ps.println( output );
            }
            ps.close();
            
            PigStorage storage = new PigStorage(":");
            LocalSeekableInputStream is = 
                new LocalSeekableInputStream( testFile );
            BufferedPositionedInputStream bpis = 
                new BufferedPositionedInputStream( is );
            storage.bindTo(testFile.getName(), bpis, 0, testFile.length());
            
            // Skip till middle of a line
            storage.skip( ( (text.length() + 1 ) 
                    * (LOOP_COUNT/2) ) + text.length()/2 );
            
            // Test if we have skipped till end of the line
            Tuple t = storage.getNext();
            String counter = new Integer( LOOP_COUNT/2 + 1 ).toString();
            String output = counter.concat( text.substring(counter.length() ) );
            assertEquals( "Invalid Data", t.get(0).toString(), output );
            
            testFile.deleteOnExit();

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

                String output = counter.concat( text.substring(counter.length() ) );
                ps.println( output );
            }
            ps.close();
            
            PigStorage storage = new PigStorage(":");
            LocalSeekableInputStream is = 
                new LocalSeekableInputStream( testFile );
            CBZip2InputStream bzis = new CBZip2InputStream( is );
            BufferedPositionedInputStream bpis = 
                new BufferedPositionedInputStream( bzis );
            storage.bindTo(testFile.getName(), bpis, 0, testFile.length());
            
            // Skip till middle of a line
            storage.skip( (text.length() + 1 ) 
                    * (LOOP_COUNT/2) + text.length()/2 );
            
            // Test if we have skipped till end of the line
            Tuple t = storage.getNext();
            String counter = new Integer( LOOP_COUNT/2 + 1 ).toString();
            String output = counter.concat( text.substring(counter.length() ) );
            assertEquals( "Invalid Data", t.get(0).toString(), output );
            
            testFile.deleteOnExit();

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

    }


    public LoadFunc getLoadFunction() {
        LoadFunc loader = null;
        if (this.parser == null) {
            loader = new PigStorage();
        } else {
            try {
                loader = (LoadFunc) PigContext
                        .instantiateFuncFromSpec(new FuncSpec(this.parser));
            } catch (Exception exp) {

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

    public static StoreFunc getStoreFunc(JobConf conf) throws ExecException {
        StoreFunc store;
        try {
            String storeFunc = conf.get("pig.storeFunc", "");
            if (storeFunc.length() == 0) {
                store = new PigStorage();
            } else {
                storeFunc = (String) ObjectSerializer.deserialize(storeFunc);
                store = (StoreFunc) PigContext
                        .instantiateFuncFromSpec(storeFunc);
            }

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

    }


    @SuppressWarnings("unchecked")
  public void init(DataStorage base) throws IOException {
        if (parser == null) {
          loader = new PigStorage();
        } else {
            try {
                loader = (LoadFunc) PigContext.instantiateFuncFromSpec(parser);
            } catch (Exception exp) {
                int errCode = 2081;

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

    public void testStoreComplexDataWithNull() throws Exception {
        Tuple inputTuple = GenRandomData.genRandSmallBagTextTupleWithNulls(new Random(), 10, 100);
        inpDB = DefaultBagFactory.getInstance().newDefaultBag();
        inpDB.add(inputTuple);
        storeAndCopyLocally(inpDB);
        PigStorage ps = new PigStorage("\t");
        BufferedReader br = new BufferedReader(new FileReader(outputFileName));
        for(String line=br.readLine();line!=null;line=br.readLine()){
            System.err.println("Complex data: ");
            System.err.println(line);
            String[] flds = line.split("\t",-1);
            Tuple t = new DefaultTuple();


            ResourceFieldSchema stringfs = new ResourceFieldSchema();
            stringfs.setType(DataType.CHARARRAY);
            ResourceFieldSchema intfs = new ResourceFieldSchema();
            intfs.setType(DataType.INTEGER);
            ResourceFieldSchema bytefs = new ResourceFieldSchema();
            bytefs.setType(DataType.BYTEARRAY);


            ResourceSchema tupleSchema = new ResourceSchema();
            tupleSchema.setFields(new ResourceFieldSchema[]{stringfs, intfs});
            ResourceFieldSchema tuplefs = new ResourceFieldSchema();
            tuplefs.setSchema(tupleSchema);
            tuplefs.setType(DataType.TUPLE);


            ResourceSchema bagSchema = new ResourceSchema();
            bagSchema.setFields(new ResourceFieldSchema[]{tuplefs});
            ResourceFieldSchema bagfs = new ResourceFieldSchema();
            bagfs.setSchema(bagSchema);
            bagfs.setType(DataType.BAG);


            ResourceSchema mapSchema = new ResourceSchema();
            mapSchema.setFields(new ResourceFieldSchema[]{bytefs});
            ResourceFieldSchema mapfs = new ResourceFieldSchema();
            mapfs.setSchema(mapSchema);
            mapfs.setType(DataType.MAP);


            t.append(flds[0].compareTo("")!=0 ? ps.getLoadCaster().bytesToBag(flds[0].getBytes(), bagfs) : null);
            t.append(flds[1].compareTo("")!=0 ? new DataByteArray(flds[1].getBytes()) : null);
            t.append(flds[2].compareTo("")!=0 ? ps.getLoadCaster().bytesToCharArray(flds[2].getBytes()) : null);
            t.append(flds[3].compareTo("")!=0 ? ps.getLoadCaster().bytesToDouble(flds[3].getBytes()) : null);
            t.append(flds[4].compareTo("")!=0 ? ps.getLoadCaster().bytesToFloat(flds[4].getBytes()) : null);
            t.append(flds[5].compareTo("")!=0 ? ps.getLoadCaster().bytesToInteger(flds[5].getBytes()) : null);
            t.append(flds[6].compareTo("")!=0 ? ps.getLoadCaster().bytesToLong(flds[6].getBytes()) : null);
            t.append(flds[7].compareTo("")!=0 ? ps.getLoadCaster().bytesToMap(flds[7].getBytes(), mapfs) : null);
            t.append(flds[8].compareTo("")!=0 ? ps.getLoadCaster().bytesToTuple(flds[8].getBytes(), tuplefs) : null);
            t.append(flds[9].compareTo("")!=0 ? ps.getLoadCaster().bytesToBoolean(flds[9].getBytes()) : null);
            t.append(flds[10].compareTo("")!=0 ? ps.getLoadCaster().bytesToDateTime(flds[10].getBytes()) : null);
            t.append(flds[11].compareTo("")!=0 ? ps.getLoadCaster().bytesToCharArray(flds[10].getBytes()) : null);
            assertTrue(TestHelper.tupleEquals(inputTuple, t));
        }
    }

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage


    @Test
    public void testStoreComplexData() throws Exception {
        inpDB = GenRandomData.genRandFullTupTextDataBag(new Random(), 10, 100);
        storeAndCopyLocally(inpDB);
        PigStorage ps = new PigStorage("\t");
        int size = 0;
        BufferedReader br = new BufferedReader(new FileReader(outputFileName));
        for(String line=br.readLine();line!=null;line=br.readLine()){
            String[] flds = line.split("\t",-1);
            Tuple t = new DefaultTuple();


            ResourceFieldSchema mapfs = GenRandomData.getRandMapFieldSchema();
            ResourceFieldSchema bagfs = GenRandomData.getSmallTupDataBagFieldSchema();
            ResourceFieldSchema tuplefs = GenRandomData.getSmallTupleFieldSchema();


            t.append(flds[0].compareTo("")!=0 ? ps.getLoadCaster().bytesToBag(flds[0].getBytes(), bagfs) : null);
            t.append(flds[1].compareTo("")!=0 ? new DataByteArray(flds[1].getBytes()) : null);
            t.append(flds[2].compareTo("")!=0 ? ps.getLoadCaster().bytesToCharArray(flds[2].getBytes()) : null);
            t.append(flds[3].compareTo("")!=0 ? ps.getLoadCaster().bytesToDouble(flds[3].getBytes()) : null);
            t.append(flds[4].compareTo("")!=0 ? ps.getLoadCaster().bytesToFloat(flds[4].getBytes()) : null);
            t.append(flds[5].compareTo("")!=0 ? ps.getLoadCaster().bytesToInteger(flds[5].getBytes()) : null);
            t.append(flds[6].compareTo("")!=0 ? ps.getLoadCaster().bytesToLong(flds[6].getBytes()) : null);
            t.append(flds[7].compareTo("")!=0 ? ps.getLoadCaster().bytesToMap(flds[7].getBytes(), mapfs) : null);
            t.append(flds[8].compareTo("")!=0 ? ps.getLoadCaster().bytesToTuple(flds[8].getBytes(), tuplefs) : null);
            t.append(flds[9].compareTo("")!=0 ? ps.getLoadCaster().bytesToBoolean(flds[9].getBytes()) : null);
            t.append(flds[10].compareTo("")!=0 ? ps.getLoadCaster().bytesToDateTime(flds[10].getBytes()) : null);
            assertEquals(true, TestHelper.bagContains(inpDB, t));
            ++size;
        }
        assertEquals(true, size==inpDB.size());
    }

View Full Code Here

Examples of org.apache.pig.builtin.PigStorage

            new Job(CONF), createMockPOLoadMapReduceOper(2L * 1000 * 1000 * 1001)));
    }


    private static MapReduceOper createMockPOLoadMapReduceOper(long size) throws Exception {
        MapReduceOper mro = new MapReduceOper(new OperatorKey());
        mro.mapPlan.add(createPOLoadWithSize(size, new PigStorage()));
        return mro;
    }

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.