Examples of StoreFuncInterface

org.apache.pig.StoreFuncInterface
StoreFuncs take records from Pig's processing and store them into a data store. Most frequently this is an HDFS file, but it could also be an HBase instance, RDBMS, etc.

Examples of org.apache.pig.StoreFuncInterface

            super(plan, new DepthFirstWalker(plan));
        }


        @Override
        public void visit(LOStore store) throws FrontendException {
            StoreFuncInterface sf = store.getStoreFunc();
            String outLoc = store.getOutputSpec().getFileName();
            int errCode = 2116;
            String validationErrStr ="Output Location Validation Failed for: '" + outLoc ;
            Job dummyJob;
            
            try {
                if(store.getSchema() != null){
                    sf.checkSchema(new ResourceSchema(Util.translateSchema(store.getSchema()), store.getSortInfo()));                
                }
                dummyJob = new Job(ConfigurationUtil.toConfiguration(pigCtx.getProperties()));
                sf.setStoreLocation(outLoc, dummyJob);
            } catch (IOException ioe) {
                if(ioe instanceof PigException){
                    errCode = ((PigException)ioe).getErrorCode();
                } 
                String exceptionMsg = ioe.getMessage();
                validationErrStr += (exceptionMsg == null) ? "" : " More info to follow:\n" +exceptionMsg;
                throw new FrontendException(validationErrStr, errCode, pigCtx.getErrorSource(), ioe);
            }
            
            validationErrStr += " More info to follow:\n";
            try {
                sf.getOutputFormat().checkOutputSpecs(dummyJob);
            } catch (IOException ioe) {
                byte errSrc = pigCtx.getErrorSource();
                switch(errSrc) {
                case PigException.BUG:
                    errCode = 2002;

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

     * to output format implementation.
     */
    @Override
    protected void visit(LOStore store) throws PlanValidationException{


        StoreFuncInterface sf = store.getStoreFunc();
        String outLoc = store.getOutputFile().getFileName();
        int errCode = 2116;
        String validationErrStr ="Output Location Validation Failed for: " + outLoc ;
        Job dummyJob;
        
        try {
            if(store.getSchema() != null){
                sf.checkSchema(new ResourceSchema(store.getSchema(), store.getSortInfo()));                
            }
            dummyJob = new Job(ConfigurationUtil.toConfiguration(pigCtx.getProperties()));
            sf.setStoreLocation(outLoc, dummyJob);
        } catch (IOException ioe) {
             if(ioe instanceof PigException){
            errCode = ((PigException)ioe).getErrorCode();
          } 
          String exceptionMsg = ioe.getMessage();
          validationErrStr += (exceptionMsg == null) ? "" : " More info to follow:\n" +exceptionMsg;
          msgCollector.collect(validationErrStr, MessageType.Error) ;
            throw new PlanValidationException(validationErrStr, errCode, pigCtx.getErrorSource(), ioe);
        }
        
        validationErrStr += " More info to follow:\n";
        try {
            sf.getOutputFormat().checkOutputSpecs(dummyJob);
        } catch (IOException ioe) {
            byte errSrc = pigCtx.getErrorSource();
            switch(errSrc) {
            case PigException.BUG:
                errCode = 2002;

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

            LinkedList<POStore> mapStores = PlanHelper.getStores(mro.mapPlan);
            LinkedList<POStore> reduceStores = PlanHelper.getStores(mro.reducePlan);
            
            for (POStore st: mapStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }


            for (POStore st: reduceStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }


            // the OutputFormat we report to Hadoop is always PigOutputFormat
            nwJob.setOutputFormatClass(PigOutputFormat.class);

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

    
    public void store(DataBag data, FuncSpec storeFuncSpec, PigContext pigContext) throws IOException {
        Configuration conf = ConfigurationUtil.toConfiguration(pigContext.getProperties());
        // create a simulated JobContext
        JobContext jc = new JobContext(conf, new JobID());
        StoreFuncInterface sfunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(
                storeFuncSpec);
        OutputFormat<?,?> of = sfunc.getOutputFormat();
        
        POStore store = new POStore(new OperatorKey());
        store.setSFile(new FileSpec(file, storeFuncSpec));
        PigOutputFormat.setLocation(jc, store);
        OutputCommitter oc;
        // create a simulated TaskAttemptContext
        TaskAttemptContext tac = new TaskAttemptContext(conf, new TaskAttemptID());
        PigOutputFormat.setLocation(tac, store);
        RecordWriter<?,?> rw ;
        try {
            of.checkOutputSpecs(jc);
            oc = of.getOutputCommitter(tac);
            oc.setupJob(jc);
            oc.setupTask(tac);
            rw = of.getRecordWriter(tac);
            sfunc.prepareToWrite(rw);
        
            for (Iterator<Tuple> it = data.iterator(); it.hasNext();) {
                Tuple row = it.next();
                sfunc.putNext(row);
            }
            rw.close(tac);
        } catch (InterruptedException e) {
            throw new IOException(e);
        }

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

            if(mapStores.size() == 1) {
                store = mapStores.get(0);
            } else {
                store = reduceStores.get(0);
            }
            StoreFuncInterface sFunc = store.getStoreFunc();
            // set output location
            PigOutputFormat.setLocation(taskattemptcontext, store);
            // The above call should have update the conf in the JobContext
            // to have the output location - now call checkOutputSpecs()
            RecordWriter writer = sFunc.getOutputFormat().getRecordWriter(
                    taskattemptcontext);
            return new PigRecordWriter(writer, sFunc, Mode.SINGLE_STORE);
        } else {
           // multi store case - in this case, all writing is done through
           // MapReducePOStoreImpl - set up a dummy RecordWriter

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

     * @throws IOException on failure
     */
    public static void setLocation(JobContext jobContext, POStore store) throws 
    IOException {
        Job storeJob = new Job(jobContext.getConfiguration());
        StoreFuncInterface storeFunc = store.getStoreFunc();
        String outputLocation = store.getSFile().getFileName();
        storeFunc.setStoreLocation(outputLocation, storeJob);
        
        // the setStoreLocation() method would indicate to the StoreFunc
        // to set the output location for its underlying OutputFormat.
        // Typically OutputFormat's store the output location in the
        // Configuration - so we need to get the modified Configuration

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

                    jobcontext.getConfiguration(), jobcontext.getJobID());
            
            // set output location
            PigOutputFormat.setLocation(jobContextCopy, store);
            
            StoreFuncInterface sFunc = store.getStoreFunc();
            OutputFormat of = sFunc.getOutputFormat();
            
            // The above call should have update the conf in the JobContext
            // to have the output location - now call checkOutputSpecs()
            of.checkOutputSpecs(jobContextCopy);
        }

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

    
    @Override
    public StoreFuncInterface createStoreFunc(POStore store) 
            throws IOException {
 
        StoreFuncInterface storeFunc = store.getStoreFunc();


        // call the setStoreLocation on the storeFunc giving it the
        // Job. Typically this will result in the OutputFormat of the
        // storeFunc storing the output location in the Configuration
        // in the Job. The PigOutFormat.setLocation() method will merge 
        // this modified Configuration into the configuration of the
        // Context we have
        PigOutputFormat.setLocation(context, store);
        OutputFormat outputFormat = storeFunc.getOutputFormat();


        // create a new record writer
        try {
            writer = outputFormat.getRecordWriter(context);
        } catch (InterruptedException e) {
            throw new IOException(e);
        }
 
        storeFunc.prepareToWrite(writer);
        
        return storeFunc;
    }

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

            TaskAttemptContext context,
            List<POStore> stores) throws IOException {
        List<Pair<OutputCommitter, POStore>> committers = 
            new ArrayList<Pair<OutputCommitter,POStore>>();
        for (POStore store : stores) {
            StoreFuncInterface sFunc = store.getStoreFunc();
            
            TaskAttemptContext updatedContext = setUpContext(context, store);
            try {
                committers.add(new Pair<OutputCommitter, POStore>(
                        sFunc.getOutputFormat().getOutputCommitter(
                                updatedContext), store));
            } catch (InterruptedException e) {
                throw new IOException(e);
            }
        }

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

        return contextCopy;   
    }


    static void storeCleanup(POStore store, Configuration conf)
            throws IOException {
        StoreFuncInterface storeFunc = store.getStoreFunc();
        if (storeFunc instanceof StoreMetadata) {
            Schema schema = store.getSchema();
            if (schema != null) {
                ((StoreMetadata) storeFunc).storeSchema(
                        new ResourceSchema(schema, store.getSortInfo()), store.getSFile()

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.