Examples of StoreFuncInterface

org.apache.pig.StoreFuncInterface
StoreFuncs take records from Pig's processing and store them into a data store. Most frequently this is an HDFS file, but it could also be an HBase instance, RDBMS, etc.

Examples of org.apache.pig.StoreFuncInterface

            LinkedList<POStore> mapStores = PlanHelper.getStores(mro.mapPlan);
            LinkedList<POStore> reduceStores = PlanHelper.getStores(mro.reducePlan);
            
            for (POStore st: mapStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }


            for (POStore st: reduceStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }


            // the OutputFormat we report to Hadoop is always PigOutputFormat
            nwJob.setOutputFormatClass(PigOutputFormat.class);

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

        if( func == null ) {
            func = PigStorage.class.getName();
        }


        FuncSpec funcSpec = new FuncSpec( func );
        StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );
        stoFunc.setStoreFuncUDFContextSignature( LOStore.constructSignature( alias, fileName, funcSpec ) );


        fileName = removeQuotes( fileName );
        FileSpec fileSpec = new FileSpec( fileName, funcSpec );
        LOStore store = new LOStore( lp, fileSpec );
        store.setAlias( alias );


        try {
            stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
        } catch (IOException ioe) {
            FrontendException e = new FrontendException(  ioe.getMessage(), ioe );
            throw e;
        }

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

        if( funcSpec == null ){
            funcSpec = new FuncSpec( PigStorage.class.getName() );
        }
        
        Object obj = PigContext.instantiateFuncFromSpec( funcSpec );
        StoreFuncInterface stoFunc = (StoreFuncInterface)obj;
        String sig = QueryParserUtils.constructSignature( inputAlias, filename, funcSpec);
        stoFunc.setStoreFuncUDFContextSignature( sig );
        String absolutePath = fileNameMap.get( sig );
        if (absolutePath == null) {
            absolutePath = stoFunc.relToAbsPathForStoreLocation( filename, 
                    QueryParserUtils.getCurrentDir( pigContext ) );
            if (absolutePath!=null) {
                QueryParserUtils.setHdfsServers( absolutePath, pigContext );
            }
            fileNameMap.put( sig, absolutePath );

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

            LinkedList<POStore> mapStores = PlanHelper.getPhysicalOperators(mro.mapPlan, POStore.class);
            LinkedList<POStore> reduceStores = PlanHelper.getPhysicalOperators(mro.reducePlan, POStore.class);


            for (POStore st: mapStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }


            for (POStore st: reduceStores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), nwJob);
            }


            // the OutputFormat we report to Hadoop is always PigOutputFormat
            nwJob.setOutputFormatClass(PigOutputFormat.class);

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

    String buildStoreOp(SourceLocation loc, String alias, String inputAlias, String filename, FuncSpec funcSpec)
    throws ParserValidationException {
        try {
            // Load StoreFunc class from default properties if funcSpec is null. Fallback on PigStorage if StoreFunc is not specified in properties.
            funcSpec = funcSpec == null ? new FuncSpec(pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName())) : funcSpec;
            StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(funcSpec);
            String fileNameKey = inputAlias + "_" + (storeIndex++) ;


            String signature = inputAlias + "_" + newOperatorKey();
            stoFunc.setStoreFuncUDFContextSignature(signature);


            String absolutePath = fileNameMap.get(fileNameKey);
            if (absolutePath == null) {
                absolutePath = stoFunc.relToAbsPathForStoreLocation(
                        filename,
                        QueryParserUtils.getCurrentDir(pigContext));
                if (absolutePath!=null) {
                    QueryParserUtils.setHdfsServers(absolutePath, pigContext);
                }

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

        if (stores.size() > 0) {


            ArrayList<POStore> storeLocations = new ArrayList<POStore>();
            for (POStore st : stores) {
                storeLocations.add(st);
                StoreFuncInterface sFunc = st.getStoreFunc();
                sFunc.setStoreLocation(st.getSFile().getFileName(), job);
            }


            Path tmpLocation = null;
            if (stores.size() == 1) {
                POStore st = stores.get(0);

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

    throws ParserValidationException {
        try {
            // Load StoreFunc class from default properties if funcSpec is null. Fallback on PigStorage if StoreFunc is not specified in properties.
            funcSpec = funcSpec == null ? new FuncSpec(pigContext.getProperties().getProperty(
                    PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName())) : funcSpec;
            StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(funcSpec);
            String fileNameKey = inputAlias + "_" + (storeIndex++) ;


            String signature = inputAlias + "_" + newOperatorKey();
            stoFunc.setStoreFuncUDFContextSignature(signature);


            String absolutePath = fileNameMap.get(fileNameKey);
            if (absolutePath == null) {
                absolutePath = stoFunc.relToAbsPathForStoreLocation(
                        filename,
                        QueryParserUtils.getCurrentDir(pigContext));
                if (absolutePath!=null) {
                    QueryParserUtils.setHdfsServers(absolutePath, pigContext);
                }

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

    public static void attachStorePlan(String scope, LogicalPlan lp, String fileName, String func,
            Operator input, String alias, PigContext pigContext) throws FrontendException {
        func = func == null ? pigContext.getProperties().getProperty(PigConfiguration.PIG_DEFAULT_STORE_FUNC, PigStorage.class.getName()) : func;


        FuncSpec funcSpec = new FuncSpec( func );
        StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec( funcSpec );


        fileName = removeQuotes( fileName );
        FileSpec fileSpec = new FileSpec( fileName, funcSpec );
        String sig = alias + "_" + LogicalPlanBuilder.newOperatorKey(scope);
        stoFunc.setStoreFuncUDFContextSignature(sig);
        LOStore store = new LOStore(lp, fileSpec, stoFunc, sig);
        store.setAlias(alias);


        try {
            stoFunc.relToAbsPathForStoreLocation( fileName, getCurrentDir( pigContext ) );
        } catch (IOException ioe) {
            FrontendException e = new FrontendException(  ioe.getMessage(), ioe );
            throw e;
        }

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

            if(mapStores.size() == 1) {
                store = mapStores.get(0);
            } else {
                store = reduceStores.get(0);
            }
            StoreFuncInterface sFunc = store.getStoreFunc();
            // set output location
            PigOutputFormat.setLocation(taskattemptcontext, store);
            // The above call should have update the conf in the JobContext
            // to have the output location - now call checkOutputSpecs()
            RecordWriter writer = sFunc.getOutputFormat().getRecordWriter(
                    taskattemptcontext);
            return new PigRecordWriter(writer, sFunc, Mode.SINGLE_STORE);
        } else {
           // multi store case - in this case, all writing is done through
           // MapReducePOStoreImpl - set up a dummy RecordWriter

View Full Code Here

Examples of org.apache.pig.StoreFuncInterface

     * @throws IOException on failure
     */
    public static void setLocation(JobContext jobContext, POStore store) throws
    IOException {
        Job storeJob = new Job(jobContext.getConfiguration());
        StoreFuncInterface storeFunc = store.getStoreFunc();
        String outputLocation = store.getSFile().getFileName();
        storeFunc.setStoreLocation(outputLocation, storeJob);


        // the setStoreLocation() method would indicate to the StoreFunc
        // to set the output location for its underlying OutputFormat.
        // Typically OutputFormat's store the output location in the
        // Configuration - so we need to get the modified Configuration

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.