Package org.apache.pig.tools.parameters

Examples of org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor


// returns the stream of final pig script to be passed to Grunt
private static BufferedReader runParamPreprocessor(BufferedReader origPigScript, ArrayList<String> params,
                                            ArrayList<String> paramFiles, String scriptFile, boolean createFile)
                                throws org.apache.pig.tools.parameters.ParseException, IOException{
    ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
    String[] type1 = new String[1];
    String[] type2 = new String[1];

    if (createFile){
        BufferedWriter fw = new BufferedWriter(new FileWriter(scriptFile));
        psp.genSubstitutedFile (origPigScript, fw, params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles.toArray(type2) : null);
        return new BufferedReader(new FileReader (scriptFile));

    } else {
        StringWriter writer = new StringWriter();
        psp.genSubstitutedFile (origPigScript, writer,  params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles.toArray(type2) : null);
        return new BufferedReader(new StringReader(writer.toString()));
    }
}
View Full Code Here


    PigTest.getCluster();

    BufferedReader pigIStream = new BufferedReader(new StringReader(this.originalTextPigScript));
    StringWriter pigOStream = new StringWriter();

    ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
    ps.genSubstitutedFile(pigIStream, pigOStream, args, argFiles);

    String substitutedPig = pigOStream.toString();
    LOG.info(substitutedPig);

    File f = File.createTempFile("tmp", "pigunit");
View Full Code Here

    private String runPreprocessor(String script, List<String> params,
                                   List<String> files)
        throws IOException, ParseException {

        ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        StringWriter writer = new StringWriter();

        try{
            psp.genSubstitutedFile(new BufferedReader(new FileReader(script)),
                                   writer, 
                                   params.size() > 0 ? params.toArray(new String[0]) : null,
                                   files.size() > 0 ? files.toArray(new String[0]) : null);
        } catch (org.apache.pig.tools.parameters.ParseException pex) {
            throw new ParseException(pex.getMessage());
View Full Code Here

    for (String param: paramFiles) {
        FileLocalizer.FetchFileRet localFileRet = FileLocalizer.fetchFile(properties, param);
        paramFiles2.add(localFileRet.file.getAbsolutePath());
    }   
   
    ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
    String[] type1 = new String[1];
    String[] type2 = new String[1];

    if (createFile){
        BufferedWriter fw = new BufferedWriter(new FileWriter(scriptFile));
        psp.genSubstitutedFile (origPigScript, fw, params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles2.toArray(type2) : null);
        return new BufferedReader(new FileReader (scriptFile));

    } else {
        StringWriter writer = new StringWriter();
        psp.genSubstitutedFile (origPigScript, writer,  params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles2.toArray(type2) : null);
        return new BufferedReader(new StringReader(writer.toString()));
    }
}
View Full Code Here

                    paramList.add(entry.getKey()+"="+entry.getValue());
                }
            }
           
            // do parameter substitution
            ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
            StringWriter writer = new StringWriter();
            psp.genSubstitutedFile(new BufferedReader(new InputStreamReader(new FileInputStream(fileName))),
                                   writer, 
                                   paramList.size() > 0 ? paramList.toArray(new String[0]) : null,
                                   paramsFiles!=null ? paramsFiles.toArray(new String[0]) : null);
           
            GruntParser grunt = new GruntParser(new StringReader(writer.toString()));
View Full Code Here

        ps.println("b = filter a by $0 == '$querystring';");
        ps.close();
     
        String[] arg = {"querystring='中文'"};
       
      ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        BufferedReader pigIStream = new BufferedReader(new FileReader(queryFile.toString()));
        StringWriter pigOStream = new StringWriter();

        psp.genSubstitutedFile(pigIStream , pigOStream , arg, null);
       
        assertTrue(pigOStream.toString().contains("中文"));
       
        queryFile.delete();
    }
View Full Code Here

        PigServer server = new PigServer(ExecType.LOCAL);
        PigContext context = server.getPigContext();
       
        String strCmd = "A = load 'bar';\nB = foreach A generate $0;";
       
        ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        BufferedReader pin = new BufferedReader(new StringReader(strCmd))
        StringWriter writer = new StringWriter();
        psp.genSubstitutedFile(pin, writer, null, null);
        pin = new BufferedReader(new StringReader(writer.toString()));
            
        Grunt grunt = new Grunt(pin, context);
        int results[] = grunt.exec();
        for (int i=0; i<results.length; i++) {
View Full Code Here

        ps.println("b = filter a by $0 == '$querystring';");
        ps.close();
     
        String[] arg = {"querystring='中文'"};
       
      ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        BufferedReader pigIStream = new BufferedReader(new FileReader(queryFile.toString()));
        StringWriter pigOStream = new StringWriter();

        psp.genSubstitutedFile(pigIStream , pigOStream , arg, null);
       
        assertTrue(pigOStream.toString().contains("中文"));
       
        queryFile.delete();
    }
View Full Code Here

    private String runPreprocessor(String script, List<String> params,
                                   List<String> files)
        throws IOException, ParseException {

        ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        StringWriter writer = new StringWriter();

        try{
            psp.genSubstitutedFile(new BufferedReader(new FileReader(script)),
                                   writer, 
                                   params.size() > 0 ? params.toArray(new String[0]) : null,
                                   files.size() > 0 ? files.toArray(new String[0]) : null);
        } catch (org.apache.pig.tools.parameters.ParseException pex) {
            throw new ParseException(pex.getMessage());
View Full Code Here

// returns the stream of final pig script to be passed to Grunt
private static BufferedReader runParamPreprocessor(BufferedReader origPigScript, ArrayList<String> params,
                                            ArrayList<String> paramFiles, String scriptFile, boolean createFile)
                                throws org.apache.pig.tools.parameters.ParseException, IOException{
    ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
    String[] type1 = new String[1];
    String[] type2 = new String[1];

    if (createFile){
        BufferedWriter fw = new BufferedWriter(new FileWriter(scriptFile));
        psp.genSubstitutedFile (origPigScript, fw, params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles.toArray(type2) : null);
        return new BufferedReader(new FileReader (scriptFile));

    } else {
        StringWriter writer = new StringWriter();
        psp.genSubstitutedFile (origPigScript, writer,  params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles.toArray(type2) : null);
        return new BufferedReader(new StringReader(writer.toString()));
    }
}
View Full Code Here

TOP

Related Classes of org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.