Package org.apache.pig.tools.parameters

Examples of org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor

482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
// returns the stream of final pig script to be passed to Grunt
private static BufferedReader runParamPreprocessor(BufferedReader origPigScript, ArrayList<String> params,
                                            ArrayList<String> paramFiles, String scriptFile, boolean createFile)
                                throws org.apache.pig.tools.parameters.ParseException, IOException{
    ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
    String[] type1 = new String[1];
    String[] type2 = new String[1];

    if (createFile){
        BufferedWriter fw = new BufferedWriter(new FileWriter(scriptFile));
        psp.genSubstitutedFile (origPigScript, fw, params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles.toArray(type2) : null);
        return new BufferedReader(new FileReader (scriptFile));

    } else {
        StringWriter writer = new StringWriter();
        psp.genSubstitutedFile (origPigScript, writer,  params.size() > 0 ? params.toArray(type1) : null,
                                paramFiles.size() > 0 ? paramFiles.toArray(type2) : null);
        return new BufferedReader(new StringReader(writer.toString()));
    }
}
View Full Code Here
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
    private String runPreprocessor(String script, List<String> params,
                                   List<String> files)
        throws IOException, ParseException {

        ParameterSubstitutionPreprocessor psp = new ParameterSubstitutionPreprocessor(50);
        StringWriter writer = new StringWriter();

        try{
            psp.genSubstitutedFile(new BufferedReader(new FileReader(script)),
                                   writer, 
                                   params.size() > 0 ? params.toArray(new String[0]) : null,
                                   files.size() > 0 ? files.toArray(new String[0]) : null);
        } catch (org.apache.pig.tools.parameters.ParseException pex) {
            throw new ParseException(pex.getMessage());
View Full Code Here
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
     * Use a parameter within a pig script and provide value on the command line.
     */
    @Test
    public void testCmdlineParam() throws Exception{
        log.info("Starting test testCmdlineParam() ...");
        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
        pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
        pigOStream = new FileWriter(basedir + "/output1.pig");

        String[] arg = {"date=20080228"};
        String[] argFiles = null;
        ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);

        FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
        pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
        BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
        BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
View Full Code Here
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
     * Use a parameter within a pig script and provide value through a file.
     */
    @Test
    public void testFileParam() throws Exception{
        log.info ("Starting test  testFileParam()");
        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
        pigIStream = new BufferedReader(new FileReader(basedir + "/input1.pig"));
        pigOStream = new FileWriter(basedir + "/output1.pig");

        String[] arg = null;
        String[] argFiles = {basedir+"/ConfFile1.txt"};
        ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);

        FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
        pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
        BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
        BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
View Full Code Here
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
     *  Use a parameter within a pig script and provide value through running a binary or script.
     */
     @Test
     public void testShellCommand() throws Exception{
        log.info("Starting test testShellCommand()");
        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
        pigIStream = WithConditionalReplacement(basedir + "/input4.pig", "sh test/org/apache/pig/test/data/generate_date.sh",
                "test/org/apache/pig/test/data/generate_date.bat", Shell.WINDOWS);
        pigOStream = new FileWriter(basedir + "/output1.pig");

        String[] arg = null; //{"date=`sh generate_date.sh`"};     //`date \\T`"};
        String[] argFiles = null; // {basedir+"/ConfFile1.txt"};
        ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);

        FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
        pigExResultStream = new FileInputStream(basedir + "/ExpectedResultDefault.pig");
        BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
        BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
View Full Code Here
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
     * Use a Pig-supported parameter like "$0" and ensure that the Pig-supported parameter is not resolved
     */
    @Test
    public void testPigParamNotResolved() throws Exception{
        log.info("Starting test testPigParamNotResolved()");
        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
        pigIStream = new BufferedReader(new FileReader(basedir + "/input3.pig"));
        pigOStream = new FileWriter(basedir + "/output1.pig");

        String[] arg = null;
        String[] argFiles = null;
        try {
            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
        } catch (RuntimeException e) {
            if (e.getMessage().equals("Undefined parameter : 4")) {
                fail("Pig supported parameter $4 should not have been resolved.");
            }
        }
View Full Code Here
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
     * Use the %declare statement after the command has been used.
     */
    @Test
    public void testUndefinedParam() throws Exception{
        log.info("Starting test testUndefinedParam()");
        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
        pigIStream = new BufferedReader(new FileReader(basedir + "/input2.pig"));
        pigOStream = new FileWriter(basedir + "/output1.pig");

        String[] arg = null;
        String[] argFiles = null;
        try {
            ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
            fail ("Should have thrown an Undefined parameter exception");
        } catch (ParseException e) {
            assertEquals(e.getMessage(), "Undefined parameter : param");
        }
        log.info("Done");
View Full Code Here
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
     *  Use a parameter in %declare that is defined in terms of other parameters
     */
    @Test
    public void testSubstitutionWithinValue() throws Exception{
        log.info("Starting test  testSubstitutionWithinValue()");
        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
        pigIStream = new BufferedReader(new FileReader(basedir + "/inputSubstitutionWithinValue.pig"));
        pigOStream = new FileWriter(basedir + "/output1.pig");

        String[] arg = null;
        String[] argFiles = {basedir+"/ConfFile1.txt"};
        ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);

        FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
        pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig");
        BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
        BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
View Full Code Here
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
     *  The script itself takes an argument that is a parameter and should be resolved
     */
    @Test
    public void testSubstitutionWithinShellCommand() throws Exception{
        log.info("Starting test testSubstitutionWithinShellCommand()");
        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
        pigIStream = WithConditionalReplacement(basedir + "/inputSubstitutionWithinShellCommand.pig", "sh test/org/apache/pig/test/data/generate_date.sh",
                "test/org/apache/pig/test/data/generate_date.bat", Shell.WINDOWS);
        pigOStream = new FileWriter(basedir + "/output1.pig");

        String[] arg = null;
        String[] argFiles = null;
        ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);

        FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
        pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig");
        BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
        BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
View Full Code Here
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
     *  Use parameters passed on the command line/file that are not resolved prior to the declare statement.
     */
    @Test
    public void testCmdlineParamPriortoDeclare() throws Exception{
        log.info("Starting test testCmdlineParamPriortoDeclare()");
        ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
        pigIStream = new BufferedReader(new FileReader(basedir + "/input2.pig"));
        pigOStream = new FileWriter(basedir + "/output1.pig");

        String[] arg = {"param='20080228'"};
        String[] argFiles = null;
        ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);

        FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
        pigExResultStream = new FileInputStream(basedir + "/ExpectedResultCmdLnPriorDeclare.pig");
        BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
        BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
View Full Code Here
TOP

Related Classes of org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor

  • Example1
  • org.apache.pig.impl.PigContext
  • org.apache.pig.Main
  • org.apache.pig.parser.PigMacro
  • org.apache.pig.PigServer
  • org.apache.pig.pigunit.PigTest
  • org.apache.pig.scripting.Pig
  • org.apache.pig.test.TestGrunt
  • org.apache.pig.test.TestParamSubPreproc
  • org.apache.pig.test.TestUTF8
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.