Package org.apache.pig

Examples of org.apache.pig.PigServer.executeBatch()

358
359
360
361
362
363
364
365
366
367
368
    PigServer server = new PigServer(ExecType.LOCAL, props);
    UDFContext.getUDFContext().setClientSystemProps();
    server.setBatchOn();
    server.registerQuery("A = load '"+ fullFileName +"' as (a:int, b:chararray);");
    server.registerQuery("store A into 'default.junit_unparted' using "+HCatStorer.class.getName()+"('');");
    server.executeBatch();
    MiniCluster.deleteFile(cluster, fileName);

    driver.run("select * from junit_unparted");
    ArrayList<String> res = new ArrayList<String>();
    driver.getResults(res);
View Full Code Here
400
401
402
403
404
405
406
407
408
409
410
    PigServer server = new PigServer(ExecType.LOCAL, props);
    UDFContext.getUDFContext().setClientSystemProps();
    server.setBatchOn();
    server.registerQuery("A = load '"+ fullFileName +"' as (a:int, b:chararray);");
    server.registerQuery("store A into 'junit_unparted' using "+HCatStorer.class.getName()+"();");
    server.executeBatch();
    MiniCluster.deleteFile(cluster, fileName);

    driver.run("select * from junit_unparted");
    ArrayList<String> res = new ArrayList<String>();
    driver.getResults(res);
View Full Code Here
443
444
445
446
447
448
449
450
451
452
453
    UDFContext.getUDFContext().setClientSystemProps();
    server.setBatchOn();
    server.registerQuery("A = load '"+fullFileName+"' as (a:int, b:chararray);");
    server.registerQuery("B = filter A by a > 100;");
    server.registerQuery("store B into 'default.junit_unparted' using "+HCatStorer.class.getName()+"('','a:int,b:chararray');");
    server.executeBatch();
    MiniCluster.deleteFile(cluster, fileName);

    driver.run("select * from junit_unparted");
    ArrayList<String> res = new ArrayList<String>();
    driver.getResults(res);
View Full Code Here
476
477
478
479
480
481
482
483
484
485
486
  UDFContext.getUDFContext().setClientSystemProps();
  server.setBatchOn();
  server.registerQuery("A = load '"+fullFileName+"' as (b:chararray, a:tuple(a1:int), arr_of_struct:bag{mytup:tuple(s1:chararray)}, arr_of_struct2:bag{mytup:tuple(s1:chararray,s2:chararray)}, arr_of_struct3:bag{t3:tuple(s3:chararray)});");
  server.registerQuery("store A into 'default.junit_unparted' using "+HCatStorer.class.getName()+"('','b:chararray, a:tuple(a1:int)," +
      " arr_of_struct:bag{mytup:tuple(s1:chararray)}, arr_of_struct2:bag{mytup:tuple(s1:chararray,s2:chararray)}, arr_of_struct3:bag{t3:tuple(s3:chararray)}');");
  server.executeBatch();



  MiniCluster.deleteFile(cluster, fileName);
View Full Code Here
516
517
518
519
520
521
522
523
524
525
526
    PigServer server = new PigServer(ExecType.LOCAL, props);
    UDFContext.getUDFContext().setClientSystemProps();
    server.setBatchOn();
    server.registerQuery("A = load '"+fullFileName+"' as (a:int, b:float, c:double, d:long, e:chararray);");
    server.registerQuery("store A into 'default.junit_unparted' using "+HCatStorer.class.getName()+"('','a:int, b:float, c:double, d:long, e:chararray');");
    server.executeBatch();
    MiniCluster.deleteFile(cluster, fileName);

    driver.run("select * from junit_unparted");
    ArrayList<String> res = new ArrayList<String>();
    driver.getResults(res);
View Full Code Here
566
567
568
569
570
571
572
573
574
575
576
    PigServer server = new PigServer(ExecType.LOCAL, props);
    UDFContext.getUDFContext().setClientSystemProps();
    server.setBatchOn();
    server.registerQuery("A = load '"+fullFileName+"' as (a:int, b:chararray);");
    server.registerQuery("store A into 'default.junit_unparted' using "+HCatStorer.class.getName()+"('','a:int,b:chararray');");
    server.executeBatch();
    MiniCluster.deleteFile(cluster, fileName);

    driver.run("select * from junit_unparted");
    ArrayList<String> res = new ArrayList<String>();
    driver.getResults(res);
View Full Code Here
161
162
163
164
165
166
167
168
169
170
171
           " STORE in INTO '" + output + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();"
            };
        for (String query: queries){
            pigServer.registerQuery(query);
        }
        pigServer.executeBatch();
        verifyResults(output, expected, "snappy");
    }

    @Test
    public void testRecordWithSplit() throws IOException {
View Full Code Here
69
70
71
72
73
74
75
76
77
78
79
  PigServer server = new PigServer(ExecType.LOCAL);
  server.setBatchOn();
  logAndRegister(server, "A = load '"+INPUT_FILE_NAME+"' as (a:int, b:chararray);");
  logAndRegister(server, "store A into 'default.junit_external' using " + HCatStorerWrapper.class.getName()
    + "('c=" + part_val + "','" + tmpExternalDir.getAbsolutePath() + "');");
  server.executeBatch();

  Assert.assertTrue(tmpExternalDir.exists());
  Assert.assertTrue(new File(tmpExternalDir.getAbsoluteFile() + "/" + "part-m-00000").exists());

  driver.run("select * from junit_external");
View Full Code Here
133
134
135
136
137
138
139
140
141
        server.setBatchOn();
        server.registerQuery("data = load '" + data +
            "' using PigStorage('\t') as (my_small_int:int, my_tiny_int:int);");
        server.registerQuery(
            "store data into 'test_tbl' using org.apache.hcatalog.pig.HCatStorer();");
        List<ExecJob> jobs = server.executeBatch();
        Assert.assertEquals(expectedStatus, jobs.get(0).getStatus());
    }
}
View Full Code Here
101
102
103
104
105
106
107
108
109
110
111
        PigServer server = new PigServer(ExecType.LOCAL);
        server.setBatchOn();
        server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);");
        server.registerQuery("store A into '" + BASIC_TABLE + "' using org.apache.hcatalog.pig.HCatStorer();");

        server.executeBatch();

        driver.run("select * from " + BASIC_TABLE);
        ArrayList<String> unpartitionedTableValuesReadFromHiveDriver = new ArrayList<String>();
        driver.getResults(unpartitionedTableValuesReadFromHiveDriver);
        assertEquals(basicInputData.size(), unpartitionedTableValuesReadFromHiveDriver.size());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.