Package org.apache.pig

Examples of org.apache.pig.PigServer.executeBatch()

419
420
421
422
423
424
425
426
427
428
429
    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:float, c:double, d:long, e:chararray, h:boolean, f:bytearray);");
    //null gets stored into column g which is a binary field.
    server.registerQuery("store A into 'default.junit_unparted' using " + HCatStorer.class.getName() + "('','a:int, b:float, c:double, d:long, e:chararray, h:boolean, f:bytearray');");
    server.executeBatch();


    driver.run("select * from junit_unparted");
    ArrayList<String> res = new ArrayList<String>();
    driver.getResults(res);
View Full Code Here
476
477
478
479
480
481
482
483
484
485
486
    HcatTestUtils.createTestDataFile(INPUT_FILE_NAME, inputData);
    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);");
    server.registerQuery("store A into 'default.junit_unparted' using " + HCatStorer.class.getName() + "('','a:int,b:chararray');");
    server.executeBatch();

    driver.run("select * from junit_unparted");
    ArrayList<String> res = new ArrayList<String>();
    driver.getResults(res);
    driver.run("drop table junit_unparted");
View Full Code Here
517
518
519
520
521
522
523
524
525
526
527
    pig.setBatchOn();
    pig.registerQuery("A = LOAD '" + INPUT_FILE_NAME + "' USING PigStorage() AS (emp_id:int,emp_name:chararray,emp_start_date:chararray," +
      "emp_gender:chararray,emp_country:chararray,emp_state:chararray);");
    pig.registerQuery("IN = FILTER A BY emp_country == 'IN';");
    pig.registerQuery("STORE IN INTO 'employee' USING " + HCatStorer.class.getName() + "('emp_country=IN');");
    pig.executeBatch();
    driver.run("select * from employee");
    ArrayList<String> results = new ArrayList<String>();
    driver.getResults(results);
    Assert.assertEquals(4, results.size());
    Collections.sort(results);
View Full Code Here
554
555
556
557
558
559
560
561
562
563
564
    pig.setBatchOn();
    pig.registerQuery("A = LOAD '" + INPUT_FILE_NAME + "' USING PigStorage() AS (emp_id:int,emp_name:chararray,emp_start_date:chararray," +
      "emp_gender:chararray,emp_country:chararray,emp_state:chararray);");
    pig.registerQuery("IN = FILTER A BY emp_country == 'IN';");
    pig.registerQuery("STORE IN INTO 'employee' USING " + HCatStorer.class.getName() + "();");
    pig.executeBatch();
    driver.run("select * from employee");
    ArrayList<String> results = new ArrayList<String>();
    driver.getResults(results);
    Assert.assertEquals(4, results.size());
    Collections.sort(results);
View Full Code Here
588
589
590
591
592
593
594
595
596
597
598
    pig.setBatchOn();
    pig.registerQuery("A = LOAD '" + INPUT_FILE_NAME + "' USING PigStorage() AS (emp_id:int,emp_name:chararray,emp_start_date:chararray," +
      "emp_gender:chararray,emp_country:chararray,emp_state:chararray);");
    pig.registerQuery("IN = FILTER A BY emp_country == 'IN';");
    pig.registerQuery("STORE IN INTO 'employee' USING " + HCatStorer.class.getName() + "();");
    pig.executeBatch();
    driver.run("select * from employee");
    ArrayList<String> results = new ArrayList<String>();
    driver.getResults(results);
    Assert.assertEquals(0, results.size());
    driver.run("drop table employee");
View Full Code Here
620
621
622
623
624
625
626
627
628
629
630
        + "' as (a:int, c:chararray);");
    server.registerQuery("B = filter A by " + FailEvalFunc.class.getName()
        + "($0);");
    server.registerQuery("store B into 'ptn_fail' using "
        + HCatStorer.class.getName() + "('b=math');");
    server.executeBatch();

    String query = "show partitions ptn_fail";
    retCode = driver.run(query).getResponseCode();

    if (retCode != 0) {
View Full Code Here
68
69
70
71
72
73
74
75
76
77
78
    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    logAndRegister(server, "A = load '"+INPUT_FILE_NAME+"' as (a:int, b:chararray);");
    logAndRegister(server, "store A into 'default.junit_external' using " + HCatStorerWrapper.class.getName()
        + "('c=" + part_val + "','" + tmpExternalDir.getPath().replaceAll("\\\\", "/") + "');");
    server.executeBatch();

    Assert.assertTrue(tmpExternalDir.exists());
    Assert.assertTrue(new File(tmpExternalDir.getPath().replaceAll("\\\\", "/") + "/" + "part-m-00000").exists());

    driver.run("select * from junit_external");
View Full Code Here
197
198
199
200
201
202
203
204
205
206
207
      Schema dumpedASchema = server.dumpSchema("A");
      server.registerQuery("STORE A into '" + tablename + "' using org.apache.hive.hcatalog.pig.HCatStorer("
        + (provideSchemaToStorer ? "'', '" + pigSchema + "'" : "")
        + ");");

      ExecJob execJob = server.executeBatch().get(0);
      if (!execJob.getStatistics().isSuccessful()) {
        throw new RuntimeException("Import failed", execJob.getException());
      }
      // test that schema was loaded correctly
      server.registerQuery("X = load '" + tablename + "' using org.apache.hive.hcatalog.pig.HCatLoader();");
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
    server.registerQuery("C2 = filter C by a >= 2;");
    server.registerQuery("store C2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=1');");

    server.registerQuery("D = load '" + COMPLEX_FILE_NAME + "' as (name:chararray, studentid:int, contact:tuple(phno:chararray,email:chararray), currently_registered_courses:bag{innertup:tuple(course:chararray)}, current_grades:map[ ] , phnos :bag{innertup:tuple(phno:chararray,type:chararray)});");
    server.registerQuery("store D into '" + COMPLEX_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();");
    server.executeBatch();

  }

  private void cleanup() throws IOException, CommandNeedRetryException {
    dropTable(BASIC_TABLE);
View Full Code Here
133
134
135
136
137
138
139
140
141
    server.setBatchOn();
    server.registerQuery("data = load '" + data +
      "' using PigStorage('\t') as (my_small_int:int, my_tiny_int:int);");
    server.registerQuery(
      "store data into 'test_tbl' using org.apache.hive.hcatalog.pig.HCatStorer();");
    List<ExecJob> jobs = server.executeBatch();
    Assert.assertEquals(expectedStatus, jobs.get(0).getStatus());
  }
}
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.