Package org.apache.pig

Examples of org.apache.pig.PigServer.executeBatch()

105
106
107
108
109
110
111
112
113
114
115
    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);");
    server.registerQuery("store A into '" + BASIC_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer();");

    server.executeBatch();

    driver.run("select * from " + BASIC_TABLE);
    ArrayList<String> unpartitionedTableValuesReadFromHiveDriver = new ArrayList<String>();
    driver.getResults(unpartitionedTableValuesReadFromHiveDriver);
    assertEquals(basicInputData.size(), unpartitionedTableValuesReadFromHiveDriver.size());
View Full Code Here
127
128
129
130
131
132
133
134
135
136
137
    server.registerQuery("B2 = filter A by a < 2;");
    server.registerQuery("store B2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=0');");
    server.registerQuery("C2 = filter A by a >= 2;");
    server.registerQuery("store C2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=1');");

    server.executeBatch();

    driver.run("select * from " + PARTITIONED_TABLE);
    ArrayList<String> partitionedTableValuesReadFromHiveDriver = new ArrayList<String>();
    driver.getResults(partitionedTableValuesReadFromHiveDriver);
    assertEquals(basicInputData.size(), partitionedTableValuesReadFromHiveDriver.size());
View Full Code Here
153
154
155
156
157
158
159
160
161
162
163
    server.registerQuery("B2 = filter A by a < 2;");
    server.registerQuery("store B2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=0');");
    server.registerQuery("C2 = filter A by a >= 2;");
    server.registerQuery("store C2 into '" + PARTITIONED_TABLE + "' using org.apache.hive.hcatalog.pig.HCatStorer('bkt=1');");

    server.executeBatch();

    driver.run("select * from " + BASIC_TABLE);
    ArrayList<String> unpartitionedTableValuesReadFromHiveDriver = new ArrayList<String>();
    driver.getResults(unpartitionedTableValuesReadFromHiveDriver);
    driver.run("select * from " + PARTITIONED_TABLE);
View Full Code Here
130
131
132
133
134
135
136
137
138
139
140
    server.registerQuery("B2 = filter A by a < 2;");
    server.registerQuery("store B2 into '" + PARTITIONED_TABLE + "' using org.apache.hcatalog.pig.HCatStorer('bkt=0');");
    server.registerQuery("C2 = filter A by a >= 2;");
    server.registerQuery("store C2 into '" + PARTITIONED_TABLE + "' using org.apache.hcatalog.pig.HCatStorer('bkt=1');");

    server.executeBatch();

    driver.run("select * from " + PARTITIONED_TABLE);
    ArrayList<String> partitionedTableValuesReadFromHiveDriver = new ArrayList<String>();
    driver.getResults(partitionedTableValuesReadFromHiveDriver);
    assertEquals(basicInputData.size(), partitionedTableValuesReadFromHiveDriver.size());
View Full Code Here
156
157
158
159
160
161
162
163
164
165
166
    server.registerQuery("B2 = filter A by a < 2;");
    server.registerQuery("store B2 into '" + PARTITIONED_TABLE + "' using org.apache.hcatalog.pig.HCatStorer('bkt=0');");
    server.registerQuery("C2 = filter A by a >= 2;");
    server.registerQuery("store C2 into '" + PARTITIONED_TABLE + "' using org.apache.hcatalog.pig.HCatStorer('bkt=1');");

    server.executeBatch();

    driver.run("select * from " + BASIC_TABLE);
    ArrayList<String> unpartitionedTableValuesReadFromHiveDriver = new ArrayList<String>();
    driver.getResults(unpartitionedTableValuesReadFromHiveDriver);
    driver.run("select * from " + PARTITIONED_TABLE);
View Full Code Here
196
197
198
199
200
201
202
203
204
205
206
      Schema dumpedASchema = server.dumpSchema("A");
      server.registerQuery("STORE A into '" + tablename + "' using org.apache.hcatalog.pig.HCatStorer("
        + (provideSchemaToStorer ? "'', '" + pigSchema + "'" : "")
        + ");");

      ExecJob execJob = server.executeBatch().get(0);
      if (!execJob.getStatistics().isSuccessful()) {
        throw new RuntimeException("Import failed", execJob.getException());
      }
      // test that schema was loaded correctly
      server.registerQuery("X = load '" + tablename + "' using org.apache.hcatalog.pig.HCatLoader();");
View Full Code Here
166
167
168
169
170
171
172
173
174
175
176
    server.registerQuery("C2 = filter C by a >= 2;");
    server.registerQuery("store C2 into '" + PARTITIONED_TABLE + "' using org.apache.hcatalog.pig.HCatStorer('bkt=1');");

    server.registerQuery("D = load '" + COMPLEX_FILE_NAME + "' as (name:chararray, studentid:int, contact:tuple(phno:chararray,email:chararray), currently_registered_courses:bag{innertup:tuple(course:chararray)}, current_grades:map[ ] , phnos :bag{innertup:tuple(phno:chararray,type:chararray)});");
    server.registerQuery("store D into '" + COMPLEX_TABLE + "' using org.apache.hcatalog.pig.HCatStorer();");
    server.executeBatch();

  }

  private void cleanup() throws IOException, CommandNeedRetryException {
    dropTable(BASIC_TABLE);
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    logAndRegister(server, "A = load '"+INPUT_FILE_NAME+"' as (a:int, b:chararray);");
    logAndRegister(server, "store A into 'default.junit_external' using " + HCatStorerWrapper.class.getName()
        + "('c=" + part_val + "','" + tmpExternalDir.getAbsolutePath() + "');");
    server.executeBatch();

    Assert.assertTrue(tmpExternalDir.exists());
    Assert.assertTrue(new File(tmpExternalDir.getAbsoluteFile() + "/" + "part-m-00000").exists());

    driver.run("select * from junit_external");
View Full Code Here
134
135
136
137
138
139
140
141
142
    server.setBatchOn();
    server.registerQuery("data = load '" + data +
      "' using PigStorage('\t') as (my_small_int:int, my_tiny_int:int);");
    server.registerQuery(
      "store data into 'test_tbl' using org.apache.hcatalog.pig.HCatStorer();");
    List<ExecJob> jobs = server.executeBatch();
    Assert.assertEquals(expectedStatus, jobs.get(0).getStatus());
  }
}
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
    PigServer server = new PigServer(ExecType.LOCAL);
    server.setBatchOn();
    server.registerQuery("A = load '" + INPUT_FILE_NAME + "' as (a:int, b:chararray);");
    server.registerQuery("store A into '" + BASIC_TABLE + "' using org.apache.hcatalog.pig.HCatStorer();");

    server.executeBatch();

    driver.run("select * from " + BASIC_TABLE);
    ArrayList<String> unpartitionedTableValuesReadFromHiveDriver = new ArrayList<String>();
    driver.getResults(unpartitionedTableValuesReadFromHiveDriver);
    assertEquals(basicInputData.size(), unpartitionedTableValuesReadFromHiveDriver.size());
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.