Examples of SqoopRecord

com.cloudera.sqoop.lib.SqoopRecord
Interface implemented by the classes generated by sqoop's orm.ClassWriter. @deprecated use org.apache.sqoop.lib.SqoopRecord instead. @see org.apache.sqoop.lib.SqoopRecord
org.apache.sqoop.lib.SqoopRecord
Interface implemented by the classes generated by sqoop's orm.ClassWriter.

Examples of com.cloudera.sqoop.lib.SqoopRecord


    try {
      // Instantiate the value record object via reflection.
      Class cls = Class.forName(className, true,
          Thread.currentThread().getContextClassLoader());
      SqoopRecord record = (SqoopRecord) ReflectionUtils.newInstance(
          cls, new Configuration());


      // Create the SequenceFile.
      Configuration conf = new Configuration();
      if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
        conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
      }
      FileSystem fs = FileSystem.get(conf);
      Path tablePath = getTablePath();
      Path filePath = new Path(tablePath, "part" + fileNum);
      fs.mkdirs(tablePath);
      SequenceFile.Writer w = SequenceFile.createWriter(
          fs, conf, filePath, LongWritable.class, cls);


      // Now write the data.
      int startId = fileNum * numRecords;
      for (int i = 0; i < numRecords; i++) {
        record.parse(getRecordLine(startId + i));
        w.append(new LongWritable(startId + i), record);
      }


      w.close();
    } catch (ClassNotFoundException cnfe) {

View Full Code Here

Examples of com.cloudera.sqoop.lib.SqoopRecord

    try {
      codeGen(dt);
      // Instantiate the value record object via reflection.
      Class cls = Class.forName(getTableName(dt), true, Thread
        .currentThread().getContextClassLoader());
      SqoopRecord record = (SqoopRecord) ReflectionUtils.newInstance(cls,
        new Configuration());


      // Create the SequenceFile.
      Configuration conf = new Configuration();
      String hdfsroot;
      hdfsroot = System.getProperty("ms.datatype.test.hdfsprefix");
      if (hdfsroot == null){
        hdfsroot ="hdfs://localhost/";
      }
      conf.set("fs.default.name", hdfsroot);
      FileSystem fs = FileSystem.get(conf);
      Path tablePath = getTablePath(dt);
      Path filePath = new Path(tablePath, getTableName(dt));
      fs.mkdirs(tablePath);
      SequenceFile.Writer w = SequenceFile.createWriter(fs, conf,
        filePath, LongWritable.class, cls);


      int cnt = 0;
      for (String tmp : data) {
        record.parse(tmp + "\n");
        w.append(new LongWritable(cnt), record);
      }


      w.close();
    } catch (ClassNotFoundException cnfe) {

View Full Code Here

Examples of com.cloudera.sqoop.lib.SqoopRecord


    try {
      // Instantiate the value record object via reflection.
      Class cls = Class.forName(className, true,
          Thread.currentThread().getContextClassLoader());
      SqoopRecord record = (SqoopRecord) ReflectionUtils.newInstance(
          cls, new Configuration());


      // Create the SequenceFile.
      Configuration conf = new Configuration();
      if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
        conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
      }
      FileSystem fs = FileSystem.get(conf);
      Path tablePath = getTablePath();
      Path filePath = new Path(tablePath, "part" + fileNum);
      fs.mkdirs(tablePath);
      SequenceFile.Writer w = SequenceFile.createWriter(
          fs, conf, filePath, LongWritable.class, cls);


      // Now write the data.
      int startId = fileNum * numRecords;
      for (int i = 0; i < numRecords; i++) {
        record.parse(getRecordLine(startId + i));
        w.append(new LongWritable(startId + i), record);
      }


      w.close();
    } catch (ClassNotFoundException cnfe) {

View Full Code Here

Examples of com.cloudera.sqoop.lib.SqoopRecord

            if (status.getPath().getName().startsWith("part-m-")) {


              SequenceFile.Reader reader =
                  new SequenceFile.Reader(hdfs, status.getPath(), hadoopConf);
              LongWritable key = new LongWritable();
              @SuppressWarnings("unchecked")
              SqoopRecord value =
                  ((Class<SqoopRecord>) reader.getValueClass())
                      .getConstructor().newInstance();
              ps =
                  conn.prepareStatement("SELECT bd, bf, b, c, cl, d, f, l, nc, "
                      + "ncl, nvc, r, u, iym, ids, t, tz, tltz, rawcol FROM "
                      + OracleUtils.SYSTEMTEST_TABLE_NAME + " WHERE id = ?");
              while (reader.next(key, value)) {
                // Compare test data from hadoop with data in oracle
                Map<String, Object> fields = value.getFieldMap();
                BigDecimal id = (BigDecimal) fields.get("ID");
                ps.setBigDecimal(1, id);
                ResultSet rs = ps.executeQuery();
                assertTrue("Did not find row with id " + id + " in oracle", rs
                    .next());

View Full Code Here

Examples of com.cloudera.sqoop.lib.SqoopRecord

            if (status.getPath().getName().startsWith("part-m-")) {


              SequenceFile.Reader reader =
                  new SequenceFile.Reader(hdfs, status.getPath(), hadoopConf);
              LongWritable key = new LongWritable();
              @SuppressWarnings("unchecked")
              SqoopRecord value =
                  ((Class<SqoopRecord>) reader.getValueClass())
                      .getConstructor().newInstance();
              ps =
                  conn.prepareStatement("SELECT bd, bf, b, c, cl, d, f, l, nc, "
                      + "ncl, nvc, r, u, iym, ids, t, tz, tltz, rawcol FROM "
                      + OracleUtils.SYSTEMTEST_TABLE_NAME + " WHERE id = ?");
              while (reader.next(key, value)) {
                // Compare test data from hadoop with data in oracle
                Map<String, Object> fields = value.getFieldMap();
                BigDecimal id = (BigDecimal) fields.get("ID");
                ps.setBigDecimal(1, id);
                ResultSet rs = ps.executeQuery();
                assertTrue("Did not find row with id " + id + " in oracle", rs
                    .next());

View Full Code Here

Examples of com.cloudera.sqoop.lib.SqoopRecord


    try {
      // Instantiate the value record object via reflection.
      Class cls = Class.forName(className, true,
          Thread.currentThread().getContextClassLoader());
      SqoopRecord record = (SqoopRecord) ReflectionUtils.newInstance(
          cls, new Configuration());


      // Create the SequenceFile.
      Configuration conf = new Configuration();
      if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
        conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
      }
      FileSystem fs = FileSystem.get(conf);
      Path tablePath = getTablePath();
      Path filePath = new Path(tablePath, "part" + fileNum);
      fs.mkdirs(tablePath);
      SequenceFile.Writer w = SequenceFile.createWriter(
          fs, conf, filePath, LongWritable.class, cls);


      // Now write the data.
      int startId = fileNum * numRecords;
      for (int i = 0; i < numRecords; i++) {
        record.parse(getRecordLine(startId + i));
        w.append(new LongWritable(startId + i), record);
      }


      w.close();
    } catch (ClassNotFoundException cnfe) {

View Full Code Here

Examples of com.cloudera.sqoop.lib.SqoopRecord

    extends Reducer<Text, MergeRecord, SqoopRecord, NullWritable> {


  @Override
  public void reduce(Text key, Iterable<MergeRecord> vals, Context c)
      throws IOException, InterruptedException {
    SqoopRecord bestRecord = null;
    try {
      for (MergeRecord val : vals) {
        if (null == bestRecord && !val.isNewRecord()) {
          // Use an old record if we don't have a new record.
          bestRecord = (SqoopRecord) val.getSqoopRecord().clone();

View Full Code Here

Examples of org.apache.sqoop.lib.SqoopRecord

  }


  @Override
  public void map(LongWritable key, HCatRecord hcr, Context context)
    throws IOException, InterruptedException {
    SqoopRecord sqr = helper.convertToSqoopRecord(hcr);
    writeSqoopRecord(sqr);
    context.progress();
  }

View Full Code Here

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.