Examples of MRTask

org.apache.hadoop.chukwa.rest.objects.MrTask
MrTask generated by hbm2java
water.MRTask
Map/Reduce style distributed computation. MRTask provides several map and reduce methods that can be overriden to specify a computation. Several instances of this class will be created to distribute the computation over F/J threads and machines. Non-transient fields are copied and serialized to instances created for map invocations. Reduce methods can store their results in fields. Results are serialized and reduced all the way back to the invoking node. When the last reduce method has been called, fields of the initial MRTask instance contains the computation results. Apart from small reduced POJO returned to the calling node, MRtask2 can produce output vector(s) as a result. These will have chunks co-located with the input dataset, however, their number of lines will generally differ, (so they won't be strictly compatible with the original). To produce output vectors, call doAll.dfork version with required number of outputs and override appropriate map call taking required number of NewChunks. MRTask will automatically close the new Appendable vecs and produce an output frame with newly created Vecs.

Examples of water.MRTask

    // cross the wire during copy. B's data could potentially cross the wire.
    Frame frbind = H2OHelper.emptyFrame(fra.numRows() + frb.numRows(), fra.numCols(),
            -1, -1, fra.anyVec().group());
    Vec keys = null;


    MRTask task = new MRTask() {
        public void map(Chunk chks[], NewChunk nc) {
          Vec A_vecs[] = fra.vecs();
          Vec B_vecs[] = frb.vecs();
          long A_rows = fra.numRows();
          long B_rows = frb.numRows();
          long start = chks[0].start();
          int chunkSize = chks[0].len();
          ValueString vstr = new ValueString();


          for (int r = 0; r < chunkSize; r++) {
            for (int c = 0; c < chks.length; c++) {
              if (r + start < A_rows) {
                chks[c].set0(r, A_vecs[c].at(r + start));
                if (keysa != null) {
                  nc.addStr(keysa.atStr(vstr, r + start));
                }
              } else {
                chks[c].set0(r, B_vecs[c].at(r + start - A_rows));
                if (keysb != null) {
                  nc.addStr(keysb.atStr(vstr, r + start - A_rows));
                }
              }
            }
          }
        }
      };


    if (keysa == null) {
      keys = task.doAll(1, frbind).outputFrame(null, null).anyVec();
    } else {
      task.doAll(frbind);
    }


    return new H2ODrm(frbind, keys);
  }

View Full Code Here

Examples of water.MRTask

    Vec keys = drmA.keys;
    int AewScalar_cols = A.numCols();


    // AewScalar is written into ncs[] with an MRTask on A, and therefore will
    // be similarly partitioned as A.
    Frame AewScalar = new MRTask() {
        private double opfn(String op, double a, double b) {
          if (a == 0.0 && b == 0.0) {
            return 0.0;
          }
          if (op.equals("+")) {

View Full Code Here

Examples of water.MRTask

      bvecs[i] = fra.anyVec().makeZero();
    }


    // Next run an MRTask on the new vectors, and fill each cell (initially 0)
    // by pulling in appropriate values from B (frb)
    new MRTask() {
      public void map(Chunk chks[]) {
        int chunkSize = chks[0].len();
        long start = chks[0].start();
        Vec vecs[] = frb.vecs();

View Full Code Here

Examples of water.MRTask

      // The new String Vec will therefore be similarly partitioned as the
      // new Frame.
      //
      // vout is finally collected by calling anyVec() on outputFrame(),
      // as it is the only column in the output frame.
      vout = new MRTask() {
          public void map(Chunk chks[], NewChunk nc) {
            int chunkSize = chks[0].len();
            Vec vins[] = frin.vecs();
            long start = chks[0].start();
            ValueString vstr = new ValueString();


            for (int r = 0; r < chunkSize; r++) {
              for (int c = 0; c < chks.length; c++) {
                chks[c].set0(r, vins[c].at(start + r));
              }
              nc.addStr(vin.atStr(vstr, start + r));
            }
          }
        }.doAll(1, frout).outputFrame(null, null).anyVec();
    } else {
      // If not String keyed, then run and MRTask on the new frame, and
      // just pull in right elements from frin
      new MRTask() {
        public void map(Chunk chks[]) {
          int chunkSize = chks[0].len();
          Vec vins[] = frin.vecs();
          long start = chks[0].start();

View Full Code Here

Examples of water.MRTask

    // and A.numRows() columns.
    Frame At = H2OHelper.emptyFrame(A.numCols(), (int) A.numRows(), -1, -1);


    // Execute MRTask on the new frame, and fill each cell (initially 0) by
    // pulling in the appropriate value from A.
    new MRTask() {
      public void map(Chunk chks[]) {
        int chunkSize = chks[0].len();
        long start = chks[0].start();
        Vec A_vecs[] = A.vecs();

View Full Code Here

Examples of water.MRTask

    // ABt is written into ncs[] with an MRTask on A, and therefore will
    // be similarly partitioned as A.
    //
    // chks.length == A.numCols() (== B.numCols())
    // ncs.length == ABt_cols (B.numRows())
    Frame ABt = new MRTask() {
        public void map(Chunk chks[], NewChunk ncs[]) {
          int chunkSize = chks[0].len();
          Vec B_vecs[] = B.vecs();


          for (int c = 0; c < ncs.length; c++) {

View Full Code Here

Examples of water.MRTask


    // Execute MRTask on the new Frame, and fill each cell (initially 0) by
    // computing appropriate values from A and B.
    //
    // chks.length == B.numCols()
    new MRTask() {
      public void map(Chunk chks[]) {
        int chunkSize = chks[0].len();
        long start = chks[0].start();
        long A_rows = A.numRows();
        Vec A_vecs[] = A.vecs();

View Full Code Here

Examples of water.MRTask


    // Execute MRTask on the new Frame, and fill each cell (initially 0) by
    // computing appropriate values from A.
    //
    // chks.length == A.numCols()
    new MRTask() {
      public void map(Chunk chks[]) {
        int chunkSize = chks[0].len();
        long start = chks[0].start();
        Vec A_vecs[] = A.vecs();
        long A_rows = A.numRows();

View Full Code Here

0 1

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.