Examples of MapOperator

eu.stratosphere.api.java.record.operators.MapOperator
MapOperator that applies a {@link MapFunction} to each record independently. @see MapFunction
org.apache.flink.api.java.record.operators.MapOperator
s.apache.org/jira/browse/FLINK-1106">FLINK-1106 for more details. MapOperator that applies a {@link MapFunction} to each record independently. @see MapFunction
org.apache.hadoop.hive.ql.exec.MapOperator
Map operator. This triggers overall map side processing. This is a little different from regular operators in that it starts off by processing a Writable data structure from a Table (instead of a Hive Object).

Examples of org.apache.flink.api.java.record.operators.MapOperator

        .input(joinWithSolutionSet)
        .name(NEXT_WORKSET_REDUCER_NAME)
        .build();
    
    if (mapBeforeSolutionDelta) {
      MapOperator mapper = MapOperator.builder(new IdentityMap())
        .input(joinWithSolutionSet)
        .name(SOLUTION_DELTA_MAPPER_NAME)
        .build();
      iteration.setSolutionSetDelta(mapper);
    } else {

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.MapOperator

        .input1(source1)
        .input2(source2)
        .name("Match 1")
        .build();
      
      MapOperator ma1 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map1").build();
      
      ReduceOperator r1 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(ma1)
        .name("Reduce 1")
        .build();
      
      ReduceOperator r2 = ReduceOperator.builder(new IdentityReduce(), IntValue.class, 0)
        .input(mat1)
        .name("Reduce 2")
        .build();
      
      MapOperator ma2 = MapOperator.builder(new IdentityMap()).input(mat1).name("Map 2").build();
      
      MapOperator ma3 = MapOperator.builder(new IdentityMap()).input(ma2).name("Map 3").build();
      
      @SuppressWarnings("unchecked")
      JoinOperator mat2 = JoinOperator.builder(new DummyMatchStub(), IntValue.class, 0, 0)
        .input1(r1, r2, ma2, ma3)
        .input2(ma2)

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.MapOperator

    
    BulkIteration iteration = new BulkIteration("Loop");
    iteration.setInput(sourceA);
    iteration.setMaximumNumberOfIterations(10);
    
    MapOperator mapper = MapOperator.builder(IdentityMap.class).name("Mapper").input(iteration.getPartialSolution()).build();
    iteration.setNextPartialSolution(mapper);
    
    FileDataSink sink1 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, iteration, "Sink 1");
    
    MapOperator postMap = MapOperator.builder(IdentityMap.class).name("Post Iteration Mapper")
        .input(iteration).build();
    
    FileDataSink sink2 = new FileDataSink(DummyOutputFormat.class, OUT_FILE, postMap, "Sink 2");
    
    List<FileDataSink> sinks = new ArrayList<FileDataSink>();

View Full Code Here

Examples of org.apache.flink.api.java.record.operators.MapOperator

    
    BulkIteration iteration = new BulkIteration("Loop");
    iteration.setInput(source2);
    iteration.setMaximumNumberOfIterations(10);
    
    MapOperator inMap = MapOperator.builder(new IdentityMap())
                               .input(source1)
                               .name("In Iteration Map")
                               .setBroadcastVariable("BC", iteration.getPartialSolution())
                               .build();
    
    iteration.setNextPartialSolution(inMap);
    
    MapOperator postMap = MapOperator.builder(new IdentityMap())
                     .input(source1)
                     .name("Post Iteration Map")
                     .setBroadcastVariable("BC", iteration)
                     .build();

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.MapOperator

    try {
      jc = job;
      execContext.setJc(jc);
      // create map and fetch operators
      MapWork mrwork = Utilities.getMapWork(job);
      mo = new MapOperator();
      mo.setConf(mrwork);
      // initialize map operator
      mo.setChildren(job);
      l4j.info(mo.dump(0));
      // initialize map local work

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.MapOperator

        Utilities.setMapWork(job, mrwork);
      }
      if (mrwork.getVectorMode()) {
        mo = new VectorMapOperator();
      } else {
        mo = new MapOperator();
      }
      mo.setConf(mrwork);
      // initialize map operator
      mo.setChildren(job);
      l4j.info(mo.dump(0));

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.MapOperator

        Utilities.setMapWork(jconf, mapWork);
      }
      if (mapWork.getVectorMode()) {
        mapOp = new VectorMapOperator();
      } else {
        mapOp = new MapOperator();
      }


      // initialize map operator
      mapOp.setConf(mapWork);
      mapOp.setChildren(jconf);

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.MapOperator

    try {


      if (mapWork.getVectorMode()) {
        mapOp = new VectorMapOperator();
      } else {
        mapOp = new MapOperator();
      }


      connectOps.clear();
      if (mergeWorkList != null) {
        MapOperator mergeMapOp = null;
        for (MapWork mergeMapWork : mergeWorkList) {
          if (mergeMapWork.getVectorMode()) {
            mergeMapOp = new VectorMapOperator();
          } else {
            mergeMapOp = new MapOperator();
          }


          mergeMapOpList.add(mergeMapOp);
          // initialize the merge operators first.
          if (mergeMapOp != null) {
            mergeMapOp.setConf(mergeMapWork);
            l4j.info("Input name is " + mergeMapWork.getName());
            jconf.set(Utilities.INPUT_NAME, mergeMapWork.getName());
            mergeMapOp.setChildren(jconf);
            if (foundCachedMergeWork == false) {
              DummyStoreOperator dummyOp = getJoinParentOp(mergeMapOp);
              connectOps.put(mergeMapWork.getTag(), dummyOp);
            }
            mergeMapOp.setExecContext(new ExecMapperContext(jconf));
            mergeMapOp.initializeLocalWork(jconf);
          }
        }
      }


      // initialize map operator
      mapOp.setConf(mapWork);
      l4j.info("Main input name is " + mapWork.getName());
      jconf.set(Utilities.INPUT_NAME, mapWork.getName());
      mapOp.setChildren(jconf);
      l4j.info(mapOp.dump(0));


      MapredContext.init(true, new JobConf(jconf));
      ((TezContext) MapredContext.get()).setInputs(inputs);
      ((TezContext) MapredContext.get()).setTezProcessorContext(processorContext);
      mapOp.setExecContext(execContext);
      mapOp.initializeLocalWork(jconf);


      initializeMapRecordSources();
      mapOp.initialize(jconf, null);
      if ((mergeMapOpList != null) && mergeMapOpList.isEmpty() == false) {
        for (MapOperator mergeMapOp : mergeMapOpList) {
          jconf.set(Utilities.INPUT_NAME, mergeMapOp.getConf().getName());
          mergeMapOp.initialize(jconf, null);
        }
      }


      // Initialization isn't finished until all parents of all operators
      // are initialized. For broadcast joins that means initializing the

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.MapOperator

        Utilities.setMapWork(job, mrwork);
      }
      if (mrwork.getVectorMode()) {
        mo = new VectorMapOperator();
      } else {
        mo = new MapOperator();
      }
      mo.setConf(mrwork);
      // initialize map operator
      mo.setChildren(job);
      l4j.info(mo.dump(0));

View Full Code Here

Examples of org.apache.hadoop.hive.ql.exec.MapOperator

        Utilities.setMapWork(job, mrwork);
      }
      if (mrwork.getVectorMode()) {
        mo = new VectorMapOperator();
      } else {
        mo = new MapOperator();
      }
      mo.setConf(mrwork);
      // initialize map operator
      mo.setChildren(job);
      l4j.info(mo.dump(0));

View Full Code Here

0 1 2 3 4 5

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.