Examples of eu.stratosphere.pact.runtime.task.util.TaskConfig

Package eu.stratosphere.pact.runtime.task.util

Examples of eu.stratosphere.pact.runtime.task.util.TaskConfig

eu.stratosphere.pact.runtime.task.util.TaskConfig
Configuration class which stores all relevant parameters required to set up the Pact tasks.

    return inputNum == buildSideIndex;
  }


  @Override
  public void initialize() throws Exception {
    TaskConfig config = this.taskContext.getTaskConfig();
    
    TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
    TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
    TypeComparator<IT1> comparator1 = this.taskContext.getInputComparator(0);
    TypeComparator<IT2> comparator2 = this.taskContext.getInputComparator(1);
    MutableObjectIterator<IT1> input1 = this.taskContext.getInput(0);
    MutableObjectIterator<IT2> input2 = this.taskContext.getInput(1);


    TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = 
        this.taskContext.getTaskConfig().getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());


    int numMemoryPages = this.taskContext.getMemoryManager().computeNumberOfPages(config.getMemoryDriver());
    List<MemorySegment> memSegments = this.taskContext.getMemoryManager().allocatePages(
      this.taskContext.getOwningNepheleTask(), numMemoryPages);


    if (buildSideIndex == 0 && probeSideIndex == 1) {
      MutableHashTable<IT1, IT2> hashJoin = new MutableHashTable<IT1, IT2>(serializer1, serializer2, comparator1, comparator2,

View Full Code Here

      this.hashTable = (CompactingHashTable<IT2>) SolutionSetBroker.instance().get(identifier);
    } else {
      throw new Exception("The task context of this driver is no iterative task context.");
    }
    
    TaskConfig config = taskContext.getTaskConfig();
    ClassLoader classLoader = taskContext.getUserCodeClassLoader();
    
    TypeSerializer<IT2> solutionSetSerializer = this.hashTable.getBuildSideSerializer();
    TypeSerializer<IT1> probeSideSerializer = taskContext.<IT1>getInputSerializer(0).getSerializer();
    
    TypeComparatorFactory<IT1> probeSideComparatorFactory = config.getDriverComparator(0, classLoader);
    TypeComparator<IT2> solutionSetComparator = this.hashTable.getBuildSideComparator().duplicate();
    this.probeSideComparator = probeSideComparatorFactory.createComparator();
    
    solutionSideRecord = solutionSetSerializer.createInstance();
    probeSideRecord = probeSideSerializer.createInstance();

View Full Code Here

    }


    // obtain task configuration (including stub parameters)
    Configuration taskConf = getTaskConfiguration();
    taskConf.setClassLoader(this.userCodeClassLoader);
    this.config = new TaskConfig(taskConf);


    // now get the operator class which drives the operation
    final Class<? extends PactDriver<S, OT>> driverClass = this.config.getDriver();
    this.driver = InstantiationUtil.instantiate(driverClass, PactDriver.class);

View Full Code Here

        catch (Exception ex) {
          throw new RuntimeException("Could not instantiate chained task driver.", ex);
        }


        // get the configuration for the task
        final TaskConfig chainedStubConf = config.getChainedStubConfig(i);
        final String taskName = config.getChainedTaskName(i);


        if (i == numChained -1) {
          // last in chain, instantiate the output collector for this task
          previous = getOutputCollector(nepheleTask, chainedStubConf, cl, eventualOutputs, chainedStubConf.getNumOutputs());
        }


        ct.setup(chainedStubConf, taskName, previous, nepheleTask, cl);
        chainedTasksTarget.add(0, ct);

View Full Code Here

    // --------------- the inputs ---------------------


    // page rank input
    JobInputVertex pageWithRankInput = JobGraphUtils.createInput(new ImprovedDanglingPageRankInputFormat(),
      pageWithRankInputPath, "DanglingPageWithRankInput", jobGraph, degreeOfParallelism, numSubTasksPerInstance);
    TaskConfig pageWithRankInputConfig = new TaskConfig(pageWithRankInput.getConfiguration());
    pageWithRankInputConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
    pageWithRankInputConfig.setOutputComparator(fieldZeroComparator, 0);
    pageWithRankInputConfig.setOutputSerializer(recSerializer);
    pageWithRankInputConfig.setStubParameter("pageRank.numVertices", String.valueOf(numVertices));


    // edges as adjacency list
    JobInputVertex adjacencyListInput = JobGraphUtils.createInput(new ImprovedAdjacencyListInputFormat(),
      adjacencyListInputPath, "AdjancencyListInput", jobGraph, degreeOfParallelism, numSubTasksPerInstance);
    TaskConfig adjacencyListInputConfig = new TaskConfig(adjacencyListInput.getConfiguration());
    adjacencyListInputConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
    adjacencyListInputConfig.setOutputSerializer(recSerializer);
    adjacencyListInputConfig.setOutputComparator(fieldZeroComparator, 0);


    // --------------- the head ---------------------
    JobTaskVertex head = JobGraphUtils.createTask(IterationHeadPactTask.class, "IterationHead", jobGraph,
      degreeOfParallelism, numSubTasksPerInstance);
    TaskConfig headConfig = new TaskConfig(head.getConfiguration());
    headConfig.setIterationId(ITERATION_ID);
    
    // initial input / partial solution
    headConfig.addInputToGroup(0);
    headConfig.setIterationHeadPartialSolutionOrWorksetInputIndex(0);
    headConfig.setInputSerializer(recSerializer, 0);
    headConfig.setInputComparator(fieldZeroComparator, 0);
    headConfig.setInputLocalStrategy(0, LocalStrategy.SORT);
    headConfig.setMemoryInput(0, minorConsumer * JobGraphUtils.MEGABYTE);
    headConfig.setFilehandlesInput(0, NUM_FILE_HANDLES_PER_SORT);
    headConfig.setSpillingThresholdInput(0, SORT_SPILL_THRESHOLD);
    
    // back channel / iterations
    headConfig.setBackChannelMemory(minorConsumer * JobGraphUtils.MEGABYTE);
    
    // output into iteration
    headConfig.setOutputSerializer(recSerializer);
    headConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    headConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    
    // final output
    TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
    headFinalOutConfig.setOutputSerializer(recSerializer);
    headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);
    
    // the sync
    headConfig.setIterationHeadIndexOfSyncOutput(3);
    headConfig.setNumberOfIterations(numIterations);
    
    // the driver 
    headConfig.setDriver(CollectorMapDriver.class);
    headConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
    headConfig.setStubWrapper(new UserCodeClassWrapper<CompensatingMap>(CompensatingMap.class));
    headConfig.setStubParameter("pageRank.numVertices", String.valueOf(numVertices));
    headConfig.setStubParameter("compensation.failingWorker", failingWorkers);
    headConfig.setStubParameter("compensation.failingIteration", String.valueOf(failingIteration));
    headConfig.setStubParameter("compensation.messageLoss", String.valueOf(messageLoss));
    headConfig.addIterationAggregator(CompensatableDotProductCoGroup.AGGREGATOR_NAME, PageRankStatsAggregator.class);


    // --------------- the join ---------------------
    
    JobTaskVertex intermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
      "IterationIntermediate", jobGraph, degreeOfParallelism, numSubTasksPerInstance);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());
    intermediateConfig.setIterationId(ITERATION_ID);
//    intermediateConfig.setDriver(RepeatableHashjoinMatchDriverWithCachedBuildside.class);
    intermediateConfig.setDriver(BuildSecondCachedMatchDriver.class);
    intermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
    intermediateConfig.setMemoryDriver(matchMemory * JobGraphUtils.MEGABYTE);
    intermediateConfig.addInputToGroup(0);
    intermediateConfig.addInputToGroup(1);
    intermediateConfig.setInputSerializer(recSerializer, 0);
    intermediateConfig.setInputSerializer(recSerializer, 1);
    intermediateConfig.setDriverComparator(fieldZeroComparator, 0);
    intermediateConfig.setDriverComparator(fieldZeroComparator, 1);
    intermediateConfig.setDriverPairComparator(pairComparatorFactory);
    
    intermediateConfig.setOutputSerializer(recSerializer);
    intermediateConfig.addOutputShipStrategy(ShipStrategyType.PARTITION_HASH);
    intermediateConfig.setOutputComparator(fieldZeroComparator, 0);
    
    intermediateConfig.setStubWrapper(new UserCodeClassWrapper<CompensatableDotProductMatch>(CompensatableDotProductMatch.class));
    intermediateConfig.setStubParameter("pageRank.numVertices", String.valueOf(numVertices));
    intermediateConfig.setStubParameter("compensation.failingWorker", failingWorkers);
    intermediateConfig.setStubParameter("compensation.failingIteration", String.valueOf(failingIteration));
    intermediateConfig.setStubParameter("compensation.messageLoss", String.valueOf(messageLoss));


    // ---------------- the tail (co group) --------------------
    
    JobTaskVertex tail = JobGraphUtils.createTask(IterationTailPactTask.class, "IterationTail", jobGraph,
      degreeOfParallelism, numSubTasksPerInstance);
    TaskConfig tailConfig = new TaskConfig(tail.getConfiguration());
    tailConfig.setIterationId(ITERATION_ID);
        tailConfig.setIsWorksetUpdate();
    // TODO we need to combine!
    
    // inputs and driver
    tailConfig.setDriver(CoGroupDriver.class);
    tailConfig.setDriverStrategy(DriverStrategy.CO_GROUP);
    tailConfig.addInputToGroup(0);
    tailConfig.addInputToGroup(1);
    tailConfig.setInputSerializer(recSerializer, 0);
    tailConfig.setInputSerializer(recSerializer, 1);
    tailConfig.setDriverComparator(fieldZeroComparator, 0);
    tailConfig.setDriverComparator(fieldZeroComparator, 1);
    tailConfig.setDriverPairComparator(pairComparatorFactory);
    tailConfig.setInputAsynchronouslyMaterialized(0, true);
    tailConfig.setInputMaterializationMemory(0, minorConsumer * JobGraphUtils.MEGABYTE);
    tailConfig.setInputLocalStrategy(1, LocalStrategy.SORT);
    tailConfig.setInputComparator(fieldZeroComparator, 1);
    tailConfig.setMemoryInput(1, coGroupSortMemory * JobGraphUtils.MEGABYTE);
    tailConfig.setFilehandlesInput(1, NUM_FILE_HANDLES_PER_SORT);
    tailConfig.setSpillingThresholdInput(1, SORT_SPILL_THRESHOLD);
    
    // output
    tailConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    tailConfig.setOutputSerializer(recSerializer);
    
    // the stub
    tailConfig.setStubWrapper(new UserCodeClassWrapper<CompensatableDotProductCoGroup>(CompensatableDotProductCoGroup.class));
    tailConfig.setStubParameter("pageRank.numVertices", String.valueOf(numVertices));
    tailConfig.setStubParameter("pageRank.numDanglingVertices", String.valueOf(numDanglingVertices));
    tailConfig.setStubParameter("compensation.failingWorker", failingWorkers);
    tailConfig.setStubParameter("compensation.failingIteration", String.valueOf(failingIteration));
    tailConfig.setStubParameter("compensation.messageLoss", String.valueOf(messageLoss));
    
    // --------------- the output ---------------------


    JobOutputVertex output = JobGraphUtils.createFileOutput(jobGraph, "FinalOutput", degreeOfParallelism,
      numSubTasksPerInstance);
    TaskConfig outputConfig = new TaskConfig(output.getConfiguration());
    outputConfig.addInputToGroup(0);
    outputConfig.setInputSerializer(recSerializer, 0);
    outputConfig.setStubWrapper(new UserCodeClassWrapper<PageWithRankOutFormat>(PageWithRankOutFormat.class));
    outputConfig.setStubParameter(FileOutputFormat.FILE_PARAMETER_KEY, outputPath);
    
    // --------------- the auxiliaries ---------------------
    
    JobOutputVertex fakeTailOutput = JobGraphUtils.createFakeOutput(jobGraph, "FakeTailOutput",
      degreeOfParallelism, numSubTasksPerInstance);


    JobOutputVertex sync = JobGraphUtils.createSync(jobGraph, degreeOfParallelism);
    TaskConfig syncConfig = new TaskConfig(sync.getConfiguration());
    syncConfig.setNumberOfIterations(numIterations);
    syncConfig.addIterationAggregator(CompensatableDotProductCoGroup.AGGREGATOR_NAME, PageRankStatsAggregator.class);
    syncConfig.setConvergenceCriterion(CompensatableDotProductCoGroup.AGGREGATOR_NAME, DiffL1NormConvergenceCriterion.class);
    syncConfig.setIterationId(ITERATION_ID);
    
    // --------------- the wiring ---------------------


    JobGraphUtils.connect(pageWithRankInput, head, ChannelType.NETWORK, DistributionPattern.BIPARTITE);

View Full Code Here

    }
    
    // obtain task configuration (including stub parameters)
    Configuration taskConf = getTaskConfiguration();
    taskConf.setClassLoader(this.userCodeClassLoader);
    this.config = new TaskConfig(taskConf);
    
    initInputFormat(this.userCodeClassLoader);
    
    try {
      initOutputs(this.userCodeClassLoader);

View Full Code Here

  }




  @Override
  public void prepare() throws Exception {
    final TaskConfig config = this.taskContext.getTaskConfig();
    final DriverStrategy ls = config.getDriverStrategy();
    
    switch (ls)
    {
    case NESTEDLOOP_BLOCKED_OUTER_FIRST:
      this.blocked = true;
      this.firstIsOuter = true;
      break;
    case NESTEDLOOP_BLOCKED_OUTER_SECOND:
      this.blocked = true;
      this.firstIsOuter = false;
      break;
    case NESTEDLOOP_STREAMED_OUTER_FIRST:
      this.blocked = false;
      this.firstIsOuter = true;
      break;
    case NESTEDLOOP_STREAMED_OUTER_SECOND:
      this.blocked = false;
      this.firstIsOuter = false;
      break;
    default:
      throw new RuntimeException("Invalid local strategy for CROSS: " + ls);
    }
    
    this.memManager = this.taskContext.getMemoryManager();
    final long totalAvailableMemory = config.getMemoryDriver();
    final int numPages = this.memManager.computeNumberOfPages(totalAvailableMemory);
    
    if (numPages < 2) {
      throw new RuntimeException(  "The Cross task was initialized with too little memory. " +
          "Cross requires at least 2 memory pages.");

View Full Code Here

    super.initOutputs();


    // at this time, the outputs to the step function are created
    // add the outputs for the final solution
    this.finalOutputWriters = new ArrayList<BufferWriter>();
    final TaskConfig finalOutConfig = this.config.getIterationHeadFinalOutputConfig();
    this.finalOutputCollector = RegularPactTask.getOutputCollector(this, finalOutConfig,
      this.userCodeClassLoader, this.finalOutputWriters, finalOutConfig.getNumOutputs());


    // sanity check the setup
    final int writersIntoStepFunction = this.eventualOutputs.size();
    final int writersIntoFinalResult = this.finalOutputWriters.size();
    final int syncGateIndex = this.config.getIterationHeadIndexOfSyncOutput();

View Full Code Here

          excludeFromReset(i);
        }
      }
    }
    
    TaskConfig config = getLastTasksConfig();
    isWorksetIteration = config.getIsWorksetIteration();
    isWorksetUpdate = config.getIsWorksetUpdate();
    isSolutionSetUpdate = config.getIsSolutionSetUpdate();


    if (isWorksetUpdate) {
      worksetBackChannel = BlockingBackChannelBroker.instance().getAndRemove(brokerKey());


      if (isWorksetIteration) {

View Full Code Here

  }


  @Override
  public void invoke() throws Exception {
    userCodeClassLoader = LibraryCacheManager.getClassLoader(getEnvironment().getJobID());
    TaskConfig taskConfig = new TaskConfig(getTaskConfiguration());
    
    // instantiate all aggregators
    this.aggregators = new HashMap<String, Aggregator<?>>();
    for (AggregatorWithName<?> aggWithName : taskConfig.getIterationAggregators()) {
      Aggregator<?> agg = InstantiationUtil.instantiate(aggWithName.getAggregator(), Aggregator.class);
      aggregators.put(aggWithName.getName(), agg);
    }
    
    // instantiate the aggregator convergence criterion
    if (taskConfig.usesConvergenceCriterion()) {
      Class<? extends ConvergenceCriterion<Value>> convClass = taskConfig.getConvergenceCriterion();
      convergenceCriterion = InstantiationUtil.instantiate(convClass, ConvergenceCriterion.class);
      convergenceAggregatorName = taskConfig.getConvergenceCriterionAggregatorName();
      Preconditions.checkNotNull(convergenceAggregatorName);
    }
    
    maxNumberOfIterations = taskConfig.getNumberOfIterations();
    
    // set up the event handler
    int numEventsTillEndOfSuperstep = taskConfig.getNumberOfEventsUntilInterruptInIterativeGate(0);
    eventHandler = new SyncEventHandler(numEventsTillEndOfSuperstep, aggregators, userCodeClassLoader);
    headEventReader.subscribeToEvent(eventHandler, WorkerDoneEvent.class);


    IntegerRecord dummy = new IntegerRecord();

View Full Code Here

0 1 2 3 4 5 6

TOP

Related Classes of eu.stratosphere.pact.runtime.task.util.TaskConfig

eu.stratosphere.api.common.distributions.DataDistribution

eu.stratosphere.compiler.plantranslate.NepheleJobGraphGenerator

eu.stratosphere.compiler.plantranslate.NepheleJobGraphGenerator$IterationDescriptor

eu.stratosphere.configuration.Configuration

eu.stratosphere.pact.runtime.iterative.task.AbstractIterativePactTask

eu.stratosphere.pact.runtime.iterative.task.IterationHeadPactTask

eu.stratosphere.pact.runtime.iterative.task.IterationSynchronizationSinkTask

eu.stratosphere.pact.runtime.task.AbstractCachedBuildSideMatchDriver

eu.stratosphere.pact.runtime.task.AllGroupReduceDriver

eu.stratosphere.pact.runtime.task.AllReduceDriver

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.