Examples of eu.stratosphere.configuration.Configuration

eu.stratosphere.configuration.Configuration
Lightweight configuration object which can store key/value pairs. Configuration objects can be extracted from or integrated into the {@link GlobalConfiguration} object. They canbe transported via Nephele's IPC system to distribute configuration data at runtime. This class is thread-safe.

    // output into iteration. broadcasting the centers
    headConfig.setOutputSerializer(serializer);
    headConfig.addOutputShipStrategy(ShipStrategyType.BROADCAST);
    
    // final output
    TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
    headFinalOutConfig.setOutputSerializer(serializer);
    headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);
    
    // the sync

View Full Code Here

  public static Collection<Object[]> getConfigurations() throws FileNotFoundException, IOException {


    LinkedList<Configuration> tConfigs = new LinkedList<Configuration>();


    for(int i=1; i <= NUM_PROGRAMS; i++) {
      Configuration config = new Configuration();
      config.setInteger("ProgramId", i);
      tConfigs.add(config);
    }
    
    return toParameterList(tConfigs);
  }

View Full Code Here

        final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        
        DataSet<Tuple3<Integer, Long, String>> ds = CollectionDataSets.get3TupleDataSet(env)
              .map(new IdentityMapper<Tuple3<Integer,Long,String>>()).setParallelism(4);
        
        Configuration cfg = new Configuration();
        cfg.setString(PactCompiler.HINT_SHIP_STRATEGY, PactCompiler.HINT_SHIP_STRATEGY_REPARTITION);
        DataSet<Tuple2<Integer, String>> reduceDs = ds.reduceGroup(new Tuple3AllGroupReduceWithCombine())
            .withParameters(cfg);
        
        reduceDs.writeAsCsv(resultPath);
        env.execute();

View Full Code Here

    final JobID jobID = new JobID();
    final ExecutionVertexID vertexID = new ExecutionVertexID();
    final String taskName = "task name";
    final int indexInSubtaskGroup = 0;
    final int currentNumberOfSubtasks = 1;
    final Configuration jobConfiguration = new Configuration();
    final Configuration taskConfiguration = new Configuration();
    final Class<? extends AbstractInvokable> invokableClass = FileLineReader.class;
    final SerializableArrayList<GateDeploymentDescriptor> outputGates = new SerializableArrayList<GateDeploymentDescriptor>(
      0);
    final SerializableArrayList<GateDeploymentDescriptor> inputGates = new SerializableArrayList<GateDeploymentDescriptor>(
      0);

View Full Code Here

    headConfig.setOutputSerializer(vertexWithRankAndDanglingSerializer);
    headConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    headConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    
    // final output
    TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
    headFinalOutConfig.setOutputSerializer(vertexWithRankAndDanglingSerializer);
    headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);
    
    // the sync

View Full Code Here

      assertEquals(2, descr.getMaximumNumberOfAvailableInstances());
      
      
      // allocate something
      JobID jobID = new JobID();
      Configuration conf = new Configuration();
      InstanceRequestMap instanceRequestMap = new InstanceRequestMap();
      instanceRequestMap.setNumberOfInstances(cm.getDefaultInstanceType(), 2);
      cm.requestInstance(jobID, conf, instanceRequestMap, null);
      
      ClusterManagerTestUtils.waitForInstances(jobID, testInstanceListener, 3, 1000);

View Full Code Here

    try {
      
      final int CLEANUP_INTERVAL = 2;
      
      // configure a short cleanup interval
      Configuration config = new Configuration();
      config.setInteger("instancemanager.cluster.cleanupinterval", CLEANUP_INTERVAL);
      GlobalConfiguration.includeConfiguration(config);
      
      ClusterManager cm = new ClusterManager();
      TestInstanceListener testInstanceListener = new TestInstanceListener();
      cm.setInstanceListener(testInstanceListener);
      
      
      int ipcPort = ConfigConstants.DEFAULT_TASK_MANAGER_IPC_PORT;
      int dataPort = ConfigConstants.DEFAULT_TASK_MANAGER_DATA_PORT;


      HardwareDescription hardwareDescription = HardwareDescriptionFactory.construct(2, 2L * 1024L * 1024L * 1024L,
                                        2L * 1024L * 1024L * 1024L);


      String hostname = "192.168.198.1";
      InetAddress address = InetAddress.getByName("192.168.198.1");
      
      InstanceConnectionInfo ici1 = new InstanceConnectionInfo(address, hostname, null, ipcPort + 0, dataPort + 0);
      InstanceConnectionInfo ici2 = new InstanceConnectionInfo(address, hostname, null, ipcPort + 15, dataPort + 15);
      InstanceConnectionInfo ici3 = new InstanceConnectionInfo(address, hostname, null, ipcPort + 30, dataPort + 30);
      
      // register three instances
      cm.reportHeartBeat(ici1, hardwareDescription);
      cm.reportHeartBeat(ici2, hardwareDescription);
      cm.reportHeartBeat(ici3, hardwareDescription);
      
      
      Map<InstanceType, InstanceTypeDescription> instanceTypeDescriptions = cm.getMapOfAvailableInstanceTypes();
      assertEquals(1, instanceTypeDescriptions.size());
      
      InstanceTypeDescription descr = instanceTypeDescriptions.entrySet().iterator().next().getValue();
      assertEquals(3, descr.getMaximumNumberOfAvailableInstances());
      
      // request some instances
      JobID jobID = new JobID();
      Configuration conf = new Configuration();


      InstanceRequestMap instancem = new InstanceRequestMap();
      instancem.setNumberOfInstances(cm.getDefaultInstanceType(), 1);
      cm.requestInstance(jobID, conf, instancem, null);

View Full Code Here

    headConfig.setOutputSerializer(vertexWithRankAndDanglingSerializer);
    headConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    headConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    
    // final output
    TaskConfig headFinalOutConfig = new TaskConfig(new Configuration());
    headFinalOutConfig.setOutputSerializer(vertexWithRankAndDanglingSerializer);
    headFinalOutConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    headConfig.setIterationHeadFinalOutputConfig(headFinalOutConfig);
    
    // the sync
    headConfig.setIterationHeadIndexOfSyncOutput(3);
    headConfig.setNumberOfIterations(numIterations);
    
    // the driver 
    headConfig.setDriver(CollectorMapDriver.class);
    headConfig.setDriverStrategy(DriverStrategy.COLLECTOR_MAP);
    headConfig.setStubWrapper(new UserCodeClassWrapper<CustomCompensatingMap>(CustomCompensatingMap.class));
    headConfig.setStubParameter("pageRank.numVertices", String.valueOf(numVertices));
    headConfig.setStubParameter("compensation.failingWorker", failingWorkers);
    headConfig.setStubParameter("compensation.failingIteration", String.valueOf(failingIteration));
    headConfig.setStubParameter("compensation.messageLoss", String.valueOf(messageLoss));
    headConfig.addIterationAggregator(CustomCompensatableDotProductCoGroup.AGGREGATOR_NAME, PageRankStatsAggregator.class);


    // --------------- the join ---------------------
    
    JobTaskVertex intermediate = JobGraphUtils.createTask(IterationIntermediatePactTask.class,
      "IterationIntermediate", jobGraph, degreeOfParallelism, numSubTasksPerInstance);
    TaskConfig intermediateConfig = new TaskConfig(intermediate.getConfiguration());
    intermediateConfig.setIterationId(ITERATION_ID);
//    intermediateConfig.setDriver(RepeatableHashjoinMatchDriverWithCachedBuildside.class);
    intermediateConfig.setDriver(BuildSecondCachedMatchDriver.class);
    intermediateConfig.setDriverStrategy(DriverStrategy.HYBRIDHASH_BUILD_SECOND);
    intermediateConfig.setMemoryDriver(matchMemory * JobGraphUtils.MEGABYTE);
    intermediateConfig.addInputToGroup(0);
    intermediateConfig.addInputToGroup(1);
    intermediateConfig.setInputSerializer(vertexWithRankAndDanglingSerializer, 0);
    intermediateConfig.setInputSerializer(vertexWithAdjacencyListSerializer, 1);
    intermediateConfig.setDriverComparator(vertexWithRankAndDanglingComparator, 0);
    intermediateConfig.setDriverComparator(vertexWithAdjacencyListComparator, 1);
    intermediateConfig.setDriverPairComparator(matchComparator);
    
    intermediateConfig.setOutputSerializer(vertexWithRankSerializer);
    intermediateConfig.addOutputShipStrategy(ShipStrategyType.FORWARD);
    
    intermediateConfig.setStubWrapper(new UserCodeClassWrapper<CustomCompensatableDotProductMatch>(CustomCompensatableDotProductMatch.class));
    intermediateConfig.setStubParameter("pageRank.numVertices", String.valueOf(numVertices));
    intermediateConfig.setStubParameter("compensation.failingWorker", failingWorkers);
    intermediateConfig.setStubParameter("compensation.failingIteration", String.valueOf(failingIteration));
    intermediateConfig.setStubParameter("compensation.messageLoss", String.valueOf(messageLoss));
    
    // the combiner and the output
    TaskConfig combinerConfig = new TaskConfig(new Configuration());
    combinerConfig.addInputToGroup(0);
    combinerConfig.setInputSerializer(vertexWithRankSerializer, 0);
    combinerConfig.setDriverStrategy(DriverStrategy.SORTED_GROUP_COMBINE);
    combinerConfig.setDriverComparator(vertexWithRankComparator, 0);
    combinerConfig.setMemoryDriver(coGroupSortMemory * JobGraphUtils.MEGABYTE);

View Full Code Here

    final JobID jobID = new JobID();
    final ExecutionVertexID vertexID = new ExecutionVertexID();
    final String taskName = "task name";
    final int indexInSubtaskGroup = 0;
    final int currentNumberOfSubtasks = 1;
    final Configuration jobConfiguration = new Configuration();
    final Configuration taskConfiguration = new Configuration();
    final Class<? extends AbstractInvokable> invokableClass = FileLineReader.class;
    final SerializableArrayList<GateDeploymentDescriptor> outputGates = new SerializableArrayList<GateDeploymentDescriptor>(
      0);
    final SerializableArrayList<GateDeploymentDescriptor> inputGates = new SerializableArrayList<GateDeploymentDescriptor>(
      0);

View Full Code Here

    final JobID jobID = new JobID();
    final ExecutionVertexID vertexID = new ExecutionVertexID();
    final String taskName = "task name";
    final int indexInSubtaskGroup = 0;
    final int currentNumberOfSubtasks = 1;
    final Configuration jobConfiguration = new Configuration();
    final Configuration taskConfiguration = new Configuration();
    final Class<? extends AbstractInvokable> invokableClass = FileLineReader.class;
    final SerializableArrayList<GateDeploymentDescriptor> outputGates = new SerializableArrayList<GateDeploymentDescriptor>(
      0);
    final SerializableArrayList<GateDeploymentDescriptor> inputGates = new SerializableArrayList<GateDeploymentDescriptor>(
      0);

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of eu.stratosphere.configuration.Configuration

eu.stratosphere.api.common.io.DelimitedInputFormatSamplingTest

eu.stratosphere.api.common.io.DelimitedInputFormatTest

eu.stratosphere.api.common.io.FileInputFormatTest

eu.stratosphere.api.common.io.FileOutputFormatTest

eu.stratosphere.api.common.io.FormatUtil

eu.stratosphere.api.common.io.GenericCsvInputFormatTest

eu.stratosphere.api.common.io.SequentialFormatTest

eu.stratosphere.api.java.io.CsvInputFormatTest

eu.stratosphere.api.java.io.TextInputFormatTest

eu.stratosphere.api.java.operators.OperatorTranslation

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.