Package eu.stratosphere.api.common.distributions

Examples of eu.stratosphere.api.common.distributions.DataDistribution


    final int NUM_BUCKETS = 137;
    final double BUCKET_WIDTH = DISTR_RANGE / ((double) NUM_BUCKETS);
   
    final int NUM_ELEMENTS = 10000000;
   
    final DataDistribution distri = new UniformIntegerDistribution(DISTR_MIN, DISTR_MAX);
   
    @SuppressWarnings("unchecked")
    final RecordComparator intComp = new RecordComparator(new int[] {0}, new Class[] {IntValue.class});
    final ChannelSelector<Record> oe = new RecordOutputEmitter(ShipStrategyType.PARTITION_RANGE, intComp, distri);
   
View Full Code Here


    } catch (ClassCastException ccex) {
      throw new CorruptConfigurationException("The class noted in the configuration as the data distribution " +
          "is no subclass of DataDistribution.");
    }
   
    final DataDistribution distribution = InstantiationUtil.instantiate(clazz, DataDistribution.class);
   
    final byte[] stateEncoded = this.config.getBytes(OUTPUT_DATA_DISTRIBUTION_PREFIX + outputNum, null);
    if (stateEncoded == null) {
      throw new CorruptConfigurationException(
            "The configuration contained the data distribution type, but no serialized state.");
    }
   
    final ByteArrayInputStream bais = new ByteArrayInputStream(stateEncoded);
    final DataInputStream in = new DataInputStream(bais);
   
    try {
      distribution.read(in);
      return distribution;
    } catch (Exception ex) {
      throw new RuntimeException("The deserialization of the encoded data distribution state caused an error"
        + ex.getMessage() == null ? "." : ": " + ex.getMessage(), ex);
    }
View Full Code Here

          @SuppressWarnings("unchecked")
          TypeComparator<Record> comparator = (TypeComparator<Record>) compFact.createComparator();
          if (!comparator.supportsCompareAgainstReference()) {
            throw new Exception("Incompatibe serializer-/comparator factories.");
          }
          final DataDistribution distribution = config.getOutputDataDistribution(i, cl);
          oe = new RecordOutputEmitter(strategy, comparator, distribution);
        }

        if (task instanceof AbstractTask) {
          writers.add(new RecordWriter<Record>((AbstractTask) task, oe));
        } else if (task instanceof AbstractInputTask<?>) {
          writers.add(new RecordWriter<Record>((AbstractInputTask<?>) task, oe));
        }
      }
      if (eventualOutputs != null) {
        eventualOutputs.addAll(writers);
      }

      @SuppressWarnings("unchecked")
      final Collector<T> outColl = (Collector<T>) new RecordOutputCollector(writers);
      return outColl;
    }
    else {
      // generic case
      final List<RecordWriter<SerializationDelegate<T>>> writers = new ArrayList<RecordWriter<SerializationDelegate<T>>>(numOutputs);

      // create a writer for each output
      for (int i = 0; i < numOutputs; i++)
      {
        // create the OutputEmitter from output ship strategy
        final ShipStrategyType strategy = config.getOutputShipStrategy(i);
        final TypeComparatorFactory<T> compFactory = config.getOutputComparator(i, cl);
        final DataDistribution dataDist = config.getOutputDataDistribution(i, cl);

        final ChannelSelector<SerializationDelegate<T>> oe;
        if (compFactory == null) {
          oe = new OutputEmitter<T>(strategy);
        } else if (dataDist == null){
View Full Code Here

      sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex);
    }
   
    if (channel.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
     
      final DataDistribution dataDistribution = channel.getDataDistribution();
      if(dataDistribution != null) {
        sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex);
      } else {
        throw new RuntimeException("Range partitioning requires data distribution");
        // TODO: inject code and configuration for automatic histogram generation
View Full Code Here

  public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
    final InterestingProperties iProps = new InterestingProperties();
   
    {
      final Ordering partitioning = getPactContract().getPartitionOrdering();
      final DataDistribution dataDist = getPactContract().getDataDistribution();
      final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties();
      if (partitioning != null) {
        if(dataDist != null) {
          partitioningProps.setRangePartitioned(partitioning, dataDist);
        } else {
View Full Code Here

TOP

Related Classes of eu.stratosphere.api.common.distributions.DataDistribution

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.