Examples of RecordComparatorFactory

eu.stratosphere.api.java.typeutils.runtime.record.RecordComparatorFactory
A factory for a {@link TypeComparator} for {@link Record}. The comparator uses a subset of the fields for the comparison. That subset of fields (positions and types) is read from the supplied configuration.
org.apache.flink.api.common.typeutils.record.RecordComparatorFactory
A factory for a {@link org.apache.flink.api.common.typeutils.TypeComparator} for {@link Record}. The comparator uses a subset of the fields for the comparison. That subset of fields (positions and types) is read from the supplied configuration.

Examples of org.apache.flink.api.common.typeutils.record.RecordComparatorFactory

  {
    // -- init -------------------------------------------------------------------------------------------------
    final TypeSerializerFactory<?> serializer = RecordSerializerFactory.get();
    @SuppressWarnings("unchecked")
    final TypeComparatorFactory<?> comparator =
      new RecordComparatorFactory(new int[] { 0 }, new Class[] { LongValue.class }, new boolean[] { true });
    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();


    JobGraph jobGraph = new JobGraph("Connected Components (Unified Tails)");


    // -- invariant vertices -----------------------------------------------------------------------------------

View Full Code Here

Examples of org.apache.flink.api.common.typeutils.record.RecordComparatorFactory

  {
    // -- init -------------------------------------------------------------------------------------------------
    final TypeSerializerFactory<?> serializer = RecordSerializerFactory.get();
    @SuppressWarnings("unchecked")
    final TypeComparatorFactory<?> comparator =
      new RecordComparatorFactory(new int[] { 0 }, new Class[] { LongValue.class }, new boolean[] { true });
    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();


    JobGraph jobGraph = new JobGraph("Connected Components (Unified Tails)");


    // input

View Full Code Here

Examples of org.apache.flink.api.common.typeutils.record.RecordComparatorFactory

  {
    // -- init -------------------------------------------------------------------------------------------------
    final TypeSerializerFactory<?> serializer = RecordSerializerFactory.get();
    @SuppressWarnings("unchecked")
    final TypeComparatorFactory<?> comparator =
      new RecordComparatorFactory(new int[] { 0 }, new Class[] { LongValue.class }, new boolean[] { true });
    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();


    JobGraph jobGraph = new JobGraph("Connected Components (Intermediate Workset Update, Solution Set Tail)");


    // input

View Full Code Here

Examples of org.apache.flink.api.common.typeutils.record.RecordComparatorFactory

  {
    // -- init -------------------------------------------------------------------------------------------------
    final TypeSerializerFactory<?> serializer = RecordSerializerFactory.get();
    @SuppressWarnings("unchecked")
    final TypeComparatorFactory<?> comparator =
      new RecordComparatorFactory(new int[] { 0 }, new Class[] { LongValue.class }, new boolean[] { true });
    final TypePairComparatorFactory<?, ?> pairComparator = RecordPairComparatorFactory.get();


    JobGraph jobGraph = new JobGraph("Connected Components (Intermediate Solution Set Update, Workset Tail)");


    // input

View Full Code Here

Examples of org.apache.flink.api.common.typeutils.record.RecordComparatorFactory

    DataSinkTask<Record> testTask = new DataSinkTask<Record>();
    
    // set sorting
    super.getTaskConfig().setInputLocalStrategy(0, LocalStrategy.SORT);
    super.getTaskConfig().setInputComparator(
        new RecordComparatorFactory(new int[]{1},((Class<? extends Key<?>>[])new Class[]{IntValue.class})),
        0);
    super.getTaskConfig().setRelativeMemoryInput(0, memoryFraction);
    super.getTaskConfig().setFilehandlesInput(0, 8);
    super.getTaskConfig().setSpillingThresholdInput(0, 0.8f);

View Full Code Here

Examples of org.apache.flink.api.common.typeutils.record.RecordComparatorFactory

    super.getTaskConfig().setStubParameters(stubParams);
    
    // set sorting
    super.getTaskConfig().setInputLocalStrategy(0, LocalStrategy.SORT);
    super.getTaskConfig().setInputComparator(
        new RecordComparatorFactory(new int[]{1},((Class<? extends Key<?>>[])new Class[]{IntValue.class})),
        0);
    super.getTaskConfig().setRelativeMemoryInput(0, memoryFraction);
    super.getTaskConfig().setFilehandlesInput(0, 8);
    super.getTaskConfig().setSpillingThresholdInput(0, 0.8f);

View Full Code Here

Examples of org.apache.flink.api.common.typeutils.record.RecordComparatorFactory

    super.getTaskConfig().setStubParameters(stubParams);
    
    // set sorting
    super.getTaskConfig().setInputLocalStrategy(0, LocalStrategy.SORT);
    super.getTaskConfig().setInputComparator(
        new RecordComparatorFactory(new int[]{1},((Class<? extends Key<?>>[])new Class[]{IntValue.class})), 
        0);
    super.getTaskConfig().setRelativeMemoryInput(0, memoryFraction);
    super.getTaskConfig().setFilehandlesInput(0, 8);
    super.getTaskConfig().setSpillingThresholdInput(0, 0.8f);

View Full Code Here

Examples of org.apache.flink.api.common.typeutils.record.RecordComparatorFactory

  
  @Test
  public void testPartitionHash() {
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {0}, new Class[] {IntValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    
    int numChans = 100;
    int numRecs = 50000;
    int[] hit = new int[numChans];


    for (int i = 0; i < numRecs; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      
      delegate.setInstance(rec);
      
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }


    int cnt = 0;
    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i] > 0);
      cnt += hit[i];
    }
    assertTrue(cnt == numRecs);


    // Test for StringValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] {0}, new Class[] {StringValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, stringComp);


    numChans = 100;
    numRecs = 10000;

View Full Code Here

Examples of org.apache.flink.api.common.typeutils.record.RecordComparatorFactory

  
  @Test
  public void testForward() {
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {0}, new Class[] {IntValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    
    int numChannels = 100;
    int numRecords = 50000;
    
    int[] hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      delegate.setInstance(rec);
      
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }


    int cnt = 0;
    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i] == (numRecords/numChannels) || hit[i] == (numRecords/numChannels)-1);
      cnt += hit[i];
    }
    assertTrue(cnt == numRecords);


    // Test for StringValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] {0}, new Class[] {StringValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, stringComp);


    numChannels = 100;
    numRecords = 10000;

View Full Code Here

Examples of org.apache.flink.api.common.typeutils.record.RecordComparatorFactory

  
  @Test
  public void testBroadcast() {
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {0}, new Class[] {IntValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.BROADCAST, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    
    int numChannels = 100;
    int numRecords = 50000;
    
    int[] hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      delegate.setInstance(rec);
      
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }


    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i]+"", hit[i] == numRecords);
    }
    
    // Test for StringValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] {0}, new Class[] {StringValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.BROADCAST, stringComp);


    numChannels = 100;
    numRecords = 5000;

View Full Code Here

0 1 2 3

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.