Examples of eu.stratosphere.api.java.typeutils.runtime.record.RecordComparatorFactory

Package eu.stratosphere.api.java.typeutils.runtime.record

Examples of eu.stratosphere.api.java.typeutils.runtime.record.RecordComparatorFactory

eu.stratosphere.api.java.typeutils.runtime.record.RecordComparatorFactory
A factory for a {@link TypeComparator} for {@link Record}. The comparator uses a subset of the fields for the comparison. That subset of fields (positions and types) is read from the supplied configuration.

    DataSinkTask<Record> testTask = new DataSinkTask<Record>();
    
    // set sorting
    super.getTaskConfig().setInputLocalStrategy(0, LocalStrategy.SORT);
    super.getTaskConfig().setInputComparator(
        new RecordComparatorFactory(new int[]{1},((Class<? extends Key<?>>[])new Class[]{IntValue.class})), 0);
    super.getTaskConfig().setMemoryInput(0, 4 * 1024 * 1024);
    super.getTaskConfig().setFilehandlesInput(0, 8);
    super.getTaskConfig().setSpillingThresholdInput(0, 0.8f);


    super.registerFileOutputTask(testTask, MockOutputFormat.class, new File(tempTestPath).toURI().toString());

View Full Code Here

    super.getTaskConfig().setStubParameters(stubParams);
    
    // set sorting
    super.getTaskConfig().setInputLocalStrategy(0, LocalStrategy.SORT);
    super.getTaskConfig().setInputComparator(
        new RecordComparatorFactory(new int[]{1},((Class<? extends Key<?>>[])new Class[]{IntValue.class})), 0);
    super.getTaskConfig().setMemoryInput(0, 4 * 1024 * 1024);
    super.getTaskConfig().setFilehandlesInput(0, 8);
    super.getTaskConfig().setSpillingThresholdInput(0, 0.8f);
    
    super.registerFileOutputTask(testTask, MockFailingOutputFormat.class, new File(tempTestPath).toURI().toString());

View Full Code Here

    super.getTaskConfig().setStubParameters(stubParams);
    
    // set sorting
    super.getTaskConfig().setInputLocalStrategy(0, LocalStrategy.SORT);
    super.getTaskConfig().setInputComparator(
        new RecordComparatorFactory(new int[]{1},((Class<? extends Key<?>>[])new Class[]{IntValue.class})), 
        0);
    super.getTaskConfig().setMemoryInput(0, 4 * 1024 * 1024);
    super.getTaskConfig().setFilehandlesInput(0, 8);
    super.getTaskConfig().setSpillingThresholdInput(0, 0.8f);

View Full Code Here

  
  @Test
  public void testPartitionHash() {
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {0}, new Class[] {IntValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    
    int numChans = 100;
    int numRecs = 50000;
    int[] hit = new int[numChans];


    for (int i = 0; i < numRecs; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      
      delegate.setInstance(rec);
      
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }


    int cnt = 0;
    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i] > 0);
      cnt += hit[i];
    }
    assertTrue(cnt == numRecs);


    // Test for StringValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] {0}, new Class[] {StringValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, stringComp);


    numChans = 100;
    numRecs = 10000;

View Full Code Here

  
  @Test
  public void testForward() {
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {0}, new Class[] {IntValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    
    int numChannels = 100;
    int numRecords = 50000;
    
    int[] hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      delegate.setInstance(rec);
      
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }


    int cnt = 0;
    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i] == (numRecords/numChannels) || hit[i] == (numRecords/numChannels)-1);
      cnt += hit[i];
    }
    assertTrue(cnt == numRecords);


    // Test for StringValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] {0}, new Class[] {StringValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.FORWARD, stringComp);


    numChannels = 100;
    numRecords = 10000;

View Full Code Here

  
  @Test
  public void testBroadcast() {
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {0}, new Class[] {IntValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.BROADCAST, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    
    int numChannels = 100;
    int numRecords = 50000;
    
    int[] hit = new int[numChannels];


    for (int i = 0; i < numRecords; i++) {
      IntValue k = new IntValue(i);
      Record rec = new Record(k);
      delegate.setInstance(rec);
      
      int[] chans = oe1.selectChannels(delegate, hit.length);
      for(int j=0; j < chans.length; j++) {
        hit[chans[j]]++;
      }
    }


    for (int i = 0; i < hit.length; i++) {
      assertTrue(hit[i]+"", hit[i] == numRecords);
    }
    
    // Test for StringValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> stringComp = new RecordComparatorFactory(new int[] {0}, new Class[] {StringValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe2 = new OutputEmitter<Record>(ShipStrategyType.BROADCAST, stringComp);


    numChannels = 100;
    numRecords = 5000;

View Full Code Here

  }
  
  @Test
  public void testMultiKeys() {
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> multiComp = new RecordComparatorFactory(new int[] {0,1,3}, new Class[] {IntValue.class, StringValue.class, DoubleValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, multiComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    
    int numChannels = 100;
    int numRecords = 5000;

View Full Code Here

  
  @Test
  public void testMissingKey() {
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {1}, new Class[] {IntValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    
    Record rec = new Record(0);
    rec.setField(0, new IntValue(1));

View Full Code Here

  
  @Test
  public void testNullKey() {
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> intComp = new RecordComparatorFactory(new int[] {0}, new Class[] {IntValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, intComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    
    Record rec = new Record(2);
    rec.setField(1, new IntValue(1));

View Full Code Here

  @Test
  public void testWrongKeyClass() {
    
    // Test for IntValue
    @SuppressWarnings("unchecked")
    final TypeComparator<Record> doubleComp = new RecordComparatorFactory(new int[] {0}, new Class[] {DoubleValue.class}).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, doubleComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    
    PipedInputStream pipedInput = new PipedInputStream(1024*1024);
    DataInputStream in = new DataInputStream(pipedInput);

View Full Code Here

0 1

TOP

Related Classes of eu.stratosphere.api.java.typeutils.runtime.record.RecordComparatorFactory

eu.stratosphere.compiler.postpass.RecordModelPostPass

eu.stratosphere.pact.runtime.task.DataSinkTaskTest

eu.stratosphere.pact.runtime.task.util.OutputEmitterTest

eu.stratosphere.test.broadcastvars.KMeansIterativeNepheleITCase

eu.stratosphere.test.iterative.nephele.ConnectedComponentsNepheleITCase

eu.stratosphere.test.iterative.nephele.IterationWithChainingNepheleITCase

eu.stratosphere.configuration.IllegalConfigurationException

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.