Examples of org.apache.hadoop.util.bloom.Filter

org.apache.hadoop.util.bloom.Filter
Defines the general behavior of a filter.
A filter is a data structure which aims at offering a lossy summary of a set A. The key idea is to map entries of A (also called keys) into several positions in a vector through the use of several hash functions.
Typically, a filter will be implemented as a Bloom filter (or a Bloom filter extension).
It must be extended in order to define the real behavior. @see Key The general behavior of a key @see HashFunction A hash function

  };
  
  /** test standalone filter without tokenfilter wrapping */
  @Test
  public void testFilter() throws IOException {
    Filter filter = getFilter(filterTokens);
    Key k = new Key();
    for (String s: filterTokens) {
      setKey(k,s);
      assertTrue("Key for string " + s + " should be filter member", filter.membershipTest(k));
    }
    
    for (String s: notFilterTokens)  {
      setKey(k,s);
      assertFalse("Key for string " + s + " should not be filter member", filter.membershipTest(k));
    }
  }

View Full Code Here

    }
    assertEquals("Analyzer produced too few terms", expected.length, pos);
  }


  private static Filter getFilter(String[] tokens) throws IOException {
    Filter filter = new BloomFilter(100,50, Hash.JENKINS_HASH);
    Key k = new Key();
    for (String s: tokens) {
      setKey(k,s);
      filter.add(k);
    }
    return filter;
  }

View Full Code Here

  };
  
  /** test standalone filter without tokenfilter wrapping */
  @Test
  public void testFilter() throws IOException {
    Filter filter = getFilter(filterTokens);
    Key k = new Key();
    for (String s: filterTokens) {
      setKey(k,s);
      assertTrue("Key for string " + s + " should be filter member", filter.membershipTest(k));
    }
    
    for (String s: notFilterTokens)  {
      setKey(k,s);
      assertFalse("Key for string " + s + " should not be filter member", filter.membershipTest(k));
    }
  }

View Full Code Here

    }
    assertEquals("Analyzer produced too few terms", expected.length, pos);
  }


  private static Filter getFilter(String[] tokens) throws IOException {
    Filter filter = new BloomFilter(100,50, Hash.JENKINS_HASH);
    Key k = new Key();
    for (String s: tokens) {
      setKey(k,s);
      filter.add(k);
    }
    return filter;
  }

View Full Code Here

  };
  
  /** test standalone filter without tokenfilter wrapping */
  @Test
  public void testFilter() throws IOException {
    Filter filter = getFilter(filterTokens);
    Key k = new Key();
    for (String s: filterTokens) {
      setKey(k,s);
      assertTrue("Key for string " + s + " should be filter member", filter.membershipTest(k));
    }
    
    for (String s: notFilterTokens)  {
      setKey(k,s);
      assertFalse("Key for string " + s + " should not be filter member", filter.membershipTest(k));
    }
  }

View Full Code Here

    }
    assertEquals("Analyzer produced too few terms", expected.length, pos);
  }


  private static Filter getFilter(String[] tokens) throws IOException {
    Filter filter = new BloomFilter(100,50, Hash.JENKINS_HASH);
    Key k = new Key();
    for (String s: tokens) {
      setKey(k,s);
      filter.add(k);
    }
    return filter;
  }

View Full Code Here

  ///@Test
  public void testBloom() {
    
    int numElems = 10*1000000;
    double pct = 0.01;
    Filter bloom = BloomFactory.NewBloomInstance( numElems, pct);
    
    for( int i = 0; i< numElems; ++i ) {
      UUID uuid  = UUID.randomUUID();
    
      Key key =  new Key(uuid.toString().getBytes() );
      bloom.add( key );
      
      Assert.assertTrue( bloom.membershipTest(key));
      if( (i % 10000) == 0 ) {
          System.out.println(" Added " + i + " elements.");
      }
    }
    
  
    int numHits = 0;
    for( int i=0; i< numElems; ++i) {
      UUID uuid  = UUID.randomUUID();
      Key key =  new Key(uuid.toString().getBytes() );
      if( bloom.membershipTest(key)) {
        numHits++;
      }
    }
    System.out.print("Number of hits = " + numHits + " out of " + numElems + " or " + ((double)numHits/(double)numElems)*100.0 + " %");
    Assert.assertTrue( numHits/numElems <= pct);

View Full Code Here

    
    int numElems = 100000;
    double pct = 0.01;
    HashSet<String> unionMap = new HashSet<String>();
    
    Filter bloom1 = BloomFactory.NewBloomInstance( numElems, pct);
    for( int i = 0; i< numElems/2; ++i ) {
      UUID uuid  = UUID.randomUUID();
    
      Key key =  new Key(uuid.toString().getBytes() );
      bloom1.add( key );
      
      Assert.assertTrue( bloom1.membershipTest(key));
        (unionMap).add( uuid.toString());
      
      if( (i % 10000) == 0 ) {
          System.out.println(" Added " + i + " elements.");
      }
    }
    Filter bloom2 = BloomFactory.NewBloomInstance( numElems, pct);
    
    for( int i = 0; i< numElems/2; ++i ) {
      UUID uuid  = UUID.randomUUID();
    
      Key key =  new Key(uuid.toString().getBytes() );
      bloom2.add( key );
      
      Assert.assertTrue( bloom2.membershipTest(key));
        (unionMap).add( uuid.toString());
      
      if( (i % 10000) == 0 ) {
          System.out.println(" Added " + i + " elements.");
      }

View Full Code Here

          bloomFilter = BloomFactory.ReadBloomFromString(partial);
          ///LOG.info(" read bloom from partial " + BloomFactory.WriteBloomToString(bloomFilter));
          return true;
        } else{
          ///LOG.info(" ORng with merged before " + BloomFactory.WriteBloomToString(bloomFilter) );
          Filter other = BloomFactory.ReadBloomFromString(partial);
          ///LOG.info("ORng with merged other " + BloomFactory.WriteBloomToString(other) );
          bloomFilter.or(other);
          ///LOG.info(" ORing with merged after " + BloomFactory.WriteBloomToString(bloomFilter) );
          return true;
        }

View Full Code Here

              "_FUNC_(string bloom) "
    )
public class BloomNotUDF extends UDF {


  public String evaluate( String bloomStr ) throws IOException {
    Filter bloom = BloomFactory.GetBloomFilter( bloomStr);
    
    /// Perform a logical not 
    bloom.not();
    
    return BloomFactory.WriteBloomToString( bloom);
  }

View Full Code Here

0 1

TOP

Related Classes of org.apache.hadoop.util.bloom.Filter

brickhouse.udf.bloom.BloomAndUDF

brickhouse.udf.bloom.BloomContainsUDF

brickhouse.udf.bloom.BloomFactory

brickhouse.udf.bloom.BloomNotUDF

brickhouse.udf.bloom.BloomOrUDF

brickhouse.udf.bloom.BloomTest

brickhouse.udf.bloom.BloomUDAF$BloomUDAFEvaluator

brickhouse.udf.bloom.DistributedBloomUDF

org.apache.mahout.utils.nlp.collocations.llr.BloomTokenFilterTest

sizzle.aggregators.UniqueAggregator

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.