Examples of edu.umd.cloud9.io.map.HMapIDW

edu.umd.cloud9.io.map.HMapIFW

Writable representing a map where keys are ints and values are doubles.

One notable feature of this class is the ability to support lazy decoding, controlled by the {@link #setLazyDecodeFlag(boolean)} method. In lazy decoding mode, when an object of thistype is deserialized, key-value pairs are not inserted into the map, but rather held in arrays. The reduces memory used in cases where random access to values is not required. In lazy decoding mode, the raw keys and values may be fetched by the {@link #getKeys()} and {@link #getValues()}methods, respectively. The map can be subsequently populated with the {@link #decode()} method.
@author Jimmy Lin

    assertTrue(n2.isDecoded());
  }


  @Test
  public void testSerializeEmpty() throws IOException {
    HMapIDW m1 = new HMapIDW();


    // make sure this does nothing
    m1.decode();


    assertTrue(m1.size() == 0);


    byte[] bytes = m1.serialize();
    HMapIDW m2 = HMapIDW.create(bytes);


    assertTrue(m2.size() == 0);
  }

View Full Code Here


public class HMapIDWTest {


  @Test
  public void testBasic() throws IOException {
    HMapIDW m = new HMapIDW();


    m.put(2, 5.0);
    m.put(1, 22.0);


    double value;


    assertEquals(m.size(), 2);


    value = m.get(2);
    assertTrue(value == 5.0);


    value = m.remove(2);
    assertEquals(m.size(), 1);


    value = m.get(1);
    assertTrue(value == 22.0);
  }

View Full Code Here

  }


  @Test
  public void testSerialize1() throws IOException {
    HMapIDW.setLazyDecodeFlag(false);
    HMapIDW m1 = new HMapIDW();


    m1.put(3, 5.0);
    m1.put(4, 22.0);


    byte[] bytes = m1.serialize();
    HMapIDW n2 = HMapIDW.create(bytes);


    double value;


    assertEquals(n2.size(), 2);


    value = n2.get(3);
    assertTrue(value == 5.0);


    value = n2.remove(3);
    assertEquals(n2.size(), 1);


    value = n2.get(4);
    assertTrue(value == 22.0);
  }

View Full Code Here

  }


  @Test
  public void testSerializeLazy1() throws IOException {
    HMapIDW.setLazyDecodeFlag(true);
    HMapIDW m1 = new HMapIDW();


    m1.put(3, 5.0);
    m1.put(4, 22.0);


    byte[] bytes = m1.serialize();
    HMapIDW m2 = HMapIDW.create(bytes);


    assertFalse(m2.isDecoded());
    assertEquals(2, m2.size());


    int[] keys = m2.getKeys();
    double[] values = m2.getValues();


    assertTrue(keys[0] == 3);
    assertTrue(keys[1] == 4);


    assertTrue(values[0] == 5.0);
    assertTrue(values[1] == 22.0);


    m2.decode();
    assertTrue(m2.isDecoded());


    double value;
    assertEquals(m2.size(), 2);


    value = m2.get(3);
    assertTrue(value == 5.0);


    value = m2.remove(3);
    assertEquals(1, m2.size());


    value = m2.get(4);
    assertTrue(value == 22.0);
  }

View Full Code Here

    learning = conf.getBoolean(Settings.PROPERTY_PREFIX + "model.train", Settings.LEARNING_MODE);


    // truncateBeta = conf.getBoolean(Settings.PROPERTY_PREFIX + "model.truncate.beta", false);


    // outputValue = new HashMap();
    outputValue = new HMapIDW();


    // approximateBeta = conf.getBoolean(Settings.PROPERTY_PREFIX + "model.truncate.beta", false);
    // if (!approximateBeta) {
    // outputValue = new HashMap();
    // } else {

View Full Code Here

      int numberOfTopics, int numberOfTerms) throws IOException {
    HMapIV<double[]> beta = new HMapIV<double[]>();


    PairOfIntFloat pairOfIntFloat = new PairOfIntFloat();


    HMapIDW hashMap = new HMapIDW();
    // HashMap hashMap = new HashMap();


    // ProbDist hashMap = null;
    // if (!approximateBeta) {
    // hashMap = new HashMap();
    // } else {
    // hashMap = new BloomMap();
    // }


    while (sequenceFileReader.next(pairOfIntFloat, hashMap)) {
      Preconditions.checkArgument(
          pairOfIntFloat.getLeftElement() > 0 && pairOfIntFloat.getLeftElement() <= numberOfTopics,
          "Invalid beta vector for term " + pairOfIntFloat.getLeftElement() + "...");


      // topic is from 1 to K
      int topicIndex = pairOfIntFloat.getLeftElement() - 1;
      double logNormalizer = pairOfIntFloat.getRightElement();
      // double logNormalizer = Math.log(pairOfIntFloat.getRightElement());
      // double logNormalizer = Math.log(hashMap.getNormalizeFactor());


      // logNormalizer = LogMath.add(pairOfIntFloat.getRightElement(),
      // Settings.DEFAULT_LOG_ETA + Math.log(numberOfTerms));


      Iterator<Integer> itr = hashMap.keySet().iterator();
      while (itr.hasNext()) {
        int termIndex = itr.next();
        double logBetaValue = hashMap.get(termIndex);
        // double logBetaValue = Math.log(hashMap.get(termIndex));


        logBetaValue -= logNormalizer;


        if (!beta.containsKey(termIndex)) {

View Full Code Here

      while (sequenceFileReader.next(intWritable, text)) {
        termIndex.put(intWritable.get(), text.toString());
      }
      PairOfIntFloat pairOfIntFloat = new PairOfIntFloat();
      // HMapIFW hmap = new HMapIFW();
      HMapIDW hmap = new HMapIDW();
      TreeMap<Double, Integer> treeMap = new TreeMap<Double, Integer>();
      sequenceFileReader = new SequenceFile.Reader(fs, betaPath, conf);
      while (sequenceFileReader.next(pairOfIntFloat, hmap)) {
        treeMap.clear();


        System.out.println("==============================");
        System.out.println("Top ranked " + topDisplay + " terms for Topic "
            + pairOfIntFloat.getLeftElement());
        System.out.println("==============================");


        Iterator<Integer> itr1 = hmap.keySet().iterator();
        int temp1 = 0;
        while (itr1.hasNext()) {
          temp1 = itr1.next();
          treeMap.put(-hmap.get(temp1), temp1);
          if (treeMap.size() > topDisplay) {
            treeMap.remove(treeMap.lastKey());
          }
        }

View Full Code Here

   *     FileSystem object
   * @return
   *     mapping from term ids to df values
   */
  public static HMapIFW readTransDfTable(Path path, FileSystem fs) {
    HMapIFW transDfTable = new HMapIFW();
    try {
      SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, fs.getConf());


      IntWritable key = (IntWritable) reader.getKeyClass().newInstance();
      FloatWritable value = (FloatWritable) reader.getValueClass().newInstance();


      while (reader.next(key, value)) {
        transDfTable.put(key.get(), value.get());
        //        logger.info(key.get()+"-->"+value.get());
        key = (IntWritable) reader.getKeyClass().newInstance();
        value = (FloatWritable) reader.getValueClass().newInstance();
      }
      reader.close();

View Full Code Here

   *     ttable E-->F (i.e., Pr(f|e))
   * @return
   *     mapping from E-terms to their computed df values
   */
  public static HMapIFW translateDFTable(Vocab eVocabSrc, Vocab fVocabTrg, TTable_monolithic_IFAs e2f_probs, FrequencySortedDictionary dict, DfTableArray dfTable){
    HMapIFW transDfTable = new HMapIFW();
    for(int e=1;e<eVocabSrc.size();e++){
      int[] fS = e2f_probs.get(e).getTranslations(0.0f);
      float df=0;
      for(int f : fS){
        float probEF = e2f_probs.get(e, f);
        String fTerm = fVocabTrg.get(f);
        int id = dict.getId(fTerm); 
        if(id != -1){
          float df_f = dfTable.getDf(id);        
          df += (probEF*df_f);
        }else{
          logger.debug(fTerm+" not in dict");
        }
      }
      transDfTable.put(e, df);
    }
    return transDfTable;
  }

View Full Code Here

   *     mapping from F-terms to their df values
   * @return
   *     mapping from E-terms to their computed df values
   */
  public static HMapIFW translateDFTable(Vocab eVocabSrc, Vocab fVocabTrg, TTable_monolithic_IFAs e2f_probs, HMapSIW dfs){
    HMapIFW transDfTable = new HMapIFW();
    for(int e=1;e<eVocabSrc.size();e++){
      int[] fS = null;
      try {
        fS = e2f_probs.get(e).getTranslations(0.0f);
      } catch (Exception e1) {
        e1.printStackTrace();
      }
      float df=0;
      for(int f : fS){
        float probEF = e2f_probs.get(e, f);
        String fTerm = fVocabTrg.get(f);
        if(!dfs.containsKey(fTerm)){  //only if word is in the collection, can it contribute to the df values.
          continue;
        }      
        float df_f = dfs.get(fTerm);
        df+=(probEF*df_f);
      }
      transDfTable.put(e, df);
    }
    return transDfTable;
  }

View Full Code Here

0 1 2 3 4 5 6 7 8 9

TOP

Related Classes of edu.umd.cloud9.io.map.HMapIDW

bak.pcj.IntIterator

edu.umd.cloud9.example.bfs.BfsNodeTest

edu.umd.cloud9.example.pagerank.PageRankNodeTest

edu.umd.cloud9.io.array.ArrayListOfIntsWritable

edu.umd.cloud9.io.fastutil.Int2FloatOpenHashMapWritableTest

edu.umd.cloud9.io.fastutil.Int2IntOpenHashMapWritableTest

edu.umd.cloud9.io.fastutil.String2FloatOpenHashMapWritableTest

edu.umd.cloud9.io.fastutil.String2IntOpenHashMapWritableTest

edu.umd.cloud9.io.map.HMapIDW

edu.umd.cloud9.io.map.HMapIDWTest

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.