Examples of HMapIIW

edu.umd.cloud9.io.map.HMapIIW

Writable representing a map where both keys and values are ints.

One notable feature of this class is the ability to support lazy decoding, controlled by the {@link #setLazyDecodeFlag(boolean)} method. In lazydecoding mode, when an object of this type is deserialized, key-value pairs are not inserted into the map, but rather held in arrays. The reduces memory used in cases where random access to values is not required. In lazy decoding mode, the raw keys and values may be fetched by the {@link #getKeys()} and{@link #getValues()} methods, respectively. The map can be subsequentlypopulated with the {@link #decode()} method.
@author Jimmy Lin

Examples of edu.umd.cloud9.io.map.HMapIIW

    assertTrue(value == 22.0f);
  }


  @Test
  public void testSerializeEmpty() throws IOException {
    HMapIIW m1 = new HMapIIW();


    // make sure this does nothing
    m1.decode();


    assertTrue(m1.size() == 0);


    HMapIFW m2 = HMapIFW.create(m1.serialize());


    assertTrue(m2.size() == 0);
  }

View Full Code Here

Examples of edu.umd.cloud9.io.map.HMapIIW

      return s.substring(s.lastIndexOf("/") + 1);
    }


    private HMapIIW readSamplesFromCache(String samplesFile, JobConf conf) throws IOException {
      Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
      HMapIIW samplesMap = null;
      for (Path localFile : localFiles) {
        if (localFile.toString().contains(samplesFile)) {
          samplesMap = new HMapIIW();
          LineReader reader = new LineReader(FileSystem.getLocal(conf).open(localFile));
          Text t = new Text();
          while (reader.readLine(t) != 0) {
            int docno = Integer.parseInt(t.toString());
            sLogger.info(docno + " --> sample");
            samplesMap.put(docno, 1);
          }
          reader.close();
          sLogger.info(samplesMap.size() + " sampled");
        }
      }
      if (samplesMap == null) throw new RuntimeException("Not found in local cache: " + samplesFile);
      return samplesMap;
    }

View Full Code Here

Examples of edu.umd.cloud9.io.map.HMapIIW

      return s.substring(s.lastIndexOf("/") + 1);
    }


    private HMapIIW readSamplesFromCache(String samplesFile, JobConf conf) throws IOException {
      Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
      HMapIIW samplesMap = null;
      for (Path localFile : localFiles) {
        if (localFile.toString().contains(samplesFile)) {
          samplesMap = new HMapIIW();
          LineReader reader = new LineReader(FileSystem.getLocal(conf).open(localFile));
          Text t = new Text();
          while (reader.readLine(t) != 0) {
            int docno = Integer.parseInt(t.toString());
            sLogger.info(docno + " --> sample");
            samplesMap.put(docno, 1);
          }
          reader.close();
          sLogger.info(samplesMap.size() + " sampled");
        }
      }
      if (samplesMap == null) throw new RuntimeException("Not found in local cache: " + samplesFile);
      return samplesMap;
    }

View Full Code Here

Examples of edu.umd.cloud9.io.map.HMapIIW

      }
    }


    private HMapIIW readSamplesFromCache(String samplesFile, JobConf conf) throws IOException {
      Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
      HMapIIW samplesMap = null;
      for (Path localFile : localFiles) {
        if (localFile.toString().contains(samplesFile)) {
          samplesMap = new HMapIIW();
          LineReader reader = new LineReader(FileSystem.getLocal(conf).open(localFile));
          Text t = new Text();
          while (reader.readLine(t) != 0) {
            int docno = Integer.parseInt(t.toString());
            sLogger.info(docno + " --> sample");
            samplesMap.put(docno, 1);
          }
          reader.close();
          sLogger.info(samplesMap.size() + " sampled");
        }
      }
      if (samplesMap == null) throw new RuntimeException("Not found in local cache: " + samplesFile);
      return samplesMap;
    }

View Full Code Here

Examples of edu.umd.cloud9.io.map.HMapIIW

    super();
  }


  private static HMapIIW readSamplesFromCache(String samplesFile, JobConf conf) throws IOException {
    Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
    HMapIIW samplesMap = null;
    for (Path localFile : localFiles) {
      if (localFile.toString().contains(samplesFile)) {
        samplesMap = new HMapIIW();
        LineReader reader = new LineReader(FileSystem.getLocal(conf).open(localFile));
        Text t = new Text();
        while (reader.readLine(t) != 0) {
          int docno = Integer.parseInt(t.toString());
          sLogger.info(docno + " --> sample");
          samplesMap.put(docno, 1);
        }
        reader.close();
        sLogger.info(samplesMap.size() + " sampled");
      }
    }
    if (samplesMap == null) throw new RuntimeException("Not found in local cache: " + samplesFile);
    return samplesMap;
  }

View Full Code Here

Examples of edu.umd.cloud9.io.map.HMapIIW


    }


    private HMapIIW readSamplesFromCache(String samplesFile, JobConf conf) throws IOException {
      Path[] localFiles = DistributedCache.getLocalCacheFiles(conf);
      HMapIIW samplesMap = null;
      for (Path localFile : localFiles) {
        if (localFile.toString().contains(samplesFile)) {
          samplesMap = new HMapIIW();
          LineReader reader = new LineReader(FileSystem.getLocal(conf).open(localFile));
          Text t = new Text();
          while (reader.readLine(t) != 0) {
            int docno = Integer.parseInt(t.toString());
            sLogger.info(docno + " --> sample");
            samplesMap.put(docno, 1);
          }
          reader.close();
          sLogger.info(samplesMap.size() + " sampled");
        }
      }
      if (samplesMap == null) throw new RuntimeException("Not found in local cache: " + samplesFile);
      return samplesMap;
    }

View Full Code Here

Examples of edu.umd.cloud9.io.map.HMapIIW

      } catch (Exception e) {
        throw new RuntimeException("Error reading doc vectors!");
      }


      if (localFiles != null && localFiles.length > 0) {
        samplesMap = new HMapIIW();
        try {
          LineReader reader = new LineReader(FileSystem.getLocal(job).open(localFiles[0]));
          Text t = new Text();
          while (reader.readLine(t) != 0) {
            int docno = Integer.parseInt(t.toString());

View Full Code Here

Examples of edu.umd.cloud9.io.map.HMapIIW

      keyInt = new IntWritable();
      valText = new Text();


      sampleDocnosFile = job.get("SampleDocnosFile");
      if (sampleDocnosFile != null) {
        samplesMap = new HMapIIW();
        try {
          LineReader reader = new LineReader(FileSystem.get(job).open(new Path(sampleDocnosFile)));
          Text t = new Text();
          while (reader.readLine(t) != 0) {
            int docno = Integer.parseInt(t.toString());

View Full Code Here

Examples of edu.umd.cloud9.io.map.HMapIIW

        // TODO Auto-generated catch block
        e2.printStackTrace();
      }
      sampleDocnosFile = job.get("SampleDocnosFile");
      if (sampleDocnosFile != null) {
        samplesMap = new HMapIIW();
        try {
          LineReader reader = new LineReader(FileSystem.get(job).open(new Path(sampleDocnosFile)));
          Text t = new Text();
          while (reader.readLine(t) != 0) {
            int docno = Integer.parseInt(t.toString());

View Full Code Here

Examples of edu.umd.cloud9.io.map.HMapIIW

      keyText = new Text();
      valText = new Text();


      sampleDocnosFile = job.get("SampleDocnosFile");
      if (sampleDocnosFile != null) {
        samplesMap = new HMapIIW();
        try {
          LineReader reader = new LineReader(FileSystem.get(job).open(new Path(sampleDocnosFile)));
          Text t = new Text();
          while (reader.readLine(t) != 0) {
            String[] docnos = t.toString().split("\t");

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.