Package org.apache.mahout.common

Examples of org.apache.mahout.common.FileLineIterable

87
88
89
90
91
92
93
94
95
96
97
        Path unqualifiedItemsFilePath = new Path(itemFilePathString);
        FileSystem fs = FileSystem.get(unqualifiedItemsFilePath.toUri(), jobConf);
        itemsToRecommendFor = new FastIDSet();
        Path itemsFilePath = unqualifiedItemsFilePath.makeQualified(fs);
        in = fs.open(itemsFilePath);
        for (String line : new FileLineIterable(in)) {
          itemsToRecommendFor.add(Long.parseLong(line));
        }
      }
    } catch (IOException ioe) {
      throw new IllegalStateException(ioe);
View Full Code Here
90
91
92
93
94
95
96
97
98
  }

  private static void loadPopulation(FileSystem fs, Path f,
                                     Collection<DummyCandidate> population) throws IOException {
    FSDataInputStream in = fs.open(f);
    for (String line : new FileLineIterable(in)) {
      population.add(StringUtils.<DummyCandidate>fromString(line));
    }
  }
View Full Code Here
857
858
859
860
861
862
863
864
865
866
867
   }


  static Map<Long,List<RecommendedItem>> readRecommendations(File file) throws IOException {
    Map<Long,List<RecommendedItem>> recommendations = new HashMap<Long,List<RecommendedItem>>();
    Iterable<String> lineIterable = new FileLineIterable(file);
    for (String line : lineIterable) {

      String[] keyValue = line.split("\t");
      long userID = Long.parseLong(keyValue[0]);
      String[] tokens = keyValue[1].replaceAll("\\[", "")
View Full Code Here
82
83
84
85
86
87
88
89
90
91
92
    Charset charset = Charset.forName("UTF-8");
    BayesFileFormatter.collapse("animal", analyzer, input, charset, new File(out, "animal"));
    files = out.listFiles();
    assertEquals("files Size: " + files.length + " is not: " + 1, 1, files.length);
    int count = 0;
    for (String line : new FileLineIterable(files[0])) {
      assertTrue("line does not start with label", line.startsWith("animal"));
      count++;
    }
    assertEquals(count + " does not equal: " + WORDS.length, count, WORDS.length);
  }
View Full Code Here
165
166
167
168
169
170
171
172
173
174
175
    if (dfs.exists(outPath)) {
      dfs.delete(outPath, true);
    }

    Set<String> categories = new HashSet<String>();
    for (String line : new FileLineIterable(new File(catFile))) {
      categories.add(line.trim().toLowerCase());
    }

    DefaultStringifier<Set<String>> setStringifier = new DefaultStringifier<Set<String>>(conf, GenericsUtil.getClass(categories));
View Full Code Here
84
85
86
87
88
89
90
91
92
93
94
      itemIDPadded.insert(0, '0');
    }
    List<Preference> prefs = new ArrayList<Preference>();
    File movieFile = new File(new File(dataDirectory, "training_set"), "mv_00" + itemIDPadded + ".txt");
    try {
      for (String line : new FileLineIterable(movieFile, true)) {
        int firstComma = line.indexOf((int) ',');
        Integer userID = Integer.valueOf(line.substring(0, firstComma));
        int secondComma = line.indexOf((int) ',', firstComma + 1);
        float rating = Float.parseFloat(line.substring(firstComma + 1, secondComma));
        prefs.add(new GenericPreference(userID, itemID, rating));
View Full Code Here
62
63
64
65
66
67
68
69
70
71
72
    Map<String, List<String>> byUserEntryCache = new FastMap<String, List<String>>(100000);

    for (File byItemFile : byItemDirectory.listFiles()) {
      log.info("Processing {}", byItemFile);
      Iterator<String> lineIterator = new FileLineIterable(byItemFile, false).iterator();
      String line = lineIterator.next();
      String movieIDString = line.substring(0, line.length() - 1);
      while (lineIterator.hasNext()) {
        line = lineIterator.next();
        int firstComma = line.indexOf((int) ',');
View Full Code Here
80
81
82
83
84
85
86
87
88
89
90
    FastByIDMap<Collection<Preference>> userIDPrefMap = new FastByIDMap<Collection<Preference>>();

    int counter = 0;
    FilenameFilter filenameFilter = new MovieFilenameFilter();
    for (File movieFile : new File(dataDirectory, "training_set").listFiles(filenameFilter)) {
      Iterator<String> lineIterator = new FileLineIterable(movieFile, false).iterator();
      String line = lineIterator.next();
      long movieID = Long.parseLong(line.substring(0, line.length() - 1)); // strip colon
      while (lineIterator.hasNext()) {
        line = lineIterator.next();
        if (++counter % 100000 == 0) {
View Full Code Here
94
95
96
97
98
99
100
101
102
  }

  private static void loadPopulation(FileSystem fs, Path f,
                                     Collection<DummyCandidate> population) throws IOException {
    FSDataInputStream in = fs.open(f);
    for (String line : new FileLineIterable(in)) {
      population.add((DummyCandidate) StringUtils.fromString(line));
    }
  }
View Full Code Here
219
220
221
222
223
224
225
226
227
228
229
230
    FPGrowth<String> fp = new FPGrowth<String>();
    Set<String> features = new HashSet<String>();

    fp.generateTopKFrequentPatterns(new StringRecordIterator(
        new FileLineIterable(new File(input), encoding, false), pattern), fp
        .generateFList(new StringRecordIterator(new FileLineIterable(new File(
            input), encoding, false), pattern), minSupport), minSupport,
        maxHeapSize, features, new StringOutputConvertor(
            new SequenceFileOutputCollector<Text, TopKStringPatterns>(writer)));
    writer.close();
View Full Code Here
TOP

Related Classes of org.apache.mahout.common.FileLineIterable

  • org.apache.mahout.cf.taste.example.bookcrossing.BookCrossingDataModel
  • org.apache.mahout.cf.taste.example.grouplens.GroupLensDataModel
  • org.apache.mahout.cf.taste.example.netflix.NetflixDataModel
  • org.apache.mahout.cf.taste.example.netflix.NetflixFileDataModel
  • org.apache.mahout.cf.taste.example.netflix.TransposeToByUser
  • org.apache.mahout.cf.taste.hadoop.item.AggregateAndRecommendReducer
  • org.apache.mahout.cf.taste.hadoop.item.RecommenderJobTest
  • org.apache.mahout.cf.taste.hadoop.item.RecommenderMapper
  • org.apache.mahout.cf.taste.hadoop.item.UserVectorSplitterMapper
  • org.apache.mahout.cf.taste.hadoop.pseudo.RecommenderReducer
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.