Package org.apache.hadoop.mapred

Examples of org.apache.hadoop.mapred.OutputLogFilter

92
93
94
95
96
97
98
99
100
101
102
      double alpha = Double.parseDouble(alpha_0);
      DirichletState<VectorWritable> state = DirichletDriver.createState(modelFactory, modelPrototype,
        Integer.parseInt(prototypeSize), Integer.parseInt(numClusters), alpha);
      Path path = new Path(statePath);
      FileSystem fs = FileSystem.get(path.toUri(), job);
      FileStatus[] status = fs.listStatus(path, new OutputLogFilter());
      for (FileStatus s : status) {
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, s.getPath(), job);
        try {
          Text key = new Text();
          DirichletCluster<VectorWritable> cluster = new DirichletCluster<VectorWritable>();
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
    if (!r_job.isSuccessful()) {
      fail("Oops! The job broke due to an unexpected error");
    }
    Path[] outputFiles = FileUtil.stat2Paths(
        getFileSystem().listStatus(outDir,
        new OutputLogFilter()));
    if (outputFiles.length > 0) {
      InputStream is = getFileSystem().open(outputFiles[0]);
      BufferedReader reader = new BufferedReader(new InputStreamReader(is));
      String line = reader.readLine();
      //make sure we get what we expect as the first line, and also
View Full Code Here
178
179
180
181
182
183
184
185
186
187
188
      assertTrue("No counters found!", (numCounters > 0));
    }

    List<String> results = new ArrayList<String>();
    for (Path p:FileUtil.stat2Paths(dfs.getFileSystem().listStatus(outputPath,
                                new OutputLogFilter()))) {
      results.add(TestMiniMRWithDFS.readOutput(p, job));
    }
    assertEquals("number of reduces is wrong",
                 expectedResults.length, results.size());
    for(int i=0; i < results.size(); i++) {
View Full Code Here
92
93
94
95
96
97
98
99
100
101
102
    if (!r_job.isSuccessful()) {
      fail("Oops! The job broke due to an unexpected error");
    }
    Path[] outputFiles = FileUtil.stat2Paths(
        getFileSystem().listStatus(outDir,
        new OutputLogFilter()));
    if (outputFiles.length > 0) {
      InputStream is = getFileSystem().open(outputFiles[0]);
      BufferedReader reader = new BufferedReader(new InputStreamReader(is));
      String line = reader.readLine();
      //make sure we get what we expect as the first line, and also
View Full Code Here
111
112
113
114
115
116
117
118
119
120
121
  public static Map<Integer, List<VectorWritable>> getRepresentativePoints(Configuration conf, Path statePath) {
    Map<Integer, List<VectorWritable>> representativePoints = new HashMap<Integer, List<VectorWritable>>();
    try {
      FileSystem fs = FileSystem.get(statePath.toUri(), conf);
      FileStatus[] status = fs.listStatus(statePath, new OutputLogFilter());
      for (FileStatus s : status) {
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, s.getPath(), conf);
        try {
          IntWritable key = new IntWritable(0);
          VectorWritable point = new VectorWritable();
View Full Code Here
88
89
90
91
92
93
94
95
96
97
98
    if (!r_job.isSuccessful()) {
      fail("Oops! The job broke due to an unexpected error");
    }
    Path[] outputFiles = FileUtil.stat2Paths(
        getFileSystem().listStatus(outDir,
        new OutputLogFilter()));
    if (outputFiles.length > 0) {
      InputStream is = getFileSystem().open(outputFiles[0]);
      BufferedReader reader = new BufferedReader(new InputStreamReader(is));
      String line = reader.readLine();
      //make sure we get what we expect as the first line, and also
View Full Code Here
177
178
179
180
181
182
183
184
185
186
187
      assertTrue("No counters found!", (numCounters > 0));
    }

    List<String> results = new ArrayList<String>();
    for (Path p:FileUtil.stat2Paths(dfs.getFileSystem().listStatus(outputPath,
                                new OutputLogFilter()))) {
      results.add(TestMiniMRWithDFS.readOutput(p, job));
    }
    assertEquals("number of reduces is wrong",
                 expectedResults.length, results.size());
    for(int i=0; i < results.size(); i++) {
View Full Code Here
116
117
118
119
120
121
122
123
124
125
126
    List<String> badRecs = new ArrayList<String>();
    badRecs.addAll(MAPPER_BAD_RECORDS);
    badRecs.addAll(REDUCER_BAD_RECORDS);
    Path[] outputFiles = FileUtil.stat2Paths(
        getFileSystem().listStatus(getOutputDir(),
        new OutputLogFilter()));
   
    if (outputFiles.length > 0) {
      InputStream is = getFileSystem().open(outputFiles[0]);
      BufferedReader reader = new BufferedReader(new InputStreamReader(is));
      String line = reader.readLine();
View Full Code Here
70
71
72
73
74
75
76
77
78
79
80
    try {
      DirichletState<Vector> state = DirichletDriver.createState(modelFactory,
          Integer.parseInt(numClusters), Double.parseDouble(alpha_0));
      Path path = new Path(statePath);
      FileSystem fs = FileSystem.get(path.toUri(), job);
      FileStatus[] status = fs.listStatus(path, new OutputLogFilter());
      for (FileStatus s : status) {
        SequenceFile.Reader reader = new SequenceFile.Reader(fs, s.getPath(),
            job);
        try {
          Text key = new Text();
View Full Code Here
67
68
69
70
71
72
73
74
75
76
77
  private static CubeQueryResult getCubeQueryResult(String outPath,
      CubeScan scan) throws Exception {
    CubeQueryResult cubeQueryResult = new CubeQueryResult();
    Configuration conf = new Configuration();
    FileSystem fs = FileSystem.get(conf);
    Path[] fileList = FileUtil.stat2Paths(fs.listStatus(new Path(outPath), new OutputLogFilter()));
    for(Path path: fileList) {
      FSDataInputStream fsin = fs.open(path);
      BufferedReader reader = new BufferedReader(new InputStreamReader(fsin));
      String s = null;
      do {
View Full Code Here
TOP

Related Classes of org.apache.hadoop.mapred.OutputLogFilter

  • org.apache.hadoop.mapred.lib.TestKeyFieldBasedComparator
  • org.apache.hadoop.mapred.pipes.TestPipes
  • org.apache.hadoop.streaming.TestStreamingBadRecords
  • org.apache.mahout.clustering.dirichlet.DirichletMapper
  • org.apache.mahout.clustering.evaluation.RepresentativePointsMapper
  • org.olap4cloud.impl.CubeScanMR
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.