Package org.apache.hadoop.mapred.LineRecordReader

Examples of org.apache.hadoop.mapred.LineRecordReader.LineReader


      long lineCounter = 0;
      for (Path indexFinalPath : paths) {
        FileSystem fs = indexFinalPath.getFileSystem(conf);
        FSDataInputStream ifile = fs.open(indexFinalPath);
        LineReader lr = new LineReader(ifile, conf);
        try {
          Text line = new Text();
          while (lr.readLine(line) > 0) {
            if (++lineCounter > maxEntriesToLoad) {
              throw new HiveException("Number of compact index entries loaded during the query exceeded the maximum of " + maxEntriesToLoad
                  + " set in " + HiveConf.ConfVars.HIVE_INDEX_COMPACT_QUERY_MAX_ENTRIES.varname);
            }
            add(line);
          }
        }
        finally {
          // this will close the input stream
          lr.close();
        }
      }
    }
  }
View Full Code Here


      this.output = output;
      this.reporter = reporter;
    }

    public void run() {
      LineReader lineReader = null;
      try {
        Text key = new Text();
        Text val = new Text();
      lineReader = new LineReader((InputStream)clientIn_, job_);
        // 3/4 Tool to Hadoop
        while ((answer = UTF8ByteArrayUtils.readLine(lineReader)) != null) {

          splitKeyVal(answer, key, val);
          if(postMapper == null) {
              output.collect(key, val);
          } else {
              postMapper.map(key, val, output, reporter);
          }

          numRecWritten_++;
          long now = System.currentTimeMillis();
          if (now-lastStdoutReport > reporterOutDelay_) {
            lastStdoutReport = now;
            String hline = "Records R/W=" + numRecRead_ + "/" + numRecWritten_;
            reporter.setStatus(hline);
            logprintln(hline);
            logflush();
          }
        }
        if (lineReader != null) {
          lineReader.close();
        }
        if (clientIn_ != null) {
          clientIn_.close();
          clientIn_ = null;
          LOG.info("MROutputThread done");
        }
      } catch (Throwable th) {
        outerrThreadsThrowable = th;
        LOG.warn(StringUtils.stringifyException(th));
        try {
          if (lineReader != null) {
            lineReader.close();
          }
          if (clientIn_ != null) {
            clientIn_.close();
            clientIn_ = null;
          }
View Full Code Here

      setDaemon(true);
    }

    public void run() {
      byte[] line;
      LineReader lineReader = null;
      long bytescopied = 0L;

      try {
        lineReader = new LineReader((InputStream)clientErr_, job_);
        while ((line = UTF8ByteArrayUtils.readLine(lineReader)) != null) {

          if((maxErrorBytes < 0) || (bytescopied < maxErrorBytes)) {
            // bound the maxmimum amount of stuff to standard error
            String lineStr = new String(line, "UTF-8");
            System.err.println(lineStr);
          }
          bytescopied += line.length;

          long now = System.currentTimeMillis();
          if (now-lastStderrReport > reporterErrDelay_) {
            lastStderrReport = now;
            reporter.progress();
          }
        }
        if (lineReader != null) {
          lineReader.close();
        }
        if (clientErr_ != null) {
          clientErr_.close();
          clientErr_ = null;
          LOG.info("MRErrorThread done");
        }
      } catch (Throwable th) {
        outerrThreadsThrowable = th;
        LOG.warn(StringUtils.stringifyException(th));
        try {
          if (lineReader != null) {
            lineReader.close();
          }
          if (clientErr_ != null) {
            clientErr_.close();
            clientErr_ = null;
          }
View Full Code Here

   
    @Override
    public void run() {
      try {
        in_ = connectInputStream();
      LineReader lineReader = new LineReader((InputStream)in_, conf_);

        while (true) {
          byte[] b = UTF8ByteArrayUtils.readLine(lineReader);
          if (b == null) {
            break;
          }
          buf_.append(new String(b, "UTF-8"));
          buf_.append('\n');
        }
        lineReader.close();
        in_.close();
      } catch (IOException io) {
        throw new RuntimeException(io);
      }
    }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.mapred.LineRecordReader.LineReader

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.