Examples of ShardWriter

Examples of com.alimama.mdrill.index.utils.ShardWriter

91
92
93
94
95
96
97
98
    String uuid=java.util.UUID.randomUUID().toString();
    indexHdfsPath = new Path(outputdir, part_xxxxx).toString();
    tmpath = new Path(outputdir + "/_tmpindex", part_xxxxx + "_"+uuid).toString();
    tmpathzip = new Path(outputdir + "/_tmpindex", part_xxxxx + "_zip_"+ uuid).toString();
   
    ShardWriter shardWriter = new ShardWriter(fs, tmpath, conf);
    return shardWriter;
  }
View Full Code Here

Examples of com.alimama.mdrill.index.utils.ShardWriter

190
191
192
193
194
195
196
197
    indexHdfsPath = new Path(outputdir, part_xxxxx).toString();

    tmpath = new Path(outputdir + "/_tmpindex", part_xxxxx + "_" + java.util.UUID.randomUUID().toString()).toString();
    localtmpath = new Path("./_tmpindex", part_xxxxx + "_" + java.util.UUID.randomUUID().toString()).toString();
   
    ShardWriter shardWriter = new ShardWriter(lfs, localtmpath, conf);
    return shardWriter;
  }
View Full Code Here

Examples of org.apache.hadoop.contrib.index.lucene.ShardWriter

76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    LOG.info("Construct a shard writer for " + key);
    FileSystem fs = FileSystem.get(iconf.getConfiguration());
    String temp =
        mapredTempDir + Path.SEPARATOR + "shard_" + System.currentTimeMillis();
    final ShardWriter writer = new ShardWriter(fs, key, temp, iconf);

    // update the shard
    while (values.hasNext()) {
      IntermediateForm form = values.next();
      writer.process(form);
      reporter.progress();
    }

    // close the shard
    final Reporter fReporter = reporter;
    new Closeable() {
      boolean closed = false;

      public void close() throws IOException {
        // spawn a thread to give progress heartbeats
        Thread prog = new Thread() {
          public void run() {
            while (!closed) {
              try {
                fReporter.setStatus("closing");
                Thread.sleep(1000);
              } catch (InterruptedException e) {
                continue;
              } catch (Throwable e) {
                return;
              }
            }
          }
        };

        try {
          prog.start();

          if (writer != null) {
            writer.close();
          }
        } finally {
          closed = true;
        }
      }
View Full Code Here

Examples of org.apache.hadoop.contrib.index.lucene.ShardWriter

76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    LOG.info("Construct a shard writer for " + key);
    FileSystem fs = FileSystem.get(iconf.getConfiguration());
    String temp =
        mapredTempDir + Path.SEPARATOR + "shard_" + System.currentTimeMillis();
    final ShardWriter writer = new ShardWriter(fs, key, temp, iconf);

    // update the shard
    while (values.hasNext()) {
      IntermediateForm form = values.next();
      writer.process(form);
      reporter.progress();
    }

    // close the shard
    final Reporter fReporter = reporter;
    new Closeable() {
      volatile boolean closed = false;

      public void close() throws IOException {
        // spawn a thread to give progress heartbeats
        Thread prog = new Thread() {
          public void run() {
            while (!closed) {
              try {
                fReporter.setStatus("closing");
                Thread.sleep(1000);
              } catch (InterruptedException e) {
                continue;
              } catch (Throwable e) {
                return;
              }
            }
          }
        };

        try {
          prog.start();

          if (writer != null) {
            writer.close();
          }
        } finally {
          closed = true;
        }
      }
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.