Package elephantdb.store

Examples of elephantdb.store.DomainStore

163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180

    public InputSplit[] getSplits(JobConf jc, int ignored) throws IOException {
        Args args = (Args) Utils.getObject(jc, ARGS_CONF);
        FileSystem fs = Utils.getFS(args.inputDirHdfs, jc);
        DomainStore store = new DomainStore(fs, args.inputDirHdfs);
        String versionPath;
        if (args.version == null) {
            versionPath = store.mostRecentVersionPath();
        } else {
            versionPath = store.versionPath(args.version);
        }
        DomainSpec spec = store.getSpec();
        List<InputSplit> ret = new ArrayList<InputSplit>();
        for (int i = 0; i < spec.getNumShards(); i++) {
            String shardPath = versionPath + "/" + i;
            if (fs.exists(new Path(shardPath))) {
                ret.add(new ElephantInputSplit(new Path(shardPath).makeQualified(fs).toString(), spec, jc));
View Full Code Here
51
52
53
54
55
56
57
58
59
60
61
    TapMode mode;

    public ElephantDBTap(String dir, DomainSpec spec, Args args, TapMode mode) throws IOException {
        domainDir = dir;
        this.args = args;
        this.spec = new DomainStore(dir, spec).getSpec();
        this.mode = mode;

        setStringPath(domainDir);
        setScheme(new ElephantScheme(this.args.sourceFields,
            this.args.sinkFields, this.spec));
View Full Code Here
63
64
65
66
67
68
69
70
71
    private transient DomainStore domainStore;

    public DomainStore getDomainStore() throws IOException {
        if (domainStore == null) {
            domainStore = new DomainStore(domainDir, spec);
        }
        return domainStore;
    }
View Full Code Here
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
        conf.setNumReduceTasks(spec.getNumShards());
        conf.setSpeculativeExecution(false);
    }

    public ElephantOutputFormat.Args outputArgs(JobConf conf) throws IOException {
        DomainStore dstore = getDomainStore();
        FileSystem fs = dstore.getFileSystem();

        if (newVersionPath == null) { //working around cascading calling sinkinit twice
            newVersionPath = dstore.createVersion();
            // make the path qualified before serializing into the jobconf
            newVersionPath = new Path(newVersionPath).makeQualified(fs).toString();
        }
        ElephantOutputFormat.Args eargs = new ElephantOutputFormat.Args(spec, newVersionPath);
View Full Code Here
132
133
134
135
136
137
138
139
        return new Path(domainDir);
    }
   
    @Override
    public long getModifiedTime(JobConf conf) throws IOException {
        DomainStore dstore = getDomainStore();
        return (mode == TapMode.SINK) ? 0 : dstore.mostRecentVersion();
    }
View Full Code Here
140
141
142
143
144
145
146
147
148
149
150
151
    @Override
    public String getIdentifier() {
        String versionString = "";
        try {
            DomainStore dstore = getDomainStore();
            versionString = ((mode == TapMode.SINK) ? "LATEST" : "" + dstore.mostRecentVersion());
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        return domainDir + Path.SEPARATOR
            + ((mode == TapMode.SINK) ? "sink" : "source")
View Full Code Here
163
164
165
166
167
168
169
170
171
172
173
174
175
176
    }

    @Override
    public boolean commitResource(JobConf conf) {
        try {
            DomainStore dstore = getDomainStore();
            dstore.getFileSystem().mkdirs(new Path(newVersionPath));
           
            dstore.succeedVersion(newVersionPath);
           
            return true;

        } catch (IOException e) {
            throw new TapException("Couldn't finalize new elephant domain version", e);
View Full Code Here
179
180
181
182
183
184
185
186
187
188
        }
    }
   
    @Override
    public boolean rollbackResource(JobConf conf) throws IOException {
        DomainStore dstore = getDomainStore();
        dstore.failVersion(newVersionPath);       
       
        return true;
    }
View Full Code Here
TOP

Related Classes of elephantdb.store.DomainStore

  • elephantdb.cascading.ElephantDBTap
  • elephantdb.hadoop.ElephantInputFormat
  • elephantdb.persistence.ShardSetImpl
  • org.apache.hadoop.conf.Configuration
  • org.apache.hadoop.fs.FileSystem
  • org.apache.hadoop.fs.Path
  • java.io.IOException
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.