Package org.apache.hadoop.io.retry

Examples of org.apache.hadoop.io.retry.RetryPolicy

517
518
519
520
521
522
523
524
525
526
527
  private static JobSubmissionProtocol createProxy(
      JobSubmissionProtocol rpcJobSubmitClient,
      Configuration conf) throws IOException {

    RetryPolicy defaultPolicy =
        RetryUtils.getDefaultRetryPolicy(
            conf,
            MAPREDUCE_CLIENT_RETRY_POLICY_ENABLED_KEY,
            MAPREDUCE_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
            MAPREDUCE_CLIENT_RETRY_POLICY_SPEC_KEY,
View Full Code Here
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
    }

  private static ClientProtocol createNamenode(ClientProtocol rpcNamenode,
      Configuration conf) throws IOException {
    //default policy
    final RetryPolicy defaultPolicy =
        RetryUtils.getDefaultRetryPolicy(
            conf,
            DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_KEY,
            DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_ENABLED_DEFAULT,
            DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_KEY,
            DFSConfigKeys.DFS_CLIENT_RETRY_POLICY_SPEC_DEFAULT,
            SafeModeException.class
            );
   
    //create policy
    RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
        5, LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
   
    Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
    remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class, createPolicy);

    Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
    exceptionToPolicyMap.put(RemoteException.class,
        RetryPolicies.retryByRemoteException(
            defaultPolicy, remoteExceptionToPolicyMap));
    RetryPolicy methodPolicy = RetryPolicies.retryByException(
        defaultPolicy, exceptionToPolicyMap);
    Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
   
    methodNameToPolicyMap.put("create", methodPolicy);
View Full Code Here
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
   * set up the retry policy */
  private static NamenodeProtocol createNamenode(Configuration conf)
    throws IOException {
    InetSocketAddress nameNodeAddr =
      DataNode.createSocketAddr(FileSystem.getDefaultUri(conf).getAuthority());
    RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(
        5, 200, TimeUnit.MILLISECONDS);
    Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
    RetryPolicy methodPolicy = RetryPolicies.retryByException(
        timeoutPolicy, exceptionToPolicyMap);
    Map<String,RetryPolicy> methodNameToPolicyMap =
        new HashMap<String, RetryPolicy>();
    methodNameToPolicyMap.put("getBlocks", methodPolicy);

View Full Code Here
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
   * @return a proxy object that will retry timeouts.
   * @throws IOException
   */
  private JobSubmissionProtocol createRetryProxy(JobSubmissionProtocol raw
                                            ) throws IOException {
    RetryPolicy backoffPolicy =
      RetryPolicies.retryUpToMaximumCountWithProportionalSleep
      (5, 10, java.util.concurrent.TimeUnit.SECONDS);
    Map<Class<? extends Exception>, RetryPolicy> handlers =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
    handlers.put(SocketTimeoutException.class, backoffPolicy);
    RetryPolicy backoffTimeOuts =
      RetryPolicies.retryByException(RetryPolicies.TRY_ONCE_THEN_FAIL,handlers);
    return (JobSubmissionProtocol)
      RetryProxy.create(JobSubmissionProtocol.class, raw, backoffTimeOuts);
  }
View Full Code Here
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
        NetUtils.getSocketFactory(conf, ClientProtocol.class));
  }

  private static ClientProtocol createNamenode(ClientProtocol rpcNamenode)
    throws IOException {
    RetryPolicy timeoutPolicy = RetryPolicies.exponentialBackoffRetry(
        5, 200, TimeUnit.MILLISECONDS);
    RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
        5, LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
   
    Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
    remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class, createPolicy);

    Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
    exceptionToPolicyMap.put(RemoteException.class,
        RetryPolicies.retryByRemoteException(
            RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
    exceptionToPolicyMap.put(SocketTimeoutException.class, timeoutPolicy);
    RetryPolicy methodPolicy = RetryPolicies.retryByException(
        RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
    Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
   
    methodNameToPolicyMap.put("open", methodPolicy);
    methodNameToPolicyMap.put("setReplication", methodPolicy);
View Full Code Here
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
  } 

  private static FileSystemStore createDefaultStore(Configuration conf) {
    FileSystemStore store = new Jets3tFileSystemStore();
   
    RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                                                                               conf.getInt("fs.s3.maxRetries", 4),
                                                                               conf.getLong("fs.s3.sleepTimeSeconds", 10), TimeUnit.SECONDS);
    Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
    exceptionToPolicyMap.put(IOException.class, basePolicy);
    exceptionToPolicyMap.put(S3Exception.class, basePolicy);
   
    RetryPolicy methodPolicy = RetryPolicies.retryByException(
                                                              RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
    Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
    methodNameToPolicyMap.put("storeBlock", methodPolicy);
    methodNameToPolicyMap.put("retrieveBlock", methodPolicy);
   
View Full Code Here
371
372
373
374
375
376
377
378
379
380
381
      super.serviceStart();
    }

    @Override
    protected ResourceTracker getRMClient() throws IOException {
      RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf);
      resourceTracker =
          (ResourceTracker) RetryProxy.create(ResourceTracker.class,
            new MyResourceTracker6(rmStartIntervalMS, rmNeverStart),
            retryPolicy);
      return resourceTracker;
View Full Code Here
404
405
406
407
408
409
410
411
412
      this.conf = conf;
    }

    @Override
    protected ResourceTracker getRMClient() {
      RetryPolicy retryPolicy = RMProxy.createRetryPolicy(conf);
      return (ResourceTracker) RetryProxy.create(ResourceTracker.class,
        resourceTracker, retryPolicy);
    }
View Full Code Here
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
    }

    private static NativeFileSystemStore createDefaultStore(Configuration conf) {
        NativeFileSystemStore store = new SwiftNativeFileSystemStore();

        RetryPolicy basePolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
                conf.getInt("fs.swift.maxRetries", 4),
                conf.getLong("fs.swift.sleepTimeSeconds", 10), TimeUnit.SECONDS);
        Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
                new HashMap<Class<? extends Exception>, RetryPolicy>();
        exceptionToPolicyMap.put(IOException.class, basePolicy);
        exceptionToPolicyMap.put(SwiftException.class, basePolicy);

        RetryPolicy methodPolicy = RetryPolicies.retryByException(
                RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
        Map<String, RetryPolicy> methodNameToPolicyMap =
                new HashMap<String, RetryPolicy>();
        methodNameToPolicyMap.put("storeFile", methodPolicy);
View Full Code Here
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
        NetUtils.getSocketFactory(conf, ClientProtocol.class));
  }

  private static ClientProtocol createNamenode(ClientProtocol rpcNamenode)
    throws IOException {
    RetryPolicy createPolicy = RetryPolicies.retryUpToMaximumCountWithFixedSleep(
        5, LEASE_SOFTLIMIT_PERIOD, TimeUnit.MILLISECONDS);
   
    Map<Class<? extends Exception>,RetryPolicy> remoteExceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
    remoteExceptionToPolicyMap.put(AlreadyBeingCreatedException.class, createPolicy);

    Map<Class<? extends Exception>,RetryPolicy> exceptionToPolicyMap =
      new HashMap<Class<? extends Exception>, RetryPolicy>();
    exceptionToPolicyMap.put(RemoteException.class,
        RetryPolicies.retryByRemoteException(
            RetryPolicies.TRY_ONCE_THEN_FAIL, remoteExceptionToPolicyMap));
    RetryPolicy methodPolicy = RetryPolicies.retryByException(
        RetryPolicies.TRY_ONCE_THEN_FAIL, exceptionToPolicyMap);
    Map<String,RetryPolicy> methodNameToPolicyMap = new HashMap<String,RetryPolicy>();
   
    methodNameToPolicyMap.put("create", methodPolicy);
View Full Code Here
TOP

Related Classes of org.apache.hadoop.io.retry.RetryPolicy

  • com.sun.grid.herd.HerdLoadSensor
  • org.apache.hadoop.dfs.Balancer
  • org.apache.hadoop.dfs.DFSClient
  • org.apache.hadoop.fs.s3.S3FileSystem
  • org.apache.hadoop.fs.s3native.NativeS3FileSystem
  • org.apache.hadoop.fs.swiftnative.NativeSwiftFileSystem
  • org.apache.hadoop.hdfs.AvatarShell
  • org.apache.hadoop.hdfs.AvatarZKShell
  • org.apache.hadoop.hdfs.DFSClient
  • org.apache.hadoop.hdfs.DFSUtil
    • Copyright © 2018 www.massapicom. All rights reserved.
    All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.