Examples of AsyncDispatcher


Examples of org.apache.hadoop.yarn.event.AsyncDispatcher

    super(ContainerManagerImpl.class.getName());
    this.context = context;
    this.dirsHandler = dirsHandler;

    // ContainerManager level dispatcher.
    dispatcher = new AsyncDispatcher();
    this.deletionService = deletionContext;
    this.metrics = metrics;

    rsrcLocalizationSrvc =
        createResourceLocalizationService(exec, deletionContext);
View Full Code Here

Examples of org.apache.hadoop.yarn.event.AsyncDispatcher

  }
 
  @Test
  public void testLocalizationInit() throws Exception {
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    AsyncDispatcher dispatcher = new AsyncDispatcher();
    dispatcher.init(new Configuration());

    ContainerExecutor exec = mock(ContainerExecutor.class);
    DeletionService delService = spy(new DeletionService(exec));
    delService.init(conf);
    delService.start();

    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
      localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
      sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);

    LocalDirsHandlerService diskhandler = new LocalDirsHandlerService();
    diskhandler.init(conf);

    ResourceLocalizationService locService =
      spy(new ResourceLocalizationService(dispatcher, exec, delService,
                                          diskhandler,
                                          new NMNullStateStoreService()));
    doReturn(lfs)
      .when(locService).getLocalFileContext(isA(Configuration.class));
    try {
      dispatcher.start();

      // initialize ResourceLocalizationService
      locService.init(conf);

      final FsPermission defaultPerm = new FsPermission((short)0755);

      // verify directory creation
      for (Path p : localDirs) {
        p = new Path((new URI(p.toString())).getPath());
        Path usercache = new Path(p, ContainerLocalizer.USERCACHE);
        verify(spylfs)
          .mkdir(eq(usercache),
              eq(defaultPerm), eq(true));
        Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
        verify(spylfs)
          .mkdir(eq(publicCache),
              eq(defaultPerm), eq(true));
        Path nmPriv = new Path(p, ResourceLocalizationService.NM_PRIVATE_DIR);
        verify(spylfs).mkdir(eq(nmPriv),
            eq(ResourceLocalizationService.NM_PRIVATE_PERM), eq(true));
      }
    } finally {
      dispatcher.stop();
      delService.stop();
    }
  }
View Full Code Here

Examples of org.apache.hadoop.yarn.event.AsyncDispatcher

    }   
    DeletionService del = createDeletionService(exec);
    addService(del);

    // NodeManager level dispatcher
    this.dispatcher = new AsyncDispatcher();

    nodeHealthChecker = new NodeHealthCheckerService();
    addService(nodeHealthChecker);
    dirsHandler = nodeHealthChecker.getDiskHandler();
View Full Code Here

Examples of org.apache.hadoop.yarn.event.AsyncDispatcher

    final String logMessage = "log message\n";
    nmContext.getApplications().put(appId, new ApplicationImpl(null, "user",
        appId, null, nmContext));
   
    MockContainer container = new MockContainer(appAttemptId,
        new AsyncDispatcher(), new Configuration(), "user", appId, 1);
    container.setState(ContainerState.RUNNING);
    nmContext.getContainers().put(containerId, container);
   
    // write out log file
    Path path = dirsHandler.getLogPathForWrite(
View Full Code Here

Examples of org.apache.hadoop.yarn.event.AsyncDispatcher

    server.start();

    // Add an application and the corresponding containers
    RecordFactory recordFactory =
        RecordFactoryProvider.getRecordFactory(conf);
    Dispatcher dispatcher = new AsyncDispatcher();
    String user = "nobody";
    long clusterTimeStamp = 1234;
    ApplicationId appId =
        BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
    Application app = mock(Application.class);
View Full Code Here

Examples of org.apache.hadoop.yarn.event.AsyncDispatcher

    assertEquals("apps isn't NULL", JSONObject.NULL, json.get("apps"));
  }

  private HashMap<String, String> addAppContainers(Application app)
      throws IOException {
    Dispatcher dispatcher = new AsyncDispatcher();
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
        app.getAppId(), 1);
    Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
        app.getUser(), app.getAppId(), 1);
    Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
View Full Code Here

Examples of org.apache.hadoop.yarn.event.AsyncDispatcher

    ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId,
        appAttemptId, 0);
    nmContext.getApplications().put(appId, app);

    MockContainer container =
        new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user,
            appId, 1);
    container.setState(ContainerState.RUNNING);
    nmContext.getContainers().put(container1, container);  
    List<File> files = null;
    files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
View Full Code Here

Examples of org.gudy.azureus2.core3.util.AsyncDispatcher

  {
    synchronized( this ){
     
      if ( add_file_install_dispatcher == null ){
       
        add_file_install_dispatcher = new AsyncDispatcher();
      }
     
      add_file_install_dispatcher.dispatch(
        new AERunnable()
        {
View Full Code Here

Examples of org.gudy.azureus2.core3.util.AsyncDispatcher

    ResultListener  listener =
      new ResultListener()
      {
          //   single thread listener calls
     
        private AsyncDispatcher dispatcher = new AsyncDispatcher( 5000 );

        final private Map<Engine,List<Result[]>>  pending_results = new HashMap<Engine,List<Result[]>>();
       
        final private Map<Engine,Set<String>>  result_hashes = new HashMap<Engine, Set<String>>();
       
        public void
        contentReceived(
          final Engine engine,
          final String content )
        {
          dispatcher.dispatch(
              new AERunnable()
              {
                public void
                runSupport()
                {
                  original_listener.contentReceived( engine, content );
                }
              });
        }
       
        public void
        matchFound(
          final Engine   engine,
          final String[]   fields )
        {
          dispatcher.dispatch(
              new AERunnable()
              {
                public void
                runSupport()
                {
                  original_listener.matchFound( engine, fields );
                }
              })
        }
       
        public void
        resultsReceived(
          final Engine   engine,
          final Result[]   results )
        {
          dispatcher.dispatch(
            new AERunnable()
            {
              public void
              runSupport()
              {
                Result[] results_to_return = null;
               
                if ( batch_millis > 0 ){
                 
                  List<Result[]> list = pending_results.get( engine );
                                   
                  if ( list == null ){
                                     
                    results_to_return = results;
                   
                    pending_results.put( engine, new ArrayList<Result[]>());
                   
                    new DelayedEvent(
                      "SearchBatcher",
                      batch_millis,
                      new AERunnable()
                      {
                        public void
                        runSupport()
                        {
                          dispatcher.dispatch(
                            new AERunnable()
                            {
                              public void
                              runSupport()
                              {
                                batchResultsComplete( engine );
                              }
                            });
                        }
                      });
                  }else{
                                                             
                    list.add( results );
                  }
                }else{
               
                  results_to_return = results;
                }
               
                if ( results_to_return != null ){
                 
                  results_to_return = truncateResults( engine, results_to_return, max_results_per_engine );
                 
                  original_listener.resultsReceived( engine, results_to_return );
                }
              }
            });
        }
     
        public void
        resultsComplete(
          final Engine engine )
        {
          dispatcher.dispatch(
              new AERunnable()
              {
                public void
                runSupport()
                {
                  if ( batch_millis > 0 ){

                    batchResultsComplete( engine );
                  }
                 
                  original_listener.resultsComplete( engine );
                }
              });
        }
     
        protected void
        batchResultsComplete(
          Engine engine )
        {
          List<Result[]> list = pending_results.remove( engine );
         
          if ( list != null ){
           
            List<Result> x = new ArrayList<Result>();
           
            for ( Result[] y: list ){
             
              x.addAll( Arrays.asList( y ));
            }
           
            Result[] results = x.toArray( new Result[ x.size()]);
         
            results = truncateResults( engine, results, max_results_per_engine );
         
            original_listener.resultsReceived( engine, results );
          }
        }
       
        protected Result[]
               truncateResults(
                 Engine    engine,
                 Result[]   a_results,
                 int      max )
               {
          Set<String>  hash_set = result_hashes.get( engine );
         
          if ( hash_set == null ){
           
            hash_set = new HashSet<String>();
           
            result_hashes.put( engine, hash_set );
          }
           
          List<Result>  results = new ArrayList<Result>( a_results.length );
         
          for ( Result r: a_results ){
             
            String name = r.getName();
             
            if ( name == null || name.trim().length() == 0 ){
               
              continue;
            }
         
            if ( rem_dups ){

              String hash = r.getHash();
             
              if (   hash == null ||
                  hash.length() == 0 ){
               
                results.add( r );
               
              }else{
               
                if ( !hash_set.contains( hash )){
                 
                  results.add( r );
                 
                  hash_set.add( hash );
                }
              }
            }else{
             
              results.add( r );
            }
          }
         
                 if ( max < results.size() ){
                
                   log( "Truncating search results for " + engine.getName() + " from " + results.size() + " to " + max );
                  
                   Collections.sort(
                     results,
                     new Comparator<Result>()
                     {
                       Map<Result,Float>  ranks = new HashMap<Result, Float>();
                      
                       public int
                       compare(
                         Result r1,
                         Result r2)
                       {           
                         Float  rank1 = (Float)ranks.get(r1);
                        
                         if ( rank1 == null ){ 
                           rank1 = new Float(r1.getRank());
                           ranks.put( r1, rank1 );
                         }
                        
                         Float  rank2 = (Float)ranks.get(r2);
                        
                         if ( rank2 == null ){ 
                           rank2 = new Float(r2.getRank());
                           ranks.put( r2, rank2 );
                         }
                        
                         return( -rank1.compareTo( rank2 ));
                       }
                     });
                
                   Result[] x = new Result[max];
                  
                   int  pos = 0;
                  
                   while( pos < max ){
                  
                     x[pos] = results.get( pos );
                    
                     pos++;
                   }
                  
                   return( x );
                  
                 }else{
                  
                   return( results.toArray( new Result[ results.size()] ));
                 }
               }
       
        public void
        engineFailed(
          final Engine   engine,
          final Throwable  e )
        {
          dispatcher.dispatch(
              new AERunnable()
              {
                public void
                runSupport()
                {
                  original_listener.engineFailed( engine, e );
                }
              });
        }
       
        public void
        engineRequiresLogin(
          final Engine   engine,
          final Throwable  e )
        {
          dispatcher.dispatch(
              new AERunnable()
              {
                public void
                runSupport()
                {
View Full Code Here

Examples of org.gudy.azureus2.core3.util.AsyncDispatcher

      }
    }
   
    if ( update_check_timer != null ){
     
      new AsyncDispatcher().dispatch(
          new AERunnable()
          {
            public void
            runSupport()
            {
View Full Code Here
TOP
Copyright © 2018 www.massapi.com. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.