Examples of MiniAvatarCluster

org.apache.hadoop.hdfs.MiniAvatarCluster
This class manages a Avatar/HDFS cluster with all nodes running locally. To synchronize the AvatarNodes, it uses a local ZooKeeper server.

Examples of org.apache.hadoop.hdfs.MiniAvatarCluster

  private void setUp(String name) throws Exception {
    LOG.info("------------------- test: " + name + " START ----------------");
    conf = new Configuration();
    conf.setLong("fs.checkpoint.period", 3600);


    cluster = new MiniAvatarCluster(conf, 2, true, null, null);
    fs = cluster.getFileSystem();


    // spy the runtime
    runtime = Runtime.getRuntime();
    runtime = spy(runtime);

View Full Code Here

Examples of org.apache.hadoop.hdfs.MiniAvatarCluster

    conf.setLong("fs.avatar.standbyfs.checkinterval", 1000);
  }
  
  @Test
  public void testMergeNamespaces() throws Exception {
    MiniAvatarCluster cluster1 = null;
    MiniAvatarCluster cluster2 = null;
    Configuration conf1 = new Configuration(conf);
    conf1.set(MiniDFSCluster.DFS_CLUSTER_ID, Long.toString(System.currentTimeMillis() + 1));
    Configuration conf2 = new Configuration(conf);
    conf2.set(MiniDFSCluster.DFS_CLUSTER_ID, Long.toString(System.currentTimeMillis() + 2));
    
    try {
      LOG.info("Start cluster1 and cluster2");
      cluster1 = new MiniAvatarCluster.Builder(conf1)
              .federation(true).enableQJM(false).build();
      cluster2 = new MiniAvatarCluster.Builder(conf2)
              .numNameNodes(2).federation(true).enableQJM(false).build();
      LOG.info("Write data to cluster2 and cluster1");
      Path p = new Path("/testFile");
      TestMergeNamespaces.createFile(cluster1.getFileSystem(0), p, 512 * 1024L);
      TestMergeNamespaces.createFile(cluster2.getFileSystem(0), p, 1024 * 1024L);
      TestMergeNamespaces.createFile(cluster2.getFileSystem(1), p, 1536 * 1024L);
      LOG.info("Add cluster2 to cluster1");
      cluster1.addCluster(cluster2, false);
      TestMergeNamespaces.verifyFile(cluster1.getFileSystem(0), p, 512 * 1024L);
      TestMergeNamespaces.verifyFile(cluster1.getFileSystem(1), p, 1024 * 1024L);
      TestMergeNamespaces.verifyFile(cluster1.getFileSystem(2), p, 1536 * 1024L);

View Full Code Here

Examples of org.apache.hadoop.hdfs.MiniAvatarCluster

  }


  @Test
  public void testIsInitializedOK() throws Exception {
    conf = new Configuration();
    cluster = new MiniAvatarCluster(conf, 1, true, null, null);
    assertTrue(cluster.getNameNode(0).avatars.get(0).avatar.isInitialized());
    assertTrue(cluster.getNameNode(0).avatars.get(1).avatar.isInitialized());
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.MiniAvatarCluster

  public void setUp() throws Exception {
    conf = new Configuration();
    conf.setInt("dfs.block.size", 1024);
    conf.setLong("dfs.heartbeat.interval", 1L);
    conf.setClass("dfs.balancer.impl", AvatarBalancer.class, Balancer.class);
    cluster = new MiniAvatarCluster(conf, 2, true, null, null, 1, false,
        new long[] { CAPACITY, CAPACITY });
    conf.setLong("dfs.balancer.movedWinWidth", 2000L);
    fs = cluster.getFileSystem();
    new DFSTestUtil("/testBasic", MAX_FILES, 5, MAX_FILE_SIZE).createFiles(fs,
        "/", (short) 2);

View Full Code Here

Examples of org.apache.hadoop.hdfs.MiniAvatarCluster

   * @throws IOException
   */
  @Test
  public void test2NNRegistration() 
      throws IOException, ConfigException, InterruptedException{
    MiniAvatarCluster cluster = new MiniAvatarCluster(conf, 1, true, null, null, 2, true); 
    try {
      NameNodeInfo nn1 = cluster.getNameNode(0);
      NameNodeInfo nn2 = cluster.getNameNode(1);
      AvatarNode nn1zero = nn1.avatars.get(0).avatar;
      AvatarNode nn1one = nn1.avatars.get(1).avatar;
      AvatarNode nn2zero = nn2.avatars.get(0).avatar;
      AvatarNode nn2one = nn2.avatars.get(1).avatar;
      assertNotNull("cannot create nn1 avatar 0", nn1zero);
      assertNotNull("cannot create nn1 avatar 1", nn1one);
      assertNotNull("cannot create nn2 avatar 0", nn2zero);
      assertNotNull("cannot create nn2 avatar 1", nn2one);
      
      int ns1zero = nn1zero.getNamespaceID();
      int ns1one = nn1one.getNamespaceID();
      assertEquals("namespace ids for namenode 1 should be the same",
          ns1zero, ns1one);
      int ns2zero = nn2zero.getNamespaceID();
      int ns2one = nn2one.getNamespaceID();
      assertEquals("namespace ids for namenode 2 should be the same",
          ns2zero, ns2one);
      int lv1zero = nn1zero.getFSImage().getLayoutVersion();
      int lv1one = nn1one.getFSImage().getLayoutVersion();
      int lv2zero = nn2zero.getFSImage().getLayoutVersion();
      int lv2one = nn2one.getFSImage().getLayoutVersion();
      assertNotSame("namespace ids should be different", ns1zero, ns2zero);
      LOG.info("nn1zero: lv=" + lv1zero + ";uri=" + nn1zero.getNameNodeAddress());
      LOG.info("nn1one: lv=" + lv1one + ";uri=" + nn1one.getNameNodeAddress());
      LOG.info("nn2zero: lv=" + lv2zero + ";uri=" + nn2zero.getNameNodeAddress());
      LOG.info("nn2one: lv=" + lv2one + ";uri=" + nn2one.getNameNodeAddress());


      // check number of volumes in fsdataset
      AvatarDataNode dn = cluster.getDataNodes().get(0);
      
      for (NamespaceService nsos : dn.getAllNamespaceServices()) {
        LOG.info("reg: nsid =" + nsos.getNamespaceId() + "; name="
            + nsos.getNsRegistration().name + "; sid="
            + nsos.getNsRegistration().storageID + "; nna="
            + nsos.getNNSocketAddress());
      }


      NamespaceService nsos1 = dn.getAllNamespaceServices()[0];
      NamespaceService nsos2 = dn.getAllNamespaceServices()[1];


      // The order of bpos is not guaranteed, so fix the order
      if (nsos1.getNNSocketAddress().equals(nn2zero.getNameNodeDNAddress())) {
        NamespaceService tmp = nsos1;
        nsos1 = nsos2;
        nsos2 = tmp;
      }


      assertEquals("wrong nn address", nsos1.getNNSocketAddress(),
          nn1zero.getNameNodeDNAddress());
      assertEquals("wrong nn address", nsos2.getNNSocketAddress(),
          nn2zero.getNameNodeDNAddress());
      assertEquals("wrong nsid", nsos1.getNamespaceId(), ns1zero);
      assertEquals("wrong nsid", nsos2.getNamespaceId(), ns2zero);
    } finally {
      cluster.shutDown();
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.MiniAvatarCluster

   * @throws IOException
   */
  @Test
  public void testFedSingleNN() 
      throws IOException, ConfigException, InterruptedException {
    MiniAvatarCluster cluster = new MiniAvatarCluster(conf, 1, true, null, null, 1, true); 
    try {
      NameNodeInfo nn1 = cluster.getNameNode(0);
      AvatarNode nn1zero = nn1.avatars.get(0).avatar;
      AvatarNode nn1one = nn1.avatars.get(1).avatar;
      assertNotNull("cannot create nn1 zero", nn1zero);
      assertNotNull("cannot create nn1 one", nn1one);


      int nsid1zero = nn1zero.getNamespaceID();
      int nsid1one = nn1one.getNamespaceID();
      assertEquals("namespace ids for namenode 1 should be the same",
          nsid1zero, nsid1one);
      int lv1zero = nn1zero.getFSImage().getLayoutVersion();
      int lv1one = nn1one.getFSImage().getLayoutVersion();
      LOG.info("nn1: lv=" + lv1zero + ";nsid=" + nsid1zero + ";uri="
          + nn1zero.getNameNodeAddress());
      LOG.info("nn1: lv=" + lv1one + ";nsid=" + nsid1one + ";uri="
          + nn1one.getNameNodeAddress());


      // check number of vlumes in fsdataset
      AvatarDataNode dn = cluster.getDataNodes().get(0);
      for (NamespaceService nsos : dn.getAllNamespaceServices()) {
        LOG.info("reg: nsid =" + nsos.getNamespaceId() + "; name="
            + nsos.getNsRegistration().name + "; sid="
            + nsos.getNsRegistration().storageID + "; nna="
            + nsos.getNNSocketAddress());
      }
      // try block report
      NamespaceService nsos1 = dn.getAllNamespaceServices()[0];
      nsos1.scheduleBlockReport(0);


      assertEquals("wrong nn address", nsos1.getNNSocketAddress(),
          nn1zero.getNameNodeDNAddress());
      assertEquals("wrong nsid", nsos1.getNamespaceId(), nsid1zero);
      cluster.shutDown();
      
      // Ensure all the BPOfferService threads are shutdown
      assertEquals(0, dn.getAllNamespaceServices().length);
      cluster = null;
    } finally {
      if (cluster != null) {
        cluster.shutDown();
      }
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.MiniAvatarCluster

  @Test
  public void testMiniAvatarClusterWithMultipleNN()
      throws IOException, ConfigException, InterruptedException {
    Configuration conf = new Configuration();
    // start Federated cluster and add a node.
    MiniAvatarCluster cluster = new MiniAvatarCluster(conf, 1, true, null, null, 2, true); 
    Assert.assertNotNull(cluster);
    Assert.assertEquals("(1)Should be 2 namenodes", 2, cluster.getNumNameNodes());
    
    // add a node
    cluster.addNameNode(conf);
    Assert.assertEquals("(1)Should be 3 namenodes", 3, cluster.getNumNameNodes());
    cluster.shutDown();
        
    // 2. start with Federation flag set
    conf = new Configuration();
    cluster = new MiniAvatarCluster(conf, 1, true, null, null, 1, true);
    Assert.assertNotNull(cluster);
    Assert.assertEquals("(2)Should be 1 namenodes", 1, cluster.getNumNameNodes());
    
    // add a node
    cluster.addNameNode(conf);   
    Assert.assertEquals("(2)Should be 2 namenodes", 2, cluster.getNumNameNodes());
    cluster.shutDown();


    // 3. start non-federated
    conf = new Configuration();
    cluster = new MiniAvatarCluster(conf, 1, true, null, null);
    Assert.assertNotNull(cluster);
    Assert.assertEquals("(2)Should be 1 namenodes", 1, cluster.getNumNameNodes());
    
    // add a node
    try {
      cluster.addNameNode(conf);
      Assert.fail("shouldn't be able to add another NN to non federated cluster");
    } catch (IOException e) {
      // correct 
      Assert.assertTrue(e.getMessage().startsWith("cannot add namenode"));
      Assert.assertEquals("(3)Should be 1 namenodes", 1, cluster.getNumNameNodes());
    } finally {
      cluster.shutDown();
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.MiniAvatarCluster

      throws IOException, ConfigException, InterruptedException {
    Configuration conf = new Configuration();
    if (simulatedStorage) {
      conf.setBoolean(SimulatedFSDataset.CONFIG_PROPERTY_SIMULATED, true);
    }
    MiniAvatarCluster cluster = new MiniAvatarCluster(conf, 1, true, null, null, 3, true);
    try {


      for (int i = 0; i < cluster.getNumNameNodes(); i++) {
        FileSystem fs = cluster.getFileSystem(i);
        //
        // check that / exists
        //
        Path path = new Path("/");
        System.out.println("Path : \"" + path.toString() + "\"");
        System.out.println(fs.getFileStatus(path).isDir()); 
        assertTrue("/ should be a directory", 
                   fs.getFileStatus(path).isDir() == true);


        //
        // Create a directory inside /, then try to overwrite it
        //
        Path dir1 = new Path("/test_dir");
        fs.mkdirs(dir1);
        System.out.println("createFile: Creating " + dir1.getName() + 
          " for overwrite of existing directory.");
        try {
          fs.create(dir1, true); // Create path, overwrite=true
          fs.close();
          assertTrue("Did not prevent directory from being overwritten.", false);
        } catch (IOException ie) {
          if (!ie.getMessage().contains("already exists as a directory."))
            throw ie;
        }
      
        // create a new file in home directory. Do not close it.
        //
        Path file1 = new Path("filestatus.dat");
        FSDataOutputStream stm = TestFileCreation.createFile(fs, file1, 1);


        // verify that file exists in FS namespace
        assertTrue(file1 + " should be a file", 
                    fs.getFileStatus(file1).isDir() == false);
        System.out.println("Path : \"" + file1 + "\"");


        // write to file
        TestFileCreation.writeFile(stm);


        // Make sure a client can read it before it is closed.
        checkFile(fs, file1, 1);


        // verify that file size has changed
        long len = fs.getFileStatus(file1).getLen();
        assertTrue(file1 + " should be of size " + (numBlocks * blockSize) +
                   " but found to be of size " + len, 
                    len == numBlocks * blockSize);


        stm.close();


        // verify that file size has changed to the full size
        len = fs.getFileStatus(file1).getLen();
        assertTrue(file1 + " should be of size " + fileSize +
                   " but found to be of size " + len, 
                    len == fileSize);
     
        // Check storage usage 
        // can't check capacities for real storage since the OS file system may be changing under us.
        if (simulatedStorage) {
          AvatarDataNode dn = cluster.getDataNodes().get(0);
          int namespaceId = cluster.getNameNode(i).avatars.get(0).avatar.getNamespaceID();
          assertEquals(fileSize, dn.getFSDataset().getNSUsed(namespaceId));
          //Because all namespaces share the same simulated dataset
          assertEquals(SimulatedFSDataset.DEFAULT_CAPACITY-fileSize*(i+1), 
              dn.getFSDataset().getRemaining());
        }
      }
    } finally {
      cluster.shutDown();
    }
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.MiniAvatarCluster


  @BeforeClass
  public static void setUpBeforeClass() throws Exception {
    MiniAvatarCluster.createAndStartZooKeeper();
    conf = new Configuration();
    cluster = new MiniAvatarCluster(conf, 3, true, null, null);
    fs = cluster.getFileSystem();
  }

View Full Code Here

Examples of org.apache.hadoop.hdfs.MiniAvatarCluster

  }


  @Before
  public void setup() throws Exception {
    conf = new Configuration();
    cluster = new MiniAvatarCluster(conf, 1, true, null, null);
  }

View Full Code Here

0 1 2

TOP

All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.