Package org.apache.hadoop.hdfs.protocol

Examples of org.apache.hadoop.hdfs.protocol.ClientProtocol


      // create two files with one block each
      DFSTestUtil util = new DFSTestUtil("testCorruptFilesCorruptedBlock", 2, 1, 512);
      util.createFiles(fs, "/srcdat10");

      // fetch bad file list from namenode. There should be none.
      ClientProtocol namenode = DFSClient.createNamenode(conf);
      FileStatus[] badFiles = namenode.getCorruptFiles();
      assertTrue("Namenode has " + badFiles.length + " corrupt files. Expecting None.",
          badFiles.length == 0);

      // Now deliberately corrupt one block
      File data_dir = new File(System.getProperty("test.build.data"),
      "dfs/data/data1/current/finalized");
      assertTrue("data directory does not exist", data_dir.exists());
      File[] blocks = data_dir.listFiles();
      assertTrue("Blocks do not exist in data-dir", (blocks != null) && (blocks.length > 0));
      for (int idx = 0; idx < blocks.length; idx++) {
        if (blocks[idx].getName().startsWith("blk_") &&
            blocks[idx].getName().endsWith(".meta")) {
          //
          // shorten .meta file
          //
          RandomAccessFile file = new RandomAccessFile(blocks[idx], "rw");
          FileChannel channel = file.getChannel();
          long position = channel.size() - 2;
          int length = 2;
          byte[] buffer = new byte[length];
          random.nextBytes(buffer);
          channel.write(ByteBuffer.wrap(buffer), position);
          file.close();
          LOG.info("Deliberately corrupting file " + blocks[idx].getName() +
              " at offset " + position + " length " + length);

          // read all files to trigger detection of corrupted replica
          try {
            util.checkFiles(fs, "/srcdat10");
          } catch (BlockMissingException e) {
            System.out.println("Received BlockMissingException as expected.");
          } catch (IOException e) {
            assertTrue("Corrupted replicas not handled properly. Expecting BlockMissingException " +
                " but received IOException " + e, false);
          }
          break;
        }
      }

      // fetch bad file list from namenode. There should be one file.
      badFiles = namenode.getCorruptFiles();
      LOG.info("Namenode has bad files. " + badFiles.length);
      assertTrue("Namenode has " + badFiles.length + " bad files. Expecting 1.",
          badFiles.length == 1);
      util.cleanup(fs, "/srcdat10");
    } finally {
View Full Code Here


          final PrintWriter out = response.getWriter();
          final XMLOutputter xml = new XMLOutputter(out, "UTF-8");
          xml.declaration();
          try {
            //get content summary
            final ClientProtocol nnproxy = createNameNodeProxy();
            final ContentSummary cs = nnproxy.getContentSummary(path);

            //write xml
            xml.startTag(ContentSummary.class.getName());
            if (cs != null) {
              xml.attribute("length"        , "" + cs.getLength());
View Full Code Here

      final Configuration conf = new HdfsConfiguration(DataNode.getDataNode().getConf());
      final int socketTimeout = conf.getInt(DFSConfigKeys.DFS_CLIENT_SOCKET_TIMEOUT_KEY, HdfsConstants.READ_TIMEOUT);
      final SocketFactory socketFactory = NetUtils.getSocketFactory(conf, ClientProtocol.class);
     
      try {
        ClientProtocol nnproxy = getUGI(request, conf).doAs
        (new PrivilegedExceptionAction<ClientProtocol>() {
          @Override
          public ClientProtocol run() throws IOException {
            return DFSClient.createNamenode(conf);
          }
View Full Code Here

    final Configuration conf =
      (Configuration) getServletContext().getAttribute("name.conf");
    final UserGroupInformation ugi = getUGI(request, conf);

    try {
      final ClientProtocol nnproxy = ugi
          .doAs(new PrivilegedExceptionAction<ClientProtocol>() {
            @Override
            public ClientProtocol run() throws IOException {
              return createNameNodeProxy();
            }
          });

      final String path = request.getPathInfo() != null ?
                                                    request.getPathInfo() : "/";
      HdfsFileStatus info = nnproxy.getFileInfo(path);
      if ((info != null) && !info.isDir()) {
        response.sendRedirect(createUri(path, info, ugi, nnproxy,
              request).toURL().toString());
      } else if (info == null){
        response.sendError(400, "cat: File not found " + path);
View Full Code Here

        DFSTestUtil.createFile(fs, filepath, FILE_SIZE, (short) 1, 0L);
        DFSTestUtil.waitReplication(fs, filepath, (short) 1);
      }

      // verify there are not corrupt files
      ClientProtocol namenode = DFSClient.createNamenode(conf);
      FileStatus[] badFiles = namenode.getCorruptFiles();
      assertTrue("There are " + badFiles.length
          + " corrupt files, but expecting none", badFiles.length == 0);

      // Check if webui agrees
      URL url = new URL("http://"
          + conf.get(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY)
          + "/corrupt_files.jsp");
      String corruptFilesPage = DFSTestUtil.urlGet(url);
      assertTrue("Corrupt files page is not showing a healthy filesystem",
          corruptFilesPage.contains("No missing blocks found at the moment."));

      // Now corrupt all the files except for the last one
      for (int idx = 0; idx < filepaths.length - 1; idx++) {
        String blockName = DFSTestUtil.getFirstBlock(fs, filepaths[idx])
            .getBlockName();
        TestDatanodeBlockScanner.corruptReplica(blockName, 0);

        // read the file so that the corrupt block is reported to NN
        FSDataInputStream in = fs.open(filepaths[idx]);
        try {
          in.readFully(new byte[FILE_SIZE]);
        } catch (ChecksumException ignored) { // checksum error is expected.
        }
        in.close();
      }

      // verify if all corrupt files were reported to NN
      badFiles = namenode.getCorruptFiles();
      assertTrue("Expecting 3 corrupt files, but got " + badFiles.length,
          badFiles.length == 3);

      // Check if webui agrees
      url = new URL("http://"
View Full Code Here

        DFSTestUtil.createFile(fs, filepath, FILE_SIZE, (short) 1, 0L);
        DFSTestUtil.waitReplication(fs, filepath, (short) 1);
      }

      // verify there are not corrupt files
      ClientProtocol namenode = DFSClient.createNamenode(conf);
      FileStatus[] badFiles = namenode.getCorruptFiles();
      assertTrue("There are " + badFiles.length
          + " corrupt files, but expecting none", badFiles.length == 0);

      // Check if fsck -list-corruptfiles agree
      String outstr = runFsck(conf, 0, true, "/", "-list-corruptfiles");
      assertTrue(outstr.contains(NamenodeFsck
          .buildSummaryResultForListCorruptFiles(0, "/")));

      // Now corrupt all the files except for the last one
      for (int idx = 0; idx < filepaths.length - 1; idx++) {
        String blockName = DFSTestUtil.getFirstBlock(fs, filepaths[idx])
            .getBlockName();
        TestDatanodeBlockScanner.corruptReplica(blockName, 0);

        // read the file so that the corrupt block is reported to NN
        FSDataInputStream in = fs.open(filepaths[idx]);
        try {
          in.readFully(new byte[FILE_SIZE]);
        } catch (ChecksumException ignored) { // checksum error is expected.
        }
        in.close();
      }

      // verify if all corrupt files were reported to NN
      badFiles = namenode.getCorruptFiles();
      assertTrue("Expecting 3 corrupt files, but got " + badFiles.length,
          badFiles.length == 3);

      // check the corrupt file
      String corruptFile = "/audiobook";
View Full Code Here

    try {
      ugi.doAs(new PrivilegedExceptionAction<Void>() {
        @Override
        public Void run() throws IOException {
          ClientProtocol nn = createNameNodeProxy();
          final String path = ServletUtil.getDecodedPath(request, "/data");
          final String encodedPath = ServletUtil.getRawPath(request, "/data");
          String delegationToken = request
              .getParameter(JspHelper.DELEGATION_PARAMETER_NAME);

          HdfsFileStatus info = nn.getFileInfo(path);
          if (info != null && !info.isDir()) {
            response.sendRedirect(createRedirectURL(path, encodedPath,
                info, ugi, nn, request, delegationToken).toString());
          } else if (info == null) {
            response.sendError(400, "File not found " + path);
View Full Code Here

    ((Log4JLogger) SaslInputStream.LOG).getLogger().setLevel(Level.ALL);
  }

  @Test
  public void testDelegationTokenRpc() throws Exception {
    ClientProtocol mockNN = mock(ClientProtocol.class);
    FSNamesystem mockNameSys = mock(FSNamesystem.class);

    DelegationTokenSecretManager sm = new DelegationTokenSecretManager(
        DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
        DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
        DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT,
        3600000, mockNameSys);
    sm.startThreads();
    final Server server = new RPC.Builder(conf)
        .setProtocol(ClientProtocol.class).setInstance(mockNN)
        .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
        .setSecretManager(sm).build();
   
    server.start();

    final UserGroupInformation current = UserGroupInformation.getCurrentUser();
    final InetSocketAddress addr = NetUtils.getConnectAddress(server);
    String user = current.getUserName();
    Text owner = new Text(user);
    DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null);
    Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
        dtId, sm);
    SecurityUtil.setTokenService(token, addr);
    LOG.info("Service for token is " + token.getService());
    current.addToken(token);
    current.doAs(new PrivilegedExceptionAction<Object>() {
      @Override
      public Object run() throws Exception {
        ClientProtocol proxy = null;
        try {
          proxy = (ClientProtocol) RPC.getProxy(ClientProtocol.class,
              ClientProtocol.versionID, addr, conf);
          proxy.getServerDefaults();
        } finally {
          server.stop();
          if (proxy != null) {
            RPC.stopProxy(proxy);
          }
View Full Code Here

    @SuppressWarnings("unchecked")
    @Override
    public long renew(Token<?> token, Configuration conf) throws IOException {
      Token<DelegationTokenIdentifier> delToken =
        (Token<DelegationTokenIdentifier>) token;
      ClientProtocol nn = getNNProxy(delToken, conf);
      try {
        return nn.renewDelegationToken(delToken);
      } catch (RemoteException re) {
        throw re.unwrapRemoteException(InvalidToken.class,
                                       AccessControlException.class);
      }
    }
View Full Code Here

    public void cancel(Token<?> token, Configuration conf) throws IOException {
      Token<DelegationTokenIdentifier> delToken =
          (Token<DelegationTokenIdentifier>) token;
      LOG.info("Cancelling " +
               DelegationTokenIdentifier.stringifyToken(delToken));
      ClientProtocol nn = getNNProxy(delToken, conf);
      try {
        nn.cancelDelegationToken(delToken);
      } catch (RemoteException re) {
        throw re.unwrapRemoteException(InvalidToken.class,
            AccessControlException.class);
      }
    }
View Full Code Here

TOP

Related Classes of org.apache.hadoop.hdfs.protocol.ClientProtocol

Copyright © 2018 www.massapicom. All rights reserved.
All source code are property of their respective owners. Java is a trademark of Sun Microsystems, Inc and owned by ORACLE Inc. Contact coftware#gmail.com.