/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
import java.io.EOFException;
import java.io.IOException;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
import org.mortbay.log.Log;
import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.ByteToken;
import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.IntToken;
import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.VIntToken;
/**
* EditsLoaderCurrent processes Hadoop EditLogs files and walks over
* them using a provided EditsVisitor, calling the visitor at each element
* enumerated below.
*
*/
class EditsLoaderCurrent implements EditsLoader {
private static int[] supportedVersions = { -18, -19, -20, -21, -22, -23, -24,
-25, -26, -27, -28, -30, -31, -32, -33, -34, -35, -36, -37, -38, -39,
-40, -41, -42, -43, -44 };
private EditsVisitor v;
private int editsVersion = 0;
/**
* Constructor
*/
public EditsLoaderCurrent(EditsVisitor visitor) {
v = visitor;
}
/**
* Checks if this EditsLoader can load given version of edits
*
* @param version version to load
* @return true if this EditsLoader can load given version of edits
*/
@Override
public boolean canLoadVersion(int version) {
for(int v : supportedVersions) { if(v == version) return true; }
return false;
}
/**
* Visit a transaction ID, if the log version supports it.
*/
private void visitTxId() throws IOException {
if (LayoutVersion.supports(Feature.STORED_TXIDS, editsVersion)) {
v.visitLong(EditsElement.TRANSACTION_ID);
}
}
/**
* Visit OP_INVALID
*/
private void visit_OP_INVALID() throws IOException {
; // nothing to do, this op code has no data
}
/**
* Visit OP_ADD
*/
private void visit_OP_ADD() throws IOException {
visit_OP_ADD_or_OP_CLOSE(FSEditLogOpCodes.OP_ADD);
}
/**
* Visit OP_CLOSE
*/
private void visit_OP_CLOSE() throws IOException {
visit_OP_ADD_or_OP_CLOSE(FSEditLogOpCodes.OP_CLOSE);
}
private void visit_Blocks() throws IOException {
IntToken numBlocksToken = v.visitInt(EditsElement.NUMBLOCKS);
for (int i = 0; i < numBlocksToken.value; i++) {
v.visitEnclosingElement(EditsElement.BLOCK);
v.visitLong(EditsElement.BLOCK_ID);
v.visitLong(EditsElement.BLOCK_NUM_BYTES);
v.visitLong(EditsElement.BLOCK_GENERATION_STAMP);
if (LayoutVersion.supports(Feature.BLOCK_CHECKSUM, editsVersion)) {
v.visitInt(EditsElement.BLOCK_CHECKSUM);
}
v.leaveEnclosingElement();
}
}
/**
* Visit OP_ADD and OP_CLOSE, they are almost the same
*
* @param editsOpCode op code to visit
*/
private void visit_OP_ADD_or_OP_CLOSE(FSEditLogOpCodes editsOpCode)
throws IOException {
visitTxId();
if (!LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
IntToken opAddLength = v.visitInt(EditsElement.LENGTH);
// this happens if the edits is not properly ended (-1 op code),
// it is padded at the end with all zeros, OP_ADD is zero so
// without this check we would treat all zeros as empty OP_ADD)
if (opAddLength.value == 0) {
throw new IOException("OpCode " + editsOpCode
+ " has zero length (corrupted edits)");
}
}
v.visitStringUTF8(EditsElement.PATH);
if (LayoutVersion.supports(Feature.ADD_INODE_ID, editsVersion)) {
v.visitLong(EditsElement.INODE_ID);
}
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitShort(EditsElement.REPLICATION);
v.visitLong(EditsElement.MTIME);
v.visitLong(EditsElement.ATIME);
v.visitLong(EditsElement.BLOCKSIZE);
} else {
v.visitStringUTF8(EditsElement.REPLICATION);
v.visitStringUTF8(EditsElement.MTIME);
v.visitStringUTF8(EditsElement.ATIME);
v.visitStringUTF8(EditsElement.BLOCKSIZE);
}
// now read blocks
visit_Blocks();
// PERMISSION_STATUS
v.visitEnclosingElement(EditsElement.PERMISSION_STATUS);
v.visitStringText( EditsElement.USERNAME);
v.visitStringText( EditsElement.GROUPNAME);
v.visitShort( EditsElement.FS_PERMISSIONS);
v.leaveEnclosingElement();
if(editsOpCode == FSEditLogOpCodes.OP_ADD) {
v.visitStringUTF8(EditsElement.CLIENT_NAME);
v.visitStringUTF8(EditsElement.CLIENT_MACHINE);
}
}
/**
* Visit OP_RENAME_OLD
*/
private void visit_OP_RENAME_OLD() throws IOException {
visitTxId();
if (!LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitInt(EditsElement.LENGTH);
}
v.visitStringUTF8( EditsElement.SOURCE);
v.visitStringUTF8( EditsElement.DESTINATION);
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitLong(EditsElement.TIMESTAMP);
} else {
v.visitStringUTF8(EditsElement.TIMESTAMP);
}
}
/**
* Visit OP_HARDLINK
*/
private void visit_OP_HARDLINK() throws IOException {
visitTxId();
v.visitStringUTF8(EditsElement.SOURCE);
v.visitStringUTF8(EditsElement.DESTINATION);
v.visitLong(EditsElement.TIMESTAMP);
}
/**
* Visit OP_APPEND
*/
private void visit_OP_APPEND() throws IOException {
visitTxId();
v.visitStringUTF8(EditsElement.PATH);
// visit blocks
visit_Blocks();
// client name & machine
v.visitStringUTF8(EditsElement.CLIENT_NAME);
v.visitStringUTF8(EditsElement.CLIENT_MACHINE);
}
/**
* Visit OP_DELETE
*/
private void visit_OP_DELETE() throws IOException {
visitTxId();
if (!LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitInt(EditsElement.LENGTH);
}
v.visitStringUTF8( EditsElement.PATH);
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitLong(EditsElement.TIMESTAMP);
} else {
v.visitStringUTF8(EditsElement.TIMESTAMP);
}
}
/**
* Visit OP_MKDIR
*/
private void visit_OP_MKDIR() throws IOException {
visitTxId();
if (!LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitInt(EditsElement.LENGTH);
}
v.visitStringUTF8( EditsElement.PATH);
if (LayoutVersion.supports(Feature.ADD_INODE_ID, editsVersion)) {
v.visitLong(EditsElement.INODE_ID);
}
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitLong(EditsElement.TIMESTAMP);
v.visitLong(EditsElement.ATIME);
} else {
v.visitStringUTF8(EditsElement.TIMESTAMP);
v.visitStringUTF8(EditsElement.ATIME);
}
// PERMISSION_STATUS
v.visitEnclosingElement( EditsElement.PERMISSION_STATUS);
v.visitStringText( EditsElement.USERNAME);
v.visitStringText( EditsElement.GROUPNAME);
v.visitShort( EditsElement.FS_PERMISSIONS);
v.leaveEnclosingElement();
}
private void visit_OP_SET_REPLICATION() throws IOException {
visitTxId();
v.visitStringUTF8(EditsElement.PATH);
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitShort(EditsElement.REPLICATION);
} else {
v.visitStringUTF8(EditsElement.REPLICATION);
}
}
/**
* Visit OP_SET_PERMISSIONS
*/
private void visit_OP_SET_PERMISSIONS() throws IOException {
visitTxId();
v.visitStringUTF8( EditsElement.PATH);
v.visitShort( EditsElement.FS_PERMISSIONS);
}
/**
* Visit OP_SET_OWNER
*/
private void visit_OP_SET_OWNER() throws IOException {
visitTxId();
v.visitStringUTF8(EditsElement.PATH);
v.visitStringUTF8(EditsElement.USERNAME);
v.visitStringUTF8(EditsElement.GROUPNAME);
}
/**
* Visit OP_SET_GENSTAMP
*/
private void visit_OP_SET_GENSTAMP() throws IOException {
visitTxId();
v.visitLong(EditsElement.GENERATION_STAMP);
}
/**
* Visit OP_TIMES
*/
private void visit_OP_TIMES() throws IOException {
visitTxId();
if (!LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitInt(EditsElement.LENGTH);
}
v.visitStringUTF8( EditsElement.PATH);
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitLong(EditsElement.MTIME);
v.visitLong(EditsElement.ATIME);
} else {
v.visitStringUTF8(EditsElement.MTIME);
v.visitStringUTF8(EditsElement.ATIME);
}
}
/**
* Visit OP_SET_QUOTA
*/
private void visit_OP_SET_QUOTA() throws IOException {
visitTxId();
v.visitStringUTF8( EditsElement.PATH);
v.visitLong( EditsElement.NS_QUOTA);
v.visitLong( EditsElement.DS_QUOTA);
}
/**
* Visit OP_RENAME
*/
private void visit_OP_RENAME() throws IOException {
visitTxId();
if (!LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitInt(EditsElement.LENGTH);
}
v.visitStringUTF8( EditsElement.SOURCE);
v.visitStringUTF8( EditsElement.DESTINATION);
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitLong(EditsElement.TIMESTAMP);
} else {
v.visitStringUTF8(EditsElement.TIMESTAMP);
}
v.visitBytesWritable( EditsElement.RENAME_OPTIONS);
}
/**
* Visit OP_CONCAT_DELETE
*/
private void visit_OP_CONCAT_DELETE() throws IOException {
visitTxId();
int sourceCount = 0;
if (!LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
IntToken lengthToken = v.visitInt(EditsElement.LENGTH);
sourceCount = lengthToken.value - 2;
Log.info("------------------------- does not support");
}
v.visitStringUTF8(EditsElement.CONCAT_TARGET);
// all except of CONCAT_TARGET and TIMESTAMP
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
IntToken lengthToken = v.visitInt(EditsElement.LENGTH);
sourceCount = lengthToken.value;
Log.info("------------------------- does support");
}
for(int i = 0; i < sourceCount; i++) {
v.visitStringUTF8(EditsElement.CONCAT_SOURCE);
}
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitLong(EditsElement.TIMESTAMP);
} else {
v.visitStringUTF8(EditsElement.TIMESTAMP);
}
}
/**
* Visit OP_SYMLINK
*/
private void visit_OP_SYMLINK() throws IOException {
visitTxId();
if (!LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitInt(EditsElement.LENGTH);
}
v.visitStringUTF8( EditsElement.SOURCE);
v.visitStringUTF8( EditsElement.DESTINATION);
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitLong(EditsElement.MTIME);
v.visitLong(EditsElement.ATIME);
} else {
v.visitStringUTF8(EditsElement.MTIME);
v.visitStringUTF8(EditsElement.ATIME);
}
// PERMISSION_STATUS
v.visitEnclosingElement(EditsElement.PERMISSION_STATUS);
v.visitStringText( EditsElement.USERNAME);
v.visitStringText( EditsElement.GROUPNAME);
v.visitShort( EditsElement.FS_PERMISSIONS);
v.leaveEnclosingElement();
}
/**
* Visit OP_GET_DELEGATION_TOKEN
*/
private void visit_OP_GET_DELEGATION_TOKEN() throws IOException {
visitTxId();
v.visitByte(EditsElement.T_VERSION);
v.visitStringText(EditsElement.T_OWNER);
v.visitStringText(EditsElement.T_RENEWER);
v.visitStringText(EditsElement.T_REAL_USER);
v.visitVLong(EditsElement.T_ISSUE_DATE);
v.visitVLong(EditsElement.T_MAX_DATE);
v.visitVInt(EditsElement.T_SEQUENCE_NUMBER);
v.visitVInt(EditsElement.T_MASTER_KEY_ID);
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitLong(EditsElement.T_EXPIRY_TIME);
} else {
v.visitStringUTF8(EditsElement.T_EXPIRY_TIME);
}
}
/**
* Visit OP_RENEW_DELEGATION_TOKEN
*/
private void visit_OP_RENEW_DELEGATION_TOKEN()
throws IOException {
visitTxId();
v.visitByte(EditsElement.T_VERSION);
v.visitStringText(EditsElement.T_OWNER);
v.visitStringText(EditsElement.T_RENEWER);
v.visitStringText(EditsElement.T_REAL_USER);
v.visitVLong(EditsElement.T_ISSUE_DATE);
v.visitVLong(EditsElement.T_MAX_DATE);
v.visitVInt(EditsElement.T_SEQUENCE_NUMBER);
v.visitVInt(EditsElement.T_MASTER_KEY_ID);
if (LayoutVersion.supports(Feature.EDITLOG_OP_OPTIMIZATION, editsVersion)) {
v.visitLong(EditsElement.T_EXPIRY_TIME);
} else {
v.visitStringUTF8(EditsElement.T_EXPIRY_TIME);
}
}
/**
* Visit OP_CANCEL_DELEGATION_TOKEN
*/
private void visit_OP_CANCEL_DELEGATION_TOKEN()
throws IOException {
visitTxId();
v.visitByte( EditsElement.T_VERSION);
v.visitStringText( EditsElement.T_OWNER);
v.visitStringText( EditsElement.T_RENEWER);
v.visitStringText( EditsElement.T_REAL_USER);
v.visitVLong( EditsElement.T_ISSUE_DATE);
v.visitVLong( EditsElement.T_MAX_DATE);
v.visitVInt( EditsElement.T_SEQUENCE_NUMBER);
v.visitVInt( EditsElement.T_MASTER_KEY_ID);
}
/**
* Visit OP_UPDATE_MASTER_KEY
*/
private void visit_OP_UPDATE_MASTER_KEY()
throws IOException {
visitTxId();
v.visitVInt( EditsElement.KEY_ID);
v.visitVLong( EditsElement.KEY_EXPIRY_DATE);
VIntToken blobLengthToken = v.visitVInt(EditsElement.KEY_LENGTH);
v.visitBlob(EditsElement.KEY_BLOB, blobLengthToken.value);
}
private void visit_OP_REASSIGN_LEASE()
throws IOException {
visitTxId();
v.visitStringUTF8(EditsElement.CLIENT_NAME);
v.visitStringUTF8(EditsElement.PATH);
v.visitStringUTF8(EditsElement.CLIENT_NAME);
}
/**
* Visit OP_BEGIN_LOG_SEGMENT
*/
private void visit_OP_BEGIN_LOG_SEGMENT()
throws IOException {
visitTxId();
}
/**
* Visit OP_END_LOG_SEGMENT
*/
private void visit_OP_END_LOG_SEGMENT()
throws IOException {
visitTxId();
}
private void visitOpCode(FSEditLogOpCodes editsOpCode)
throws IOException {
switch(editsOpCode) {
case OP_INVALID: // -1
visit_OP_INVALID();
break;
case OP_ADD: // 0
visit_OP_ADD();
break;
case OP_CLOSE: // 9
visit_OP_CLOSE();
break;
case OP_RENAME:
visit_OP_RENAME_OLD();
break;
case OP_DELETE: // 2
visit_OP_DELETE();
break;
case OP_MKDIR: // 3
visit_OP_MKDIR();
break;
case OP_SET_REPLICATION: // 4
visit_OP_SET_REPLICATION();
break;
case OP_SET_PERMISSIONS: // 7
visit_OP_SET_PERMISSIONS();
break;
case OP_SET_OWNER: // 8
visit_OP_SET_OWNER();
break;
case OP_SET_GENSTAMP: // 10
visit_OP_SET_GENSTAMP();
break;
case OP_TIMES: // 13
visit_OP_TIMES();
break;
case OP_SET_QUOTA: // 14
visit_OP_SET_QUOTA();
break;
case OP_RENAME_15: // 15
visit_OP_RENAME();
break;
case OP_CONCAT_DELETE: // 16
visit_OP_CONCAT_DELETE();
break;
case OP_SYMLINK:
visit_OP_SYMLINK();
break;
case OP_GET_DELEGATION_TOKEN:
visit_OP_GET_DELEGATION_TOKEN();
break;
case OP_RENEW_DELEGATION_TOKEN:
visit_OP_RENEW_DELEGATION_TOKEN();
break;
case OP_CANCEL_DELEGATION_TOKEN:
visit_OP_CANCEL_DELEGATION_TOKEN();
break;
case OP_UPDATE_MASTER_KEY:
visit_OP_UPDATE_MASTER_KEY();
break;
case OP_REASSIGN_LEASE:
visit_OP_REASSIGN_LEASE();
break;
case OP_END_LOG_SEGMENT: // 23
visit_OP_END_LOG_SEGMENT();
break;
case OP_START_LOG_SEGMENT: // 24
visit_OP_BEGIN_LOG_SEGMENT();
break;
case OP_HARDLINK: // 25
visit_OP_HARDLINK();
break;
case OP_APPEND: //26
visit_OP_APPEND();
break;
default:
{
throw new IOException("Unknown op code " + editsOpCode);
}
}
}
/**
* Loads edits file, uses visitor to process all elements
*/
@Override
public void loadEdits() throws IOException {
try {
v.start();
v.visitEnclosingElement(EditsElement.EDITS);
IntToken editsVersionToken = v.visitInt(EditsElement.EDITS_VERSION);
editsVersion = editsVersionToken.value;
if(!canLoadVersion(editsVersion)) {
throw new IOException("Cannot process editLog version " +
editsVersionToken.value);
}
FSEditLogOpCodes editsOpCode;
do {
v.visitEnclosingElement(EditsElement.RECORD);
ByteToken opCodeToken;
try {
opCodeToken = v.visitByte(EditsElement.OPCODE);
} catch (EOFException eof) {
// Getting EOF when reading the opcode is fine --
// it's just a finalized edits file
// Just fake the OP_INVALID here.
opCodeToken = new ByteToken(EditsElement.OPCODE);
opCodeToken.fromByte(FSEditLogOpCodes.OP_INVALID.getOpCode());
v.visit(opCodeToken);
}
editsOpCode = FSEditLogOpCodes.fromByte(opCodeToken.value);
v.visitEnclosingElement(EditsElement.DATA);
visitOpCode(editsOpCode);
v.leaveEnclosingElement(); // DATA
if (editsOpCode != FSEditLogOpCodes.OP_INVALID &&
LayoutVersion.supports(Feature.EDITS_CHESKUM, editsVersion)) {
v.visitInt(EditsElement.CHECKSUM);
}
v.leaveEnclosingElement(); // RECORD
} while(editsOpCode != FSEditLogOpCodes.OP_INVALID);
v.leaveEnclosingElement(); // EDITS
v.finish();
} catch(IOException e) {
// Tell the visitor to clean up, then re-throw the exception
v.finishAbnormally();
throw e;
}
}
}