HDFS报错解决：Operation category JOURNAL is not supported in state standby

向右看齐 2023-10-03 02:46 100阅读 0赞

### 现象： ###

org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.ipc.StandbyException): Operation category JOURNAL is not supported in state standby. Visit https://s.apache.org/sbnn-error
    	at org.apache.hadoop.hdfs.server.namenode.ha.StandbyState.checkOperation(StandbyState.java:88)
    	at org.apache.hadoop.hdfs.server.namenode.NameNode$NameNodeHAContext.checkOperation(NameNode.java:1954)
    	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkOperation(FSNamesystem.java:1442)
    	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.rollEditLog(FSNamesystem.java:4716)
    	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.rollEditLog(NameNodeRpcServer.java:1293)
    	at org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolServerSideTranslatorPB.rollEditLog(NamenodeProtocolServerSideTranslatorPB.java:148)
    	at org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos$NamenodeProtocolService$2.callBlockingMethod(NamenodeProtocolProtos.java:14726)
    	at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:524)
    	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1025)
    	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:876)
    	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:822)
    	at java.security.AccessController.doPrivileged(Native Method)
    	at javax.security.auth.Subject.doAs(Subject.java:422)
    	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730)
    	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2682)
     
    	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1511)
    	at org.apache.hadoop.ipc.Client.call(Client.java:1457)
    	at org.apache.hadoop.ipc.Client.call(Client.java:1367)
    	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:228)
    	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:116)
    	at com.sun.proxy.$Proxy16.rollEditLog(Unknown Source)
    	at org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB.rollEditLog(NamenodeProtocolTranslatorPB.java:152)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$2.doWork(EditLogTailer.java:365)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$2.doWork(EditLogTailer.java:362)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$MultipleNameNodeProxy.call(EditLogTailer.java:504)
    	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    	at java.lang.Thread.run(Thread.java:748)
    2019-06-28 03:27:27,782 WARN org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer: Exception from remote name node RemoteNameNodeInfo [nnId=nns, ipcAddress=nns/192.168.56.14:8020, httpAddress=http://nns:50070], try next.
    org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.ipc.StandbyException): Operation category JOURNAL is not supported in state standby. Visit https://s.apache.org/sbnn-error
    	at org.apache.hadoop.hdfs.server.namenode.ha.StandbyState.checkOperation(StandbyState.java:88)
    	at org.apache.hadoop.hdfs.server.namenode.NameNode$NameNodeHAContext.checkOperation(NameNode.java:1954)
    	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkOperation(FSNamesystem.java:1442)
    	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.rollEditLog(FSNamesystem.java:4716)
    	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.rollEditLog(NameNodeRpcServer.java:1293)
    	at org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolServerSideTranslatorPB.rollEditLog(NamenodeProtocolServerSideTranslatorPB.java:148)
    	at org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos$NamenodeProtocolService$2.callBlockingMethod(NamenodeProtocolProtos.java:14726)
    	at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:524)
    	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1025)
    	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:876)
    	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:822)
    	at java.security.AccessController.doPrivileged(Native Method)
    	at javax.security.auth.Subject.doAs(Subject.java:422)
    	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730)
    	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2682)
     
    	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1511)
    	at org.apache.hadoop.ipc.Client.call(Client.java:1457)
    	at org.apache.hadoop.ipc.Client.call(Client.java:1367)
    	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:228)
    	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:116)
    	at com.sun.proxy.$Proxy16.rollEditLog(Unknown Source)
    	at org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB.rollEditLog(NamenodeProtocolTranslatorPB.java:152)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$2.doWork(EditLogTailer.java:365)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$2.doWork(EditLogTailer.java:362)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$MultipleNameNodeProxy.call(EditLogTailer.java:504)
    	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    	at java.lang.Thread.run(Thread.java:748)
    2019-06-28 03:27:27,829 WARN org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer: Exception from remote name node RemoteNameNodeInfo [nnId=nns, ipcAddress=nns/192.168.56.14:8020, httpAddress=http://nns:50070], try next.
    org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.ipc.StandbyException): Operation category JOURNAL is not supported in state standby. Visit https://s.apache.org/sbnn-error
    	at org.apache.hadoop.hdfs.server.namenode.ha.StandbyState.checkOperation(StandbyState.java:88)
    	at org.apache.hadoop.hdfs.server.namenode.NameNode$NameNodeHAContext.checkOperation(NameNode.java:1954)
    	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkOperation(FSNamesystem.java:1442)
    	at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.rollEditLog(FSNamesystem.java:4716)
    	at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.rollEditLog(NameNodeRpcServer.java:1293)
    	at org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolServerSideTranslatorPB.rollEditLog(NamenodeProtocolServerSideTranslatorPB.java:148)
    	at org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos$NamenodeProtocolService$2.callBlockingMethod(NamenodeProtocolProtos.java:14726)
    	at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:524)
    	at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1025)
    	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:876)
    	at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:822)
    	at java.security.AccessController.doPrivileged(Native Method)
    	at javax.security.auth.Subject.doAs(Subject.java:422)
    	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1730)
    	at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2682)
     
    	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1511)
    	at org.apache.hadoop.ipc.Client.call(Client.java:1457)
    	at org.apache.hadoop.ipc.Client.call(Client.java:1367)
    	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:228)
    	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:116)
    	at com.sun.proxy.$Proxy16.rollEditLog(Unknown Source)
    	at org.apache.hadoop.hdfs.protocolPB.NamenodeProtocolTranslatorPB.rollEditLog(NamenodeProtocolTranslatorPB.java:152)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$2.doWork(EditLogTailer.java:365)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$2.doWork(EditLogTailer.java:362)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$MultipleNameNodeProxy.call(EditLogTailer.java:504)
    	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    	at java.lang.Thread.run(Thread.java:748)
    2019-06-28 03:27:27,830 WARN org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer: Unable to trigger a roll of the active NN
    java.util.concurrent.ExecutionException: java.io.IOException: Cannot find any valid remote NN to service request!
    	at java.util.concurrent.FutureTask.report(FutureTask.java:122)
    	at java.util.concurrent.FutureTask.get(FutureTask.java:206)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer.triggerActiveLogRoll(EditLogTailer.java:380)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$EditLogTailerThread.doWork(EditLogTailer.java:430)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$EditLogTailerThread.access$300(EditLogTailer.java:399)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$EditLogTailerThread$1.run(EditLogTailer.java:416)
    	at org.apache.hadoop.security.SecurityUtil.doAsLoginUserOrFatal(SecurityUtil.java:484)
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$EditLogTailerThread.run(EditLogTailer.java:412)
    Caused by: java.io.IOException: Cannot find any valid remote NN to service request!
    	at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$MultipleNameNodeProxy.call(EditLogTailer.java:515)
    	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    	at java.lang.Thread.run(Thread.java:748)
    2019-06-28 03:27:29,843 INFO org.apache.hadoop.util.JvmPauseMonitor: Detected pause in JVM or host machine (eg GC): pause of approximately 1605ms
    No GCs detected
    2019-06-28 03:28:30,033 INFO org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer: Triggering log roll on remote NameNode
    2019-06-28 03:28:30,278 INFO org.apache.hadoop.hdfs.server.namenode.FSImage: Reading org.apache.hadoop.hdfs.server.namenode.RedundantEditLogInputStream@6d00c3eb expecting start txid #1
    2019-06-28 03:28:30,279 INFO org.apache.hadoop.hdfs.server.namenode.FSImage: Start loading edits file http://dn3:8480/getJournal?jid=mycluster&segmentTxId=1&storageInfo=-65%3A1565240010%3A1561706674134%3ACID-d60aab16-2a77-48ed-8a5f-5699ca87b6bc&inProgressOk=true, http://dn1:8480/getJournal?jid=mycluster&segmentTxId=1&storageInfo=-65%3A1565240010%3A1561706674134%3ACID-d60aab16-2a77-48ed-8a5f-5699ca87b6bc&inProgressOk=true maxTxnsToRead = 9223372036854775807
    2019-06-28 03:28:30,288 INFO org.apache.hadoop.hdfs.server.namenode.RedundantEditLogInputStream: Fast-forwarding stream 'http://dn3:8480/getJournal?jid=mycluster&segmentTxId=1&storageInfo=-65%3A1565240010%3A1561706674134%3ACID-d60aab16-2a77-48ed-8a5f-5699ca87b6bc&inProgressOk=true, http://dn1:8480/getJournal?jid=mycluster&segmentTxId=1&storageInfo=-65%3A1565240010%3A1561706674134%3ACID-d60aab16-2a77-48ed-8a5f-5699ca87b6bc&inProgressOk=true' to transaction ID 1
    2019-06-28 03:28:30,288 INFO org.apache.hadoop.hdfs.server.namenode.RedundantEditLogInputStream: Fast-forwarding stream 'http://dn3:8480/getJournal?jid=mycluster&segmentTxId=1&storageInfo=-65%3A1565240010%3A1561706674134%3ACID-d60aab16-2a77-48ed-8a5f-5699ca87b6bc&inProgressOk=true' to transaction ID 1
    2019-06-28 03:28:30,625 INFO org.apache.hadoop.hdfs.server.namenode.FSImage: Edits file http://dn3:8480/getJournal?jid=mycluster&segmentTxId=1&storageInfo=-65%3A1565240010%3A1561706674134%3ACID-d60aab16-2a77-48ed-8a5f-5699ca87b6bc&inProgressOk=true, http://dn1:8480/getJournal?jid=mycluster&segmentTxId=1&storageInfo=-65%3A1565240010%3A1561706674134%3ACID-d60aab16-2a77-48ed-8a5f-5699ca87b6bc&inProgressOk=true of size 42 edits # 2 loaded in 0 seconds

### 解决方法： ###

1、原因：两个namenode节点均处于standby状态，没有active状态的节点\*

2、解决办法：`hdfs haadmin -transitionToActive --forcemanual nn1`（想要转换为active状态的namenode的名称）