From 27cc33babfe98e155d28b2b5e31153e1d6b1a0ae Mon Sep 17 00:00:00 2001 From: wubin01 Date: Thu, 27 Oct 2022 14:54:37 +0800 Subject: [PATCH 01/13] feat(net): tron system integrates libp2p module --- .gitignore | 1 - .../tron/common/overlay/message/Message.java | 4 + .../common/overlay/discover/node/Node.java | 135 -------- .../common/parameter/CommonParameter.java | 10 +- framework/build.gradle | 2 + .../common/application/ApplicationImpl.java | 9 +- .../application/TronApplicationContext.java | 12 +- .../org/tron/common/backup/BackupManager.java | 12 +- .../message}/KeepAliveMessage.java | 7 +- .../{net/udp => backup}/message/Message.java | 17 +- .../message/UdpMessageTypeEnum.java | 10 +- .../backup/{ => socket}/BackupServer.java | 12 +- .../socket}/EventHandler.java | 2 +- .../socket}/MessageHandler.java | 2 +- .../socket}/PacketDecoder.java | 4 +- .../handler => backup/socket}/UdpEvent.java | 4 +- .../client/DatabaseGrpcClient.java | 3 +- .../client/WalletGrpcClient.java | 2 +- .../discover/DiscoverMessageInspector.java | 86 ----- .../udp/message/discover/FindNodeMessage.java | 55 --- .../message/discover/NeighborsMessage.java | 77 ----- .../net/udp/message/discover/PingMessage.java | 69 ---- .../net/udp/message/discover/PongMessage.java | 55 --- .../common/overlay/client/PeerClient.java | 99 ------ .../overlay/discover/DiscoverListener.java | 48 --- .../overlay/discover/DiscoverServer.java | 143 -------- .../common/overlay/discover/DiscoverTask.java | 63 ---- .../overlay/discover/DiscoveryExecutor.java | 52 --- .../common/overlay/discover/RefreshTask.java | 37 --- .../common/overlay/discover/dht/Bucket.java | 158 --------- .../common/overlay/discover/dht/DHTUtils.java | 40 --- .../common/overlay/discover/dht/Peer.java | 98 ------ .../common/overlay/discover/node/DBNode.java | 14 - .../overlay/discover/node/DBNodeStats.java | 34 -- .../overlay/discover/node/NodeHandler.java | 305 ----------------- .../overlay/discover/node/NodeManager.java | 314 ------------------ .../node/statistics/NodeStatistics.java | 214 ------------ .../discover/node/statistics/Reputation.java | 48 --- .../discover/table/DistanceComparator.java | 47 --- .../discover/table/KademliaOptions.java | 34 -- .../overlay/discover/table/NodeBucket.java | 75 ----- .../overlay/discover/table/NodeEntry.java | 106 ------ .../overlay/discover/table/NodeTable.java | 131 -------- .../discover/table/TimeComparator.java | 41 --- .../common/overlay/message/MessageCodec.java | 65 ---- .../overlay/message/MessageFactory.java | 22 -- .../common/overlay/message/P2pMessage.java | 34 -- .../overlay/message/P2pMessageFactory.java | 67 ---- .../overlay/message/StaticMessages.java | 10 - .../tron/common/overlay/server/Channel.java | 279 ---------------- .../common/overlay/server/ChannelManager.java | 206 ------------ .../overlay/server/HandshakeHandler.java | 223 ------------- .../common/overlay/server/MessageQueue.java | 217 ------------ .../overlay/server/MessageRoundTrip.java | 39 --- .../common/overlay/server/P2pHandler.java | 103 ------ .../server/PeerConnectionCheckService.java | 65 ---- .../common/overlay/server/PeerServer.java | 84 ----- .../tron/common/overlay/server/SyncPool.java | 265 --------------- .../server/TronChannelInitializer.java | 62 ---- .../TrxProtobufVarint32FrameDecoder.java | 99 ------ .../overlay/server/WireTrafficStats.java | 95 ------ .../src/main/java/org/tron/core/Wallet.java | 30 +- .../java/org/tron/core/config/args/Args.java | 47 ++- .../tron/core/consensus/BlockHandleImpl.java | 2 +- .../org/tron/core/consensus/PbftBaseImpl.java | 17 +- .../tron/core/net/P2pEventHandlerImpl.java | 239 +++++++++++++ .../org/tron/core/net/TronNetDelegate.java | 16 +- .../org/tron/core/net/TronNetHandler.java | 43 --- .../org/tron/core/net/TronNetService.java | 211 +++++------- .../tron/core/net/message/BlocksMessage.java | 43 --- .../net/message/FetchBlockHeadersMessage.java | 17 - .../tron/core/net/message/ItemNotFound.java | 30 -- .../core/net/message/PbftMessageFactory.java | 11 +- .../message/TransactionInventoryMessage.java | 21 -- .../core/net/message/TronMessageFactory.java | 40 ++- .../net/message/{ => adv}/BlockMessage.java | 4 +- .../{ => adv}/FetchInvDataMessage.java | 3 +- .../message/{ => adv}/InventoryMessage.java | 4 +- .../message/{ => adv}/TransactionMessage.java | 4 +- .../{ => adv}/TransactionsMessage.java | 4 +- .../net/message/base}/DisconnectMessage.java | 14 +- .../net/message/handshake}/HelloMessage.java | 26 +- .../net/message/keepalive}/PingMessage.java | 9 +- .../net/message/keepalive}/PongMessage.java | 9 +- .../message/{ => pbft}/PbftCommitMessage.java | 4 +- .../{ => sync}/BlockInventoryMessage.java | 4 +- .../{ => sync}/ChainInventoryMessage.java | 4 +- .../{ => sync}/SyncBlockChainMessage.java | 3 +- .../net/messagehandler/BlockMsgHandler.java | 18 +- .../ChainInventoryMsgHandler.java | 13 +- .../FetchInvDataMsgHandler.java | 33 +- .../messagehandler/InventoryMsgHandler.java | 6 +- .../messagehandler/PbftDataSyncHandler.java | 2 +- .../PbftMsgHandler.java} | 49 +-- .../SyncBlockChainMsgHandler.java | 4 +- .../TransactionsMsgHandler.java | 6 +- .../tron/core/net/peer/PeerConnection.java | 155 +++++++-- .../org/tron/core/net/peer/PeerManager.java | 154 +++++++++ .../org/tron/core/net/peer/TronState.java | 7 + .../tron/core/net/service/RelayService.java | 63 ---- .../net/service/{ => adv}/AdvService.java | 22 +- .../{ => fetchblock}/FetchBlockService.java | 12 +- .../service/handshake/HandshakeService.java | 119 +++++++ .../service/keepalive/KeepAliveService.java | 66 ++++ .../core/net/service/nodepersist/DBNode.java | 24 ++ .../core/net/service/nodepersist/DBNodes.java | 13 + .../nodepersist/NodePersistService.java | 91 +++++ .../net/service/relay/RelayService.java} | 91 +++-- .../net/service}/statistics/MessageCount.java | 4 +- .../statistics/MessageStatistics.java | 70 +--- .../service/statistics/NodeStatistics.java | 54 +++ .../service/statistics/PeerStatistics.java | 5 + .../service/statistics/TronStatsManager.java | 83 +++++ .../net/service/{ => sync}/SyncService.java | 15 +- .../tron/core/services/NodeInfoService.java | 61 ++-- .../org/tron/core/services/RpcApiService.java | 26 +- .../main/java/org/tron/program/FullNode.java | 2 +- .../java/org/tron/program/SolidityNode.java | 13 +- .../discover/node/NodeHandlerTest.java | 100 ------ .../discover/node/NodeManagerTest.java | 226 ------------- .../discover/node/NodeStatisticsTest.java | 150 --------- .../node/statistics/ReputationTest.java | 33 -- .../overlay/discover/table/NodeEntryTest.java | 69 ---- .../overlay/discover/table/NodeTableTest.java | 206 ------------ .../discover/table/TimeComparatorTest.java | 21 -- .../org/tron/common/utils/JsonUtilTest.java | 38 ++- .../test/java/org/tron/core/net/BaseNet.java | 2 +- .../java/org/tron/core/net/BaseNetTest.java | 4 +- .../tron/core/net/DisconnectMessageTest.java | 2 - .../java/org/tron/core/net/MessageTest.java | 2 +- .../test/java/org/tron/core/net/TcpTest.java | 294 ---------------- .../test/java/org/tron/core/net/UdpTest.java | 127 ------- .../messagehandler/BlockMsgHandlerTest.java | 3 +- .../ChainInventoryMsgHandlerTest.java | 2 +- .../InventoryMsgHandlerTest.java | 28 +- .../SyncBlockChainMsgHandlerTest.java | 2 +- .../core/net/services/AdvServiceTest.java | 47 +-- .../core/net/services/RelayServiceTest.java | 18 +- .../org/tron/program/SolidityNodeTest.java | 2 +- 139 files changed, 1502 insertions(+), 6640 deletions(-) delete mode 100644 common/src/main/java/org/tron/common/overlay/discover/node/Node.java rename framework/src/main/java/org/tron/common/{net/udp/message/backup => backup/message}/KeepAliveMessage.java (77%) rename framework/src/main/java/org/tron/common/{net/udp => backup}/message/Message.java (72%) rename framework/src/main/java/org/tron/common/{net/udp => backup}/message/UdpMessageTypeEnum.java (78%) rename framework/src/main/java/org/tron/common/backup/{ => socket}/BackupServer.java (91%) rename framework/src/main/java/org/tron/common/{net/udp/handler => backup/socket}/EventHandler.java (71%) rename framework/src/main/java/org/tron/common/{net/udp/handler => backup/socket}/MessageHandler.java (98%) rename framework/src/main/java/org/tron/common/{net/udp/handler => backup/socket}/PacketDecoder.java (95%) rename framework/src/main/java/org/tron/common/{net/udp/handler => backup/socket}/UdpEvent.java (93%) rename framework/src/main/java/org/tron/common/{overlay => }/client/DatabaseGrpcClient.java (97%) rename framework/src/main/java/org/tron/common/{overlay => }/client/WalletGrpcClient.java (99%) delete mode 100644 framework/src/main/java/org/tron/common/net/udp/message/discover/DiscoverMessageInspector.java delete mode 100755 framework/src/main/java/org/tron/common/net/udp/message/discover/FindNodeMessage.java delete mode 100755 framework/src/main/java/org/tron/common/net/udp/message/discover/NeighborsMessage.java delete mode 100755 framework/src/main/java/org/tron/common/net/udp/message/discover/PingMessage.java delete mode 100755 framework/src/main/java/org/tron/common/net/udp/message/discover/PongMessage.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/client/PeerClient.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/DiscoverListener.java delete mode 100755 framework/src/main/java/org/tron/common/overlay/discover/DiscoverServer.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/DiscoverTask.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/DiscoveryExecutor.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/RefreshTask.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/dht/Bucket.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/dht/DHTUtils.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/dht/Peer.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/node/DBNode.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/node/DBNodeStats.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/node/NodeHandler.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/node/NodeManager.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/node/statistics/NodeStatistics.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/node/statistics/Reputation.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/table/DistanceComparator.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/table/KademliaOptions.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/table/NodeBucket.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/table/NodeEntry.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/table/NodeTable.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/discover/table/TimeComparator.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/message/MessageCodec.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/message/MessageFactory.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/message/P2pMessage.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/message/P2pMessageFactory.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/message/StaticMessages.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/server/Channel.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/server/HandshakeHandler.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/server/MessageRoundTrip.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/server/P2pHandler.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/server/PeerServer.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/server/SyncPool.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/server/TronChannelInitializer.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/server/TrxProtobufVarint32FrameDecoder.java delete mode 100644 framework/src/main/java/org/tron/common/overlay/server/WireTrafficStats.java create mode 100644 framework/src/main/java/org/tron/core/net/P2pEventHandlerImpl.java delete mode 100644 framework/src/main/java/org/tron/core/net/TronNetHandler.java delete mode 100644 framework/src/main/java/org/tron/core/net/message/BlocksMessage.java delete mode 100644 framework/src/main/java/org/tron/core/net/message/FetchBlockHeadersMessage.java delete mode 100644 framework/src/main/java/org/tron/core/net/message/ItemNotFound.java delete mode 100644 framework/src/main/java/org/tron/core/net/message/TransactionInventoryMessage.java rename framework/src/main/java/org/tron/core/net/message/{ => adv}/BlockMessage.java (92%) rename framework/src/main/java/org/tron/core/net/message/{ => adv}/FetchInvDataMessage.java (88%) rename framework/src/main/java/org/tron/core/net/message/{ => adv}/InventoryMessage.java (94%) rename framework/src/main/java/org/tron/core/net/message/{ => adv}/TransactionMessage.java (91%) rename framework/src/main/java/org/tron/core/net/message/{ => adv}/TransactionsMessage.java (90%) rename framework/src/main/java/org/tron/{common/overlay/message => core/net/message/base}/DisconnectMessage.java (70%) rename framework/src/main/java/org/tron/{common/overlay/message => core/net/message/handshake}/HelloMessage.java (88%) rename framework/src/main/java/org/tron/{common/overlay/message => core/net/message/keepalive}/PingMessage.java (74%) rename framework/src/main/java/org/tron/{common/overlay/message => core/net/message/keepalive}/PongMessage.java (74%) rename framework/src/main/java/org/tron/core/net/message/{ => pbft}/PbftCommitMessage.java (86%) rename framework/src/main/java/org/tron/core/net/message/{ => sync}/BlockInventoryMessage.java (91%) rename framework/src/main/java/org/tron/core/net/message/{ => sync}/ChainInventoryMessage.java (94%) rename framework/src/main/java/org/tron/core/net/message/{ => sync}/SyncBlockChainMessage.java (92%) rename framework/src/main/java/org/tron/core/net/{PbftHandler.java => messagehandler/PbftMsgHandler.java} (56%) create mode 100644 framework/src/main/java/org/tron/core/net/peer/PeerManager.java create mode 100644 framework/src/main/java/org/tron/core/net/peer/TronState.java delete mode 100644 framework/src/main/java/org/tron/core/net/service/RelayService.java rename framework/src/main/java/org/tron/core/net/service/{ => adv}/AdvService.java (95%) rename framework/src/main/java/org/tron/core/net/service/{ => fetchblock}/FetchBlockService.java (94%) create mode 100644 framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java create mode 100644 framework/src/main/java/org/tron/core/net/service/keepalive/KeepAliveService.java create mode 100644 framework/src/main/java/org/tron/core/net/service/nodepersist/DBNode.java create mode 100644 framework/src/main/java/org/tron/core/net/service/nodepersist/DBNodes.java create mode 100644 framework/src/main/java/org/tron/core/net/service/nodepersist/NodePersistService.java rename framework/src/main/java/org/tron/{common/overlay/server/FastForward.java => core/net/service/relay/RelayService.java} (67%) rename framework/src/main/java/org/tron/{common/overlay/discover/node => core/net/service}/statistics/MessageCount.java (93%) rename framework/src/main/java/org/tron/{common/overlay/discover/node => core/net/service}/statistics/MessageStatistics.java (78%) create mode 100644 framework/src/main/java/org/tron/core/net/service/statistics/NodeStatistics.java create mode 100644 framework/src/main/java/org/tron/core/net/service/statistics/PeerStatistics.java create mode 100644 framework/src/main/java/org/tron/core/net/service/statistics/TronStatsManager.java rename framework/src/main/java/org/tron/core/net/service/{ => sync}/SyncService.java (96%) delete mode 100644 framework/src/test/java/org/tron/common/overlay/discover/node/NodeHandlerTest.java delete mode 100644 framework/src/test/java/org/tron/common/overlay/discover/node/NodeManagerTest.java delete mode 100644 framework/src/test/java/org/tron/common/overlay/discover/node/NodeStatisticsTest.java delete mode 100644 framework/src/test/java/org/tron/common/overlay/discover/node/statistics/ReputationTest.java delete mode 100644 framework/src/test/java/org/tron/common/overlay/discover/table/NodeEntryTest.java delete mode 100644 framework/src/test/java/org/tron/common/overlay/discover/table/NodeTableTest.java delete mode 100644 framework/src/test/java/org/tron/common/overlay/discover/table/TimeComparatorTest.java delete mode 100644 framework/src/test/java/org/tron/core/net/TcpTest.java delete mode 100644 framework/src/test/java/org/tron/core/net/UdpTest.java diff --git a/.gitignore b/.gitignore index b980800f353..9b19aa5de68 100644 --- a/.gitignore +++ b/.gitignore @@ -31,7 +31,6 @@ shareddata.* # protobuf generated classes src/main/gen - src/main/java/org/tron/core/bftconsensus src/test/java/org/tron/consensus2 src/main/java/META-INF/ diff --git a/chainbase/src/main/java/org/tron/common/overlay/message/Message.java b/chainbase/src/main/java/org/tron/common/overlay/message/Message.java index a0269d0481f..84c3f695686 100644 --- a/chainbase/src/main/java/org/tron/common/overlay/message/Message.java +++ b/chainbase/src/main/java/org/tron/common/overlay/message/Message.java @@ -69,6 +69,10 @@ public ByteBuf getSendData() { return Unpooled.wrappedBuffer(ArrayUtils.add(this.getData(), 0, type)); } + public byte[] getSendBytes() { + return ArrayUtils.add(this.getData(), 0, type); + } + public Sha256Hash getMessageId() { return Sha256Hash.of(CommonParameter.getInstance().isECKeyCryptoEngine(), getData()); diff --git a/common/src/main/java/org/tron/common/overlay/discover/node/Node.java b/common/src/main/java/org/tron/common/overlay/discover/node/Node.java deleted file mode 100644 index 9115b9c9414..00000000000 --- a/common/src/main/java/org/tron/common/overlay/discover/node/Node.java +++ /dev/null @@ -1,135 +0,0 @@ -package org.tron.common.overlay.discover.node; - -import java.io.Serializable; -import java.util.Random; -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.bouncycastle.util.encoders.Hex; -import org.tron.common.utils.ByteArray; -import org.tron.common.utils.Utils; - -@Slf4j(topic = "discover") -public class Node implements Serializable { - - private static final long serialVersionUID = -4267600517925770636L; - - private byte[] id; - - private String host; - - private int port; - - @Getter - private int bindPort; - - @Setter - private int p2pVersion; - - private boolean isFakeNodeId = false; - - public Node(byte[] id, String host, int port) { - this.id = id; - this.host = host; - this.port = port; - this.isFakeNodeId = true; - } - - public Node(byte[] id, String host, int port, int bindPort) { - this.id = id; - this.host = host; - this.port = port; - this.bindPort = bindPort; - } - - public static Node instanceOf(String hostPort) { - try { - String[] sz = hostPort.split(":"); - int port = Integer.parseInt(sz[1]); - return new Node(Node.getNodeId(), sz[0], port); - } catch (Exception e) { - logger.error("Parse node failed, {}", hostPort); - throw e; - } - } - - public static byte[] getNodeId() { - int NODE_ID_LENGTH = 64; - Random gen = new Random(); - byte[] id = new byte[NODE_ID_LENGTH]; - gen.nextBytes(id); - return id; - } - - public boolean isConnectible(int argsP2PVersion) { - return port == bindPort && p2pVersion == argsP2PVersion; - } - - public String getHexId() { - return Hex.toHexString(id); - } - - public String getHexIdShort() { - return Utils.getIdShort(getHexId()); - } - - public boolean isDiscoveryNode() { - return isFakeNodeId; - } - - public byte[] getId() { - return id; - } - - public void setId(byte[] id) { - this.id = id; - } - - public String getHost() { - return host; - } - - public int getPort() { - return port; - } - - public void setPort(int port) { - this.port = port; - } - - public String getIdString() { - if (id == null) { - return null; - } - return new String(id); - } - - @Override - public String toString() { - return "Node{" + " host='" + host + '\'' + ", port=" + port - + ", id=" + ByteArray.toHexString(id) + '}'; - } - - @Override - public int hashCode() { - return this.toString().hashCode(); - } - - @Override - public boolean equals(Object o) { - if (o == null) { - return false; - } - - if (o == this) { - return true; - } - - if (o.getClass() == getClass()) { - return StringUtils.equals(getIdString(), ((Node) o).getIdString()); - } - - return false; - } -} diff --git a/common/src/main/java/org/tron/common/parameter/CommonParameter.java b/common/src/main/java/org/tron/common/parameter/CommonParameter.java index accf4766ea4..870c697473c 100644 --- a/common/src/main/java/org/tron/common/parameter/CommonParameter.java +++ b/common/src/main/java/org/tron/common/parameter/CommonParameter.java @@ -1,6 +1,9 @@ package org.tron.common.parameter; import com.beust.jcommander.Parameter; + +import java.net.InetAddress; +import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -11,7 +14,6 @@ import org.tron.common.config.DbBackupConfig; import org.tron.common.logsfilter.EventPluginConfig; import org.tron.common.logsfilter.FilterQuery; -import org.tron.common.overlay.discover.node.Node; import org.tron.common.setting.RocksDbSettings; import org.tron.core.Constant; import org.tron.core.config.args.Overlay; @@ -390,12 +392,12 @@ public class CommonParameter { public GenesisBlock genesisBlock; @Getter @Setter - public List activeNodes; + public List activeNodes; @Getter @Setter - public List passiveNodes; + public List passiveNodes; @Getter - public List fastForwardNodes; + public List fastForwardNodes; @Getter public int maxFastForwardNum; @Getter diff --git a/framework/build.gradle b/framework/build.gradle index 7df08a90eb9..9e1e57413ea 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -44,6 +44,8 @@ dependencies { testCompile group: 'org.testng', name: 'testng', version: '6.14.3' + compile group: 'com.github.tronprotocol', name: 'libp2p', version: 'test-v0.1.3' + compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' compile group: 'com.typesafe', name: 'config', version: '1.3.2' diff --git a/framework/src/main/java/org/tron/common/application/ApplicationImpl.java b/framework/src/main/java/org/tron/common/application/ApplicationImpl.java index 1be119f1be3..235ece75835 100644 --- a/framework/src/main/java/org/tron/common/application/ApplicationImpl.java +++ b/framework/src/main/java/org/tron/common/application/ApplicationImpl.java @@ -12,6 +12,7 @@ import org.tron.core.metrics.MetricsUtil; import org.tron.core.net.TronNetService; import org.tron.program.FullNode; +import org.tron.program.SolidityNode; @Slf4j(topic = "app") @Component @@ -56,7 +57,9 @@ public void initServices(CommonParameter parameter) { * start up the app. */ public void startup() { - tronNetService.start(); + if (!Args.getInstance().isSolidityNode()) { + tronNetService.start(); + } consensusService.start(); MetricsUtil.init(); } @@ -64,7 +67,9 @@ public void startup() { @Override public void shutdown() { logger.info("******** start to shutdown ********"); - tronNetService.stop(); + if (!Args.getInstance().isSolidityNode()) { + tronNetService.close(); + } consensusService.stop(); synchronized (dbManager.getRevokingStore()) { dbManager.getSession().reset(); diff --git a/framework/src/main/java/org/tron/common/application/TronApplicationContext.java b/framework/src/main/java/org/tron/common/application/TronApplicationContext.java index 204b5dc5a68..7f0aea813a3 100644 --- a/framework/src/main/java/org/tron/common/application/TronApplicationContext.java +++ b/framework/src/main/java/org/tron/common/application/TronApplicationContext.java @@ -2,10 +2,8 @@ import org.springframework.beans.factory.support.DefaultListableBeanFactory; import org.springframework.context.annotation.AnnotationConfigApplicationContext; -import org.tron.common.overlay.discover.DiscoverServer; -import org.tron.common.overlay.discover.node.NodeManager; -import org.tron.common.overlay.server.ChannelManager; import org.tron.core.db.Manager; +import org.tron.core.net.TronNetService; public class TronApplicationContext extends AnnotationConfigApplicationContext { @@ -31,12 +29,8 @@ public void destroy() { appT.shutdownServices(); appT.shutdown(); - DiscoverServer discoverServer = getBean(DiscoverServer.class); - discoverServer.close(); - ChannelManager channelManager = getBean(ChannelManager.class); - channelManager.close(); - NodeManager nodeManager = getBean(NodeManager.class); - nodeManager.close(); + TronNetService tronNetService = getBean(TronNetService.class); + tronNetService.close(); Manager dbManager = getBean(Manager.class); dbManager.stopRePushThread(); diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index 1d78dd234ad..ef304164f1a 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -3,7 +3,7 @@ import static org.tron.common.backup.BackupManager.BackupStatusEnum.INIT; import static org.tron.common.backup.BackupManager.BackupStatusEnum.MASTER; import static org.tron.common.backup.BackupManager.BackupStatusEnum.SLAVER; -import static org.tron.common.net.udp.message.UdpMessageTypeEnum.BACKUP_KEEP_ALIVE; +import static org.tron.common.backup.message.UdpMessageTypeEnum.BACKUP_KEEP_ALIVE; import io.netty.util.internal.ConcurrentSet; import java.net.InetAddress; @@ -14,11 +14,11 @@ import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; -import org.tron.common.net.udp.handler.EventHandler; -import org.tron.common.net.udp.handler.MessageHandler; -import org.tron.common.net.udp.handler.UdpEvent; -import org.tron.common.net.udp.message.Message; -import org.tron.common.net.udp.message.backup.KeepAliveMessage; +import org.tron.common.backup.message.KeepAliveMessage; +import org.tron.common.backup.message.Message; +import org.tron.common.backup.socket.EventHandler; +import org.tron.common.backup.socket.MessageHandler; +import org.tron.common.backup.socket.UdpEvent; import org.tron.common.parameter.CommonParameter; @Slf4j(topic = "backup") diff --git a/framework/src/main/java/org/tron/common/net/udp/message/backup/KeepAliveMessage.java b/framework/src/main/java/org/tron/common/backup/message/KeepAliveMessage.java similarity index 77% rename from framework/src/main/java/org/tron/common/net/udp/message/backup/KeepAliveMessage.java rename to framework/src/main/java/org/tron/common/backup/message/KeepAliveMessage.java index 7e3ed37684a..fca63451107 100644 --- a/framework/src/main/java/org/tron/common/net/udp/message/backup/KeepAliveMessage.java +++ b/framework/src/main/java/org/tron/common/backup/message/KeepAliveMessage.java @@ -1,9 +1,8 @@ -package org.tron.common.net.udp.message.backup; +package org.tron.common.backup.message; -import static org.tron.common.net.udp.message.UdpMessageTypeEnum.BACKUP_KEEP_ALIVE; +import static org.tron.common.backup.message.UdpMessageTypeEnum.BACKUP_KEEP_ALIVE; -import org.tron.common.net.udp.message.Message; -import org.tron.common.overlay.discover.node.Node; +import org.tron.p2p.discover.Node; import org.tron.protos.Discover; public class KeepAliveMessage extends Message { diff --git a/framework/src/main/java/org/tron/common/net/udp/message/Message.java b/framework/src/main/java/org/tron/common/backup/message/Message.java similarity index 72% rename from framework/src/main/java/org/tron/common/net/udp/message/Message.java rename to framework/src/main/java/org/tron/common/backup/message/Message.java index bda851f9a5f..8f09a452877 100644 --- a/framework/src/main/java/org/tron/common/net/udp/message/Message.java +++ b/framework/src/main/java/org/tron/common/backup/message/Message.java @@ -1,16 +1,11 @@ -package org.tron.common.net.udp.message; +package org.tron.common.backup.message; import org.apache.commons.lang3.ArrayUtils; -import org.tron.common.net.udp.message.backup.KeepAliveMessage; -import org.tron.common.net.udp.message.discover.FindNodeMessage; -import org.tron.common.net.udp.message.discover.NeighborsMessage; -import org.tron.common.net.udp.message.discover.PingMessage; -import org.tron.common.net.udp.message.discover.PongMessage; -import org.tron.common.overlay.discover.node.Node; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.ByteArray; import org.tron.common.utils.Sha256Hash; import org.tron.core.exception.P2pException; +import org.tron.p2p.discover.Node; import org.tron.protos.Discover.Endpoint; public abstract class Message { @@ -33,14 +28,6 @@ public static Message parse(byte[] encode) throws Exception { byte type = encode[0]; byte[] data = ArrayUtils.subarray(encode, 1, encode.length); switch (UdpMessageTypeEnum.fromByte(type)) { - case DISCOVER_PING: - return new PingMessage(data); - case DISCOVER_PONG: - return new PongMessage(data); - case DISCOVER_FIND_NODE: - return new FindNodeMessage(data); - case DISCOVER_NEIGHBORS: - return new NeighborsMessage(data); case BACKUP_KEEP_ALIVE: return new KeepAliveMessage(data); default: diff --git a/framework/src/main/java/org/tron/common/net/udp/message/UdpMessageTypeEnum.java b/framework/src/main/java/org/tron/common/backup/message/UdpMessageTypeEnum.java similarity index 78% rename from framework/src/main/java/org/tron/common/net/udp/message/UdpMessageTypeEnum.java rename to framework/src/main/java/org/tron/common/backup/message/UdpMessageTypeEnum.java index 1af4aa882d5..d4a5a9dd64b 100644 --- a/framework/src/main/java/org/tron/common/net/udp/message/UdpMessageTypeEnum.java +++ b/framework/src/main/java/org/tron/common/backup/message/UdpMessageTypeEnum.java @@ -1,18 +1,10 @@ -package org.tron.common.net.udp.message; +package org.tron.common.backup.message; import java.util.HashMap; import java.util.Map; public enum UdpMessageTypeEnum { - DISCOVER_PING((byte) 0x01), - - DISCOVER_PONG((byte) 0x02), - - DISCOVER_FIND_NODE((byte) 0x03), - - DISCOVER_NEIGHBORS((byte) 0x04), - BACKUP_KEEP_ALIVE((byte) 0x05), UNKNOWN((byte) 0xFF); diff --git a/framework/src/main/java/org/tron/common/backup/BackupServer.java b/framework/src/main/java/org/tron/common/backup/socket/BackupServer.java similarity index 91% rename from framework/src/main/java/org/tron/common/backup/BackupServer.java rename to framework/src/main/java/org/tron/common/backup/socket/BackupServer.java index f95191a295d..e3b1de31736 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupServer.java +++ b/framework/src/main/java/org/tron/common/backup/socket/BackupServer.java @@ -1,4 +1,4 @@ -package org.tron.common.backup; +package org.tron.common.backup.socket; import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; @@ -11,10 +11,9 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.tron.common.net.udp.handler.MessageHandler; -import org.tron.common.net.udp.handler.PacketDecoder; -import org.tron.common.overlay.server.WireTrafficStats; +import org.tron.common.backup.BackupManager; import org.tron.common.parameter.CommonParameter; +import org.tron.p2p.stats.TrafficStats; @Slf4j(topic = "backup") @Component @@ -30,9 +29,6 @@ public class BackupServer { private volatile boolean shutdown = false; - @Autowired - private WireTrafficStats stats; - @Autowired public BackupServer(final BackupManager backupManager) { this.backupManager = backupManager; @@ -61,7 +57,7 @@ private void start() throws Exception { @Override public void initChannel(NioDatagramChannel ch) throws Exception { - ch.pipeline().addLast(stats.udp); + ch.pipeline().addLast(TrafficStats.udp); ch.pipeline().addLast(new ProtobufVarint32LengthFieldPrepender()); ch.pipeline().addLast(new ProtobufVarint32FrameDecoder()); ch.pipeline().addLast(new PacketDecoder()); diff --git a/framework/src/main/java/org/tron/common/net/udp/handler/EventHandler.java b/framework/src/main/java/org/tron/common/backup/socket/EventHandler.java similarity index 71% rename from framework/src/main/java/org/tron/common/net/udp/handler/EventHandler.java rename to framework/src/main/java/org/tron/common/backup/socket/EventHandler.java index db7ddae9d35..699d4cc0cba 100644 --- a/framework/src/main/java/org/tron/common/net/udp/handler/EventHandler.java +++ b/framework/src/main/java/org/tron/common/backup/socket/EventHandler.java @@ -1,4 +1,4 @@ -package org.tron.common.net.udp.handler; +package org.tron.common.backup.socket; public interface EventHandler { diff --git a/framework/src/main/java/org/tron/common/net/udp/handler/MessageHandler.java b/framework/src/main/java/org/tron/common/backup/socket/MessageHandler.java similarity index 98% rename from framework/src/main/java/org/tron/common/net/udp/handler/MessageHandler.java rename to framework/src/main/java/org/tron/common/backup/socket/MessageHandler.java index 656dd9fb667..81ad88c6547 100644 --- a/framework/src/main/java/org/tron/common/net/udp/handler/MessageHandler.java +++ b/framework/src/main/java/org/tron/common/backup/socket/MessageHandler.java @@ -16,7 +16,7 @@ * along with the ethereumJ library. If not, see . */ -package org.tron.common.net.udp.handler; +package org.tron.common.backup.socket; import io.netty.buffer.Unpooled; import io.netty.channel.Channel; diff --git a/framework/src/main/java/org/tron/common/net/udp/handler/PacketDecoder.java b/framework/src/main/java/org/tron/common/backup/socket/PacketDecoder.java similarity index 95% rename from framework/src/main/java/org/tron/common/net/udp/handler/PacketDecoder.java rename to framework/src/main/java/org/tron/common/backup/socket/PacketDecoder.java index 4859dcd04f6..4a66bf3253b 100644 --- a/framework/src/main/java/org/tron/common/net/udp/handler/PacketDecoder.java +++ b/framework/src/main/java/org/tron/common/backup/socket/PacketDecoder.java @@ -16,7 +16,7 @@ * along with the ethereumJ library. If not, see . */ -package org.tron.common.net.udp.handler; +package org.tron.common.backup.socket; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; @@ -24,7 +24,7 @@ import io.netty.handler.codec.MessageToMessageDecoder; import java.util.List; import lombok.extern.slf4j.Slf4j; -import org.tron.common.net.udp.message.Message; +import org.tron.common.backup.message.Message; @Slf4j(topic = "net") public class PacketDecoder extends MessageToMessageDecoder { diff --git a/framework/src/main/java/org/tron/common/net/udp/handler/UdpEvent.java b/framework/src/main/java/org/tron/common/backup/socket/UdpEvent.java similarity index 93% rename from framework/src/main/java/org/tron/common/net/udp/handler/UdpEvent.java rename to framework/src/main/java/org/tron/common/backup/socket/UdpEvent.java index 969dcd0e1c7..f9755d15781 100644 --- a/framework/src/main/java/org/tron/common/net/udp/handler/UdpEvent.java +++ b/framework/src/main/java/org/tron/common/backup/socket/UdpEvent.java @@ -16,10 +16,10 @@ * along with the ethereumJ library. If not, see . */ -package org.tron.common.net.udp.handler; +package org.tron.common.backup.socket; import java.net.InetSocketAddress; -import org.tron.common.net.udp.message.Message; +import org.tron.common.backup.message.Message; public class UdpEvent { diff --git a/framework/src/main/java/org/tron/common/overlay/client/DatabaseGrpcClient.java b/framework/src/main/java/org/tron/common/client/DatabaseGrpcClient.java similarity index 97% rename from framework/src/main/java/org/tron/common/overlay/client/DatabaseGrpcClient.java rename to framework/src/main/java/org/tron/common/client/DatabaseGrpcClient.java index 7a2da659275..b83c7235900 100644 --- a/framework/src/main/java/org/tron/common/overlay/client/DatabaseGrpcClient.java +++ b/framework/src/main/java/org/tron/common/client/DatabaseGrpcClient.java @@ -1,4 +1,4 @@ -package org.tron.common.overlay.client; +package org.tron.common.client; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; @@ -27,7 +27,6 @@ public DatabaseGrpcClient(String host) { databaseBlockingStub = DatabaseGrpc.newBlockingStub(channel); } - public Block getBlock(long blockNum) { if (blockNum < 0) { return databaseBlockingStub.getNowBlock(EmptyMessage.newBuilder().build()); diff --git a/framework/src/main/java/org/tron/common/overlay/client/WalletGrpcClient.java b/framework/src/main/java/org/tron/common/client/WalletGrpcClient.java similarity index 99% rename from framework/src/main/java/org/tron/common/overlay/client/WalletGrpcClient.java rename to framework/src/main/java/org/tron/common/client/WalletGrpcClient.java index 6bff4810eec..9d3b5797e20 100644 --- a/framework/src/main/java/org/tron/common/overlay/client/WalletGrpcClient.java +++ b/framework/src/main/java/org/tron/common/client/WalletGrpcClient.java @@ -1,4 +1,4 @@ -package org.tron.common.overlay.client; +package org.tron.common.client; import com.google.protobuf.ByteString; import io.grpc.ManagedChannel; diff --git a/framework/src/main/java/org/tron/common/net/udp/message/discover/DiscoverMessageInspector.java b/framework/src/main/java/org/tron/common/net/udp/message/discover/DiscoverMessageInspector.java deleted file mode 100644 index 6c8afdd5625..00000000000 --- a/framework/src/main/java/org/tron/common/net/udp/message/discover/DiscoverMessageInspector.java +++ /dev/null @@ -1,86 +0,0 @@ -package org.tron.common.net.udp.message.discover; - -import java.util.regex.Pattern; -import org.springframework.util.StringUtils; -import org.tron.common.net.udp.message.Message; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.overlay.discover.table.KademliaOptions; - -public class DiscoverMessageInspector { - - public static final Pattern PATTERN_IP = - Pattern.compile("^(1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|[1-9])\\" - + ".(1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|\\d)\\" - + ".(1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|\\d)\\" - + ".(1\\d{2}|2[0-4]\\d|25[0-5]|[1-9]\\d|\\d)$"); - - private static boolean isFound(String str, Pattern pattern) { - if (str == null || pattern == null) { - return false; - } - return pattern.matcher(str).find(); - } - - private static boolean validNode(Node node) { - if (node == null) { - return false; - } - if (!isFound(node.getHost(), PATTERN_IP) - || node.getId().length != KademliaOptions.NODE_ID_LEN) { - return false; - } - return true; - } - - private static boolean valid(PingMessage message) { - return validNode(message.getFrom()) && validNode(message.getTo()); - } - - private static boolean valid(PongMessage message) { - return validNode(message.getFrom()); - } - - private static boolean valid(FindNodeMessage message) { - return validNode(message.getFrom()) - && message.getTargetId().length == KademliaOptions.NODE_ID_LEN; - } - - private static boolean valid(NeighborsMessage message) { - if (!validNode(message.getFrom())) { - return false; - } - if (!StringUtils.isEmpty(message.getNodes())) { - if (message.getNodes().size() > KademliaOptions.BUCKET_SIZE) { - return false; - } - for (Node node : message.getNodes()) { - if (!validNode(node)) { - return false; - } - } - } - return true; - } - - public static boolean valid(Message message) { - boolean flag = false; - switch (message.getType()) { - case DISCOVER_PING: - flag = valid((PingMessage) message); - break; - case DISCOVER_PONG: - flag = valid((PongMessage) message); - break; - case DISCOVER_FIND_NODE: - flag = valid((FindNodeMessage) message); - break; - case DISCOVER_NEIGHBORS: - flag = valid((NeighborsMessage) message); - break; - default: - break; - } - return flag; - } - -} diff --git a/framework/src/main/java/org/tron/common/net/udp/message/discover/FindNodeMessage.java b/framework/src/main/java/org/tron/common/net/udp/message/discover/FindNodeMessage.java deleted file mode 100755 index 1507366a9c5..00000000000 --- a/framework/src/main/java/org/tron/common/net/udp/message/discover/FindNodeMessage.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.tron.common.net.udp.message.discover; - -import static org.tron.common.net.udp.message.UdpMessageTypeEnum.DISCOVER_FIND_NODE; - -import com.google.protobuf.ByteString; -import org.tron.common.net.udp.message.Message; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.utils.ByteArray; -import org.tron.protos.Discover; -import org.tron.protos.Discover.Endpoint; -import org.tron.protos.Discover.FindNeighbours; - -public class FindNodeMessage extends Message { - - private Discover.FindNeighbours findNeighbours; - - public FindNodeMessage(byte[] data) throws Exception { - super(DISCOVER_FIND_NODE, data); - this.findNeighbours = Discover.FindNeighbours.parseFrom(data); - } - - public FindNodeMessage(Node from, byte[] targetId) { - super(DISCOVER_FIND_NODE, null); - Endpoint fromEndpoint = Endpoint.newBuilder() - .setAddress(ByteString.copyFrom(ByteArray.fromString(from.getHost()))) - .setPort(from.getPort()) - .setNodeId(ByteString.copyFrom(from.getId())) - .build(); - this.findNeighbours = FindNeighbours.newBuilder() - .setFrom(fromEndpoint) - .setTargetId(ByteString.copyFrom(targetId)) - .setTimestamp(System.currentTimeMillis()) - .build(); - this.data = this.findNeighbours.toByteArray(); - } - - public byte[] getTargetId() { - return this.findNeighbours.getTargetId().toByteArray(); - } - - @Override - public long getTimestamp() { - return this.findNeighbours.getTimestamp(); - } - - @Override - public Node getFrom() { - return Message.getNode(findNeighbours.getFrom()); - } - - @Override - public String toString() { - return "[findNeighbours: " + findNeighbours; - } -} diff --git a/framework/src/main/java/org/tron/common/net/udp/message/discover/NeighborsMessage.java b/framework/src/main/java/org/tron/common/net/udp/message/discover/NeighborsMessage.java deleted file mode 100755 index 61f7bddd18e..00000000000 --- a/framework/src/main/java/org/tron/common/net/udp/message/discover/NeighborsMessage.java +++ /dev/null @@ -1,77 +0,0 @@ -package org.tron.common.net.udp.message.discover; - -import static org.tron.common.net.udp.message.UdpMessageTypeEnum.DISCOVER_NEIGHBORS; - -import com.google.protobuf.ByteString; -import java.util.ArrayList; -import java.util.List; -import org.tron.common.net.udp.message.Message; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.utils.ByteArray; -import org.tron.protos.Discover; -import org.tron.protos.Discover.Endpoint; -import org.tron.protos.Discover.Neighbours; -import org.tron.protos.Discover.Neighbours.Builder; - -public class NeighborsMessage extends Message { - - private Discover.Neighbours neighbours; - - public NeighborsMessage(byte[] data) throws Exception { - super(DISCOVER_NEIGHBORS, data); - this.neighbours = Discover.Neighbours.parseFrom(data); - } - - public NeighborsMessage(Node from, List neighbours, long sequence) { - super(DISCOVER_NEIGHBORS, null); - Builder builder = Neighbours.newBuilder() - .setTimestamp(sequence); - - neighbours.forEach(neighbour -> { - Endpoint endpoint = Endpoint.newBuilder() - .setAddress(ByteString.copyFrom(ByteArray.fromString(neighbour.getHost()))) - .setPort(neighbour.getPort()) - .setNodeId(ByteString.copyFrom(neighbour.getId())) - .build(); - - builder.addNeighbours(endpoint); - }); - - Endpoint fromEndpoint = Endpoint.newBuilder() - .setAddress(ByteString.copyFrom(ByteArray.fromString(from.getHost()))) - .setPort(from.getPort()) - .setNodeId(ByteString.copyFrom(from.getId())) - .build(); - - builder.setFrom(fromEndpoint); - - this.neighbours = builder.build(); - - this.data = this.neighbours.toByteArray(); - } - - public List getNodes() { - List nodes = new ArrayList<>(); - neighbours.getNeighboursList().forEach(neighbour -> nodes.add( - new Node(neighbour.getNodeId().toByteArray(), - ByteArray.toStr(neighbour.getAddress().toByteArray()), - neighbour.getPort()))); - return nodes; - } - - @Override - public long getTimestamp() { - return this.neighbours.getTimestamp(); - } - - @Override - public Node getFrom() { - return Message.getNode(neighbours.getFrom()); - } - - @Override - public String toString() { - return "[neighbours: " + neighbours; - } - -} diff --git a/framework/src/main/java/org/tron/common/net/udp/message/discover/PingMessage.java b/framework/src/main/java/org/tron/common/net/udp/message/discover/PingMessage.java deleted file mode 100755 index e7510414163..00000000000 --- a/framework/src/main/java/org/tron/common/net/udp/message/discover/PingMessage.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.tron.common.net.udp.message.discover; - -import static org.tron.common.net.udp.message.UdpMessageTypeEnum.DISCOVER_PING; - -import com.google.protobuf.ByteString; -import org.tron.common.net.udp.message.Message; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.utils.ByteArray; -import org.tron.core.config.args.Args; -import org.tron.protos.Discover; -import org.tron.protos.Discover.Endpoint; - -public class PingMessage extends Message { - - private Discover.PingMessage pingMessage; - - public PingMessage(byte[] data) throws Exception { - super(DISCOVER_PING, data); - this.pingMessage = Discover.PingMessage.parseFrom(data); - } - - public PingMessage(Node from, Node to) { - super(DISCOVER_PING, null); - Endpoint fromEndpoint = Endpoint.newBuilder() - .setNodeId(ByteString.copyFrom(from.getId())) - .setPort(from.getPort()) - .setAddress(ByteString.copyFrom(ByteArray.fromString(from.getHost()))) - .build(); - Endpoint toEndpoint = Endpoint.newBuilder() - .setNodeId(ByteString.copyFrom(to.getId())) - .setPort(to.getPort()) - .setAddress(ByteString.copyFrom(ByteArray.fromString(to.getHost()))) - .build(); - this.pingMessage = Discover.PingMessage.newBuilder() - .setVersion(Args.getInstance().getNodeP2pVersion()) - .setFrom(fromEndpoint) - .setTo(toEndpoint) - .setTimestamp(System.currentTimeMillis()) - .build(); - this.data = this.pingMessage.toByteArray(); - } - - public int getVersion() { - return this.pingMessage.getVersion(); - } - - public Node getTo() { - Endpoint to = this.pingMessage.getTo(); - Node node = new Node(to.getNodeId().toByteArray(), - ByteArray.toStr(to.getAddress().toByteArray()), to.getPort()); - return node; - } - - @Override - public long getTimestamp() { - return this.pingMessage.getTimestamp(); - } - - @Override - public Node getFrom() { - return Message.getNode(pingMessage.getFrom()); - } - - @Override - public String toString() { - return "[pingMessage: " + pingMessage; - } - -} diff --git a/framework/src/main/java/org/tron/common/net/udp/message/discover/PongMessage.java b/framework/src/main/java/org/tron/common/net/udp/message/discover/PongMessage.java deleted file mode 100755 index 735434966ef..00000000000 --- a/framework/src/main/java/org/tron/common/net/udp/message/discover/PongMessage.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.tron.common.net.udp.message.discover; - -import static org.tron.common.net.udp.message.UdpMessageTypeEnum.DISCOVER_PONG; - -import com.google.protobuf.ByteString; -import org.tron.common.net.udp.message.Message; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.utils.ByteArray; -import org.tron.core.config.args.Args; -import org.tron.protos.Discover; -import org.tron.protos.Discover.Endpoint; - -public class PongMessage extends Message { - - private Discover.PongMessage pongMessage; - - public PongMessage(byte[] data) throws Exception { - super(DISCOVER_PONG, data); - this.pongMessage = Discover.PongMessage.parseFrom(data); - } - - public PongMessage(Node from) { - super(DISCOVER_PONG, null); - Endpoint toEndpoint = Endpoint.newBuilder() - .setAddress(ByteString.copyFrom(ByteArray.fromString(from.getHost()))) - .setPort(from.getPort()) - .setNodeId(ByteString.copyFrom(from.getId())) - .build(); - this.pongMessage = Discover.PongMessage.newBuilder() - .setFrom(toEndpoint) - .setEcho(Args.getInstance().getNodeP2pVersion()) - .setTimestamp(System.currentTimeMillis()) - .build(); - this.data = this.pongMessage.toByteArray(); - } - - public int getVersion() { - return this.pongMessage.getEcho(); - } - - @Override - public long getTimestamp() { - return this.pongMessage.getTimestamp(); - } - - @Override - public Node getFrom() { - return Message.getNode(pongMessage.getFrom()); - } - - @Override - public String toString() { - return "[pongMessage: " + pongMessage; - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/client/PeerClient.java b/framework/src/main/java/org/tron/common/overlay/client/PeerClient.java deleted file mode 100644 index d6a8b224c84..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/client/PeerClient.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.tron.common.overlay.client; - -import io.netty.bootstrap.Bootstrap; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelOption; -import io.netty.channel.DefaultMessageSizeEstimator; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.nio.NioSocketChannel; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.atomic.AtomicInteger; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.stereotype.Component; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.overlay.discover.node.NodeHandler; -import org.tron.common.overlay.server.TronChannelInitializer; -import org.tron.core.config.args.Args; -import org.tron.protos.Protocol.ReasonCode; - -@Slf4j(topic = "net") -@Component -public class PeerClient { - - @Autowired - private ApplicationContext ctx; - - private EventLoopGroup workerGroup; - - public PeerClient() { - workerGroup = new NioEventLoopGroup(0, new ThreadFactory() { - private AtomicInteger cnt = new AtomicInteger(0); - - @Override - public Thread newThread(Runnable r) { - return new Thread(r, "TronJClientWorker-" + cnt.getAndIncrement()); - } - }); - } - - public void connect(String host, int port, String remoteId) { - try { - ChannelFuture f = connectAsync(host, port, remoteId, false); - f.sync().channel().closeFuture().sync(); - } catch (Exception e) { - logger.warn("Can't connect to {}:{}, cause:{})", host, port, e.getMessage()); - } - } - - public void connectAsync(NodeHandler nodeHandler, boolean discoveryMode) { - Node node = nodeHandler.getNode(); - try { - connectAsync(node.getHost(), node.getPort(), node.getHexId(), discoveryMode) - .addListener((ChannelFutureListener) future -> { - if (!future.isSuccess()) { - logger.warn("Connect to {}:{} fail, cause:{}", node.getHost(), node.getPort(), - future.cause().getMessage()); - nodeHandler.getNodeStatistics().nodeDisconnectedLocal(ReasonCode.CONNECT_FAIL); - nodeHandler.getNodeStatistics().notifyDisconnect(); - future.channel().close(); - } - }); - } catch (Exception e) { - logger.warn("Connect to peer {} failed, reason: {}", - node.getHost(), e.getMessage()); - } - } - - private ChannelFuture connectAsync(String host, int port, String remoteId, - boolean discoveryMode) { - - logger.info("Connect peer {}:{}, remoteId:{}", host, port, remoteId); - - TronChannelInitializer tronChannelInitializer = ctx - .getBean(TronChannelInitializer.class, remoteId); - tronChannelInitializer.setPeerDiscoveryMode(discoveryMode); - - Bootstrap b = new Bootstrap(); - b.group(workerGroup); - b.channel(NioSocketChannel.class); - - b.option(ChannelOption.SO_KEEPALIVE, true); - b.option(ChannelOption.MESSAGE_SIZE_ESTIMATOR, DefaultMessageSizeEstimator.DEFAULT); - b.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, Args.getInstance().getNodeConnectionTimeout()); - b.remoteAddress(host, port); - - b.handler(tronChannelInitializer); - - // Start the client. - return b.connect(); - } - - public void close() { - workerGroup.shutdownGracefully(); - workerGroup.terminationFuture().syncUninterruptibly(); - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/DiscoverListener.java b/framework/src/main/java/org/tron/common/overlay/discover/DiscoverListener.java deleted file mode 100644 index e5760dbc2c6..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/DiscoverListener.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover; - -import org.tron.common.overlay.discover.node.NodeHandler; -import org.tron.common.overlay.discover.node.NodeManager; - -/** - * Allows to handle discovered nodes state changes Created by Anton Nashatyrev on 21.07.2015. - */ -public interface DiscoverListener { - - /** - * Invoked whenever a new node appeared which meets criteria specified in the {@link - * NodeManager#addDiscoverListener} method - */ - void nodeAppeared(NodeHandler handler); - - /** - * Invoked whenever a node stops meeting criteria. - */ - void nodeDisappeared(NodeHandler handler); - - class Adapter implements DiscoverListener { - - public void nodeAppeared(NodeHandler handler) { - } - - public void nodeDisappeared(NodeHandler handler) { - } - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/DiscoverServer.java b/framework/src/main/java/org/tron/common/overlay/discover/DiscoverServer.java deleted file mode 100755 index ce8d577caf2..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/DiscoverServer.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover; - -import io.netty.bootstrap.Bootstrap; -import io.netty.channel.Channel; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.nio.NioDatagramChannel; -import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder; -import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender; -import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.tron.common.net.udp.handler.MessageHandler; -import org.tron.common.net.udp.handler.PacketDecoder; -import org.tron.common.overlay.discover.node.NodeManager; -import org.tron.common.overlay.server.WireTrafficStats; -import org.tron.common.parameter.CommonParameter; -import org.tron.core.config.args.Args; - -@Slf4j(topic = "discover") -@Component -public class DiscoverServer { - - @Autowired - private NodeManager nodeManager; - - @Autowired - private WireTrafficStats stats; - - private CommonParameter parameter = Args.getInstance(); - - private int port = parameter.getNodeListenPort(); - - private Channel channel; - - private DiscoveryExecutor discoveryExecutor; - - private volatile boolean shutdown = false; - - @Autowired - public DiscoverServer(final NodeManager nodeManager) { - this.nodeManager = nodeManager; - if (parameter.isNodeDiscoveryEnable() && !parameter.isFastForward() - && !parameter.isSolidityNode()) { - if (port == 0) { - logger.error("Discovery can't be started while listen port == 0"); - } else { - new Thread(() -> { - try { - start(); - } catch (Exception e) { - logger.error("Discovery server start failed", e); - } - }, "DiscoverServer").start(); - } - } - } - - public void start() throws Exception { - NioEventLoopGroup group = new NioEventLoopGroup(parameter.getUdpNettyWorkThreadNum()); - try { - discoveryExecutor = new DiscoveryExecutor(nodeManager); - discoveryExecutor.start(); - while (!shutdown) { - Bootstrap b = new Bootstrap(); - b.group(group) - .channel(NioDatagramChannel.class) - .handler(new ChannelInitializer() { - @Override - public void initChannel(NioDatagramChannel ch) - throws Exception { - ch.pipeline().addLast(stats.udp); - ch.pipeline().addLast(new ProtobufVarint32LengthFieldPrepender()); - ch.pipeline().addLast(new ProtobufVarint32FrameDecoder()); - ch.pipeline().addLast(new PacketDecoder()); - MessageHandler messageHandler = new MessageHandler(ch, nodeManager); - nodeManager.setMessageSender(messageHandler); - ch.pipeline().addLast(messageHandler); - } - }); - - channel = b.bind(port).sync().channel(); - - logger.info("Discovery server started, bind port {}", port); - - channel.closeFuture().sync(); - if (shutdown) { - logger.info("Shutdown discovery server"); - break; - } - logger.warn("Restart discovery server after 5 sec pause..."); - Thread.sleep(5000); - } - } catch (InterruptedException e) { - logger.warn("Discover server interrupted"); - Thread.currentThread().interrupt(); - } catch (Exception e) { - logger.error("Start discovery server with port {} failed", port, e); - } finally { - group.shutdownGracefully().sync(); - } - } - - public void close() { - logger.info("Closing discovery server..."); - shutdown = true; - if (channel != null) { - try { - channel.close().await(10, TimeUnit.SECONDS); - } catch (Exception e) { - logger.error("Closing discovery server failed", e); - } - } - - if (discoveryExecutor != null) { - try { - discoveryExecutor.close(); - } catch (Exception e) { - logger.error("Closing discovery executor failed", e); - } - } - } - -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/DiscoverTask.java b/framework/src/main/java/org/tron/common/overlay/discover/DiscoverTask.java deleted file mode 100644 index 8f105b6ee89..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/DiscoverTask.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.tron.common.overlay.discover; - -import java.util.ArrayList; -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.overlay.discover.node.NodeManager; -import org.tron.common.overlay.discover.table.KademliaOptions; - -@Slf4j(topic = "discover") -public class DiscoverTask implements Runnable { - - private NodeManager nodeManager; - - private byte[] nodeId; - - public DiscoverTask(NodeManager nodeManager) { - this.nodeManager = nodeManager; - this.nodeId = nodeManager.getPublicHomeNode().getId(); - } - - @Override - public void run() { - discover(nodeId, 0, new ArrayList<>()); - } - - public synchronized void discover(byte[] nodeId, int round, List prevTried) { - - try { - if (round == KademliaOptions.MAX_STEPS) { - return; - } - - List closest = nodeManager.getTable().getClosestNodes(nodeId); - List tried = new ArrayList<>(); - for (Node n : closest) { - if (!tried.contains(n) && !prevTried.contains(n)) { - try { - nodeManager.getNodeHandler(n).sendFindNode(nodeId); - tried.add(n); - wait(50); - } catch (Exception ex) { - logger.error("Unexpected Exception", ex); - } - } - if (tried.size() == KademliaOptions.ALPHA) { - break; - } - } - - if (tried.isEmpty()) { - return; - } - - tried.addAll(prevTried); - - discover(nodeId, round + 1, tried); - } catch (Exception ex) { - logger.error("{}", ex); - } - } - -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/DiscoveryExecutor.java b/framework/src/main/java/org/tron/common/overlay/discover/DiscoveryExecutor.java deleted file mode 100644 index c15a362f9de..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/DiscoveryExecutor.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover; - -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import org.tron.common.overlay.discover.node.NodeManager; -import org.tron.common.overlay.discover.table.KademliaOptions; - -public class DiscoveryExecutor { - - private ScheduledExecutorService discoverer = Executors.newSingleThreadScheduledExecutor(); - private ScheduledExecutorService refresher = Executors.newSingleThreadScheduledExecutor(); - - private NodeManager nodeManager; - - public DiscoveryExecutor(NodeManager nodeManager) { - this.nodeManager = nodeManager; - } - - public void start() { - discoverer.scheduleWithFixedDelay( - new DiscoverTask(nodeManager), - 1, KademliaOptions.DISCOVER_CYCLE, TimeUnit.SECONDS); - - refresher.scheduleWithFixedDelay( - new RefreshTask(nodeManager), - 1, KademliaOptions.BUCKET_REFRESH, TimeUnit.MILLISECONDS); - } - - public void close() { - discoverer.shutdownNow(); - refresher.shutdownNow(); - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/RefreshTask.java b/framework/src/main/java/org/tron/common/overlay/discover/RefreshTask.java deleted file mode 100644 index d090cd82dcc..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/RefreshTask.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover; - -import java.util.ArrayList; -import lombok.extern.slf4j.Slf4j; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.overlay.discover.node.NodeManager; - -@Slf4j(topic = "discover") -public class RefreshTask extends DiscoverTask { - - public RefreshTask(NodeManager nodeManager) { - super(nodeManager); - } - - @Override - public void run() { - discover(Node.getNodeId(), 0, new ArrayList<>()); - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/dht/Bucket.java b/framework/src/main/java/org/tron/common/overlay/discover/dht/Bucket.java deleted file mode 100644 index bc523b46d4d..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/dht/Bucket.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover.dht; - -import java.util.ArrayList; -import java.util.List; - -public class Bucket { - - public static final int MAX_KADEMLIA_K = 5; - - // if bit = 1 go left - private Bucket left; - - // if bit = 0 go right - private Bucket right; - - private String name; - - private List peers = new ArrayList<>(); - - - public Bucket(String name) { - this.name = name; - } - - - public void add(Peer peer) { - - if (peer == null) { - throw new Error("Not a leaf"); - } - - if (peers == null) { - - if (peer.nextBit(name) == 1) { - left.add(peer); - } else { - right.add(peer); - } - - return; - } - - peers.add(peer); - - if (peers.size() > MAX_KADEMLIA_K) { - splitBucket(); - } - } - - public void splitBucket() { - left = new Bucket(name + "1"); - right = new Bucket(name + "0"); - - for (Peer id : peers) { - if (id.nextBit(name) == 1) { - left.add(id); - } else { - right.add(id); - } - } - - this.peers = null; - } - - - public Bucket left() { - return left; - } - - public Bucket right() { - return right; - } - - - @Override - public String toString() { - - StringBuilder sb = new StringBuilder(); - - sb.append(name).append("\n"); - - if (peers == null) { - return sb.toString(); - } - - for (Peer id : peers) { - sb.append(id.toBinaryString()).append("\n"); - } - - return sb.toString(); - } - - - public void traverseTree(DoOnTree doOnTree) { - - if (left != null) { - left.traverseTree(doOnTree); - } - if (right != null) { - right.traverseTree(doOnTree); - } - - doOnTree.call(this); - } - - //tree operations - - public String getName() { - return name; - } - - public List getPeers() { - return peers; - } - - public interface DoOnTree { - - void call(Bucket bucket); - } - - public static class SaveLeaf implements DoOnTree { - - private List leafs = new ArrayList<>(); - - @Override - public void call(Bucket bucket) { - if (bucket.peers != null) { - leafs.add(bucket); - } - } - - public List getLeafs() { - return leafs; - } - - public void setLeafs(List leafs) { - this.leafs = leafs; - } - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/dht/DHTUtils.java b/framework/src/main/java/org/tron/common/overlay/discover/dht/DHTUtils.java deleted file mode 100644 index 77c5874ae4d..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/dht/DHTUtils.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover.dht; - -import java.util.List; - -public class DHTUtils { - - public static void printAllLeafs(Bucket root) { - Bucket.SaveLeaf saveLeaf = new Bucket.SaveLeaf(); - root.traverseTree(saveLeaf); - - for (Bucket bucket : saveLeaf.getLeafs()) { - System.out.println(bucket); - } - } - - public static List getAllLeafs(Bucket root) { - Bucket.SaveLeaf saveLeaf = new Bucket.SaveLeaf(); - root.traverseTree(saveLeaf); - - return saveLeaf.getLeafs(); - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/dht/Peer.java b/framework/src/main/java/org/tron/common/overlay/discover/dht/Peer.java deleted file mode 100644 index 41d2964aa61..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/dht/Peer.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover.dht; - -import java.math.BigInteger; -import org.bouncycastle.util.BigIntegers; -import org.bouncycastle.util.encoders.Hex; -import org.tron.common.utils.Utils; -import org.tron.core.Constant; - -public class Peer { - - private byte[] id; - private String host = Constant.LOCAL_HOST; - private int port = 0; - - public Peer(byte[] id, String host, int port) { - this.id = id; - this.host = host; - this.port = port; - } - - public Peer(byte[] ip) { - this.id = ip; - } - - public static byte[] randomPeerId() { - - byte[] peerIdBytes = new BigInteger(512, Utils.getRandom()).toByteArray(); - - final String peerId; - if (peerIdBytes.length > 64) { - peerId = Hex.toHexString(peerIdBytes, 1, 64); - } else { - peerId = Hex.toHexString(peerIdBytes); - } - - return Hex.decode(peerId); - } - - public byte nextBit(String startPattern) { - - if (this.toBinaryString().startsWith(startPattern + "1")) { - return 1; - } else { - return 0; - } - } - - public byte[] calcDistance(Peer toPeer) { - - BigInteger aaPeer = new BigInteger(getId()); - BigInteger bbPeer = new BigInteger(toPeer.getId()); - - BigInteger distance = aaPeer.xor(bbPeer); - return BigIntegers.asUnsignedByteArray(distance); - } - - public byte[] getId() { - return id; - } - - public void setId(byte[] id) { - this.id = id; - } - - @Override - public String toString() { - return String - .format("Peer {\n id=%s, \n host=%s, \n port=%d\n}", Hex.toHexString(id), host, port); - } - - public String toBinaryString() { - - BigInteger bi = new BigInteger(1, id); - String out = String.format("%512s", bi.toString(2)); - out = out.replace(' ', '0'); - - return out; - } - -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/node/DBNode.java b/framework/src/main/java/org/tron/common/overlay/discover/node/DBNode.java deleted file mode 100644 index 2a5af767ecd..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/node/DBNode.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.tron.common.overlay.discover.node; - -import java.util.ArrayList; -import java.util.List; -import lombok.Getter; -import lombok.Setter; - -public class DBNode { - - @Getter - @Setter - private List nodes = new ArrayList<>(); - -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/node/DBNodeStats.java b/framework/src/main/java/org/tron/common/overlay/discover/node/DBNodeStats.java deleted file mode 100644 index 52b2b6bdc9b..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/node/DBNodeStats.java +++ /dev/null @@ -1,34 +0,0 @@ -package org.tron.common.overlay.discover.node; - -import lombok.Getter; -import lombok.Setter; - -public class DBNodeStats { - - @Getter - @Setter - private byte[] id; - - @Getter - @Setter - private String host; - - @Getter - @Setter - private int port; - - @Getter - @Setter - private int reputation; - - public DBNodeStats() { - } - - public DBNodeStats(byte[] id, String host, int port, int reputation) { - this.id = id; - this.host = host; - this.port = port; - this.reputation = reputation; - } - -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/node/NodeHandler.java b/framework/src/main/java/org/tron/common/overlay/discover/node/NodeHandler.java deleted file mode 100644 index 44e8761c29a..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/node/NodeHandler.java +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover.node; - -import java.net.InetSocketAddress; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import org.tron.common.net.udp.handler.UdpEvent; -import org.tron.common.net.udp.message.Message; -import org.tron.common.net.udp.message.discover.FindNodeMessage; -import org.tron.common.net.udp.message.discover.NeighborsMessage; -import org.tron.common.net.udp.message.discover.PingMessage; -import org.tron.common.net.udp.message.discover.PongMessage; -import org.tron.common.overlay.discover.node.statistics.NodeStatistics; -import org.tron.common.prometheus.MetricKeys; -import org.tron.common.prometheus.Metrics; -import org.tron.core.config.args.Args; - -@Slf4j(topic = "discover") -public class NodeHandler { - - private static long pingTimeout = Args.getInstance().getNodeDiscoveryPingTimeout(); - private Node sourceNode; - private Node node; - private State state; - private NodeManager nodeManager; - private NodeStatistics nodeStatistics; - private NodeHandler replaceCandidate; - private InetSocketAddress inetSocketAddress; - private AtomicInteger pingTrials = new AtomicInteger(3); - private volatile boolean waitForPong = false; - private volatile boolean waitForNeighbors = false; - private volatile long pingSent; - @Getter - @Setter - private int reputation; - - - public NodeHandler(Node node, NodeManager nodeManager) { - this.node = node; - this.nodeManager = nodeManager; - this.inetSocketAddress = new InetSocketAddress(node.getHost(), node.getPort()); - this.nodeStatistics = new NodeStatistics(); - changeState(State.DISCOVERED); - } - - public InetSocketAddress getInetSocketAddress() { - return inetSocketAddress; - } - - public Node getSourceNode() { - return sourceNode; - } - - public void setSourceNode(Node sourceNode) { - this.sourceNode = sourceNode; - } - - public Node getNode() { - return node; - } - - public void setNode(Node node) { - this.node = node; - } - - public State getState() { - return state; - } - - public NodeStatistics getNodeStatistics() { - return nodeStatistics; - } - - private void challengeWith(NodeHandler replaceCandidate) { - this.replaceCandidate = replaceCandidate; - changeState(State.EVICTCANDIDATE); - } - - // Manages state transfers - public void changeState(State newState) { - State oldState = state; - if (newState == State.DISCOVERED) { - if (sourceNode != null && sourceNode.getPort() != node.getPort()) { - changeState(State.DEAD); - } else { - sendPing(); - } - } - if (!node.isDiscoveryNode()) { - if (newState == State.ALIVE) { - Node evictCandidate = nodeManager.getTable().addNode(this.node); - if (evictCandidate == null) { - newState = State.ACTIVE; - } else { - NodeHandler evictHandler = nodeManager.getNodeHandler(evictCandidate); - if (evictHandler.state != State.EVICTCANDIDATE) { - evictHandler.challengeWith(this); - } - } - } - if (newState == State.ACTIVE) { - if (oldState == State.ALIVE) { - // new node won the challenge - nodeManager.getTable().addNode(node); - } else if (oldState == State.EVICTCANDIDATE) { - // nothing to do here the node is already in the table - } else { - // wrong state transition - } - } - - if (newState == State.NONACTIVE) { - if (oldState == State.EVICTCANDIDATE) { - // lost the challenge - // Removing ourselves from the table - nodeManager.getTable().dropNode(node); - // Congratulate the winner - replaceCandidate.changeState(State.ACTIVE); - } else if (oldState == State.ALIVE) { - // ok the old node was better, nothing to do here - } else { - // wrong state transition - } - } - } - - if (newState == State.EVICTCANDIDATE) { - // trying to survive, sending ping and waiting for pong - sendPing(); - } - state = newState; - } - - public void handlePing(PingMessage msg) { - if (!nodeManager.getTable().getNode().equals(node)) { - sendPong(); - } - node.setP2pVersion(msg.getVersion()); - if (!node.isConnectible(Args.getInstance().getNodeP2pVersion())) { - changeState(State.NONACTIVE); - } else if (state.equals(State.NONACTIVE) || state.equals(State.DEAD)) { - changeState(State.DISCOVERED); - } - } - - public void handlePong(PongMessage msg) { - if (waitForPong) { - waitForPong = false; - getNodeStatistics().discoverMessageLatency.add(System.currentTimeMillis() - pingSent); - Metrics.histogramObserve(MetricKeys.Histogram.PING_PONG_LATENCY, - (System.currentTimeMillis() - pingSent) / Metrics.MILLISECONDS_PER_SECOND); - getNodeStatistics().lastPongReplyTime.set(System.currentTimeMillis()); - node.setId(msg.getFrom().getId()); - node.setP2pVersion(msg.getVersion()); - if (!node.isConnectible(Args.getInstance().getNodeP2pVersion())) { - changeState(State.NONACTIVE); - } else { - changeState(State.ALIVE); - } - } - } - - public void handleNeighbours(NeighborsMessage msg) { - if (!waitForNeighbors) { - logger.warn("Receive neighbors from {} without send find nodes", node.getHost()); - return; - } - waitForNeighbors = false; - for (Node n : msg.getNodes()) { - if (!nodeManager.getPublicHomeNode().getHexId().equals(n.getHexId())) { - nodeManager.getNodeHandler(n); - } - } - } - - public void handleFindNode(FindNodeMessage msg) { - List closest = nodeManager.getTable().getClosestNodes(msg.getTargetId()); - sendNeighbours(closest, msg.getTimestamp()); - } - - public void handleTimedOut() { - waitForPong = false; - if (pingTrials.getAndDecrement() > 0) { - sendPing(); - } else { - if (state == State.DISCOVERED) { - changeState(State.DEAD); - } else if (state == State.EVICTCANDIDATE) { - changeState(State.NONACTIVE); - } else { - // TODO just influence to reputation - } - } - } - - public void sendPing() { - PingMessage msg = new PingMessage(nodeManager.getPublicHomeNode(), getNode()); - waitForPong = true; - pingSent = System.currentTimeMillis(); - sendMessage(msg); - - if (nodeManager.getPongTimer().isShutdown()) { - return; - } - nodeManager.getPongTimer().schedule(() -> { - try { - if (waitForPong) { - waitForPong = false; - handleTimedOut(); - } - } catch (Exception e) { - logger.error("Unhandled exception in pong timer schedule", e); - } - }, pingTimeout, TimeUnit.MILLISECONDS); - } - - public void sendPong() { - Message pong = new PongMessage(nodeManager.getPublicHomeNode()); - sendMessage(pong); - } - - public void sendFindNode(byte[] target) { - waitForNeighbors = true; - FindNodeMessage msg = new FindNodeMessage(nodeManager.getPublicHomeNode(), target); - sendMessage(msg); - } - - public void sendNeighbours(List neighbours, long sequence) { - Message msg = new NeighborsMessage(nodeManager.getPublicHomeNode(), neighbours, sequence); - sendMessage(msg); - } - - private void sendMessage(Message msg) { - nodeManager.sendOutbound(new UdpEvent(msg, getInetSocketAddress())); - nodeStatistics.messageStatistics.addUdpOutMessage(msg.getType()); - } - - @Override - public String toString() { - return "NodeHandler[state: " + state + ", node: " + node.getHost() + ":" + node.getPort() + "]"; - } - - public enum State { - /** - * The new node was just discovered either by receiving it with Neighbours message or by - * receiving Ping from a new node In either case we are sending Ping and waiting for Pong If the - * Pong is received the node becomes {@link #ALIVE} If the Pong was timed out the node becomes - * {@link #DEAD} - */ - DISCOVERED, - /** - * The node didn't send the Pong message back withing acceptable timeout This is the final - * state - */ - DEAD, - /** - * The node responded with Pong and is now the candidate for inclusion to the table If the table - * has bucket space for this node it is added to table and becomes {@link #ACTIVE} If the table - * bucket is full this node is challenging with the old node from the bucket if it wins then old - * node is dropped, and this node is added and becomes {@link #ACTIVE} else this node becomes - * {@link #NONACTIVE} - */ - ALIVE, - /** - * The node is included in the table. It may become {@link #EVICTCANDIDATE} if a new node wants - * to become Active but the table bucket is full. - */ - ACTIVE, - /** - * This node is in the table but is currently challenging with a new Node candidate to survive - * in the table bucket If it wins then returns back to {@link #ACTIVE} state, else is evicted - * from the table and becomes {@link #NONACTIVE} - */ - EVICTCANDIDATE, - /** - * Veteran. It was Alive and even Active but is now retired due to loosing the challenge with - * another Node. For no this is the final state It's an option for future to return veterans - * back to the table - */ - NONACTIVE - } - -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/node/NodeManager.java b/framework/src/main/java/org/tron/common/overlay/discover/node/NodeManager.java deleted file mode 100644 index e8f2870fe98..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/node/NodeManager.java +++ /dev/null @@ -1,314 +0,0 @@ -package org.tron.common.overlay.discover.node; - -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; -import java.util.Map; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.function.Consumer; -import java.util.function.Predicate; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.tron.common.net.udp.handler.EventHandler; -import org.tron.common.net.udp.handler.UdpEvent; -import org.tron.common.net.udp.message.Message; -import org.tron.common.net.udp.message.discover.DiscoverMessageInspector; -import org.tron.common.net.udp.message.discover.FindNodeMessage; -import org.tron.common.net.udp.message.discover.NeighborsMessage; -import org.tron.common.net.udp.message.discover.PingMessage; -import org.tron.common.net.udp.message.discover.PongMessage; -import org.tron.common.overlay.discover.node.NodeHandler.State; -import org.tron.common.overlay.discover.node.statistics.NodeStatistics; -import org.tron.common.overlay.discover.table.NodeTable; -import org.tron.common.parameter.CommonParameter; -import org.tron.common.prometheus.MetricKeys; -import org.tron.common.prometheus.MetricLabels; -import org.tron.common.prometheus.Metrics; -import org.tron.common.utils.ByteArray; -import org.tron.common.utils.CollectionUtils; -import org.tron.common.utils.JsonUtil; -import org.tron.core.ChainBaseManager; -import org.tron.core.capsule.BytesCapsule; -import org.tron.core.config.args.Args; -import org.tron.core.metrics.MetricsKey; -import org.tron.core.metrics.MetricsUtil; - -@Slf4j(topic = "discover") -@Component -public class NodeManager implements EventHandler { - - private static final byte[] DB_KEY_PEERS = "peers".getBytes(); - private static final long DB_COMMIT_RATE = 1 * 60 * 1000L; - private static final int MAX_NODES = 2000; - private static final int MAX_NODES_WRITE_TO_DB = 30; - private static final int NODES_TRIM_THRESHOLD = 3000; - private CommonParameter commonParameter = Args.getInstance(); - private ChainBaseManager chainBaseManager; - private Consumer messageSender; - - private NodeTable table; - private Node homeNode; - private Map nodeHandlerMap = new ConcurrentHashMap<>(); - private List bootNodes = new ArrayList<>(); - - private volatile boolean discoveryEnabled; - - private volatile boolean inited = false; - - private Timer nodeManagerTasksTimer = new Timer("NodeManagerTasks"); - - private ScheduledExecutorService pongTimer; - - @Autowired - public NodeManager(ChainBaseManager chainBaseManager) { - this.chainBaseManager = chainBaseManager; - discoveryEnabled = commonParameter.isNodeDiscoveryEnable(); - - homeNode = new Node(Node.getNodeId(), commonParameter.getNodeExternalIp(), - commonParameter.getNodeListenPort()); - - for (String boot : commonParameter.getSeedNode().getIpList()) { - bootNodes.add(Node.instanceOf(boot)); - } - - logger.info("Home node is {}", homeNode); - - table = new NodeTable(homeNode); - - this.pongTimer = Executors.newSingleThreadScheduledExecutor(); - } - - public ScheduledExecutorService getPongTimer() { - return pongTimer; - } - - @Override - public void channelActivated() { - if (!inited) { - inited = true; - - if (commonParameter.isNodeDiscoveryPersist()) { - dbRead(); - nodeManagerTasksTimer.scheduleAtFixedRate(new TimerTask() { - @Override - public void run() { - dbWrite(); - } - }, DB_COMMIT_RATE, DB_COMMIT_RATE); - } - - for (Node node : bootNodes) { - getNodeHandler(node); - } - } - } - - public boolean isNodeAlive(NodeHandler nodeHandler) { - return nodeHandler.getState().equals(State.ALIVE) - || nodeHandler.getState().equals(State.ACTIVE) - || nodeHandler.getState().equals(State.EVICTCANDIDATE); - } - - private void dbRead() { - try { - byte[] nodeBytes = chainBaseManager.getCommonStore().get(DB_KEY_PEERS).getData(); - if (ByteArray.isEmpty(nodeBytes)) { - return; - } - DBNode dbNode = JsonUtil.json2Obj(new String(nodeBytes), DBNode.class); - logger.info("Reading node statistics from store: {} nodes", dbNode.getNodes().size()); - dbNode.getNodes().forEach(n -> { - Node node = new Node(n.getId(), n.getHost(), n.getPort()); - getNodeHandler(node).getNodeStatistics().setPersistedReputation(n.getReputation()); - }); - } catch (Exception e) { - logger.error("DB read node failed", e); - } - } - - private void dbWrite() { - try { - List batch = new ArrayList<>(); - DBNode dbNode = new DBNode(); - for (NodeHandler nodeHandler : nodeHandlerMap.values()) { - Node node = nodeHandler.getNode(); - if (node.isConnectible(Args.getInstance().getNodeP2pVersion())) { - DBNodeStats nodeStatic = new DBNodeStats(node.getId(), node.getHost(), - node.getPort(), nodeHandler.getNodeStatistics().getReputation()); - batch.add(nodeStatic); - } - } - int size = batch.size(); - batch.sort(Comparator.comparingInt(value -> -value.getReputation())); - if (batch.size() > MAX_NODES_WRITE_TO_DB) { - batch = batch.subList(0, MAX_NODES_WRITE_TO_DB); - } - - dbNode.setNodes(batch); - - logger.info("Write node statistics to store: m:{}/t:{}/{}/{} nodes", - nodeHandlerMap.size(), getTable().getAllNodes().size(), size, batch.size()); - - chainBaseManager.getCommonStore() - .put(DB_KEY_PEERS, new BytesCapsule(JsonUtil.obj2Json(dbNode).getBytes())); - } catch (Exception e) { - logger.error("DB write node failed", e); - } - } - - public void setMessageSender(Consumer messageSender) { - this.messageSender = messageSender; - } - - private String getKey(Node n) { - return getKey(new InetSocketAddress(n.getHost(), n.getPort())); - } - - private String getKey(InetSocketAddress address) { - InetAddress inetAddress = address.getAddress(); - return (inetAddress == null ? address.getHostString() : inetAddress.getHostAddress()) + ":" - + address.getPort(); - } - - public NodeHandler getNodeHandler(Node n) { - String key = getKey(n); - NodeHandler ret = nodeHandlerMap.get(key); - if (ret == null) { - trimTable(); - ret = new NodeHandler(n, this); - nodeHandlerMap.put(key, ret); - } else if (ret.getNode().isDiscoveryNode() && !n.isDiscoveryNode()) { - ret.setNode(n); - } - return ret; - } - - private void trimTable() { - if (nodeHandlerMap.size() > NODES_TRIM_THRESHOLD) { - nodeHandlerMap.values().forEach(handler -> { - if (!handler.getNode().isConnectible(Args.getInstance().getNodeP2pVersion())) { - nodeHandlerMap.values().remove(handler); - } - }); - } - if (nodeHandlerMap.size() > NODES_TRIM_THRESHOLD) { - List sorted = new ArrayList<>(nodeHandlerMap.values()); - sorted.sort(Comparator.comparingInt(o -> o.getNodeStatistics().getReputation())); - for (NodeHandler handler : sorted) { - nodeHandlerMap.values().remove(handler); - if (nodeHandlerMap.size() <= MAX_NODES) { - break; - } - } - } - } - - public boolean hasNodeHandler(Node n) { - return nodeHandlerMap.containsKey(getKey(n)); - } - - public NodeTable getTable() { - return table; - } - - public NodeStatistics getNodeStatistics(Node n) { - return getNodeHandler(n).getNodeStatistics(); - } - - @Override - public void handleEvent(UdpEvent udpEvent) { - Message m = udpEvent.getMessage(); - if (!DiscoverMessageInspector.valid(m)) { - return; - } - - InetSocketAddress sender = udpEvent.getAddress(); - - Node n = new Node(m.getFrom().getId(), sender.getHostString(), sender.getPort(), - m.getFrom().getPort()); - - NodeHandler nodeHandler = getNodeHandler(n); - nodeHandler.getNodeStatistics().messageStatistics.addUdpInMessage(m.getType()); - int length = udpEvent.getMessage().getData().length + 1; - MetricsUtil.meterMark(MetricsKey.NET_UDP_IN_TRAFFIC, length); - Metrics.histogramObserve(MetricKeys.Histogram.UDP_BYTES, length, - MetricLabels.Histogram.TRAFFIC_IN); - - switch (m.getType()) { - case DISCOVER_PING: - nodeHandler.handlePing((PingMessage) m); - break; - case DISCOVER_PONG: - nodeHandler.handlePong((PongMessage) m); - break; - case DISCOVER_FIND_NODE: - nodeHandler.handleFindNode((FindNodeMessage) m); - break; - case DISCOVER_NEIGHBORS: - nodeHandler.handleNeighbours((NeighborsMessage) m); - break; - default: - break; - } - } - - public void sendOutbound(UdpEvent udpEvent) { - if (discoveryEnabled && messageSender != null) { - messageSender.accept(udpEvent); - int length = udpEvent.getMessage().getSendData().length; - MetricsUtil.meterMark(MetricsKey.NET_UDP_OUT_TRAFFIC, length); - Metrics.histogramObserve(MetricKeys.Histogram.UDP_BYTES, length, - MetricLabels.Histogram.TRAFFIC_OUT); - - } - } - - public List getNodes(Predicate predicate, int limit) { - List filtered = new ArrayList<>(); - for (NodeHandler handler : nodeHandlerMap.values()) { - if (handler.getNode().isConnectible(Args.getInstance().getNodeP2pVersion()) - && predicate.test(handler)) { - handler.setReputation(handler.getNodeStatistics().getReputation()); - filtered.add(handler); - } - } - filtered.sort(Comparator.comparingInt(handler -> -handler.getReputation())); - return CollectionUtils.truncate(filtered, limit); - } - - public List dumpActiveNodes() { - List handlers = new ArrayList<>(); - for (NodeHandler handler : this.nodeHandlerMap.values()) { - if (isNodeAlive(handler)) { - handlers.add(handler); - } - } - return handlers; - } - - public Node getPublicHomeNode() { - return homeNode; - } - - // just for test - public void clearNodeHandlerMap() { - nodeHandlerMap.clear(); - } - - public void close() { - try { - nodeManagerTasksTimer.cancel(); - pongTimer.shutdownNow(); - } catch (Exception e) { - logger.error("Close nodeManagerTasksTimer or pongTimer failed", e); - } - } - -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/node/statistics/NodeStatistics.java b/framework/src/main/java/org/tron/common/overlay/discover/node/statistics/NodeStatistics.java deleted file mode 100644 index 82b18b2071c..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/node/statistics/NodeStatistics.java +++ /dev/null @@ -1,214 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover.node.statistics; - -import java.util.concurrent.atomic.AtomicLong; -import lombok.Getter; -import org.tron.core.config.args.Args; -import org.tron.protos.Protocol.ReasonCode; - -public class NodeStatistics { - - public static final int REPUTATION_PREDEFINED = 100000; - public static final long TOO_MANY_PEERS_PENALIZE_TIMEOUT = 60 * 1000L; - private static final long CLEAR_CYCLE_TIME = 60 * 60 * 1000L; - public final MessageStatistics messageStatistics = new MessageStatistics(); - public final MessageCount p2pHandShake = new MessageCount(); - public final MessageCount tcpFlow = new MessageCount(); - public final SimpleStatter discoverMessageLatency; - public final SimpleStatter pingMessageLatency; - public final AtomicLong lastPongReplyTime = new AtomicLong(0L); // in milliseconds - private final long MIN_DATA_LENGTH = Args.getInstance().getReceiveTcpMinDataLength(); - private boolean isPredefined = false; - private int persistedReputation = 0; - @Getter - private int disconnectTimes = 0; - @Getter - private ReasonCode tronLastRemoteDisconnectReason = null; - @Getter - private ReasonCode tronLastLocalDisconnectReason = null; - private long lastDisconnectedTime = 0; - private long firstDisconnectedTime = 0; - private Reputation reputation; - - public NodeStatistics() { - discoverMessageLatency = new SimpleStatter(); - pingMessageLatency = new SimpleStatter(); - reputation = new Reputation(this); - } - - public int getReputation() { - int score = 0; - if (!isReputationPenalized()) { - score += persistedReputation / 5 + reputation.getScore(); - } - if (isPredefined) { - score += REPUTATION_PREDEFINED; - } - return score; - } - - public ReasonCode getDisconnectReason() { - if (tronLastLocalDisconnectReason != null) { - return tronLastLocalDisconnectReason; - } - if (tronLastRemoteDisconnectReason != null) { - return tronLastRemoteDisconnectReason; - } - return ReasonCode.UNKNOWN; - } - - public boolean isReputationPenalized() { - - if (wasDisconnected() && tronLastRemoteDisconnectReason == ReasonCode.TOO_MANY_PEERS - && System.currentTimeMillis() - lastDisconnectedTime < TOO_MANY_PEERS_PENALIZE_TIMEOUT) { - return true; - } - - if (wasDisconnected() && tronLastRemoteDisconnectReason == ReasonCode.DUPLICATE_PEER - && System.currentTimeMillis() - lastDisconnectedTime < TOO_MANY_PEERS_PENALIZE_TIMEOUT) { - return true; - } - - if (firstDisconnectedTime > 0 - && (System.currentTimeMillis() - firstDisconnectedTime) > CLEAR_CYCLE_TIME) { - tronLastLocalDisconnectReason = null; - tronLastRemoteDisconnectReason = null; - disconnectTimes = 0; - persistedReputation = 0; - firstDisconnectedTime = 0; - } - - if (tronLastLocalDisconnectReason == ReasonCode.INCOMPATIBLE_PROTOCOL - || tronLastRemoteDisconnectReason == ReasonCode.INCOMPATIBLE_PROTOCOL - || tronLastLocalDisconnectReason == ReasonCode.BAD_PROTOCOL - || tronLastRemoteDisconnectReason == ReasonCode.BAD_PROTOCOL - || tronLastLocalDisconnectReason == ReasonCode.BAD_BLOCK - || tronLastRemoteDisconnectReason == ReasonCode.BAD_BLOCK - || tronLastLocalDisconnectReason == ReasonCode.BAD_TX - || tronLastRemoteDisconnectReason == ReasonCode.BAD_TX - || tronLastLocalDisconnectReason == ReasonCode.FORKED - || tronLastRemoteDisconnectReason == ReasonCode.FORKED - || tronLastLocalDisconnectReason == ReasonCode.UNLINKABLE - || tronLastRemoteDisconnectReason == ReasonCode.UNLINKABLE - || tronLastLocalDisconnectReason == ReasonCode.INCOMPATIBLE_CHAIN - || tronLastRemoteDisconnectReason == ReasonCode.INCOMPATIBLE_CHAIN - || tronLastRemoteDisconnectReason == ReasonCode.SYNC_FAIL - || tronLastLocalDisconnectReason == ReasonCode.SYNC_FAIL - || tronLastRemoteDisconnectReason == ReasonCode.INCOMPATIBLE_VERSION - || tronLastLocalDisconnectReason == ReasonCode.INCOMPATIBLE_VERSION) { - persistedReputation = 0; - return true; - } - return false; - } - - public void nodeDisconnectedRemote(ReasonCode reason) { - lastDisconnectedTime = System.currentTimeMillis(); - tronLastRemoteDisconnectReason = reason; - } - - public void nodeDisconnectedLocal(ReasonCode reason) { - lastDisconnectedTime = System.currentTimeMillis(); - tronLastLocalDisconnectReason = reason; - } - - public void notifyDisconnect() { - lastDisconnectedTime = System.currentTimeMillis(); - if (firstDisconnectedTime <= 0) { - firstDisconnectedTime = lastDisconnectedTime; - } - if (tronLastLocalDisconnectReason == ReasonCode.RESET) { - return; - } - disconnectTimes++; - persistedReputation = persistedReputation / 2; - } - - public boolean wasDisconnected() { - return lastDisconnectedTime > 0; - } - - public boolean isPredefined() { - return isPredefined; - } - - public void setPredefined(boolean isPredefined) { - this.isPredefined = isPredefined; - } - - public void setPersistedReputation(int persistedReputation) { - this.persistedReputation = persistedReputation; - } - - @Override - public String toString() { - return "NodeStat[reput: " + getReputation() + "(" + persistedReputation + "), discover: " - + messageStatistics.discoverInPong + "/" + messageStatistics.discoverOutPing + " " - + messageStatistics.discoverOutPong + "/" + messageStatistics.discoverInPing + " " - + messageStatistics.discoverInNeighbours + "/" + messageStatistics.discoverOutFindNode - + " " - + messageStatistics.discoverOutNeighbours + "/" + messageStatistics.discoverInFindNode - + " " - + ((int) discoverMessageLatency.getAvg()) + "ms" - + ", p2p: " + p2pHandShake + "/" + messageStatistics.p2pInHello + "/" - + messageStatistics.p2pOutHello + " " - + ", tron: " + messageStatistics.tronInMessage + "/" + messageStatistics.tronOutMessage - + " " - + (wasDisconnected() ? "X " + disconnectTimes : "") - + (tronLastLocalDisconnectReason != null ? ("<=" + tronLastLocalDisconnectReason) : " ") - + (tronLastRemoteDisconnectReason != null ? ("=>" + tronLastRemoteDisconnectReason) : " ") - + ", tcp flow: " + tcpFlow.getTotalCount(); - } - - public boolean nodeIsHaveDataTransfer() { - return tcpFlow.getTotalCount() > MIN_DATA_LENGTH; - } - - public void resetTcpFlow() { - tcpFlow.reset(); - } - - public class SimpleStatter { - - private long sum; - @Getter - private long count; - @Getter - private long last; - @Getter - private long min; - @Getter - private long max; - - public void add(long value) { - last = value; - sum += value; - min = min == 0 ? value : Math.min(min, value); - max = Math.max(max, value); - count++; - } - - public long getAvg() { - return count == 0 ? 0 : sum / count; - } - - } - -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/node/statistics/Reputation.java b/framework/src/main/java/org/tron/common/overlay/discover/node/statistics/Reputation.java deleted file mode 100644 index 685407c11ed..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/node/statistics/Reputation.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.tron.common.overlay.discover.node.statistics; - -import static java.lang.Math.min; - -public class Reputation { - - private NodeStatistics nodeStatistics; - - public Reputation(NodeStatistics nodeStatistics) { - this.nodeStatistics = nodeStatistics; - - } - - public int getScore() { - return getNodeActiveScore() + getPacketLossRateScore() + getNetLatencyScore() - + getHandshakeScore() + getTcpFlowScore() + getDisconnectionScore(); - } - - private int getNodeActiveScore() { - long inPongTotalCount = nodeStatistics.messageStatistics.discoverInPong.getTotalCount(); - return inPongTotalCount == 0 ? 0 : 100; - } - - private int getPacketLossRateScore() { - MessageStatistics s = nodeStatistics.messageStatistics; - long in = s.discoverInPong.getTotalCount() + s.discoverInNeighbours.getTotalCount(); - long out = s.discoverOutPing.getTotalCount() + s.discoverOutFindNode.getTotalCount(); - return out == 0 ? 0 : 100 - min((int) ((1 - (double) in / out) * 200), 100); - } - - private int getNetLatencyScore() { - return (int) (nodeStatistics.discoverMessageLatency.getAvg() == 0 ? 0 - : min(1000 / nodeStatistics.discoverMessageLatency.getAvg(), 20)); - } - - private int getHandshakeScore() { - return nodeStatistics.p2pHandShake.getTotalCount() > 0 ? 20 : 0; - } - - private int getTcpFlowScore() { - return (int) min(nodeStatistics.tcpFlow.getTotalCount() / 10240, 20); - } - - private int getDisconnectionScore() { - return -10 * nodeStatistics.getDisconnectTimes(); - } - -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/table/DistanceComparator.java b/framework/src/main/java/org/tron/common/overlay/discover/table/DistanceComparator.java deleted file mode 100644 index 38e0ff4b6a3..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/table/DistanceComparator.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover.table; - -import java.util.Comparator; - -/** - * Created by kest on 5/26/15. - */ -public class DistanceComparator implements Comparator { - - private byte[] targetId; - - DistanceComparator(byte[] targetId) { - this.targetId = targetId; - } - - @Override - public int compare(NodeEntry e1, NodeEntry e2) { - int d1 = NodeEntry.distance(targetId, e1.getNode().getId()); - int d2 = NodeEntry.distance(targetId, e2.getNode().getId()); - - if (d1 > d2) { - return 1; - } else if (d1 < d2) { - return -1; - } else { - return 0; - } - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/table/KademliaOptions.java b/framework/src/main/java/org/tron/common/overlay/discover/table/KademliaOptions.java deleted file mode 100644 index c5c37dd646e..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/table/KademliaOptions.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover.table; - -/** - * Created by kest on 5/25/15. - */ -public class KademliaOptions { - - public static final int NODE_ID_LEN = 64; - public static final int BUCKET_SIZE = 16; - public static final int ALPHA = 3; - public static final int BINS = 256; - public static final int MAX_STEPS = 8; - - public static final long BUCKET_REFRESH = 7200; //bucket refreshing interval in millis - public static final long DISCOVER_CYCLE = 30; //discovery cycle interval in seconds -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/table/NodeBucket.java b/framework/src/main/java/org/tron/common/overlay/discover/table/NodeBucket.java deleted file mode 100644 index 5742310802a..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/table/NodeBucket.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover.table; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Created by kest on 5/25/15. - */ -public class NodeBucket { - - private final int depth; - private List nodes = new ArrayList<>(); - - NodeBucket(int depth) { - this.depth = depth; - } - - public int getDepth() { - return depth; - } - - public synchronized NodeEntry addNode(NodeEntry e) { - if (!nodes.contains(e)) { - if (nodes.size() >= KademliaOptions.BUCKET_SIZE) { - return getLastSeen(); - } else { - nodes.add(e); - } - } - - return null; - } - - private NodeEntry getLastSeen() { - List sorted = nodes; - Collections.sort(sorted, new TimeComparator()); - return sorted.get(0); - } - - public synchronized void dropNode(NodeEntry entry) { - for (NodeEntry e : nodes) { - if (e.getId().equals(entry.getId())) { - nodes.remove(e); - break; - } - } - } - - public int getNodesCount() { - return nodes.size(); - } - - public List getNodes() { - return nodes; - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/table/NodeEntry.java b/framework/src/main/java/org/tron/common/overlay/discover/table/NodeEntry.java deleted file mode 100644 index 4d83f58734d..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/table/NodeEntry.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover.table; - -import org.tron.common.overlay.discover.node.Node; - -public class NodeEntry { - private Node node; - private String entryId; - private int distance; - private long modified; - - public NodeEntry(byte[] ownerId, Node n) { - this.node = n; - entryId = n.getHost(); - distance = distance(ownerId, n.getId()); - touch(); - } - - public static int distance(byte[] ownerId, byte[] targetId) { - byte[] h1 = targetId; - byte[] h2 = ownerId; - - byte[] hash = new byte[Math.min(h1.length, h2.length)]; - - for (int i = 0; i < hash.length; i++) { - hash[i] = (byte) (h1[i] ^ h2[i]); - } - - int d = KademliaOptions.BINS; - - for (byte b : hash) { - if (b == 0) { - d -= 8; - } else { - int count = 0; - for (int i = 7; i >= 0; i--) { - boolean a = ((b & 0xff) & (1 << i)) == 0; - if (a) { - count++; - } else { - break; - } - } - - d -= count; - - break; - } - } - return d; - } - - public void touch() { - modified = System.currentTimeMillis(); - } - - public int getDistance() { - return distance; - } - - public String getId() { - return entryId; - } - - public Node getNode() { - return node; - } - - public long getModified() { - return modified; - } - - @Override - public boolean equals(Object o) { - boolean ret = false; - - if (o != null && this.getClass() == o.getClass()) { - NodeEntry e = (NodeEntry) o; - ret = this.getId().equals(e.getId()); - } - - return ret; - } - - @Override - public int hashCode() { - return this.entryId.hashCode(); - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/table/NodeTable.java b/framework/src/main/java/org/tron/common/overlay/discover/table/NodeTable.java deleted file mode 100644 index 3846dbaf94d..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/table/NodeTable.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover.table; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import lombok.extern.slf4j.Slf4j; -import org.tron.common.overlay.discover.node.Node; - -@Slf4j(topic = "discover") -public class NodeTable { - - private final Node node; // our node - private transient NodeBucket[] buckets; - private transient Map nodes; - - public NodeTable(Node n) { - this.node = n; - initialize(); - } - - public Node getNode() { - return node; - } - - public final void initialize() { - nodes = new HashMap<>(); - buckets = new NodeBucket[KademliaOptions.BINS]; - for (int i = 0; i < KademliaOptions.BINS; i++) { - buckets[i] = new NodeBucket(i); - } - } - - public synchronized Node addNode(Node n) { - if (n.getHost().equals(node.getHost())) { - return null; - } - - NodeEntry entry = nodes.get(n.getHost()); - if (entry != null) { - entry.touch(); - return null; - } - - NodeEntry e = new NodeEntry(node.getId(), n); - NodeEntry lastSeen = buckets[getBucketId(e)].addNode(e); - if (lastSeen != null) { - return lastSeen.getNode(); - } - nodes.put(n.getHost(), e); - return null; - } - - public synchronized void dropNode(Node n) { - NodeEntry entry = nodes.get(n.getHost()); - if (entry != null) { - nodes.remove(n.getHost()); - buckets[getBucketId(entry)].dropNode(entry); - } - } - - public synchronized boolean contains(Node n) { - return nodes.containsKey(n.getHost()); - } - - public synchronized void touchNode(Node n) { - NodeEntry entry = nodes.get(n.getHost()); - if (entry != null) { - entry.touch(); - } - } - - public int getBucketsCount() { - int i = 0; - for (NodeBucket b : buckets) { - if (b.getNodesCount() > 0) { - i++; - } - } - return i; - } - - public int getBucketId(NodeEntry e) { - int id = e.getDistance() - 1; - return id < 0 ? 0 : id; - } - - public synchronized int getNodesCount() { - return nodes.size(); - } - - public synchronized List getAllNodes() { - return new ArrayList<>(nodes.values()); - } - - public synchronized List getClosestNodes(byte[] targetId) { - List closestEntries = getAllNodes(); - List closestNodes = new ArrayList<>(); - Collections.sort(closestEntries, new DistanceComparator(targetId)); - if (closestEntries.size() > KademliaOptions.BUCKET_SIZE) { - closestEntries = closestEntries.subList(0, KademliaOptions.BUCKET_SIZE); - } - for (NodeEntry e : closestEntries) { - if (!e.getNode().isDiscoveryNode()) { - closestNodes.add(e.getNode()); - } - } - return closestNodes; - } - -} diff --git a/framework/src/main/java/org/tron/common/overlay/discover/table/TimeComparator.java b/framework/src/main/java/org/tron/common/overlay/discover/table/TimeComparator.java deleted file mode 100644 index 056d913e781..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/discover/table/TimeComparator.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.discover.table; - -import java.util.Comparator; - -/** - * Created by kest on 5/26/15. - */ -public class TimeComparator implements Comparator { - - @Override - public int compare(NodeEntry e1, NodeEntry e2) { - long t1 = e1.getModified(); - long t2 = e2.getModified(); - - if (t1 < t2) { - return 1; - } else if (t1 > t2) { - return -1; - } else { - return 0; - } - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/message/MessageCodec.java b/framework/src/main/java/org/tron/common/overlay/message/MessageCodec.java deleted file mode 100644 index 30a2329131e..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/message/MessageCodec.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.tron.common.overlay.message; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ByteToMessageDecoder; -import java.util.List; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import org.tron.common.overlay.server.Channel; -import org.tron.common.prometheus.MetricKeys; -import org.tron.common.prometheus.MetricLabels; -import org.tron.common.prometheus.Metrics; -import org.tron.core.exception.P2pException; -import org.tron.core.metrics.MetricsKey; -import org.tron.core.metrics.MetricsUtil; -import org.tron.core.net.message.MessageTypes; -import org.tron.core.net.message.PbftMessageFactory; -import org.tron.core.net.message.TronMessageFactory; - -@Component -@Scope("prototype") -public class MessageCodec extends ByteToMessageDecoder { - - private Channel channel; - private P2pMessageFactory p2pMessageFactory = new P2pMessageFactory(); - private TronMessageFactory tronMessageFactory = new TronMessageFactory(); - private PbftMessageFactory pbftMessageFactory = new PbftMessageFactory(); - - @Override - protected void decode(ChannelHandlerContext ctx, ByteBuf buffer, List out) - throws Exception { - int length = buffer.readableBytes(); - byte[] encoded = new byte[length]; - buffer.readBytes(encoded); - try { - Message msg = createMessage(encoded); - channel.getNodeStatistics().tcpFlow.add(length); - MetricsUtil.meterMark(MetricsKey.NET_TCP_IN_TRAFFIC, length); - Metrics.histogramObserve(MetricKeys.Histogram.TCP_BYTES, length, - MetricLabels.Histogram.TRAFFIC_IN); - out.add(msg); - } catch (Exception e) { - channel.processException(e); - } - } - - public void setChannel(Channel channel) { - this.channel = channel; - } - - private Message createMessage(byte[] encoded) throws Exception { - byte type = encoded[0]; - if (MessageTypes.inP2pRange(type)) { - return p2pMessageFactory.create(encoded); - } - if (MessageTypes.inTronRange(type)) { - return tronMessageFactory.create(encoded); - } - if (MessageTypes.inPbftRange(type)) { - return pbftMessageFactory.create(encoded); - } - throw new P2pException(P2pException.TypeEnum.NO_SUCH_MESSAGE, "type=" + encoded[0]); - } - -} \ No newline at end of file diff --git a/framework/src/main/java/org/tron/common/overlay/message/MessageFactory.java b/framework/src/main/java/org/tron/common/overlay/message/MessageFactory.java deleted file mode 100644 index 6aa41ba085f..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/message/MessageFactory.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * java-tron is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * java-tron is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.tron.common.overlay.message; - -public abstract class MessageFactory { - - protected abstract Message create(byte[] data) throws Exception; - -} diff --git a/framework/src/main/java/org/tron/common/overlay/message/P2pMessage.java b/framework/src/main/java/org/tron/common/overlay/message/P2pMessage.java deleted file mode 100644 index a7827733837..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/message/P2pMessage.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.message; - -public abstract class P2pMessage extends Message { - - public P2pMessage() { - } - - public P2pMessage(byte[] rawData) { - super(rawData); - } - - public P2pMessage(byte type, byte[] rawData) { - super(type, rawData); - } - -} diff --git a/framework/src/main/java/org/tron/common/overlay/message/P2pMessageFactory.java b/framework/src/main/java/org/tron/common/overlay/message/P2pMessageFactory.java deleted file mode 100644 index e403b349984..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/message/P2pMessageFactory.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.message; - -import org.apache.commons.lang3.ArrayUtils; -import org.tron.core.exception.P2pException; -import org.tron.core.exception.P2pException.TypeEnum; -import org.tron.core.net.message.MessageTypes; - -public class P2pMessageFactory extends MessageFactory { - - @Override - public P2pMessage create(byte[] data) throws Exception { - if (data.length <= 1) { - throw new P2pException(TypeEnum.MESSAGE_WITH_WRONG_LENGTH, - "messageType=" + (data.length == 1 ? data[0] : "unknown")); - } - try { - byte type = data[0]; - byte[] rawData = ArrayUtils.subarray(data, 1, data.length); - return create(type, rawData); - } catch (Exception e) { - if (e instanceof P2pException) { - throw e; - } else { - throw new P2pException(P2pException.TypeEnum.PARSE_MESSAGE_FAILED, - "type=" + data[0] + ", len=" + data.length); - } - } - } - - private P2pMessage create(byte type, byte[] rawData) throws Exception { - MessageTypes messageType = MessageTypes.fromByte(type); - if (messageType == null) { - throw new P2pException(P2pException.TypeEnum.NO_SUCH_MESSAGE, - "type=" + type + ", len=" + rawData.length); - } - switch (messageType) { - case P2P_HELLO: - return new HelloMessage(type, rawData); - case P2P_DISCONNECT: - return new DisconnectMessage(type, rawData); - case P2P_PING: - return new PingMessage(type, rawData); - case P2P_PONG: - return new PongMessage(type, rawData); - default: - throw new P2pException(P2pException.TypeEnum.NO_SUCH_MESSAGE, messageType.toString()); - } - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/message/StaticMessages.java b/framework/src/main/java/org/tron/common/overlay/message/StaticMessages.java deleted file mode 100644 index 46904ad1d5d..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/message/StaticMessages.java +++ /dev/null @@ -1,10 +0,0 @@ -package org.tron.common.overlay.message; - -import org.springframework.stereotype.Component; - -@Component -public class StaticMessages { - - public static final PingMessage PING_MESSAGE = new PingMessage(); - public static final PongMessage PONG_MESSAGE = new PongMessage(); -} diff --git a/framework/src/main/java/org/tron/common/overlay/server/Channel.java b/framework/src/main/java/org/tron/common/overlay/server/Channel.java deleted file mode 100644 index f67cd8ba43b..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/server/Channel.java +++ /dev/null @@ -1,279 +0,0 @@ -package org.tron.common.overlay.server; - -import com.google.common.base.Throwables; -import com.google.protobuf.ByteString; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelPipeline; -import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender; -import io.netty.handler.timeout.ReadTimeoutException; -import io.netty.handler.timeout.ReadTimeoutHandler; -import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.SocketAddress; -import java.util.concurrent.TimeUnit; - -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.overlay.discover.node.NodeHandler; -import org.tron.common.overlay.discover.node.NodeManager; -import org.tron.common.overlay.discover.node.statistics.NodeStatistics; -import org.tron.common.overlay.message.DisconnectMessage; -import org.tron.common.overlay.message.HelloMessage; -import org.tron.common.overlay.message.MessageCodec; -import org.tron.common.overlay.message.StaticMessages; -import org.tron.core.db.ByteArrayWrapper; -import org.tron.core.exception.P2pException; -import org.tron.core.net.PbftHandler; -import org.tron.core.net.TronNetHandler; -import org.tron.protos.Protocol.ReasonCode; - -@Slf4j(topic = "net") -@Component -@Scope("prototype") -public class Channel { - - @Autowired - protected MessageQueue msgQueue; - protected NodeStatistics nodeStatistics; - @Autowired - private MessageCodec messageCodec; - @Autowired - private NodeManager nodeManager; - @Autowired - private StaticMessages staticMessages; - @Autowired - private WireTrafficStats stats; - @Autowired - private HandshakeHandler handshakeHandler; - @Autowired - private P2pHandler p2pHandler; - @Autowired - private TronNetHandler tronNetHandler; - @Autowired - private PbftHandler pbftHandler; - private ChannelManager channelManager; - private ChannelHandlerContext ctx; - private InetSocketAddress inetSocketAddress; - private Node node; - private long startTime; - @Getter - private TronState tronState = TronState.INIT; - private boolean isActive; - @Getter - @Setter - private ByteString address; - - private volatile boolean isDisconnect; - - @Getter - private volatile long disconnectTime; - - private boolean isTrustPeer; - - private boolean isFastForwardPeer; - - public void init(ChannelPipeline pipeline, String remoteId, boolean discoveryMode, - ChannelManager channelManager) { - - this.channelManager = channelManager; - - isActive = remoteId != null && !remoteId.isEmpty(); - - startTime = System.currentTimeMillis(); - - //TODO: use config here - pipeline.addLast("readTimeoutHandler", new ReadTimeoutHandler(60, TimeUnit.SECONDS)); - pipeline.addLast(stats.tcp); - pipeline.addLast("protoPender", new ProtobufVarint32LengthFieldPrepender()); - pipeline.addLast("lengthDecode", new TrxProtobufVarint32FrameDecoder(this)); - - //handshake first - pipeline.addLast("handshakeHandler", handshakeHandler); - - messageCodec.setChannel(this); - msgQueue.setChannel(this); - handshakeHandler.setChannel(this, remoteId); - p2pHandler.setChannel(this); - tronNetHandler.setChannel(this); - pbftHandler.setChannel(this); - - p2pHandler.setMsgQueue(msgQueue); - tronNetHandler.setMsgQueue(msgQueue); - pbftHandler.setMsgQueue(msgQueue); - } - - public void publicHandshakeFinished(ChannelHandlerContext ctx, HelloMessage msg) { - isTrustPeer = channelManager.getTrustNodes().getIfPresent(getInetAddress()) != null; - isFastForwardPeer = channelManager.getFastForwardNodes().containsKey(getInetAddress()); - ctx.pipeline().remove(handshakeHandler); - msgQueue.activate(ctx); - ctx.pipeline().addLast("messageCodec", messageCodec); - ctx.pipeline().addLast("p2p", p2pHandler); - ctx.pipeline().addLast("data", tronNetHandler); - ctx.pipeline().addLast("pbft", pbftHandler); - setStartTime(msg.getTimestamp()); - setTronState(TronState.HANDSHAKE_FINISHED); - getNodeStatistics().p2pHandShake.add(); - logger.info("Finish handshake with {}", ctx.channel().remoteAddress()); - } - - /** - * Set node and register it in NodeManager if it is not registered yet. - */ - public void initNode(byte[] nodeId, int remotePort) { - Node n = new Node(nodeId, inetSocketAddress.getHostString(), remotePort); - NodeHandler handler = nodeManager.getNodeHandler(n); - node = handler.getNode(); - nodeStatistics = handler.getNodeStatistics(); - handler.getNode().setId(nodeId); - } - - public void disconnect(ReasonCode reason) { - this.isDisconnect = true; - this.disconnectTime = System.currentTimeMillis(); - channelManager.processDisconnect(this, reason); - DisconnectMessage msg = new DisconnectMessage(reason); - logger.info("Send to {} online-time {}s, {}", - ctx.channel().remoteAddress(), - (System.currentTimeMillis() - startTime) / 1000, - msg); - getNodeStatistics().nodeDisconnectedLocal(reason); - ctx.writeAndFlush(msg.getSendData()).addListener(future -> close()); - } - - public void processException(Throwable throwable) { - Throwable baseThrowable = throwable; - try { - baseThrowable = Throwables.getRootCause(baseThrowable); - } catch (IllegalArgumentException e) { - baseThrowable = e.getCause(); - logger.warn("Loop in causal chain detected"); - } - SocketAddress address = ctx.channel().remoteAddress(); - if (throwable instanceof ReadTimeoutException - || throwable instanceof IOException) { - logger.warn("Close peer {}, reason: {}", address, throwable.getMessage()); - } else if (baseThrowable instanceof P2pException) { - logger.warn("Close peer {}, type: {}, info: {}", - address, ((P2pException) baseThrowable).getType(), baseThrowable.getMessage()); - } else { - logger.error("Close peer {}, exception caught", address, throwable); - } - close(); - } - - public void close() { - this.isDisconnect = true; - this.disconnectTime = System.currentTimeMillis(); - p2pHandler.close(); - msgQueue.close(); - ctx.close(); - } - - public Node getNode() { - return node; - } - - public byte[] getNodeId() { - return node == null ? null : node.getId(); - } - - public ByteArrayWrapper getNodeIdWrapper() { - return node == null ? null : new ByteArrayWrapper(node.getId()); - } - - public String getPeerId() { - return node == null ? "" : node.getHexId(); - } - - public void setChannelHandlerContext(ChannelHandlerContext ctx) { - this.ctx = ctx; - this.inetSocketAddress = ctx == null ? null : (InetSocketAddress) ctx.channel().remoteAddress(); - } - - public InetAddress getInetAddress() { - return ctx == null ? null : ((InetSocketAddress) ctx.channel().remoteAddress()).getAddress(); - } - - public NodeStatistics getNodeStatistics() { - return nodeStatistics; - } - - public long getStartTime() { - return startTime; - } - - public void setStartTime(long startTime) { - this.startTime = startTime; - } - - public void setTronState(TronState tronState) { - this.tronState = tronState; - logger.info("Peer {} status change to {}", inetSocketAddress, tronState); - } - - public boolean isActive() { - return isActive; - } - - public boolean isDisconnect() { - return isDisconnect; - } - - public boolean isTrustPeer() { - return isTrustPeer; - } - - public boolean isFastForwardPeer() { - return isFastForwardPeer; - } - - @Override - public boolean equals(Object o) { - - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Channel channel = (Channel) o; - if (inetSocketAddress != null ? !inetSocketAddress.equals(channel.inetSocketAddress) - : channel.inetSocketAddress != null) { - return false; - } - if (node != null ? !node.equals(channel.node) : channel.node != null) { - return false; - } - return this == channel; - } - - @Override - public int hashCode() { - int result = inetSocketAddress != null ? inetSocketAddress.hashCode() : 0; - result = 31 * result + (node != null ? node.hashCode() : 0); - return result; - } - - @Override - public String toString() { - return String.format("%s | %s", inetSocketAddress, getPeerId()); - } - - public enum TronState { - INIT, - HANDSHAKE_FINISHED, - START_TO_SYNC, - SYNCING, - SYNC_COMPLETED, - SYNC_FAILED - } - -} - diff --git a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java b/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java deleted file mode 100644 index 68c5ee78d97..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java +++ /dev/null @@ -1,206 +0,0 @@ -package org.tron.common.overlay.server; - -import static org.tron.protos.Protocol.ReasonCode.DUPLICATE_PEER; -import static org.tron.protos.Protocol.ReasonCode.TOO_MANY_PEERS; -import static org.tron.protos.Protocol.ReasonCode.TOO_MANY_PEERS_WITH_SAME_IP; -import static org.tron.protos.Protocol.ReasonCode.UNKNOWN; - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.util.Collection; -import java.util.Locale; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.tron.common.overlay.client.PeerClient; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.parameter.CommonParameter; -import org.tron.common.prometheus.MetricKeys; -import org.tron.common.prometheus.Metrics; -import org.tron.core.config.args.Args; -import org.tron.core.db.ByteArrayWrapper; -import org.tron.core.metrics.MetricsKey; -import org.tron.core.metrics.MetricsUtil; -import org.tron.protos.Protocol; -import org.tron.protos.Protocol.ReasonCode; - -@Slf4j(topic = "net") -@Component -public class ChannelManager { - - private final Map activeChannels = new ConcurrentHashMap<>(); - @Autowired - private PeerServer peerServer; - @Autowired - private PeerClient peerClient; - @Autowired - private SyncPool syncPool; - @Autowired - private PeerConnectionCheckService peerConnectionCheckService; - @Autowired - private FastForward fastForward; - private CommonParameter parameter = CommonParameter.getInstance(); - private Cache badPeers = CacheBuilder.newBuilder().maximumSize(10000) - .expireAfterWrite(1, TimeUnit.HOURS).recordStats().build(); - - private Cache recentlyDisconnected = CacheBuilder.newBuilder() - .maximumSize(1000).expireAfterWrite(30, TimeUnit.SECONDS).recordStats().build(); - - @Getter - private Cache helloMessageCache = CacheBuilder.newBuilder() - .maximumSize(2000).expireAfterWrite(24, TimeUnit.HOURS).recordStats().build(); - - @Getter - private Cache trustNodes = CacheBuilder.newBuilder().maximumSize(100).build(); - - @Getter - private Map activeNodes = new ConcurrentHashMap(); - - @Getter - private Map fastForwardNodes = new ConcurrentHashMap(); - - private int maxConnections = parameter.getMaxConnections(); - - private int maxConnectionsWithSameIp = parameter.getMaxConnectionsWithSameIp(); - - public void init() { - if (this.parameter.getNodeListenPort() > 0 && !this.parameter.isSolidityNode()) { - new Thread(() -> peerServer.start(Args.getInstance().getNodeListenPort()), - "PeerServerThread").start(); - } - - InetAddress address; - for (Node node : parameter.getPassiveNodes()) { - address = new InetSocketAddress(node.getHost(), node.getPort()).getAddress(); - trustNodes.put(address, node); - } - - for (Node node : parameter.getActiveNodes()) { - address = new InetSocketAddress(node.getHost(), node.getPort()).getAddress(); - trustNodes.put(address, node); - activeNodes.put(address, node); - } - - for (Node node : parameter.getFastForwardNodes()) { - address = new InetSocketAddress(node.getHost(), node.getPort()).getAddress(); - trustNodes.put(address, node); - fastForwardNodes.put(address, node); - } - - logger.info("Node config, trust {}, active {}, forward {}", - trustNodes.size(), activeNodes.size(), fastForwardNodes.size()); - - peerConnectionCheckService.init(); - syncPool.init(); - fastForward.init(); - } - - public void processDisconnect(Channel channel, ReasonCode reason) { - InetAddress inetAddress = channel.getInetAddress(); - if (inetAddress == null) { - return; - } - switch (reason) { - case BAD_PROTOCOL: - case BAD_BLOCK: - case BAD_TX: - badPeers.put(channel.getInetAddress(), reason); - break; - default: - recentlyDisconnected.put(channel.getInetAddress(), reason); - break; - } - MetricsUtil.counterInc(MetricsKey.NET_DISCONNECTION_COUNT); - MetricsUtil.counterInc(MetricsKey.NET_DISCONNECTION_DETAIL + reason); - Metrics.counterInc(MetricKeys.Counter.P2P_DISCONNECT, 1, - reason.name().toLowerCase(Locale.ROOT)); - } - - public void notifyDisconnect(Channel channel) { - syncPool.onDisconnect(channel); - activeChannels.values().remove(channel); - if (channel != null) { - if (channel.getNodeStatistics() != null) { - channel.getNodeStatistics().notifyDisconnect(); - } - InetAddress inetAddress = channel.getInetAddress(); - if (inetAddress != null && recentlyDisconnected.getIfPresent(inetAddress) == null) { - recentlyDisconnected.put(channel.getInetAddress(), UNKNOWN); - } - } - } - - public synchronized boolean processPeer(Channel peer) { - - if (trustNodes.getIfPresent(peer.getInetAddress()) == null) { - if (recentlyDisconnected.getIfPresent(peer) != null) { - logger.info("Peer {} recently disconnected", peer.getInetAddress()); - return false; - } - - if (badPeers.getIfPresent(peer) != null) { - peer.disconnect(peer.getNodeStatistics().getDisconnectReason()); - return false; - } - - if (!peer.isActive() && activeChannels.size() >= maxConnections) { - peer.disconnect(TOO_MANY_PEERS); - return false; - } - - if (getConnectionNum(peer.getInetAddress()) >= maxConnectionsWithSameIp) { - peer.disconnect(TOO_MANY_PEERS_WITH_SAME_IP); - return false; - } - } - - Channel channel = activeChannels.get(peer.getNodeIdWrapper()); - if (channel != null) { - if (channel.getStartTime() > peer.getStartTime()) { - logger.info("Disconnect connection established later, {}", channel.getNode()); - channel.disconnect(DUPLICATE_PEER); - } else { - peer.disconnect(DUPLICATE_PEER); - return false; - } - } - activeChannels.put(peer.getNodeIdWrapper(), peer); - logger.info("Add active peer {}, total active peers: {}", peer, activeChannels.size()); - return true; - } - - public int getConnectionNum(InetAddress inetAddress) { - int cnt = 0; - for (Channel channel : activeChannels.values()) { - if (channel.getInetAddress().equals(inetAddress)) { - cnt++; - } - } - return cnt; - } - - public Collection getActivePeers() { - return activeChannels.values(); - } - - public Cache getRecentlyDisconnected() { - return this.recentlyDisconnected; - } - - public Cache getBadPeers() { - return this.badPeers; - } - - public void close() { - peerConnectionCheckService.close(); - syncPool.close(); - peerServer.close(); - peerClient.close(); - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/server/HandshakeHandler.java b/framework/src/main/java/org/tron/common/overlay/server/HandshakeHandler.java deleted file mode 100644 index b36c7f31b5d..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/server/HandshakeHandler.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.server; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ByteToMessageDecoder; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.bouncycastle.util.encoders.Hex; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import org.tron.common.overlay.discover.node.NodeManager; -import org.tron.common.overlay.message.DisconnectMessage; -import org.tron.common.overlay.message.HelloMessage; -import org.tron.common.overlay.message.P2pMessage; -import org.tron.common.overlay.message.P2pMessageFactory; -import org.tron.common.prometheus.MetricKeys; -import org.tron.common.prometheus.MetricLabels; -import org.tron.common.prometheus.Metrics; -import org.tron.common.utils.ByteArray; -import org.tron.core.ChainBaseManager; -import org.tron.core.config.args.Args; -import org.tron.core.db.Manager; -import org.tron.core.metrics.MetricsKey; -import org.tron.core.metrics.MetricsUtil; -import org.tron.core.net.peer.PeerConnection; -import org.tron.protos.Protocol.ReasonCode; - -@Slf4j(topic = "net") -@Component -@Scope("prototype") -public class HandshakeHandler extends ByteToMessageDecoder { - - private Channel channel; - - @Autowired - private NodeManager nodeManager; - - @Autowired - private ChannelManager channelManager; - - @Autowired - private Manager manager; - - @Autowired - private ChainBaseManager chainBaseManager; - - @Autowired - private FastForward fastForward; - - private byte[] remoteId; - - private P2pMessageFactory messageFactory = new P2pMessageFactory(); - - @Autowired - private SyncPool syncPool; - - @Override - public void channelActive(ChannelHandlerContext ctx) throws Exception { - logger.info("Channel active, {}", ctx.channel().remoteAddress()); - channel.setChannelHandlerContext(ctx); - if (remoteId.length == 64) { - channel.initNode(remoteId, ((InetSocketAddress) ctx.channel().remoteAddress()).getPort()); - sendHelloMsg(ctx, System.currentTimeMillis()); - } - } - - @Override - protected void decode(ChannelHandlerContext ctx, ByteBuf buffer, List out) - throws Exception { - byte[] encoded = new byte[buffer.readableBytes()]; - buffer.readBytes(encoded); - P2pMessage msg = messageFactory.create(encoded); - - logger.info("Handshake receive from {}, {}", ctx.channel().remoteAddress(), msg); - - switch (msg.getType()) { - case P2P_HELLO: - handleHelloMsg(ctx, (HelloMessage) msg); - break; - case P2P_DISCONNECT: - if (channel.getNodeStatistics() != null) { - channel.getNodeStatistics() - .nodeDisconnectedRemote(((DisconnectMessage) msg).getReasonCode()); - } - channel.close(); - break; - default: - channel.close(); - break; - } - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { - channel.processException(cause); - } - - public void setChannel(Channel channel, String remoteId) { - this.channel = channel; - this.remoteId = Hex.decode(remoteId); - } - - protected void sendHelloMsg(ChannelHandlerContext ctx, long time) { - HelloMessage message = new HelloMessage( - nodeManager.getPublicHomeNode(), time, chainBaseManager); - fastForward.fillHelloMessage(message, channel); - ((PeerConnection) channel).setHelloMessageSend(message); - ctx.writeAndFlush(message.getSendData()); - int length = message.getSendData().readableBytes(); - channel.getNodeStatistics().messageStatistics.addTcpOutMessage(message); - MetricsUtil.meterMark(MetricsKey.NET_TCP_OUT_TRAFFIC, length); - Metrics.histogramObserve(MetricKeys.Histogram.TCP_BYTES, length, - MetricLabels.Histogram.TRAFFIC_OUT); - - logger.info("Handshake send to {}, {} ", ctx.channel().remoteAddress(), message); - } - - private void handleHelloMsg(ChannelHandlerContext ctx, HelloMessage msg) { - channel.initNode(msg.getFrom().getId(), msg.getFrom().getPort()); - - if (!msg.valid()) { - logger.warn("Peer {} invalid hello message parameters, " - + "GenesisBlockId: {}, SolidBlockId: {}, HeadBlockId: {}", - ctx.channel().remoteAddress(), - ByteArray.toHexString(msg.getInstance().getGenesisBlockId().getHash().toByteArray()), - ByteArray.toHexString(msg.getInstance().getSolidBlockId().getHash().toByteArray()), - ByteArray.toHexString(msg.getInstance().getHeadBlockId().getHash().toByteArray())); - channel.disconnect(ReasonCode.UNEXPECTED_IDENTITY); - return; - } - - channel.setAddress(msg.getHelloMessage().getAddress()); - - if (!fastForward.checkHelloMessage(msg, channel)) { - channel.disconnect(ReasonCode.UNEXPECTED_IDENTITY); - return; - } - - InetAddress address = ((InetSocketAddress) ctx.channel().remoteAddress()).getAddress(); - if (remoteId.length != 64 - && channelManager.getTrustNodes().getIfPresent(address) == null - && !syncPool.isCanConnect()) { - channel.disconnect(ReasonCode.TOO_MANY_PEERS); - return; - } - - long headBlockNum = chainBaseManager.getHeadBlockNum(); - long lowestBlockNum = msg.getLowestBlockNum(); - if (lowestBlockNum > headBlockNum) { - logger.info("Peer {} miss block, lowestBlockNum:{}, headBlockNum:{}", - ctx.channel().remoteAddress(), lowestBlockNum, headBlockNum); - channel.disconnect(ReasonCode.LIGHT_NODE_SYNC_FAIL); - return; - } - - if (msg.getVersion() != Args.getInstance().getNodeP2pVersion()) { - logger.info("Peer {} different p2p version, peer->{}, me->{}", - ctx.channel().remoteAddress(), msg.getVersion(), Args.getInstance().getNodeP2pVersion()); - channel.disconnect(ReasonCode.INCOMPATIBLE_VERSION); - return; - } - - if (!Arrays - .equals(chainBaseManager.getGenesisBlockId().getBytes(), - msg.getGenesisBlockId().getBytes())) { - logger - .info("Peer {} different genesis block, peer->{}, me->{}", ctx.channel().remoteAddress(), - msg.getGenesisBlockId().getString(), - chainBaseManager.getGenesisBlockId().getString()); - channel.disconnect(ReasonCode.INCOMPATIBLE_CHAIN); - return; - } - - if (chainBaseManager.getSolidBlockId().getNum() >= msg.getSolidBlockId().getNum() - && !chainBaseManager.containBlockInMainChain(msg.getSolidBlockId())) { - logger.info("Peer {} different solid block, peer->{}, me->{}", ctx.channel().remoteAddress(), - msg.getSolidBlockId().getString(), chainBaseManager.getSolidBlockId().getString()); - channel.disconnect(ReasonCode.FORKED); - return; - } - - if (msg.getFrom().getHost().equals(address.getHostAddress())) { - channelManager.getHelloMessageCache().put(msg.getFrom().getHost(), msg.getHelloMessage()); - } - - ((PeerConnection) channel).setHelloMessageReceive(msg); - - channel.getNodeStatistics().messageStatistics.addTcpInMessage(msg); - - channel.publicHandshakeFinished(ctx, msg); - if (!channelManager.processPeer(channel)) { - return; - } - - if (remoteId.length != 64) { - sendHelloMsg(ctx, msg.getTimestamp()); - } - - syncPool.onConnect(channel); - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java deleted file mode 100644 index de59cb85736..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java +++ /dev/null @@ -1,217 +0,0 @@ -package org.tron.common.overlay.server; - -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import java.util.Queue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import org.tron.common.overlay.message.Message; -import org.tron.common.overlay.message.PingMessage; -import org.tron.common.overlay.message.PongMessage; -import org.tron.common.prometheus.MetricKeys; -import org.tron.common.prometheus.MetricLabels; -import org.tron.common.prometheus.Metrics; -import org.tron.consensus.pbft.message.PbftBaseMessage; -import org.tron.core.metrics.MetricsKey; -import org.tron.core.metrics.MetricsUtil; -import org.tron.core.net.message.InventoryMessage; -import org.tron.core.net.message.TransactionsMessage; -import org.tron.protos.Protocol.Inventory.InventoryType; -import org.tron.protos.Protocol.ReasonCode; - -@Slf4j(topic = "net") -@Component -@Scope("prototype") -public class MessageQueue { - - private static ScheduledExecutorService sendTimer = - Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "sendTimer")); - private volatile boolean sendMsgFlag = false; - private volatile long sendTime; - private volatile long sendPing; - private Thread sendMsgThread; - private Channel channel; - private ChannelHandlerContext ctx = null; - private Queue requestQueue = new ConcurrentLinkedQueue<>(); - private BlockingQueue msgQueue = new LinkedBlockingQueue<>(); - private ScheduledFuture sendTask; - - - public void activate(ChannelHandlerContext ctx) { - - this.ctx = ctx; - - sendMsgFlag = true; - - sendTask = sendTimer.scheduleAtFixedRate(() -> { - try { - if (sendMsgFlag) { - send(); - } - } catch (Exception e) { - logger.error("Unhandled exception", e); - } - }, 10, 10, TimeUnit.MILLISECONDS); - - sendMsgThread = new Thread(() -> { - while (sendMsgFlag) { - try { - if (msgQueue.isEmpty()) { - Thread.sleep(10); - continue; - } - Message msg = msgQueue.take(); - if (channel.isDisconnect()) { - logger.warn("Failed to send to {} as channel has closed, {}", - ctx.channel().remoteAddress(), msg); - msgQueue.clear(); - return; - } - ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { - if (!future.isSuccess() && !channel.isDisconnect()) { - logger.warn("Failed to send to {}, {} ", ctx.channel().remoteAddress(), msg); - } - }); - } catch (InterruptedException e) { - logger.warn("Send message server interrupted"); - Thread.currentThread().interrupt(); - } catch (Exception e) { - logger.error("Failed to send to {} ", ctx.channel().remoteAddress(), e); - } - } - }); - sendMsgThread.setName("sendMsgThread-" + ctx.channel().remoteAddress()); - sendMsgThread.start(); - } - - public void setChannel(Channel channel) { - this.channel = channel; - } - - public void fastSend(Message msg) { - if (channel.isDisconnect()) { - logger.warn("Fast send to {} failed as channel has closed, {} ", - ctx.channel().remoteAddress(), msg); - return; - } - logger.info("Fast send to {}, {} ", ctx.channel().remoteAddress(), msg); - ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { - if (!future.isSuccess() && !channel.isDisconnect()) { - logger.warn("Fast send to {} failed, {}", ctx.channel().remoteAddress(), msg); - } - }); - } - - public boolean sendMessage(Message msg) { - long now = System.currentTimeMillis(); - if (msg instanceof PingMessage) { - if (now - sendTime < 10_000 && now - sendPing < 60_000) { - return false; - } - sendPing = now; - } - if (needToLog(msg)) { - logger.info("Send to {}, {} ", ctx.channel().remoteAddress(), msg); - } - channel.getNodeStatistics().messageStatistics.addTcpOutMessage(msg); - int length = msg.getSendData().readableBytes(); - MetricsUtil.meterMark(MetricsKey.NET_TCP_OUT_TRAFFIC, length); - Metrics.histogramObserve(MetricKeys.Histogram.TCP_BYTES, length, - MetricLabels.Histogram.TRAFFIC_OUT); - - sendTime = System.currentTimeMillis(); - if (msg.getAnswerMessage() != null) { - requestQueue.add(new MessageRoundTrip(msg)); - } else { - msgQueue.offer(msg); - } - return true; - } - - public void receivedMessage(Message msg) { - if (needToLog(msg)) { - logger.info("Receive from {}, {}", ctx.channel().remoteAddress(), msg); - } - channel.getNodeStatistics().messageStatistics.addTcpInMessage(msg); - MessageRoundTrip rt = requestQueue.peek(); - if (rt != null && rt.getMsg().getAnswerMessage() == msg.getClass()) { - requestQueue.remove(); - if (rt.getMsg() instanceof PingMessage) { - channel.getNodeStatistics().pingMessageLatency - .add(System.currentTimeMillis() - rt.getTime()); - } - } - } - - public void close() { - sendMsgFlag = false; - if (sendTask != null && !sendTask.isCancelled()) { - sendTask.cancel(true); - sendTask = null; - } - if (sendMsgThread != null) { - try { - sendMsgThread.join(20); - sendMsgThread = null; - } catch (InterruptedException e) { - logger.warn("Send message join interrupted"); - Thread.currentThread().interrupt(); - } catch (Exception e) { - logger.warn("Join send thread failed, peer {}", ctx.channel().remoteAddress()); - } - } - } - - private boolean needToLog(Message msg) { - if (msg instanceof PingMessage - || msg instanceof PongMessage - || msg instanceof TransactionsMessage - || msg instanceof PbftBaseMessage) { - return false; - } - - if (msg instanceof InventoryMessage - && ((InventoryMessage) msg).getInventoryType().equals(InventoryType.TRX)) { - return false; - } - - return true; - } - - private void send() { - MessageRoundTrip rt = requestQueue.peek(); - if (!sendMsgFlag || rt == null) { - return; - } - if (rt.getRetryTimes() > 0 && !rt.hasToRetry()) { - return; - } - if (rt.getRetryTimes() > 0) { - channel.getNodeStatistics().nodeDisconnectedLocal(ReasonCode.PING_TIMEOUT); - logger.warn("Wait {} timeout. close channel {}", - rt.getMsg().getAnswerMessage(), ctx.channel().remoteAddress()); - channel.close(); - return; - } - - Message msg = rt.getMsg(); - - ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { - if (!future.isSuccess()) { - logger.warn("Fail send to {}, {}", ctx.channel().remoteAddress(), msg); - } - }); - - rt.incRetryTimes(); - rt.saveTime(); - } - -} diff --git a/framework/src/main/java/org/tron/common/overlay/server/MessageRoundTrip.java b/framework/src/main/java/org/tron/common/overlay/server/MessageRoundTrip.java deleted file mode 100644 index d14954c4bf9..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/server/MessageRoundTrip.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.tron.common.overlay.server; - -import org.tron.common.overlay.message.Message; - -public class MessageRoundTrip { - - private final Message msg; - private long time = 0; - private long retryTimes = 0; - - public MessageRoundTrip(Message msg) { - this.msg = msg; - saveTime(); - } - - public long getRetryTimes() { - return retryTimes; - } - - public void incRetryTimes() { - ++retryTimes; - } - - public void saveTime() { - time = System.currentTimeMillis(); - } - - public long getTime() { - return time; - } - - public boolean hasToRetry() { - return 20000 < System.currentTimeMillis() - time; - } - - public Message getMsg() { - return msg; - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/server/P2pHandler.java b/framework/src/main/java/org/tron/common/overlay/server/P2pHandler.java deleted file mode 100644 index 3346506396d..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/server/P2pHandler.java +++ /dev/null @@ -1,103 +0,0 @@ -package org.tron.common.overlay.server; - -import static org.tron.common.overlay.message.StaticMessages.PING_MESSAGE; -import static org.tron.common.overlay.message.StaticMessages.PONG_MESSAGE; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import org.tron.common.overlay.discover.node.statistics.MessageStatistics; -import org.tron.common.overlay.message.DisconnectMessage; -import org.tron.common.overlay.message.P2pMessage; -import org.tron.protos.Protocol.ReasonCode; - -@Slf4j(topic = "net") -@Component -@Scope("prototype") -public class P2pHandler extends SimpleChannelInboundHandler { - - private static ScheduledExecutorService pingTimer = - Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "P2pPingTimer")); - - private MessageQueue msgQueue; - - private Channel channel; - - private ScheduledFuture pingTask; - - private volatile boolean hasPing = false; - - @Override - public void handlerAdded(ChannelHandlerContext ctx) { - pingTask = pingTimer.scheduleAtFixedRate(() -> { - if (!hasPing) { - hasPing = msgQueue.sendMessage(PING_MESSAGE); - } - }, 10, 10, TimeUnit.SECONDS); - } - - @Override - public void channelRead0(final ChannelHandlerContext ctx, P2pMessage msg) { - - msgQueue.receivedMessage(msg); - MessageStatistics messageStatistics = channel.getNodeStatistics().messageStatistics; - switch (msg.getType()) { - case P2P_PING: - int count = messageStatistics.p2pInPing.getCount(10); - if (count > 3) { - logger.warn("TCP attack found: {} with ping count({})", ctx.channel().remoteAddress(), - count); - channel.disconnect(ReasonCode.BAD_PROTOCOL); - return; - } - msgQueue.sendMessage(PONG_MESSAGE); - break; - case P2P_PONG: - if (messageStatistics.p2pInPong.getTotalCount() > messageStatistics.p2pOutPing - .getTotalCount()) { - logger.warn("TCP attack found: {} with ping count({}), pong count({})", - ctx.channel().remoteAddress(), - messageStatistics.p2pOutPing.getTotalCount(), - messageStatistics.p2pInPong.getTotalCount()); - channel.disconnect(ReasonCode.BAD_PROTOCOL); - return; - } - hasPing = false; - channel.getNodeStatistics().lastPongReplyTime.set(System.currentTimeMillis()); - break; - case P2P_DISCONNECT: - channel.getNodeStatistics() - .nodeDisconnectedRemote(((DisconnectMessage) msg).getReasonCode()); - channel.close(); - break; - default: - channel.close(); - break; - } - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - channel.processException(cause); - } - - public void setMsgQueue(MessageQueue msgQueue) { - this.msgQueue = msgQueue; - } - - public void setChannel(Channel channel) { - this.channel = channel; - } - - public void close() { - if (pingTask != null && !pingTask.isCancelled()) { - pingTask.cancel(false); - } - } -} \ No newline at end of file diff --git a/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java b/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java deleted file mode 100644 index e0d9d39b4a7..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.tron.common.overlay.server; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Random; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.tron.core.config.args.Args; -import org.tron.core.net.peer.PeerConnection; -import org.tron.protos.Protocol; - -@Slf4j(topic = "net") -@Service -public class PeerConnectionCheckService { - private int maxConnections = Args.getInstance().getMaxConnections(); - private boolean isFastForward = Args.getInstance().isFastForward(); - private boolean isOpenFullTcpDisconnect = Args.getInstance().isOpenFullTcpDisconnect(); - private ScheduledExecutorService poolLoopExecutor = Executors.newSingleThreadScheduledExecutor(); - - @Autowired - private SyncPool syncPool; - - public void init() { - if (isFastForward || !isOpenFullTcpDisconnect) { - return; - } - logger.info("Start peer connection check service"); - poolLoopExecutor.scheduleWithFixedDelay(() -> { - try { - check(); - } catch (Throwable t) { - logger.error("Exception in peer connection check", t); - } - }, 10, 30, TimeUnit.SECONDS); - } - - public void close() { - logger.info("Close peer connection check service"); - poolLoopExecutor.shutdown(); - } - - public void check() { - if (syncPool.getActivePeers().size() < maxConnections) { - return; - } - Collection peers = syncPool.getActivePeers().stream() - .filter(peer -> peer.isIdle()) - .filter(peer -> !peer.isTrustPeer()) - .filter(peer -> !peer.isActive()) - .collect(Collectors.toList()); - if (peers.size() == 0) { - return; - } - List list = new ArrayList(); - peers.forEach(p -> list.add(p)); - PeerConnection peer = list.get(new Random().nextInt(peers.size())); - peer.disconnect(Protocol.ReasonCode.RANDOM_ELIMINATION); - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/server/PeerServer.java b/framework/src/main/java/org/tron/common/overlay/server/PeerServer.java deleted file mode 100644 index e00372e9a5d..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/server/PeerServer.java +++ /dev/null @@ -1,84 +0,0 @@ -package org.tron.common.overlay.server; - -import io.netty.bootstrap.ServerBootstrap; -import io.netty.channel.ChannelFuture; -import io.netty.channel.ChannelOption; -import io.netty.channel.DefaultMessageSizeEstimator; -import io.netty.channel.EventLoopGroup; -import io.netty.channel.nio.NioEventLoopGroup; -import io.netty.channel.socket.nio.NioServerSocketChannel; -import io.netty.handler.logging.LoggingHandler; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.stereotype.Component; -import org.tron.common.parameter.CommonParameter; -import org.tron.core.config.args.Args; - -@Slf4j(topic = "net") -@Component -public class PeerServer { - - private CommonParameter parameter = CommonParameter.getInstance(); - - private ApplicationContext ctx; - - private boolean listening; - - private ChannelFuture channelFuture; - - @Autowired - public PeerServer(final Args args, final ApplicationContext ctx) { - this.ctx = ctx; - } - - public void start(int port) { - - EventLoopGroup bossGroup = new NioEventLoopGroup(1); - EventLoopGroup workerGroup = new NioEventLoopGroup(parameter.getTcpNettyWorkThreadNum()); - TronChannelInitializer tronChannelInitializer = ctx.getBean(TronChannelInitializer.class, ""); - - try { - ServerBootstrap b = new ServerBootstrap(); - - b.group(bossGroup, workerGroup); - b.channel(NioServerSocketChannel.class); - - b.option(ChannelOption.MESSAGE_SIZE_ESTIMATOR, DefaultMessageSizeEstimator.DEFAULT); - b.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, this.parameter.getNodeConnectionTimeout()); - - b.handler(new LoggingHandler()); - b.childHandler(tronChannelInitializer); - - // Start the client. - logger.info("TCP listener started, bind port {}", port); - - channelFuture = b.bind(port).sync(); - - listening = true; - - // Wait until the connection is closed. - channelFuture.channel().closeFuture().sync(); - - logger.info("TCP listener closed"); - - } catch (Exception e) { - logger.error("Start TCP server failed", e); - } finally { - workerGroup.shutdownGracefully(); - bossGroup.shutdownGracefully(); - listening = false; - } - } - - public void close() { - if (listening && channelFuture != null && channelFuture.channel().isOpen()) { - try { - logger.info("Closing TCP server..."); - channelFuture.channel().close().sync(); - } catch (Exception e) { - logger.error("Closing TCP server failed", e); - } - } - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java b/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java deleted file mode 100644 index e4b28494404..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java +++ /dev/null @@ -1,265 +0,0 @@ -package org.tron.common.overlay.server; - -import com.codahale.metrics.Snapshot; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.google.common.collect.Lists; -import java.net.InetAddress; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Predicate; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.stereotype.Component; -import org.tron.common.overlay.client.PeerClient; -import org.tron.common.overlay.discover.node.NodeHandler; -import org.tron.common.overlay.discover.node.NodeManager; -import org.tron.common.parameter.CommonParameter; -import org.tron.common.prometheus.MetricKeys; -import org.tron.common.prometheus.MetricLabels; -import org.tron.common.prometheus.Metrics; -import org.tron.core.ChainBaseManager; -import org.tron.core.config.args.Args; -import org.tron.core.metrics.MetricsKey; -import org.tron.core.metrics.MetricsUtil; -import org.tron.core.net.peer.PeerConnection; -import org.tron.protos.Protocol; - -@Slf4j(topic = "net") -@Component -public class SyncPool { - private final List activePeers = Collections - .synchronizedList(new ArrayList<>()); - private Cache nodeHandlerCache = CacheBuilder.newBuilder() - .maximumSize(1000).expireAfterWrite(180, TimeUnit.SECONDS).recordStats().build(); - private final AtomicInteger passivePeersCount = new AtomicInteger(0); - private final AtomicInteger activePeersCount = new AtomicInteger(0); - - @Autowired - private NodeManager nodeManager; - - @Autowired - private ApplicationContext ctx; - - @Autowired - private ChainBaseManager chainBaseManager; - - private ChannelManager channelManager; - - private CommonParameter commonParameter = CommonParameter.getInstance(); - - private int maxConnectionsWithSameIp = commonParameter.getMaxConnectionsWithSameIp(); - - private ScheduledExecutorService poolLoopExecutor = Executors.newSingleThreadScheduledExecutor(); - - private ScheduledExecutorService logExecutor = Executors.newSingleThreadScheduledExecutor(); - - private PeerClient peerClient; - - private int disconnectTimeout = 60_000; - - private int maxConnections = Args.getInstance().getMaxConnections(); - private int minConnections = Args.getInstance().getMinConnections(); - private int minActiveConnections = Args.getInstance().getMinActiveConnections(); - - public void init() { - - channelManager = ctx.getBean(ChannelManager.class); - - peerClient = ctx.getBean(PeerClient.class); - - poolLoopExecutor.scheduleWithFixedDelay(() -> { - try { - check(); - fillUp(); - } catch (Throwable t) { - logger.error("Exception in sync worker", t); - } - }, 100, 3600, TimeUnit.MILLISECONDS); - - logExecutor.scheduleWithFixedDelay(() -> { - try { - logActivePeers(); - } catch (Throwable t) { - logger.error("Exception in sync worker", t); - } - }, 30, 10, TimeUnit.SECONDS); - } - - private void check() { - for (PeerConnection peer : new ArrayList<>(activePeers)) { - long now = System.currentTimeMillis(); - long disconnectTime = peer.getDisconnectTime(); - if (disconnectTime != 0 && now - disconnectTime > disconnectTimeout) { - logger.warn("Notify disconnect peer {}.", peer.getInetAddress()); - channelManager.notifyDisconnect(peer); - } - } - } - - private void fillUp() { - List connectNodes = new ArrayList<>(); - Set addressInUse = new HashSet<>(); - Set nodesInUse = new HashSet<>(); - channelManager.getActivePeers().forEach(channel -> { - nodesInUse.add(channel.getPeerId()); - addressInUse.add(channel.getInetAddress()); - }); - - channelManager.getActiveNodes().forEach((address, node) -> { - nodesInUse.add(node.getHexId()); - if (!addressInUse.contains(address)) { - connectNodes.add(nodeManager.getNodeHandler(node)); - } - }); - - int size = Math.max(minConnections - activePeers.size(), - minActiveConnections - activePeersCount.get()); - int lackSize = size - connectNodes.size(); - if (lackSize > 0) { - nodesInUse.add(nodeManager.getPublicHomeNode().getHexId()); - List newNodes = nodeManager.getNodes(new NodeSelector(nodesInUse), lackSize); - connectNodes.addAll(newNodes); - } - - connectNodes.forEach(n -> { - peerClient.connectAsync(n, false); - nodeHandlerCache.put(n, System.currentTimeMillis()); - }); - } - - synchronized void logActivePeers() { - String str = String.format("\n\n============ Peer stats: all %d, active %d, passive %d\n\n", - channelManager.getActivePeers().size(), activePeersCount.get(), passivePeersCount.get()); - metric(channelManager.getActivePeers().size(), MetricLabels.Gauge.PEERS_ALL); - metric(activePeersCount.get(), MetricLabels.Gauge.PEERS_ACTIVE); - metric(passivePeersCount.get(), MetricLabels.Gauge.PEERS_PASSIVE); - StringBuilder sb = new StringBuilder(str); - int valid = 0; - for (PeerConnection peer : new ArrayList<>(activePeers)) { - sb.append(peer.log()); - appendPeerLatencyLog(sb, peer); - sb.append("\n"); - if (!(peer.isNeedSyncFromUs() || peer.isNeedSyncFromPeer())) { - valid++; - } - } - metric(valid, MetricLabels.Gauge.PEERS_VALID); - logger.info(sb.toString()); - } - - private void metric(double amt, String peerType) { - Metrics.gaugeSet(MetricKeys.Gauge.PEERS, amt, peerType); - } - - private void appendPeerLatencyLog(StringBuilder builder, PeerConnection peer) { - Snapshot peerSnapshot = MetricsUtil.getHistogram(MetricsKey.NET_LATENCY_FETCH_BLOCK - + peer.getNode().getHost()).getSnapshot(); - builder.append(String.format( - "top99 : %f, top95 : %f, top75 : %f, max : %d, min : %d, mean : %f, median : %f", - peerSnapshot.get99thPercentile(), peerSnapshot.get95thPercentile(), - peerSnapshot.get75thPercentile(), peerSnapshot.getMax(), peerSnapshot.getMin(), - peerSnapshot.getMean(), peerSnapshot.getMedian())).append("\n"); - } - - public List getActivePeers() { - List peers = Lists.newArrayList(); - for (PeerConnection peer : new ArrayList<>(activePeers)) { - if (!peer.isDisconnect()) { - peers.add(peer); - } - } - return peers; - } - - public synchronized void onConnect(Channel peer) { - PeerConnection peerConnection = (PeerConnection) peer; - if (!activePeers.contains(peerConnection)) { - if (!peerConnection.isActive()) { - passivePeersCount.incrementAndGet(); - } else { - activePeersCount.incrementAndGet(); - } - activePeers.add(peerConnection); - activePeers - .sort(Comparator.comparingDouble( - c -> c.getNodeStatistics().pingMessageLatency.getAvg())); - peerConnection.onConnect(); - } - } - - public synchronized void onDisconnect(Channel peer) { - PeerConnection peerConnection = (PeerConnection) peer; - if (activePeers.contains(peerConnection)) { - if (!peerConnection.isActive()) { - passivePeersCount.decrementAndGet(); - } else { - activePeersCount.decrementAndGet(); - } - activePeers.remove(peerConnection); - peerConnection.onDisconnect(); - } - } - - public boolean isCanConnect() { - return passivePeersCount.get() < maxConnections - minActiveConnections; - } - - public void close() { - try { - activePeers.forEach(p -> { - if (!p.isDisconnect()) { - p.close(); - } - }); - poolLoopExecutor.shutdownNow(); - logExecutor.shutdownNow(); - } catch (Exception e) { - logger.error("Problems shutting down executor", e); - } - } - - public AtomicInteger getPassivePeersCount() { - return passivePeersCount; - } - - public AtomicInteger getActivePeersCount() { - return activePeersCount; - } - - class NodeSelector implements Predicate { - - private Set nodesInUse; - - public NodeSelector(Set nodesInUse) { - this.nodesInUse = nodesInUse; - } - - @Override - public boolean test(NodeHandler handler) { - long headNum = chainBaseManager.getHeadBlockNum(); - InetAddress inetAddress = handler.getInetSocketAddress().getAddress(); - Protocol.HelloMessage message = channelManager.getHelloMessageCache() - .getIfPresent(inetAddress.getHostAddress()); - return !((handler.getNode().getHost().equals(nodeManager.getPublicHomeNode().getHost()) - && handler.getNode().getPort() == nodeManager.getPublicHomeNode().getPort()) - || (channelManager.getRecentlyDisconnected().getIfPresent(inetAddress) != null) - || (channelManager.getBadPeers().getIfPresent(inetAddress) != null) - || (channelManager.getConnectionNum(inetAddress) >= maxConnectionsWithSameIp) - || (nodesInUse.contains(handler.getNode().getHexId())) - || (nodeHandlerCache.getIfPresent(handler) != null) - || (message != null && headNum < message.getLowestBlockNum())); - } - } - -} diff --git a/framework/src/main/java/org/tron/common/overlay/server/TronChannelInitializer.java b/framework/src/main/java/org/tron/common/overlay/server/TronChannelInitializer.java deleted file mode 100644 index f6732adadd5..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/server/TronChannelInitializer.java +++ /dev/null @@ -1,62 +0,0 @@ -package org.tron.common.overlay.server; - -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelInitializer; -import io.netty.channel.ChannelOption; -import io.netty.channel.FixedRecvByteBufAllocator; -import io.netty.channel.socket.nio.NioSocketChannel; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import org.tron.core.net.peer.PeerConnection; - -@Slf4j(topic = "net") -@Component -@Scope("prototype") -public class TronChannelInitializer extends ChannelInitializer { - - @Autowired - private ApplicationContext ctx; - - @Autowired - private ChannelManager channelManager; - - private String remoteId; - - private boolean peerDiscoveryMode = false; - - public TronChannelInitializer(String remoteId) { - this.remoteId = remoteId; - } - - @Override - public void initChannel(NioSocketChannel ch) throws Exception { - try { - final Channel channel = ctx.getBean(PeerConnection.class); - - channel.init(ch.pipeline(), remoteId, peerDiscoveryMode, channelManager); - - // limit the size of receiving buffer to 1024 - ch.config().setRecvByteBufAllocator(new FixedRecvByteBufAllocator(256 * 1024)); - ch.config().setOption(ChannelOption.SO_RCVBUF, 256 * 1024); - ch.config().setOption(ChannelOption.SO_BACKLOG, 1024); - - // be aware of channel closing - ch.closeFuture().addListener((ChannelFutureListener) future -> { - logger.info("Close channel: {}", channel); - if (!peerDiscoveryMode) { - channelManager.notifyDisconnect(channel); - } - }); - - } catch (Exception e) { - logger.error("Unexpected error: ", e); - } - } - - public void setPeerDiscoveryMode(boolean peerDiscoveryMode) { - this.peerDiscoveryMode = peerDiscoveryMode; - } -} diff --git a/framework/src/main/java/org/tron/common/overlay/server/TrxProtobufVarint32FrameDecoder.java b/framework/src/main/java/org/tron/common/overlay/server/TrxProtobufVarint32FrameDecoder.java deleted file mode 100644 index b6696200aff..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/server/TrxProtobufVarint32FrameDecoder.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.tron.common.overlay.server; - -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ByteToMessageDecoder; -import io.netty.handler.codec.CorruptedFrameException; -import java.util.List; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class TrxProtobufVarint32FrameDecoder extends ByteToMessageDecoder { - - private static final Logger logger = LoggerFactory - .getLogger(TrxProtobufVarint32FrameDecoder.class); - - private static final int maxMsgLength = 5 * 1024 * 1024;//5M - - private Channel channel; - - public TrxProtobufVarint32FrameDecoder(Channel channel) { - this.channel = channel; - } - - private static int readRawVarint32(ByteBuf buffer) { - if (!buffer.isReadable()) { - return 0; - } - buffer.markReaderIndex(); - byte tmp = buffer.readByte(); - if (tmp >= 0) { - return tmp; - } else { - int result = tmp & 127; - if (!buffer.isReadable()) { - buffer.resetReaderIndex(); - return 0; - } - if ((tmp = buffer.readByte()) >= 0) { - result |= tmp << 7; - } else { - result |= (tmp & 127) << 7; - if (!buffer.isReadable()) { - buffer.resetReaderIndex(); - return 0; - } - if ((tmp = buffer.readByte()) >= 0) { - result |= tmp << 14; - } else { - result |= (tmp & 127) << 14; - if (!buffer.isReadable()) { - buffer.resetReaderIndex(); - return 0; - } - if ((tmp = buffer.readByte()) >= 0) { - result |= tmp << 21; - } else { - result |= (tmp & 127) << 21; - if (!buffer.isReadable()) { - buffer.resetReaderIndex(); - return 0; - } - result |= (tmp = buffer.readByte()) << 28; - if (tmp < 0) { - throw new CorruptedFrameException("malformed varint."); - } - } - } - } - return result; - } - } - - @Override - protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) throws Exception { - in.markReaderIndex(); - int preIndex = in.readerIndex(); - int length = readRawVarint32(in); - if (length >= maxMsgLength) { - logger.warn("Recv a big msg, host : {}, msg length is : {}", ctx.channel().remoteAddress(), - length); - in.clear(); - channel.close(); - return; - } - if (preIndex == in.readerIndex()) { - return; - } - if (length < 0) { - throw new CorruptedFrameException("negative length: " + length); - } - - if (in.readableBytes() < length) { - in.resetReaderIndex(); - } else { - out.add(in.readRetainedSlice(length)); - } - } -} - diff --git a/framework/src/main/java/org/tron/common/overlay/server/WireTrafficStats.java b/framework/src/main/java/org/tron/common/overlay/server/WireTrafficStats.java deleted file mode 100644 index 757e302626c..00000000000 --- a/framework/src/main/java/org/tron/common/overlay/server/WireTrafficStats.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Copyright (c) [2016] [ ] - * This file is part of the ethereumJ library. - * - * The ethereumJ library is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * The ethereumJ library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with the ethereumJ library. If not, see . - */ - -package org.tron.common.overlay.server; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelDuplexHandler; -import io.netty.channel.ChannelHandler; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.ChannelPromise; -import io.netty.channel.socket.DatagramPacket; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import javax.annotation.PreDestroy; -import lombok.extern.slf4j.Slf4j; -import org.springframework.stereotype.Component; - -@Slf4j(topic = "net") -@Component -public class WireTrafficStats implements Runnable { - - public final TrafficStatHandler tcp = new TrafficStatHandler(); - public final TrafficStatHandler udp = new TrafficStatHandler(); - private ScheduledExecutorService executor; - - public WireTrafficStats() { - executor = Executors.newSingleThreadScheduledExecutor( - new ThreadFactoryBuilder().setNameFormat("WireTrafficStats-%d").build()); - executor.scheduleAtFixedRate(this, 10, 10, TimeUnit.SECONDS); - } - - @Override - public void run() { - } - - @PreDestroy - public void close() { - executor.shutdownNow(); - } - - @ChannelHandler.Sharable - static class TrafficStatHandler extends ChannelDuplexHandler { - - private AtomicLong outSize = new AtomicLong(); - private AtomicLong inSize = new AtomicLong(); - private AtomicLong outPackets = new AtomicLong(); - private AtomicLong inPackets = new AtomicLong(); - - public String stats() { - return ""; - } - - - @Override - public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception { - inPackets.incrementAndGet(); - if (msg instanceof ByteBuf) { - inSize.addAndGet(((ByteBuf) msg).readableBytes()); - } else if (msg instanceof DatagramPacket) { - inSize.addAndGet(((DatagramPacket) msg).content().readableBytes()); - } - super.channelRead(ctx, msg); - } - - @Override - public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) - throws Exception { - outPackets.incrementAndGet(); - if (msg instanceof ByteBuf) { - outSize.addAndGet(((ByteBuf) msg).readableBytes()); - } else if (msg instanceof DatagramPacket) { - outSize.addAndGet(((DatagramPacket) msg).content().readableBytes()); - } - super.write(ctx, msg, promise); - } - } -} diff --git a/framework/src/main/java/org/tron/core/Wallet.java b/framework/src/main/java/org/tron/core/Wallet.java index 6cb7e9b9d01..3079f5764d4 100755 --- a/framework/src/main/java/org/tron/core/Wallet.java +++ b/framework/src/main/java/org/tron/core/Wallet.java @@ -102,8 +102,6 @@ import org.tron.common.crypto.Hash; import org.tron.common.crypto.SignInterface; import org.tron.common.crypto.SignUtils; -import org.tron.common.overlay.discover.node.NodeHandler; -import org.tron.common.overlay.discover.node.NodeManager; import org.tron.common.parameter.CommonParameter; import org.tron.common.runtime.ProgramResult; import org.tron.common.runtime.vm.LogInfo; @@ -177,7 +175,7 @@ import org.tron.core.exception.ZksnarkException; import org.tron.core.net.TronNetDelegate; import org.tron.core.net.TronNetService; -import org.tron.core.net.message.TransactionMessage; +import org.tron.core.net.message.adv.TransactionMessage; import org.tron.core.store.AccountIdIndexStore; import org.tron.core.store.AccountStore; import org.tron.core.store.AccountTraceStore; @@ -266,8 +264,6 @@ public class Wallet { @Autowired private ChainBaseManager chainBaseManager; - @Autowired - private NodeManager nodeManager; private int minEffectiveConnection = Args.getInstance().getMinEffectiveConnection(); private boolean trxCacheEnable = Args.getInstance().isTrxCacheEnable(); public static final String CONTRACT_VALIDATE_EXCEPTION = "ContractValidateException: {}"; @@ -2426,26 +2422,14 @@ public TransactionInfoList getTransactionInfoByBlockNum(long blockNum) { } public NodeList listNodes() { - List handlerList = nodeManager.dumpActiveNodes(); - - Map nodeHandlerMap = new HashMap<>(); - for (NodeHandler handler : handlerList) { - String key = handler.getNode().getHexId() + handler.getNode().getHost(); - nodeHandlerMap.put(key, handler); - } - NodeList.Builder nodeListBuilder = NodeList.newBuilder(); - - nodeHandlerMap.entrySet().stream() - .forEach(v -> { - org.tron.common.overlay.discover.node.Node node = v.getValue() - .getNode(); - nodeListBuilder.addNodes(Node.newBuilder().setAddress( + TronNetService.getP2pService().getConnectableNodes().forEach(node -> { + nodeListBuilder.addNodes(Node.newBuilder().setAddress( Address.newBuilder() - .setHost(ByteString - .copyFrom(ByteArray.fromString(node.getHost()))) - .setPort(node.getPort()))); - }); + .setHost(ByteString + .copyFrom(ByteArray.fromString(node.getHost()))) + .setPort(node.getPort()))); + }); return nodeListBuilder.build(); } diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 1625a4c684d..c926fe29724 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -19,6 +19,7 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.net.InetAddress; +import java.net.InetSocketAddress; import java.net.Socket; import java.net.URL; import java.nio.file.Paths; @@ -56,7 +57,6 @@ import org.tron.common.logsfilter.TriggerConfig; import org.tron.common.logsfilter.trigger.ContractEventTrigger; import org.tron.common.logsfilter.trigger.ContractLogTrigger; -import org.tron.common.overlay.discover.node.Node; import org.tron.common.parameter.CommonParameter; import org.tron.common.parameter.RateLimiterInitialization; import org.tron.common.setting.RocksDbSettings; @@ -886,11 +886,11 @@ public static void setParam(final String[] args, final String confFileName) { .getInt(Constant.NODE_VALID_CONTRACT_PROTO_THREADS) : Runtime.getRuntime().availableProcessors(); - PARAMETER.activeNodes = getNodes(config, Constant.NODE_ACTIVE); + PARAMETER.activeNodes = getInetSocketAddress(config, Constant.NODE_ACTIVE); - PARAMETER.passiveNodes = getNodes(config, Constant.NODE_PASSIVE); + PARAMETER.passiveNodes = getInetAddress(config, Constant.NODE_PASSIVE); - PARAMETER.fastForwardNodes = getNodes(config, Constant.NODE_FAST_FORWARD); + PARAMETER.fastForwardNodes = getInetSocketAddress(config, Constant.NODE_FAST_FORWARD); PARAMETER.maxFastForwardNum = config.hasPath(Constant.NODE_MAX_FAST_FORWARD_NUM) ? config .getInt(Constant.NODE_MAX_FAST_FORWARD_NUM) : 3; @@ -1108,25 +1108,46 @@ private static RateLimiterInitialization getRateLimiterFromConfig( return initialization; } - private static List getNodes(final com.typesafe.config.Config config, String path) { + private static List getInetSocketAddress( + final com.typesafe.config.Config config, String path) { if (!config.hasPath(path)) { return Collections.emptyList(); } - List ret = new ArrayList<>(); + List ret = new ArrayList<>(); List list = config.getStringList(path); for (String configString : list) { - Node n = Node.instanceOf(configString); - if (!(PARAMETER.nodeDiscoveryBindIp.equals(n.getHost()) - || PARAMETER.nodeExternalIp.equals(n.getHost()) - || Constant.LOCAL_HOST.equals(n.getHost())) - || PARAMETER.nodeListenPort != n.getPort()) { - ret.add(n); + String[] sz = configString.split(":"); + String ip = sz[0]; + int port = Integer.parseInt(sz[1]); + if (!(PARAMETER.nodeDiscoveryBindIp.equals(ip) + || PARAMETER.nodeExternalIp.equals(ip) + || Constant.LOCAL_HOST.equals(ip)) + || PARAMETER.nodeListenPort != port) { + ret.add(new InetSocketAddress(ip, port)); } } return ret; } - private static EventPluginConfig getEventPluginConfig(final com.typesafe.config.Config config) { + private static List getInetAddress( + final com.typesafe.config.Config config, String path) { + if (!config.hasPath(path)) { + return Collections.emptyList(); + } + List ret = new ArrayList<>(); + List list = config.getStringList(path); + for (String configString : list) { + try { + ret.add(InetAddress.getByName(configString.split(":")[0])); + } catch (Exception e) { + logger.warn("Get inet address failed, {}", e.getMessage()); + } + } + return ret; + } + + private static EventPluginConfig getEventPluginConfig( + final com.typesafe.config.Config config) { EventPluginConfig eventPluginConfig = new EventPluginConfig(); boolean useNativeQueue = false; diff --git a/framework/src/main/java/org/tron/core/consensus/BlockHandleImpl.java b/framework/src/main/java/org/tron/core/consensus/BlockHandleImpl.java index fcfb801d1ce..b905b42381f 100644 --- a/framework/src/main/java/org/tron/core/consensus/BlockHandleImpl.java +++ b/framework/src/main/java/org/tron/core/consensus/BlockHandleImpl.java @@ -12,7 +12,7 @@ import org.tron.core.capsule.BlockCapsule; import org.tron.core.db.Manager; import org.tron.core.net.TronNetService; -import org.tron.core.net.message.BlockMessage; +import org.tron.core.net.message.adv.BlockMessage; @Slf4j(topic = "consensus") @Component diff --git a/framework/src/main/java/org/tron/core/consensus/PbftBaseImpl.java b/framework/src/main/java/org/tron/core/consensus/PbftBaseImpl.java index 07f7ea5d0e2..61538e966ed 100644 --- a/framework/src/main/java/org/tron/core/consensus/PbftBaseImpl.java +++ b/framework/src/main/java/org/tron/core/consensus/PbftBaseImpl.java @@ -1,32 +1,32 @@ package org.tron.core.consensus; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.tron.common.overlay.server.SyncPool; import org.tron.consensus.base.PbftInterface; import org.tron.consensus.pbft.message.PbftBaseMessage; import org.tron.core.capsule.BlockCapsule; import org.tron.core.db.Manager; import org.tron.core.exception.BadItemException; import org.tron.core.exception.ItemNotFoundException; +import org.tron.core.net.peer.PeerConnection; +import org.tron.core.net.peer.PeerManager; @Component public class PbftBaseImpl implements PbftInterface { - @Autowired - private SyncPool syncPool; - @Autowired private Manager manager; @Override public boolean isSyncing() { - if (syncPool == null) { + List peers = PeerManager.getPeers(); + if (peers.isEmpty()) { return true; } AtomicBoolean result = new AtomicBoolean(false); - syncPool.getActivePeers().forEach(peerConnection -> { + peers.forEach(peerConnection -> { if (peerConnection.isNeedSyncFromPeer()) { result.set(true); return; @@ -37,10 +37,11 @@ public boolean isSyncing() { @Override public void forwardMessage(PbftBaseMessage message) { - if (syncPool == null) { + List peers = PeerManager.getPeers(); + if (peers.isEmpty()) { return; } - syncPool.getActivePeers().forEach(peerConnection -> { + peers.forEach(peerConnection -> { peerConnection.sendMessage(message); }); } diff --git a/framework/src/main/java/org/tron/core/net/P2pEventHandlerImpl.java b/framework/src/main/java/org/tron/core/net/P2pEventHandlerImpl.java new file mode 100644 index 00000000000..6a50ccb0a2e --- /dev/null +++ b/framework/src/main/java/org/tron/core/net/P2pEventHandlerImpl.java @@ -0,0 +1,239 @@ +package org.tron.core.net; + +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.ApplicationContext; +import org.springframework.stereotype.Component; +import org.tron.common.prometheus.MetricKeys; +import org.tron.common.prometheus.Metrics; +import org.tron.consensus.pbft.message.PbftMessage; +import org.tron.core.exception.P2pException; +import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.PbftMessageFactory; +import org.tron.core.net.message.TronMessage; +import org.tron.core.net.message.TronMessageFactory; +import org.tron.core.net.message.base.DisconnectMessage; +import org.tron.core.net.message.handshake.HelloMessage; +import org.tron.core.net.messagehandler.BlockMsgHandler; +import org.tron.core.net.messagehandler.ChainInventoryMsgHandler; +import org.tron.core.net.messagehandler.FetchInvDataMsgHandler; +import org.tron.core.net.messagehandler.InventoryMsgHandler; +import org.tron.core.net.messagehandler.PbftDataSyncHandler; +import org.tron.core.net.messagehandler.PbftMsgHandler; +import org.tron.core.net.messagehandler.SyncBlockChainMsgHandler; +import org.tron.core.net.messagehandler.TransactionsMsgHandler; +import org.tron.core.net.peer.PeerConnection; +import org.tron.core.net.peer.PeerManager; +import org.tron.core.net.service.handshake.HandshakeService; +import org.tron.core.net.service.keepalive.KeepAliveService; +import org.tron.p2p.P2pEventHandler; +import org.tron.p2p.connection.Channel; +import org.tron.protos.Protocol; + +@Slf4j(topic = "net") +@Component +public class P2pEventHandlerImpl extends P2pEventHandler { + + private static final String TAG = "~"; + private static final int DURATION_STEP = 50; + @Getter + private static AtomicInteger passivePeersCount = new AtomicInteger(0); + @Getter + private final AtomicInteger activePeersCount = new AtomicInteger(0); + + @Autowired + private ApplicationContext ctx; + + @Autowired + private SyncBlockChainMsgHandler syncBlockChainMsgHandler; + + @Autowired + private ChainInventoryMsgHandler chainInventoryMsgHandler; + + @Autowired + private InventoryMsgHandler inventoryMsgHandler; + + @Autowired + private FetchInvDataMsgHandler fetchInvDataMsgHandler; + + @Autowired + private BlockMsgHandler blockMsgHandler; + + @Autowired + private TransactionsMsgHandler transactionsMsgHandler; + + @Autowired + private PbftDataSyncHandler pbftDataSyncHandler; + + @Autowired + private HandshakeService handshakeService; + + @Autowired + private PbftMsgHandler pbftMsgHandler; + + @Autowired + private KeepAliveService keepAliveService; + + private byte MESSAGE_MAX_TYPE = 127; + + public P2pEventHandlerImpl() { + Set set = new HashSet<>(); + for (byte i = 0; i < MESSAGE_MAX_TYPE; i++) { + set.add(i); + } + messageTypes = set; + } + + @Override + public synchronized void onConnect(Channel channel) { + PeerConnection peerConnection = PeerManager.add(ctx, channel); + if (peerConnection != null) { + handshakeService.startHandshake(peerConnection); + } + } + + @Override + public synchronized void onDisconnect(Channel channel) { + PeerConnection peerConnection = PeerManager.remove(channel); + if (peerConnection != null) { + peerConnection.onDisconnect(); + } + } + + @Override + public void onMessage(Channel c, byte[] data) { + PeerConnection peerConnection = PeerManager.getPeerConnection(c); + if (peerConnection == null) { + logger.warn("Receive msg from unknown peer {}", c.getInetSocketAddress()); + return; + } + + if (MessageTypes.PBFT_MSG.asByte() == data[0]) { + PbftMessage message = null; + try { + message = (PbftMessage) PbftMessageFactory.create(data); + pbftMsgHandler.processMessage(peerConnection, message); + } catch (Exception e) { + logger.warn("PBFT Message from {} process failed, {}", + peerConnection.getInetSocketAddress(), message, e); + peerConnection.disconnect(Protocol.ReasonCode.BAD_PROTOCOL); + } + return; + } + + processMessage(peerConnection, data); + } + + private void processMessage(PeerConnection peer, byte[] data) { + long startTime = System.currentTimeMillis(); + TronMessage msg = null; + try { + msg = TronMessageFactory.create(data); + peer.getPeerStatistics().messageStatistics.addTcpInMessage(msg); + logger.info("Receive message from peer: {}, {}", + peer.getInetSocketAddress(), msg); + switch (msg.getType()) { + case P2P_PING: + case P2P_PONG: + keepAliveService.processMessage(peer, msg); + break; + case P2P_HELLO: + handshakeService.processHelloMessage(peer, (HelloMessage) msg); + break; + case P2P_DISCONNECT: + peer.getChannel().close(); + peer.getNodeStatistics() + .nodeDisconnectedRemote(((DisconnectMessage)msg).getReason()); + break; + case SYNC_BLOCK_CHAIN: + syncBlockChainMsgHandler.processMessage(peer, msg); + break; + case BLOCK_CHAIN_INVENTORY: + chainInventoryMsgHandler.processMessage(peer, msg); + break; + case INVENTORY: + inventoryMsgHandler.processMessage(peer, msg); + break; + case FETCH_INV_DATA: + fetchInvDataMsgHandler.processMessage(peer, msg); + break; + case BLOCK: + blockMsgHandler.processMessage(peer, msg); + break; + case TRXS: + transactionsMsgHandler.processMessage(peer, msg); + break; + case PBFT_COMMIT_MSG: + pbftDataSyncHandler.processMessage(peer, msg); + break; + default: + throw new P2pException(P2pException.TypeEnum.NO_SUCH_MESSAGE, msg.getType().toString()); + } + } catch (Exception e) { + processException(peer, msg, e); + } finally { + long costs = System.currentTimeMillis() - startTime; + if (costs > 50) { + logger.info("Message processing costs {} ms, peer: {}, type: {}, time tag: {}", + costs, peer.getInetSocketAddress(), msg.getType(), getTimeTag(costs)); + Metrics.histogramObserve(MetricKeys.Histogram.MESSAGE_PROCESS_LATENCY, + costs / Metrics.MILLISECONDS_PER_SECOND, msg.getType().name()); + } + } + } + + private void processException(PeerConnection peer, TronMessage msg, Exception ex) { + Protocol.ReasonCode code; + + if (ex instanceof P2pException) { + P2pException.TypeEnum type = ((P2pException) ex).getType(); + switch (type) { + case BAD_TRX: + code = Protocol.ReasonCode.BAD_TX; + break; + case BAD_BLOCK: + code = Protocol.ReasonCode.BAD_BLOCK; + break; + case NO_SUCH_MESSAGE: + case MESSAGE_WITH_WRONG_LENGTH: + case BAD_MESSAGE: + code = Protocol.ReasonCode.BAD_PROTOCOL; + break; + case SYNC_FAILED: + code = Protocol.ReasonCode.SYNC_FAIL; + break; + case UNLINK_BLOCK: + code = Protocol.ReasonCode.UNLINKABLE; + break; + case DB_ITEM_NOT_FOUND: + code = Protocol.ReasonCode.FETCH_FAIL; + break; + default: + code = Protocol.ReasonCode.UNKNOWN; + break; + } + logger.warn("Message from {} process failed, {} \n type: {}, detail: {}", + peer.getInetSocketAddress(), msg, type, ex.getMessage()); + } else { + code = Protocol.ReasonCode.UNKNOWN; + logger.warn("Message from {} process failed, {}", + peer.getInetSocketAddress(), msg, ex); + } + + peer.disconnect(code); + } + + private String getTimeTag(long duration) { + StringBuilder tag = new StringBuilder(TAG); + long tagCount = duration / DURATION_STEP; + for (; tagCount > 0; tagCount--) { + tag.append(TAG); + } + return tag.toString(); + } + +} diff --git a/framework/src/main/java/org/tron/core/net/TronNetDelegate.java b/framework/src/main/java/org/tron/core/net/TronNetDelegate.java index 7a0c361165e..eafa9ff3766 100644 --- a/framework/src/main/java/org/tron/core/net/TronNetDelegate.java +++ b/framework/src/main/java/org/tron/core/net/TronNetDelegate.java @@ -16,10 +16,8 @@ import org.bouncycastle.util.encoders.Hex; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.tron.common.backup.BackupServer; +import org.tron.common.backup.socket.BackupServer; import org.tron.common.overlay.message.Message; -import org.tron.common.overlay.server.ChannelManager; -import org.tron.common.overlay.server.SyncPool; import org.tron.common.prometheus.MetricKeys; import org.tron.common.prometheus.MetricLabels; import org.tron.common.prometheus.Metrics; @@ -55,9 +53,9 @@ import org.tron.core.exception.ValidateSignatureException; import org.tron.core.exception.ZksnarkException; import org.tron.core.metrics.MetricsService; -import org.tron.core.net.message.BlockMessage; import org.tron.core.net.message.MessageTypes; -import org.tron.core.net.message.TransactionMessage; +import org.tron.core.net.message.adv.BlockMessage; +import org.tron.core.net.message.adv.TransactionMessage; import org.tron.core.net.peer.PeerConnection; import org.tron.core.store.WitnessScheduleStore; import org.tron.protos.Protocol.Inventory.InventoryType; @@ -66,12 +64,6 @@ @Component public class TronNetDelegate { - @Autowired - private SyncPool syncPool; - - @Autowired - private ChannelManager channelManager; - @Autowired private Manager dbManager; @@ -123,7 +115,7 @@ public void init() { } public Collection getActivePeer() { - return syncPool.getActivePeers(); + return TronNetService.getPeers(); } public long getSyncBeginNumber() { diff --git a/framework/src/main/java/org/tron/core/net/TronNetHandler.java b/framework/src/main/java/org/tron/core/net/TronNetHandler.java deleted file mode 100644 index 32c687244af..00000000000 --- a/framework/src/main/java/org/tron/core/net/TronNetHandler.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.tron.core.net; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; -import org.tron.common.overlay.server.Channel; -import org.tron.common.overlay.server.MessageQueue; -import org.tron.core.net.message.TronMessage; -import org.tron.core.net.peer.PeerConnection; - -@Component -@Scope("prototype") -public class TronNetHandler extends SimpleChannelInboundHandler { - - protected PeerConnection peer; - - private MessageQueue msgQueue; - - @Autowired - private TronNetService tronNetService; - - @Override - public void channelRead0(final ChannelHandlerContext ctx, TronMessage msg) throws Exception { - msgQueue.receivedMessage(msg); - tronNetService.onMessage(peer, msg); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - peer.processException(cause); - } - - public void setMsgQueue(MessageQueue msgQueue) { - this.msgQueue = msgQueue; - } - - public void setChannel(Channel channel) { - this.peer = (PeerConnection) channel; - } - -} \ No newline at end of file diff --git a/framework/src/main/java/org/tron/core/net/TronNetService.java b/framework/src/main/java/org/tron/core/net/TronNetService.java index 82568d6a7c0..c46c4688071 100644 --- a/framework/src/main/java/org/tron/core/net/TronNetService.java +++ b/framework/src/main/java/org/tron/core/net/TronNetService.java @@ -1,37 +1,38 @@ package org.tron.core.net; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.tron.common.overlay.message.Message; -import org.tron.common.overlay.server.ChannelManager; -import org.tron.common.prometheus.MetricKeys; -import org.tron.common.prometheus.Metrics; -import org.tron.core.exception.P2pException; -import org.tron.core.exception.P2pException.TypeEnum; -import org.tron.core.net.message.BlockMessage; -import org.tron.core.net.message.TransactionMessage; -import org.tron.core.net.message.TronMessage; -import org.tron.core.net.messagehandler.BlockMsgHandler; -import org.tron.core.net.messagehandler.ChainInventoryMsgHandler; -import org.tron.core.net.messagehandler.FetchInvDataMsgHandler; -import org.tron.core.net.messagehandler.InventoryMsgHandler; -import org.tron.core.net.messagehandler.PbftDataSyncHandler; -import org.tron.core.net.messagehandler.SyncBlockChainMsgHandler; +import org.tron.common.parameter.CommonParameter; +import org.tron.core.config.args.Args; +import org.tron.core.net.message.adv.TransactionMessage; import org.tron.core.net.messagehandler.TransactionsMsgHandler; import org.tron.core.net.peer.PeerConnection; +import org.tron.core.net.peer.PeerManager; import org.tron.core.net.peer.PeerStatusCheck; -import org.tron.core.net.service.AdvService; -import org.tron.core.net.service.FetchBlockService; -import org.tron.core.net.service.SyncService; -import org.tron.protos.Protocol.ReasonCode; +import org.tron.core.net.service.adv.AdvService; +import org.tron.core.net.service.fetchblock.FetchBlockService; +import org.tron.core.net.service.keepalive.KeepAliveService; +import org.tron.core.net.service.nodepersist.NodePersistService; +import org.tron.core.net.service.statistics.TronStatsManager; +import org.tron.core.net.service.sync.SyncService; +import org.tron.p2p.P2pConfig; +import org.tron.p2p.P2pService; @Slf4j(topic = "net") @Component public class TronNetService { - @Autowired - private ChannelManager channelManager; + @Getter + private static P2pConfig p2pConfig; + + @Getter + private static P2pService p2pService; @Autowired private AdvService advService; @@ -43,150 +44,92 @@ public class TronNetService { private PeerStatusCheck peerStatusCheck; @Autowired - private SyncBlockChainMsgHandler syncBlockChainMsgHandler; - - @Autowired - private ChainInventoryMsgHandler chainInventoryMsgHandler; + private TransactionsMsgHandler transactionsMsgHandler; @Autowired - private InventoryMsgHandler inventoryMsgHandler; - + private FetchBlockService fetchBlockService; @Autowired - private FetchInvDataMsgHandler fetchInvDataMsgHandler; + private KeepAliveService keepAliveService; - @Autowired - private BlockMsgHandler blockMsgHandler; + private CommonParameter parameter = Args.getInstance(); @Autowired - private TransactionsMsgHandler transactionsMsgHandler; + private P2pEventHandlerImpl p2pEventHandler; @Autowired - private PbftDataSyncHandler pbftDataSyncHandler; + private NodePersistService nodePersistService; @Autowired - private FetchBlockService fetchBlockService; - - private static final String TAG = "~"; - private static final int DURATION_STEP = 50; + private TronStatsManager tronStatsManager; public void start() { - channelManager.init(); - advService.init(); - syncService.init(); - peerStatusCheck.init(); - transactionsMsgHandler.init(); - fetchBlockService.init(); - logger.info("TronNetService start successfully"); + try { + p2pConfig = getConfig(); + p2pService = new P2pService(); + p2pService.start(p2pConfig); + p2pService.register(p2pEventHandler); + advService.init(); + syncService.init(); + peerStatusCheck.init(); + transactionsMsgHandler.init(); + fetchBlockService.init(); + keepAliveService.init(); + nodePersistService.init(); + tronStatsManager.init(); + PeerManager.init(); + logger.info("Net service start successfully"); + } catch (Exception e) { + logger.error("Net service start failed", e); + } } - public void stop() { - logger.info("TronNetService closed start"); - channelManager.close(); + public void close() { + PeerManager.close(); + tronStatsManager.close(); + nodePersistService.close(); + keepAliveService.close(); advService.close(); syncService.close(); peerStatusCheck.close(); transactionsMsgHandler.close(); fetchBlockService.close(); - logger.info("TronNetService closed successfully"); + p2pService.close(); + logger.info("Net service closed successfully"); } - public int fastBroadcastTransaction(TransactionMessage msg) { - return advService.fastBroadcastTransaction(msg); + public static List getPeers() { + return PeerManager.getPeers(); } public void broadcast(Message msg) { advService.broadcast(msg); } - protected void onMessage(PeerConnection peer, TronMessage msg) { - long startTime = System.currentTimeMillis(); - try { - switch (msg.getType()) { - case SYNC_BLOCK_CHAIN: - syncBlockChainMsgHandler.processMessage(peer, msg); - break; - case BLOCK_CHAIN_INVENTORY: - chainInventoryMsgHandler.processMessage(peer, msg); - break; - case INVENTORY: - inventoryMsgHandler.processMessage(peer, msg); - break; - case FETCH_INV_DATA: - fetchInvDataMsgHandler.processMessage(peer, msg); - break; - case BLOCK: - blockMsgHandler.processMessage(peer, msg); - break; - case TRXS: - transactionsMsgHandler.processMessage(peer, msg); - break; - case PBFT_COMMIT_MSG: - pbftDataSyncHandler.processMessage(peer, msg); - break; - default: - throw new P2pException(TypeEnum.NO_SUCH_MESSAGE, msg.getType().toString()); - } - } catch (Exception e) { - processException(peer, msg, e); - } finally { - long costs = System.currentTimeMillis() - startTime; - if (costs > DURATION_STEP) { - logger.info("Message processing costs {} ms, peer: {}, type: {}, time tag: {}", - costs, peer.getInetAddress(), msg.getType(), getTimeTag(costs)); - Metrics.histogramObserve(MetricKeys.Histogram.MESSAGE_PROCESS_LATENCY, - costs / Metrics.MILLISECONDS_PER_SECOND, msg.getType().name()); - } - } + public int fastBroadcastTransaction(TransactionMessage msg) { + return advService.fastBroadcastTransaction(msg); } - private void processException(PeerConnection peer, TronMessage msg, Exception ex) { - ReasonCode code; - - if (ex instanceof P2pException) { - TypeEnum type = ((P2pException) ex).getType(); - switch (type) { - case BAD_TRX: - code = ReasonCode.BAD_TX; - break; - case BAD_BLOCK: - code = ReasonCode.BAD_BLOCK; - break; - case NO_SUCH_MESSAGE: - case MESSAGE_WITH_WRONG_LENGTH: - case BAD_MESSAGE: - code = ReasonCode.BAD_PROTOCOL; - break; - case SYNC_FAILED: - code = ReasonCode.SYNC_FAIL; - break; - case UNLINK_BLOCK: - code = ReasonCode.UNLINKABLE; - break; - case DB_ITEM_NOT_FOUND: - code = ReasonCode.FETCH_FAIL; - break; - default: - code = ReasonCode.UNKNOWN; - break; - } - logger.warn("Message from {} process failed, {} \n type: {}, detail: {}", - peer.getInetAddress(), msg, type, ex.getMessage()); - } else { - code = ReasonCode.UNKNOWN; - logger.warn("Message from {} process failed, {}", - peer.getInetAddress(), msg, ex); + private P2pConfig getConfig() { + List seeds = new ArrayList<>(); + seeds.addAll(nodePersistService.dbRead()); + for (String s : parameter.getSeedNode().getIpList()) { + String[] sz = s.split(":"); + seeds.add(new InetSocketAddress(sz[0], Integer.parseInt(sz[1]))); } - peer.disconnect(code); - } - - private String getTimeTag(long duration) { - StringBuilder tag = new StringBuilder(TAG); - long tagCount = duration / DURATION_STEP; - for (; tagCount > 0; tagCount--) { - tag.append(TAG); - } - return tag.toString(); + P2pConfig config = new P2pConfig(); + config.setSeedNodes(seeds); + config.setActiveNodes(parameter.getActiveNodes()); + config.setTrustNodes(parameter.getPassiveNodes()); + config.getActiveNodes().forEach(n -> config.getTrustNodes().add(n.getAddress())); + config.setMaxConnections(parameter.getMaxConnections()); + config.setMinConnections(parameter.getMinConnections()); + config.setMaxConnectionsWithSameIp(parameter.getMaxConnectionsWithSameIp()); + config.setPort(parameter.getNodeListenPort()); + config.setVersion(parameter.getNodeP2pVersion()); + config.setDisconnectionPolicyEnable(parameter.isOpenFullTcpDisconnect()); + config.setDiscoverEnable(parameter.isNodeDiscoveryEnable()); + return config; } -} +} \ No newline at end of file diff --git a/framework/src/main/java/org/tron/core/net/message/BlocksMessage.java b/framework/src/main/java/org/tron/core/net/message/BlocksMessage.java deleted file mode 100644 index 8705ff12bb7..00000000000 --- a/framework/src/main/java/org/tron/core/net/message/BlocksMessage.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.tron.core.net.message; - -import java.util.List; -import org.apache.commons.collections4.CollectionUtils; -import org.tron.core.capsule.TransactionCapsule; -import org.tron.protos.Protocol.Block; -import org.tron.protos.Protocol.Items; - -public class BlocksMessage extends TronMessage { - - private List blocks; - - public BlocksMessage(byte[] data) throws Exception { - super(data); - this.type = MessageTypes.BLOCKS.asByte(); - Items items = Items.parseFrom(getCodedInputStream(data)); - if (items.getType() == Items.ItemType.BLOCK) { - blocks = items.getBlocksList(); - } - if (isFilter() && CollectionUtils.isNotEmpty(blocks)) { - compareBytes(data, items.toByteArray()); - for (Block block : blocks) { - TransactionCapsule.validContractProto(block.getTransactionsList()); - } - } - } - - public List getBlocks() { - return blocks; - } - - @Override - public String toString() { - return super.toString() + "size: " + (CollectionUtils.isNotEmpty(blocks) ? blocks - .size() : 0); - } - - @Override - public Class getAnswerMessage() { - return null; - } - -} diff --git a/framework/src/main/java/org/tron/core/net/message/FetchBlockHeadersMessage.java b/framework/src/main/java/org/tron/core/net/message/FetchBlockHeadersMessage.java deleted file mode 100644 index 0e6e3f396b2..00000000000 --- a/framework/src/main/java/org/tron/core/net/message/FetchBlockHeadersMessage.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.tron.core.net.message; - -import org.tron.protos.Protocol; - -public class FetchBlockHeadersMessage extends InventoryMessage { - - public FetchBlockHeadersMessage(byte[] packed) throws Exception { - super(packed); - this.type = MessageTypes.FETCH_BLOCK_HEADERS.asByte(); - } - - public FetchBlockHeadersMessage(Protocol.Inventory inv) { - super(inv); - this.type = MessageTypes.FETCH_BLOCK_HEADERS.asByte(); - } - -} \ No newline at end of file diff --git a/framework/src/main/java/org/tron/core/net/message/ItemNotFound.java b/framework/src/main/java/org/tron/core/net/message/ItemNotFound.java deleted file mode 100644 index b4694048db0..00000000000 --- a/framework/src/main/java/org/tron/core/net/message/ItemNotFound.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.tron.core.net.message; - -import org.tron.protos.Protocol; - -public class ItemNotFound extends TronMessage { - - private org.tron.protos.Protocol.Items notFound; - - /** - * means can not find this block or trx. - */ - public ItemNotFound() { - Protocol.Items.Builder itemsBuilder = Protocol.Items.newBuilder(); - itemsBuilder.setType(Protocol.Items.ItemType.ERR); - notFound = itemsBuilder.build(); - this.type = MessageTypes.ITEM_NOT_FOUND.asByte(); - this.data = notFound.toByteArray(); - } - - @Override - public String toString() { - return "item not found"; - } - - @Override - public Class getAnswerMessage() { - return null; - } - -} diff --git a/framework/src/main/java/org/tron/core/net/message/PbftMessageFactory.java b/framework/src/main/java/org/tron/core/net/message/PbftMessageFactory.java index b40e9b8fa30..ea6b25e6ab1 100644 --- a/framework/src/main/java/org/tron/core/net/message/PbftMessageFactory.java +++ b/framework/src/main/java/org/tron/core/net/message/PbftMessageFactory.java @@ -1,21 +1,16 @@ package org.tron.core.net.message; import org.apache.commons.lang3.ArrayUtils; -import org.tron.common.overlay.message.MessageFactory; import org.tron.consensus.pbft.message.PbftBaseMessage; import org.tron.consensus.pbft.message.PbftMessage; import org.tron.core.exception.P2pException; -/** - * msg factory. - */ -public class PbftMessageFactory extends MessageFactory { +public class PbftMessageFactory { private static String LEN = ", len="; private static String TYPE = "type="; - @Override - public PbftBaseMessage create(byte[] data) throws Exception { + public static PbftBaseMessage create(byte[] data) throws Exception { try { byte type = data[0]; byte[] rawData = ArrayUtils.subarray(data, 1, data.length); @@ -28,7 +23,7 @@ public PbftBaseMessage create(byte[] data) throws Exception { } } - private PbftBaseMessage create(byte type, byte[] packed) throws Exception { + private static PbftBaseMessage create(byte type, byte[] packed) throws Exception { MessageTypes receivedTypes = MessageTypes.fromByte(type); if (receivedTypes == null) { throw new P2pException(P2pException.TypeEnum.NO_SUCH_MESSAGE, diff --git a/framework/src/main/java/org/tron/core/net/message/TransactionInventoryMessage.java b/framework/src/main/java/org/tron/core/net/message/TransactionInventoryMessage.java deleted file mode 100644 index 44fea28c88f..00000000000 --- a/framework/src/main/java/org/tron/core/net/message/TransactionInventoryMessage.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.tron.core.net.message; - -import java.util.List; -import org.tron.common.utils.Sha256Hash; -import org.tron.protos.Protocol.Inventory; -import org.tron.protos.Protocol.Inventory.InventoryType; - -public class TransactionInventoryMessage extends InventoryMessage { - - public TransactionInventoryMessage(byte[] packed) throws Exception { - super(packed); - } - - public TransactionInventoryMessage(Inventory inv) { - super(inv); - } - - public TransactionInventoryMessage(List hashList) { - super(hashList, InventoryType.TRX); - } -} diff --git a/framework/src/main/java/org/tron/core/net/message/TronMessageFactory.java b/framework/src/main/java/org/tron/core/net/message/TronMessageFactory.java index a4e9089dce0..3cc1b35e506 100644 --- a/framework/src/main/java/org/tron/core/net/message/TronMessageFactory.java +++ b/framework/src/main/java/org/tron/core/net/message/TronMessageFactory.java @@ -1,20 +1,28 @@ package org.tron.core.net.message; import org.apache.commons.lang3.ArrayUtils; -import org.tron.common.overlay.message.MessageFactory; +import org.tron.consensus.pbft.message.PbftMessage; import org.tron.core.exception.P2pException; import org.tron.core.metrics.MetricsKey; import org.tron.core.metrics.MetricsUtil; +import org.tron.core.net.message.adv.BlockMessage; +import org.tron.core.net.message.adv.FetchInvDataMessage; +import org.tron.core.net.message.adv.InventoryMessage; +import org.tron.core.net.message.adv.TransactionMessage; +import org.tron.core.net.message.adv.TransactionsMessage; +import org.tron.core.net.message.base.DisconnectMessage; +import org.tron.core.net.message.handshake.HelloMessage; +import org.tron.core.net.message.keepalive.PingMessage; +import org.tron.core.net.message.keepalive.PongMessage; +import org.tron.core.net.message.pbft.PbftCommitMessage; +import org.tron.core.net.message.sync.ChainInventoryMessage; +import org.tron.core.net.message.sync.SyncBlockChainMessage; -/** - * msg factory. - */ -public class TronMessageFactory extends MessageFactory { +public class TronMessageFactory { private static final String DATA_LEN = ", len="; - @Override - public TronMessage create(byte[] data) throws Exception { + public static TronMessage create(byte[] data) throws Exception { boolean isException = false; try { byte type = data[0]; @@ -34,21 +42,27 @@ public TronMessage create(byte[] data) throws Exception { } } - private TronMessage create(byte type, byte[] packed) throws Exception { + private static TronMessage create(byte type, byte[] packed) throws Exception { MessageTypes receivedTypes = MessageTypes.fromByte(type); if (receivedTypes == null) { throw new P2pException(P2pException.TypeEnum.NO_SUCH_MESSAGE, "type=" + type + DATA_LEN + packed.length); } switch (receivedTypes) { + case P2P_HELLO: + return new HelloMessage(packed); + case P2P_DISCONNECT: + return new DisconnectMessage(packed); + case P2P_PING: + return new PingMessage(packed); + case P2P_PONG: + return new PongMessage(packed); case TRX: return new TransactionMessage(packed); case BLOCK: return new BlockMessage(packed); case TRXS: return new TransactionsMessage(packed); - case BLOCKS: - return new BlocksMessage(packed); case INVENTORY: return new InventoryMessage(packed); case FETCH_INV_DATA: @@ -57,12 +71,6 @@ private TronMessage create(byte type, byte[] packed) throws Exception { return new SyncBlockChainMessage(packed); case BLOCK_CHAIN_INVENTORY: return new ChainInventoryMessage(packed); - case ITEM_NOT_FOUND: - return new ItemNotFound(); - case FETCH_BLOCK_HEADERS: - return new FetchBlockHeadersMessage(packed); - case TRX_INVENTORY: - return new TransactionInventoryMessage(packed); case PBFT_COMMIT_MSG: return new PbftCommitMessage(packed); default: diff --git a/framework/src/main/java/org/tron/core/net/message/BlockMessage.java b/framework/src/main/java/org/tron/core/net/message/adv/BlockMessage.java similarity index 92% rename from framework/src/main/java/org/tron/core/net/message/BlockMessage.java rename to framework/src/main/java/org/tron/core/net/message/adv/BlockMessage.java index 1b4c42e8060..d5aad2cd5c4 100644 --- a/framework/src/main/java/org/tron/core/net/message/BlockMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/adv/BlockMessage.java @@ -1,10 +1,12 @@ -package org.tron.core.net.message; +package org.tron.core.net.message.adv; import org.tron.common.overlay.message.Message; import org.tron.common.utils.Sha256Hash; import org.tron.core.capsule.BlockCapsule; import org.tron.core.capsule.BlockCapsule.BlockId; import org.tron.core.capsule.TransactionCapsule; +import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.TronMessage; public class BlockMessage extends TronMessage { diff --git a/framework/src/main/java/org/tron/core/net/message/FetchInvDataMessage.java b/framework/src/main/java/org/tron/core/net/message/adv/FetchInvDataMessage.java similarity index 88% rename from framework/src/main/java/org/tron/core/net/message/FetchInvDataMessage.java rename to framework/src/main/java/org/tron/core/net/message/adv/FetchInvDataMessage.java index 16caf8795d3..0a22461edce 100644 --- a/framework/src/main/java/org/tron/core/net/message/FetchInvDataMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/adv/FetchInvDataMessage.java @@ -1,7 +1,8 @@ -package org.tron.core.net.message; +package org.tron.core.net.message.adv; import java.util.List; import org.tron.common.utils.Sha256Hash; +import org.tron.core.net.message.MessageTypes; import org.tron.protos.Protocol.Inventory; import org.tron.protos.Protocol.Inventory.InventoryType; diff --git a/framework/src/main/java/org/tron/core/net/message/InventoryMessage.java b/framework/src/main/java/org/tron/core/net/message/adv/InventoryMessage.java similarity index 94% rename from framework/src/main/java/org/tron/core/net/message/InventoryMessage.java rename to framework/src/main/java/org/tron/core/net/message/adv/InventoryMessage.java index 701270a6626..d0991a9dcd3 100644 --- a/framework/src/main/java/org/tron/core/net/message/InventoryMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/adv/InventoryMessage.java @@ -1,10 +1,12 @@ -package org.tron.core.net.message; +package org.tron.core.net.message.adv; import java.util.Deque; import java.util.LinkedList; import java.util.List; import java.util.stream.Collectors; import org.tron.common.utils.Sha256Hash; +import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.TronMessage; import org.tron.protos.Protocol; import org.tron.protos.Protocol.Inventory; import org.tron.protos.Protocol.Inventory.InventoryType; diff --git a/framework/src/main/java/org/tron/core/net/message/TransactionMessage.java b/framework/src/main/java/org/tron/core/net/message/adv/TransactionMessage.java similarity index 91% rename from framework/src/main/java/org/tron/core/net/message/TransactionMessage.java rename to framework/src/main/java/org/tron/core/net/message/adv/TransactionMessage.java index eebbe0980b0..3ffd65e0386 100644 --- a/framework/src/main/java/org/tron/core/net/message/TransactionMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/adv/TransactionMessage.java @@ -1,8 +1,10 @@ -package org.tron.core.net.message; +package org.tron.core.net.message.adv; import org.tron.common.overlay.message.Message; import org.tron.common.utils.Sha256Hash; import org.tron.core.capsule.TransactionCapsule; +import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.TronMessage; import org.tron.protos.Protocol.Transaction; public class TransactionMessage extends TronMessage { diff --git a/framework/src/main/java/org/tron/core/net/message/TransactionsMessage.java b/framework/src/main/java/org/tron/core/net/message/adv/TransactionsMessage.java similarity index 90% rename from framework/src/main/java/org/tron/core/net/message/TransactionsMessage.java rename to framework/src/main/java/org/tron/core/net/message/adv/TransactionsMessage.java index 72110041e87..2193ac6b546 100644 --- a/framework/src/main/java/org/tron/core/net/message/TransactionsMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/adv/TransactionsMessage.java @@ -1,7 +1,9 @@ -package org.tron.core.net.message; +package org.tron.core.net.message.adv; import java.util.List; import org.tron.core.capsule.TransactionCapsule; +import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.TronMessage; import org.tron.protos.Protocol; import org.tron.protos.Protocol.Transaction; diff --git a/framework/src/main/java/org/tron/common/overlay/message/DisconnectMessage.java b/framework/src/main/java/org/tron/core/net/message/base/DisconnectMessage.java similarity index 70% rename from framework/src/main/java/org/tron/common/overlay/message/DisconnectMessage.java rename to framework/src/main/java/org/tron/core/net/message/base/DisconnectMessage.java index 576dd0ef2a0..c8d0d2485ff 100755 --- a/framework/src/main/java/org/tron/common/overlay/message/DisconnectMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/base/DisconnectMessage.java @@ -1,10 +1,11 @@ -package org.tron.common.overlay.message; +package org.tron.core.net.message.base; import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.TronMessage; import org.tron.protos.Protocol; import org.tron.protos.Protocol.ReasonCode; -public class DisconnectMessage extends P2pMessage { +public class DisconnectMessage extends TronMessage { private Protocol.DisconnectMessage disconnectMessage; @@ -13,6 +14,11 @@ public DisconnectMessage(byte type, byte[] rawData) throws Exception { this.disconnectMessage = Protocol.DisconnectMessage.parseFrom(this.data); } + public DisconnectMessage(byte[] data) throws Exception { + super(MessageTypes.P2P_DISCONNECT.asByte(), data); + this.disconnectMessage = Protocol.DisconnectMessage.parseFrom(data); + } + public DisconnectMessage(ReasonCode reasonCode) { this.disconnectMessage = Protocol.DisconnectMessage .newBuilder() @@ -22,8 +28,8 @@ public DisconnectMessage(ReasonCode reasonCode) { this.data = this.disconnectMessage.toByteArray(); } - public int getReason() { - return this.disconnectMessage.getReason().getNumber(); + public ReasonCode getReason() { + return this.disconnectMessage.getReason(); } public ReasonCode getReasonCode() { diff --git a/framework/src/main/java/org/tron/common/overlay/message/HelloMessage.java b/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java similarity index 88% rename from framework/src/main/java/org/tron/common/overlay/message/HelloMessage.java rename to framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java index 86c01725aed..01cf19b2a0b 100755 --- a/framework/src/main/java/org/tron/common/overlay/message/HelloMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java @@ -1,10 +1,7 @@ -package org.tron.common.overlay.message; +package org.tron.core.net.message.handshake; import com.google.protobuf.ByteString; -import io.netty.channel.ChannelHandlerContext; import lombok.Getter; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.overlay.server.HandshakeHandler; import org.tron.common.utils.ByteArray; import org.tron.core.ChainBaseManager; import org.tron.core.Constant; @@ -12,11 +9,13 @@ import org.tron.core.config.args.Args; import org.tron.core.db.CommonStore; import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.TronMessage; +import org.tron.p2p.discover.Node; import org.tron.protos.Discover.Endpoint; import org.tron.protos.Protocol; import org.tron.protos.Protocol.HelloMessage.Builder; -public class HelloMessage extends P2pMessage { +public class HelloMessage extends TronMessage { @Getter private Protocol.HelloMessage helloMessage; @@ -26,6 +25,11 @@ public HelloMessage(byte type, byte[] rawData) throws Exception { this.helloMessage = Protocol.HelloMessage.parseFrom(rawData); } + public HelloMessage(byte[] data) throws Exception { + super(MessageTypes.P2P_HELLO.asByte(), data); + this.helloMessage = Protocol.HelloMessage.parseFrom(data); + } + public HelloMessage(Node from, long timestamp, ChainBaseManager chainBaseManager) { Endpoint fromEndpoint = Endpoint.newBuilder() @@ -124,12 +128,12 @@ public Class getAnswerMessage() { @Override public String toString() { - return new StringBuilder().append(super.toString()).append(", ") - .append("from: ").append(getFrom()).append(", ") - .append("timestamp: ").append(getTimestamp()).append(", ") - .append("headBlockId: {").append(getHeadBlockId().getString()).append("}, ") - .append("nodeType: ").append(helloMessage.getNodeType()).append(", ") - .append("lowestBlockNum: ").append(helloMessage.getLowestBlockNum()) + return new StringBuilder().append(super.toString()) + .append("from: ").append(getFrom().getInetSocketAddress()).append("\n") + .append("timestamp: ").append(getTimestamp()).append("\n") + .append("headBlockId: ").append(getHeadBlockId().getString()).append("\n") + .append("nodeType: ").append(helloMessage.getNodeType()).append("\n") + .append("lowestBlockNum: ").append(helloMessage.getLowestBlockNum()).append("\n") .toString(); } diff --git a/framework/src/main/java/org/tron/common/overlay/message/PingMessage.java b/framework/src/main/java/org/tron/core/net/message/keepalive/PingMessage.java similarity index 74% rename from framework/src/main/java/org/tron/common/overlay/message/PingMessage.java rename to framework/src/main/java/org/tron/core/net/message/keepalive/PingMessage.java index dbdbe956c5a..209ece12ddd 100644 --- a/framework/src/main/java/org/tron/common/overlay/message/PingMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/keepalive/PingMessage.java @@ -1,9 +1,10 @@ -package org.tron.common.overlay.message; +package org.tron.core.net.message.keepalive; import org.bouncycastle.util.encoders.Hex; import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.TronMessage; -public class PingMessage extends P2pMessage { +public class PingMessage extends TronMessage { private static final byte[] FIXED_PAYLOAD = Hex.decode("C0"); @@ -16,6 +17,10 @@ public PingMessage(byte type, byte[] rawData) { super(type, rawData); } + public PingMessage(byte[] data) { + super(MessageTypes.P2P_PING.asByte(), data); + } + @Override public byte[] getData() { return FIXED_PAYLOAD; diff --git a/framework/src/main/java/org/tron/common/overlay/message/PongMessage.java b/framework/src/main/java/org/tron/core/net/message/keepalive/PongMessage.java similarity index 74% rename from framework/src/main/java/org/tron/common/overlay/message/PongMessage.java rename to framework/src/main/java/org/tron/core/net/message/keepalive/PongMessage.java index c17755125b2..d585a8e46d3 100644 --- a/framework/src/main/java/org/tron/common/overlay/message/PongMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/keepalive/PongMessage.java @@ -1,9 +1,10 @@ -package org.tron.common.overlay.message; +package org.tron.core.net.message.keepalive; import org.bouncycastle.util.encoders.Hex; import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.TronMessage; -public class PongMessage extends P2pMessage { +public class PongMessage extends TronMessage { private static final byte[] FIXED_PAYLOAD = Hex.decode("C0"); @@ -16,6 +17,10 @@ public PongMessage(byte type, byte[] rawData) { super(type, rawData); } + public PongMessage(byte[] data) { + super(MessageTypes.P2P_PONG.asByte(), data); + } + @Override public byte[] getData() { return FIXED_PAYLOAD; diff --git a/framework/src/main/java/org/tron/core/net/message/PbftCommitMessage.java b/framework/src/main/java/org/tron/core/net/message/pbft/PbftCommitMessage.java similarity index 86% rename from framework/src/main/java/org/tron/core/net/message/PbftCommitMessage.java rename to framework/src/main/java/org/tron/core/net/message/pbft/PbftCommitMessage.java index 720f577676e..bfa34367f88 100644 --- a/framework/src/main/java/org/tron/core/net/message/PbftCommitMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/pbft/PbftCommitMessage.java @@ -1,6 +1,8 @@ -package org.tron.core.net.message; +package org.tron.core.net.message.pbft; import org.tron.core.capsule.PbftSignCapsule; +import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.TronMessage; import org.tron.protos.Protocol.PBFTCommitResult; public class PbftCommitMessage extends TronMessage { diff --git a/framework/src/main/java/org/tron/core/net/message/BlockInventoryMessage.java b/framework/src/main/java/org/tron/core/net/message/sync/BlockInventoryMessage.java similarity index 91% rename from framework/src/main/java/org/tron/core/net/message/BlockInventoryMessage.java rename to framework/src/main/java/org/tron/core/net/message/sync/BlockInventoryMessage.java index daeab8e678e..53a736dcb90 100644 --- a/framework/src/main/java/org/tron/core/net/message/BlockInventoryMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/sync/BlockInventoryMessage.java @@ -1,9 +1,11 @@ -package org.tron.core.net.message; +package org.tron.core.net.message.sync; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; import org.tron.core.capsule.BlockCapsule.BlockId; +import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.TronMessage; import org.tron.protos.Protocol; import org.tron.protos.Protocol.BlockInventory; diff --git a/framework/src/main/java/org/tron/core/net/message/ChainInventoryMessage.java b/framework/src/main/java/org/tron/core/net/message/sync/ChainInventoryMessage.java similarity index 94% rename from framework/src/main/java/org/tron/core/net/message/ChainInventoryMessage.java rename to framework/src/main/java/org/tron/core/net/message/sync/ChainInventoryMessage.java index b38961d2e51..4179544ebf7 100644 --- a/framework/src/main/java/org/tron/core/net/message/ChainInventoryMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/sync/ChainInventoryMessage.java @@ -1,4 +1,4 @@ -package org.tron.core.net.message; +package org.tron.core.net.message.sync; import java.util.ArrayList; import java.util.Deque; @@ -6,6 +6,8 @@ import java.util.List; import java.util.stream.Collectors; import org.tron.core.capsule.BlockCapsule.BlockId; +import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.TronMessage; import org.tron.protos.Protocol; import org.tron.protos.Protocol.ChainInventory; diff --git a/framework/src/main/java/org/tron/core/net/message/SyncBlockChainMessage.java b/framework/src/main/java/org/tron/core/net/message/sync/SyncBlockChainMessage.java similarity index 92% rename from framework/src/main/java/org/tron/core/net/message/SyncBlockChainMessage.java rename to framework/src/main/java/org/tron/core/net/message/sync/SyncBlockChainMessage.java index 3163660ecb2..d2e46a5aa70 100644 --- a/framework/src/main/java/org/tron/core/net/message/SyncBlockChainMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/sync/SyncBlockChainMessage.java @@ -1,7 +1,8 @@ -package org.tron.core.net.message; +package org.tron.core.net.message.sync; import java.util.List; import org.tron.core.capsule.BlockCapsule.BlockId; +import org.tron.core.net.message.MessageTypes; import org.tron.protos.Protocol.BlockInventory.Type; public class SyncBlockChainMessage extends BlockInventoryMessage { diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/BlockMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/BlockMsgHandler.java index 5d6d4eb7f75..115e97db50d 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/BlockMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/BlockMsgHandler.java @@ -18,14 +18,14 @@ import org.tron.core.metrics.MetricsKey; import org.tron.core.metrics.MetricsUtil; import org.tron.core.net.TronNetDelegate; -import org.tron.core.net.message.BlockMessage; import org.tron.core.net.message.TronMessage; +import org.tron.core.net.message.adv.BlockMessage; import org.tron.core.net.peer.Item; import org.tron.core.net.peer.PeerConnection; -import org.tron.core.net.service.AdvService; -import org.tron.core.net.service.FetchBlockService; -import org.tron.core.net.service.RelayService; -import org.tron.core.net.service.SyncService; +import org.tron.core.net.service.adv.AdvService; +import org.tron.core.net.service.fetchblock.FetchBlockService; +import org.tron.core.net.service.relay.RelayService; +import org.tron.core.net.service.sync.SyncService; import org.tron.core.services.WitnessProductBlockService; import org.tron.protos.Protocol.Inventory.InventoryType; @@ -61,7 +61,7 @@ public void processMessage(PeerConnection peer, TronMessage msg) throws P2pExcep BlockMessage blockMessage = (BlockMessage) msg; BlockId blockId = blockMessage.getBlockId(); - if (!fastForward && !peer.isFastForwardPeer()) { + if (!fastForward && !peer.isRelayPeer()) { check(peer, blockMessage); } @@ -71,13 +71,13 @@ public void processMessage(PeerConnection peer, TronMessage msg) throws P2pExcep } else { Item item = new Item(blockId, InventoryType.BLOCK); long now = System.currentTimeMillis(); - if (peer.isFastForwardPeer()) { + if (peer.isRelayPeer()) { peer.getAdvInvSpread().put(item, now); } Long time = peer.getAdvInvRequest().remove(item); if (null != time) { MetricsUtil.histogramUpdateUnCheck(MetricsKey.NET_LATENCY_FETCH_BLOCK - + peer.getNode().getHost(), now - time); + + peer.getInetAddress(), now - time); Metrics.histogramObserve(MetricKeys.Histogram.BLOCK_FETCH_LATENCY, (now - time) / Metrics.MILLISECONDS_PER_SECOND); } @@ -91,7 +91,7 @@ public void processMessage(PeerConnection peer, TronMessage msg) throws P2pExcep + "txs/process {}/{}ms, witness: {}", blockId.getNum(), interval, - peer.getInetAddress(), + peer.getInetSocketAddress(), time == null ? 0 : now - time, now - blockMessage.getBlockCapsule().getTimeStamp(), ((BlockMessage) msg).getBlockCapsule().getTransactions().size(), diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/ChainInventoryMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/ChainInventoryMsgHandler.java index 81f0d2290a1..9b71f6f185a 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/ChainInventoryMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/ChainInventoryMsgHandler.java @@ -6,22 +6,21 @@ import java.util.LinkedList; import java.util.List; import java.util.NoSuchElementException; - import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.tron.common.overlay.server.Channel; import org.tron.core.capsule.BlockCapsule.BlockId; import org.tron.core.config.Parameter.ChainConstant; import org.tron.core.config.Parameter.NetConstants; import org.tron.core.exception.P2pException; import org.tron.core.exception.P2pException.TypeEnum; import org.tron.core.net.TronNetDelegate; -import org.tron.core.net.message.ChainInventoryMessage; import org.tron.core.net.message.TronMessage; +import org.tron.core.net.message.sync.ChainInventoryMessage; import org.tron.core.net.peer.PeerConnection; -import org.tron.core.net.service.SyncService; +import org.tron.core.net.peer.TronState; +import org.tron.core.net.service.sync.SyncService; @Slf4j(topic = "net") @Component @@ -47,7 +46,7 @@ public void processMessage(PeerConnection peer, TronMessage msg) throws P2pExcep Deque blockIdWeGet = new LinkedList<>(chainInventoryMessage.getBlockIds()); if (blockIdWeGet.size() == 1 && tronNetDelegate.containBlock(blockIdWeGet.peek())) { - peer.setTronState(Channel.TronState.SYNC_COMPLETED); + peer.setTronState(TronState.SYNC_COMPLETED); peer.setNeedSyncFromPeer(false); return; } @@ -74,11 +73,11 @@ public void processMessage(PeerConnection peer, TronMessage msg) throws P2pExcep } if (blockId != null) { logger.info("Block {} from {} is processed", - blockId.getString(), peer.getNode().getHost()); + blockId.getString(), peer.getInetAddress()); } } catch (NoSuchElementException e) { logger.warn("Process ChainInventoryMessage failed, peer {}, isDisconnect:{}", - peer.getNode().getHost(), peer.isDisconnect()); + peer.getInetAddress(), peer.isDisconnect()); return; } } diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/FetchInvDataMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/FetchInvDataMsgHandler.java index c719674966d..6b5b68b1d11 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/FetchInvDataMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/FetchInvDataMsgHandler.java @@ -1,7 +1,5 @@ package org.tron.core.net.messagehandler; -import static org.tron.core.config.Parameter.ChainConstant.BLOCK_PRODUCED_INTERVAL; - import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.collect.Lists; @@ -10,7 +8,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.tron.common.overlay.discover.node.statistics.MessageCount; import org.tron.common.overlay.message.Message; import org.tron.common.utils.Sha256Hash; import org.tron.consensus.ConsensusDelegate; @@ -21,17 +18,17 @@ import org.tron.core.exception.P2pException; import org.tron.core.exception.P2pException.TypeEnum; import org.tron.core.net.TronNetDelegate; -import org.tron.core.net.message.BlockMessage; -import org.tron.core.net.message.FetchInvDataMessage; import org.tron.core.net.message.MessageTypes; -import org.tron.core.net.message.PbftCommitMessage; -import org.tron.core.net.message.TransactionMessage; -import org.tron.core.net.message.TransactionsMessage; import org.tron.core.net.message.TronMessage; +import org.tron.core.net.message.adv.BlockMessage; +import org.tron.core.net.message.adv.FetchInvDataMessage; +import org.tron.core.net.message.adv.TransactionMessage; +import org.tron.core.net.message.adv.TransactionsMessage; +import org.tron.core.net.message.pbft.PbftCommitMessage; import org.tron.core.net.peer.Item; import org.tron.core.net.peer.PeerConnection; -import org.tron.core.net.service.AdvService; -import org.tron.core.net.service.SyncService; +import org.tron.core.net.service.adv.AdvService; +import org.tron.core.net.service.sync.SyncService; import org.tron.protos.Protocol.Inventory.InventoryType; import org.tron.protos.Protocol.PBFTMessage.Raw; import org.tron.protos.Protocol.Transaction; @@ -139,8 +136,8 @@ private void check(PeerConnection peer, FetchInvDataMessage fetchInvDataMsg) thr throw new P2pException(TypeEnum.BAD_MESSAGE, "not spread inv: {}" + hash); } } - int fetchCount = peer.getNodeStatistics().messageStatistics.tronInTrxFetchInvDataElement - .getCount(10); + int fetchCount = peer.getPeerStatistics().messageStatistics.tronInTrxFetchInvDataElement + .getCount(10); int maxCount = advService.getTrxCount().getCount(60); if (fetchCount > maxCount) { logger.warn("Peer fetch too more transactions in 10 seconds, " @@ -155,17 +152,7 @@ private void check(PeerConnection peer, FetchInvDataMessage fetchInvDataMsg) thr break; } } - if (isAdv) { - MessageCount tronOutAdvBlock = peer.getNodeStatistics().messageStatistics.tronOutAdvBlock; - tronOutAdvBlock.add(fetchInvDataMsg.getHashList().size()); - int outBlockCountIn1min = tronOutAdvBlock.getCount(60); - int producedBlockIn2min = 120_000 / BLOCK_PRODUCED_INTERVAL; - if (outBlockCountIn1min > producedBlockIn2min) { - logger.warn("Peer fetch too more blocks in a minute, producedBlockIn2min: {}," - + " outBlockCountIn1min: {}, peer: {}", - producedBlockIn2min, outBlockCountIn1min, peer.getInetAddress()); - } - } else { + if (!isAdv) { if (!peer.isNeedSyncFromUs()) { throw new P2pException(TypeEnum.BAD_MESSAGE, "no need sync"); } diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/InventoryMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/InventoryMsgHandler.java index 96f022543f2..65fa09128db 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/InventoryMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/InventoryMsgHandler.java @@ -6,11 +6,11 @@ import org.tron.common.utils.Sha256Hash; import org.tron.core.config.args.Args; import org.tron.core.net.TronNetDelegate; -import org.tron.core.net.message.InventoryMessage; import org.tron.core.net.message.TronMessage; +import org.tron.core.net.message.adv.InventoryMessage; import org.tron.core.net.peer.Item; import org.tron.core.net.peer.PeerConnection; -import org.tron.core.net.service.AdvService; +import org.tron.core.net.service.adv.AdvService; import org.tron.protos.Protocol.Inventory.InventoryType; @Slf4j(topic = "net") @@ -55,7 +55,7 @@ private boolean check(PeerConnection peer, InventoryMessage inventoryMessage) { } if (type.equals(InventoryType.TRX)) { - int count = peer.getNodeStatistics().messageStatistics.tronInTrxInventoryElement.getCount(10); + int count = peer.getPeerStatistics().messageStatistics.tronInTrxInventoryElement.getCount(10); if (count > maxCountIn10s) { logger.warn("Drop inv: {} size: {} from Peer {}, Inv count: {} is overload", type, size, peer.getInetAddress(), count); diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/PbftDataSyncHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/PbftDataSyncHandler.java index 5f107c9958d..238d131abe8 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/PbftDataSyncHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/PbftDataSyncHandler.java @@ -26,8 +26,8 @@ import org.tron.core.capsule.TransactionCapsule; import org.tron.core.db.PbftSignDataStore; import org.tron.core.exception.P2pException; -import org.tron.core.net.message.PbftCommitMessage; import org.tron.core.net.message.TronMessage; +import org.tron.core.net.message.pbft.PbftCommitMessage; import org.tron.core.net.peer.PeerConnection; import org.tron.protos.Protocol.PBFTMessage.DataType; import org.tron.protos.Protocol.PBFTMessage.Raw; diff --git a/framework/src/main/java/org/tron/core/net/PbftHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/PbftMsgHandler.java similarity index 56% rename from framework/src/main/java/org/tron/core/net/PbftHandler.java rename to framework/src/main/java/org/tron/core/net/messagehandler/PbftMsgHandler.java index 4120edce707..44eed8d1c2f 100644 --- a/framework/src/main/java/org/tron/core/net/PbftHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/PbftMsgHandler.java @@ -1,32 +1,22 @@ -package org.tron.core.net; +package org.tron.core.net.messagehandler; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.util.concurrent.Striped; -import io.netty.channel.ChannelHandlerContext; -import io.netty.channel.SimpleChannelInboundHandler; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.Lock; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; -import org.tron.common.overlay.server.Channel; -import org.tron.common.overlay.server.MessageQueue; -import org.tron.common.overlay.server.SyncPool; import org.tron.consensus.base.Param; import org.tron.consensus.pbft.PbftManager; import org.tron.consensus.pbft.message.PbftBaseMessage; import org.tron.consensus.pbft.message.PbftMessage; import org.tron.core.exception.P2pException; +import org.tron.core.net.TronNetService; import org.tron.core.net.peer.PeerConnection; @Component -@Scope("prototype") -public class PbftHandler extends SimpleChannelInboundHandler { - - protected PeerConnection peer; - - private MessageQueue msgQueue; +public class PbftMsgHandler { private static final Striped striped = Striped.lazyWeakLock(1024); @@ -36,12 +26,7 @@ public class PbftHandler extends SimpleChannelInboundHandler { @Autowired private PbftManager pbftManager; - @Autowired - private SyncPool syncPool; - - @Override - public void channelRead0(final ChannelHandlerContext ctx, PbftMessage msg) throws Exception { - msgQueue.receivedMessage(msg); + public void processMessage(PeerConnection peer, PbftMessage msg) throws Exception { if (Param.getInstance().getPbftInterface().isSyncing()) { return; } @@ -57,7 +42,7 @@ public void channelRead0(final ChannelHandlerContext ctx, PbftMessage msg) throw throw new P2pException(P2pException.TypeEnum.BAD_MESSAGE, msg.toString()); } msgCache.put(key, true); - forwardMessage(msg); + forwardMessage(peer, msg); pbftManager.doAction(msg); } finally { lock.unlock(); @@ -65,27 +50,9 @@ public void channelRead0(final ChannelHandlerContext ctx, PbftMessage msg) throw } - public void forwardMessage(PbftBaseMessage message) { - if (syncPool == null) { - return; - } - syncPool.getActivePeers().stream().filter(peerConnection -> !peerConnection.equals(peer)) - .forEach(peerConnection -> { - peerConnection.sendMessage(message); - }); - } - - @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - peer.processException(cause); - } - - public void setMsgQueue(MessageQueue msgQueue) { - this.msgQueue = msgQueue; - } - - public void setChannel(Channel channel) { - this.peer = (PeerConnection) channel; + public void forwardMessage(PeerConnection peer, PbftBaseMessage message) { + TronNetService.getPeers().stream().filter(peerConnection -> !peerConnection.equals(peer)) + .forEach(peerConnection -> peerConnection.sendMessage(message)); } private String buildKey(PbftBaseMessage msg) { diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java index d9fc767ac6c..37d46f6a8f3 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java @@ -11,9 +11,9 @@ import org.tron.core.exception.P2pException; import org.tron.core.exception.P2pException.TypeEnum; import org.tron.core.net.TronNetDelegate; -import org.tron.core.net.message.ChainInventoryMessage; -import org.tron.core.net.message.SyncBlockChainMessage; import org.tron.core.net.message.TronMessage; +import org.tron.core.net.message.sync.ChainInventoryMessage; +import org.tron.core.net.message.sync.SyncBlockChainMessage; import org.tron.core.net.peer.PeerConnection; import org.tron.protos.Protocol; diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/TransactionsMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/TransactionsMsgHandler.java index 28cc68c2e44..df46c448e4d 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/TransactionsMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/TransactionsMsgHandler.java @@ -15,12 +15,12 @@ import org.tron.core.exception.P2pException; import org.tron.core.exception.P2pException.TypeEnum; import org.tron.core.net.TronNetDelegate; -import org.tron.core.net.message.TransactionMessage; -import org.tron.core.net.message.TransactionsMessage; import org.tron.core.net.message.TronMessage; +import org.tron.core.net.message.adv.TransactionMessage; +import org.tron.core.net.message.adv.TransactionsMessage; import org.tron.core.net.peer.Item; import org.tron.core.net.peer.PeerConnection; -import org.tron.core.net.service.AdvService; +import org.tron.core.net.service.adv.AdvService; import org.tron.protos.Protocol.Inventory.InventoryType; import org.tron.protos.Protocol.ReasonCode; import org.tron.protos.Protocol.Transaction; diff --git a/framework/src/main/java/org/tron/core/net/peer/PeerConnection.java b/framework/src/main/java/org/tron/core/net/peer/PeerConnection.java index c23db4baa28..3c35e08dd0a 100644 --- a/framework/src/main/java/org/tron/core/net/peer/PeerConnection.java +++ b/framework/src/main/java/org/tron/core/net/peer/PeerConnection.java @@ -2,8 +2,12 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import com.google.protobuf.ByteString; +import java.net.InetAddress; +import java.net.InetSocketAddress; import java.util.Deque; import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -15,22 +19,57 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; -import org.tron.common.overlay.message.HelloMessage; import org.tron.common.overlay.message.Message; -import org.tron.common.overlay.server.Channel; +import org.tron.common.prometheus.MetricKeys; +import org.tron.common.prometheus.Metrics; import org.tron.common.utils.Pair; import org.tron.common.utils.Sha256Hash; +import org.tron.consensus.pbft.message.PbftBaseMessage; import org.tron.core.Constant; import org.tron.core.capsule.BlockCapsule.BlockId; import org.tron.core.config.Parameter.NetConstants; +import org.tron.core.metrics.MetricsKey; +import org.tron.core.metrics.MetricsUtil; import org.tron.core.net.TronNetDelegate; -import org.tron.core.net.service.AdvService; -import org.tron.core.net.service.SyncService; +import org.tron.core.net.message.adv.InventoryMessage; +import org.tron.core.net.message.adv.TransactionsMessage; +import org.tron.core.net.message.base.DisconnectMessage; +import org.tron.core.net.message.handshake.HelloMessage; +import org.tron.core.net.message.keepalive.PingMessage; +import org.tron.core.net.message.keepalive.PongMessage; +import org.tron.core.net.service.adv.AdvService; +import org.tron.core.net.service.statistics.NodeStatistics; +import org.tron.core.net.service.statistics.PeerStatistics; +import org.tron.core.net.service.statistics.TronStatsManager; +import org.tron.core.net.service.sync.SyncService; +import org.tron.p2p.connection.Channel; +import org.tron.protos.Protocol; @Slf4j(topic = "net") @Component @Scope("prototype") -public class PeerConnection extends Channel { +public class PeerConnection { + + @Getter + private PeerStatistics peerStatistics = new PeerStatistics(); + + @Getter + private NodeStatistics nodeStatistics; + + @Getter + private Channel channel; + + @Getter + @Setter + private boolean isRelayPeer; + + @Getter + @Setter + private ByteString address; + + @Getter + @Setter + private TronState tronState = TronState.INIT; @Autowired private TronNetDelegate tronNetDelegate; @@ -51,6 +90,8 @@ public class PeerConnection extends Channel { private int invCacheSize = 20_000; + private long BAD_PEER_BAN_TIME = 3600 * 1000; + @Setter @Getter private Cache advInvReceive = CacheBuilder.newBuilder().maximumSize(invCacheSize) @@ -100,6 +141,11 @@ public class PeerConnection extends Channel { @Getter private volatile boolean needSyncFromUs = true; + public void setChannel(Channel channel) { + this.channel = channel; + this.nodeStatistics = TronStatsManager.getNodeStatistics(channel.getInetAddress()); + } + public void setBlockBothHave(BlockId blockId) { this.blockBothHave = blockId; this.blockBothHaveUpdateTime = System.currentTimeMillis(); @@ -110,11 +156,11 @@ public boolean isIdle() { } public void sendMessage(Message message) { - msgQueue.sendMessage(message); - } - - public void fastSend(Message message) { - msgQueue.fastSend(message); + if (needToLog(message)) { + logger.info("Send peer {} message {}", channel.getInetSocketAddress(), message); + } + channel.send(message.getSendBytes()); + peerStatistics.messageStatistics.addTcpOutMessage(message); } public void onConnect() { @@ -149,9 +195,8 @@ public void onDisconnect() { public String log() { long now = System.currentTimeMillis(); return String.format( - "Peer %s [%8s]\n" - + "ping msg: count %d, max-average-min-last: %d %d %d %d\n" - + "connect time: %ds\n" + "Peer %s\n" + + "connect time: %ds [%sms]\n" + "last know block num: %s\n" + "needSyncFromPeer:%b\n" + "needSyncFromUs:%b\n" @@ -161,16 +206,9 @@ public String log() { + "remainNum:%d\n" + "syncChainRequested:%d\n" + "blockInProcess:%d\n", - getNode().getHost() + ":" + getNode().getPort(), - getNode().getHexIdShort(), - - getNodeStatistics().pingMessageLatency.getCount(), - getNodeStatistics().pingMessageLatency.getMax(), - getNodeStatistics().pingMessageLatency.getAvg(), - getNodeStatistics().pingMessageLatency.getMin(), - getNodeStatistics().pingMessageLatency.getLast(), - - (now - getStartTime()) / Constant.ONE_THOUSAND, + channel.getInetAddress(), + (now - channel.getStartTime()) / Constant.ONE_THOUSAND, + channel.getLatency(), fastForwardBlock != null ? fastForwardBlock.getNum() : blockBothHave.getNum(), isNeedSyncFromPeer(), isNeedSyncFromUs(), @@ -180,12 +218,79 @@ public String log() { remainNum, syncChainRequested == null ? 0 : (now - syncChainRequested.getValue()) / Constant.ONE_THOUSAND, - syncBlockInProcess.size()) - + nodeStatistics.toString() + "\n"; + syncBlockInProcess.size()); } public boolean isSyncFinish() { return !(needSyncFromPeer || needSyncFromUs); } + public void disconnect(Protocol.ReasonCode code) { + sendMessage(new DisconnectMessage(code)); + processDisconnect(code); + nodeStatistics.nodeDisconnectedLocal(code); + } + + public InetSocketAddress getInetSocketAddress() { + return channel.getInetSocketAddress(); + } + + public InetAddress getInetAddress() { + return channel.getInetAddress(); + } + + public boolean isDisconnect() { + return channel.isDisconnect(); + } + + private void processDisconnect(Protocol.ReasonCode reason) { + InetAddress inetAddress = channel.getInetAddress(); + if (inetAddress == null) { + return; + } + switch (reason) { + case BAD_PROTOCOL: + case BAD_BLOCK: + case BAD_TX: + channel.close(BAD_PEER_BAN_TIME); + break; + default: + channel.close(); + break; + } + MetricsUtil.counterInc(MetricsKey.NET_DISCONNECTION_COUNT); + MetricsUtil.counterInc(MetricsKey.NET_DISCONNECTION_DETAIL + reason); + Metrics.counterInc(MetricKeys.Counter.P2P_DISCONNECT, 1, + reason.name().toLowerCase(Locale.ROOT)); + } + + private boolean needToLog(Message msg) { + if (msg instanceof PingMessage + || msg instanceof PongMessage + || msg instanceof TransactionsMessage + || msg instanceof PbftBaseMessage) { + return false; + } + + if (msg instanceof InventoryMessage && ((InventoryMessage) msg) + .getInventoryType().equals(Protocol.Inventory.InventoryType.TRX)) { + return false; + } + + return true; + } + + @Override + public boolean equals(Object o) { + if (o == null || !(o instanceof PeerConnection)) { + return false; + } + return this.channel.equals(((PeerConnection) o).getChannel()); + } + + @Override + public int hashCode() { + return this.channel.hashCode(); + } + } diff --git a/framework/src/main/java/org/tron/core/net/peer/PeerManager.java b/framework/src/main/java/org/tron/core/net/peer/PeerManager.java new file mode 100644 index 00000000000..dfcb0c0e599 --- /dev/null +++ b/framework/src/main/java/org/tron/core/net/peer/PeerManager.java @@ -0,0 +1,154 @@ +package org.tron.core.net.peer; + +import com.google.common.collect.Lists; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationContext; +import org.tron.common.prometheus.MetricKeys; +import org.tron.common.prometheus.MetricLabels; +import org.tron.common.prometheus.Metrics; +import org.tron.p2p.connection.Channel; + +@Slf4j(topic = "net") +public class PeerManager { + + private static List peers = Collections.synchronizedList(new ArrayList<>()); + @Getter + private static AtomicInteger passivePeersCount = new AtomicInteger(0); + @Getter + private static AtomicInteger activePeersCount = new AtomicInteger(0); + + private static ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + + private static long DISCONNECTION_TIME_OUT = 60_000; + + public static void init() { + executor.scheduleWithFixedDelay(() -> { + try { + check(); + logPeerStats(); + } catch (Throwable t) { + logger.error("Exception in peer manager", t); + } + }, 30, 10, TimeUnit.SECONDS); + } + + public static void close() { + try { + peers.forEach(p -> { + if (!p.isDisconnect()) { + p.getChannel().close(); + } + }); + executor.shutdownNow(); + } catch (Exception e) { + logger.error("Peer manager shutdown failed", e); + } + } + + public static synchronized PeerConnection add(ApplicationContext ctx, Channel channel) { + PeerConnection peerConnection = getPeerConnection(channel); + if (peerConnection != null) { + return null; + } + peerConnection = ctx.getBean(PeerConnection.class); + peerConnection.setChannel(channel); + peers.add(peerConnection); + if (channel.isActive()) { + activePeersCount.incrementAndGet(); + } else { + passivePeersCount.incrementAndGet(); + } + return peerConnection; + } + + public static synchronized PeerConnection remove(Channel channel) { + PeerConnection peerConnection = getPeerConnection(channel); + if (peerConnection == null) { + return null; + } + remove(peerConnection); + return peerConnection; + } + + private static void remove(PeerConnection peerConnection) { + peers.remove(peerConnection); + if (peerConnection.getChannel().isActive()) { + activePeersCount.decrementAndGet(); + } else { + passivePeersCount.decrementAndGet(); + } + } + + public static synchronized void sortPeers() { + peers.sort(Comparator.comparingDouble(c -> c.getChannel().getLatency())); + } + + public static PeerConnection getPeerConnection(Channel channel) { + for (PeerConnection peer : new ArrayList<>(peers)) { + if (peer.getChannel().equals(channel)) { + return peer; + } + } + return null; + } + + public static List getPeers() { + List peers = Lists.newArrayList(); + for (PeerConnection peer : new ArrayList<>(PeerManager.peers)) { + if (!peer.isDisconnect()) { + peers.add(peer); + } + } + return peers; + } + + private static void check() { + long now = System.currentTimeMillis(); + for (PeerConnection peer : new ArrayList<>(peers)) { + long disconnectTime = peer.getChannel().getDisconnectTime(); + if (disconnectTime != 0 && now - disconnectTime > DISCONNECTION_TIME_OUT) { + logger.warn("Notify disconnect peer {}.", peer.getInetSocketAddress()); + peers.remove(peer); + if (peer.getChannel().isActive()) { + activePeersCount.decrementAndGet(); + } else { + passivePeersCount.decrementAndGet(); + } + peer.onDisconnect(); + } + } + } + + private static synchronized void logPeerStats() { + String str = String.format("\n\n============ Peer stats: all %d, active %d, passive %d\n\n", + peers.size(), activePeersCount.get(), passivePeersCount.get()); + metric(peers.size(), MetricLabels.Gauge.PEERS_ALL); + metric(activePeersCount.get(), MetricLabels.Gauge.PEERS_ACTIVE); + metric(passivePeersCount.get(), MetricLabels.Gauge.PEERS_PASSIVE); + StringBuilder sb = new StringBuilder(str); + int valid = 0; + for (PeerConnection peer : new ArrayList<>(peers)) { + sb.append(peer.log()); + sb.append("\n"); + if (!(peer.isNeedSyncFromUs() || peer.isNeedSyncFromPeer())) { + valid++; + } + } + metric(valid, MetricLabels.Gauge.PEERS_VALID); + logger.info(sb.toString()); + } + + private static void metric(double amt, String peerType) { + Metrics.gaugeSet(MetricKeys.Gauge.PEERS, amt, peerType); + } + +} diff --git a/framework/src/main/java/org/tron/core/net/peer/TronState.java b/framework/src/main/java/org/tron/core/net/peer/TronState.java new file mode 100644 index 00000000000..34bd4a39288 --- /dev/null +++ b/framework/src/main/java/org/tron/core/net/peer/TronState.java @@ -0,0 +1,7 @@ +package org.tron.core.net.peer; + +public enum TronState { + INIT, + SYNCING, + SYNC_COMPLETED +} diff --git a/framework/src/main/java/org/tron/core/net/service/RelayService.java b/framework/src/main/java/org/tron/core/net/service/RelayService.java deleted file mode 100644 index b4abfc835ea..00000000000 --- a/framework/src/main/java/org/tron/core/net/service/RelayService.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.tron.core.net.service; - -import com.google.protobuf.ByteString; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.tron.core.ChainBaseManager; -import org.tron.core.config.args.Args; -import org.tron.core.net.TronNetDelegate; -import org.tron.core.net.message.BlockMessage; -import org.tron.core.net.peer.Item; -import org.tron.core.net.peer.PeerConnection; -import org.tron.protos.Protocol; - -@Slf4j(topic = "net") -@Component -public class RelayService { - - @Autowired - private ChainBaseManager chainBaseManager; - - @Autowired - private TronNetDelegate tronNetDelegate; - - private int maxFastForwardNum = Args.getInstance().getMaxFastForwardNum(); - - private Set getNextWitnesses(ByteString key, Integer count) { - List list = chainBaseManager.getWitnessScheduleStore().getActiveWitnesses(); - int index = list.indexOf(key); - if (index < 0) { - return new HashSet<>(list); - } - Set set = new HashSet<>(); - for (; count > 0; count--) { - set.add(list.get(++index % list.size())); - } - return set; - } - - public void broadcast(BlockMessage msg) { - Set witnesses = getNextWitnesses( - msg.getBlockCapsule().getWitnessAddress(), maxFastForwardNum); - Item item = new Item(msg.getBlockId(), Protocol.Inventory.InventoryType.BLOCK); - List peers = tronNetDelegate.getActivePeer().stream() - .filter(peer -> !peer.isNeedSyncFromPeer() && !peer.isNeedSyncFromUs()) - .filter(peer -> peer.getAdvInvReceive().getIfPresent(item) == null - && peer.getAdvInvSpread().getIfPresent(item) == null) - .filter(peer -> peer.getAddress() != null && witnesses.contains(peer.getAddress())) - .collect(Collectors.toList()); - - peers.forEach(peer -> { - peer.fastSend(msg); - peer.getAdvInvSpread().put(item, System.currentTimeMillis()); - peer.setFastForwardBlock(msg.getBlockId()); - }); - } - -} - diff --git a/framework/src/main/java/org/tron/core/net/service/AdvService.java b/framework/src/main/java/org/tron/core/net/service/adv/AdvService.java similarity index 95% rename from framework/src/main/java/org/tron/core/net/service/AdvService.java rename to framework/src/main/java/org/tron/core/net/service/adv/AdvService.java index b10e0542919..b6ea7a3445f 100644 --- a/framework/src/main/java/org/tron/core/net/service/AdvService.java +++ b/framework/src/main/java/org/tron/core/net/service/adv/AdvService.java @@ -1,4 +1,4 @@ -package org.tron.core.net.service; +package org.tron.core.net.service.adv; import static org.tron.core.config.Parameter.ChainConstant.BLOCK_PRODUCED_INTERVAL; import static org.tron.core.config.Parameter.NetConstants.MAX_TRX_FETCH_PER_PEER; @@ -19,23 +19,25 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; + import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.tron.common.overlay.discover.node.statistics.MessageCount; import org.tron.common.overlay.message.Message; import org.tron.common.utils.Sha256Hash; import org.tron.common.utils.Time; import org.tron.core.capsule.BlockCapsule.BlockId; import org.tron.core.config.args.Args; import org.tron.core.net.TronNetDelegate; -import org.tron.core.net.message.BlockMessage; -import org.tron.core.net.message.FetchInvDataMessage; -import org.tron.core.net.message.InventoryMessage; -import org.tron.core.net.message.TransactionMessage; +import org.tron.core.net.message.adv.BlockMessage; +import org.tron.core.net.message.adv.FetchInvDataMessage; +import org.tron.core.net.message.adv.InventoryMessage; +import org.tron.core.net.message.adv.TransactionMessage; import org.tron.core.net.peer.Item; import org.tron.core.net.peer.PeerConnection; +import org.tron.core.net.service.fetchblock.FetchBlockService; +import org.tron.core.net.service.statistics.MessageCount; import org.tron.protos.Protocol.Inventory.InventoryType; @Slf4j(topic = "net") @@ -169,7 +171,7 @@ public int fastBroadcastTransaction(TransactionMessage msg) { && peer.getAdvInvSpread().getIfPresent(item) == null) { peersCount++; peer.getAdvInvSpread().put(item, Time.getCurrentMillis()); - peer.fastSend(inventoryMessage); + peer.sendMessage(inventoryMessage); } } if (peersCount == 0) { @@ -359,12 +361,12 @@ public int getSize(PeerConnection peer) { public void sendInv() { send.forEach((peer, ids) -> ids.forEach((key, value) -> { - if (peer.isFastForwardPeer() && key.equals(InventoryType.TRX)) { + if (peer.isRelayPeer() && key.equals(InventoryType.TRX)) { return; } if (key.equals(InventoryType.BLOCK)) { value.sort(Comparator.comparingLong(value1 -> new BlockId(value1).getNum())); - peer.fastSend(new InventoryMessage(value, key)); + peer.sendMessage(new InventoryMessage(value, key)); } else { peer.sendMessage(new InventoryMessage(value, key)); } @@ -375,7 +377,7 @@ void sendFetch() { send.forEach((peer, ids) -> ids.forEach((key, value) -> { if (key.equals(InventoryType.BLOCK)) { value.sort(Comparator.comparingLong(value1 -> new BlockId(value1).getNum())); - peer.fastSend(new FetchInvDataMessage(value, key)); + peer.sendMessage(new FetchInvDataMessage(value, key)); fetchBlockService.fetchBlock(value, peer); } else { peer.sendMessage(new FetchInvDataMessage(value, key)); diff --git a/framework/src/main/java/org/tron/core/net/service/FetchBlockService.java b/framework/src/main/java/org/tron/core/net/service/fetchblock/FetchBlockService.java similarity index 94% rename from framework/src/main/java/org/tron/core/net/service/FetchBlockService.java rename to framework/src/main/java/org/tron/core/net/service/fetchblock/FetchBlockService.java index ef541eb3559..f3699c3be2e 100644 --- a/framework/src/main/java/org/tron/core/net/service/FetchBlockService.java +++ b/framework/src/main/java/org/tron/core/net/service/fetchblock/FetchBlockService.java @@ -1,4 +1,4 @@ -package org.tron.core.net.service; +package org.tron.core.net.service.fetchblock; import java.util.Collections; import java.util.Comparator; @@ -22,15 +22,11 @@ import org.tron.core.metrics.MetricsKey; import org.tron.core.metrics.MetricsUtil; import org.tron.core.net.TronNetDelegate; -import org.tron.core.net.message.FetchInvDataMessage; +import org.tron.core.net.message.adv.FetchInvDataMessage; import org.tron.core.net.peer.Item; import org.tron.core.net.peer.PeerConnection; import org.tron.protos.Protocol.Inventory.InventoryType; -/** - * @author kiven.miao - * @date 2022/3/17 3:50 下午 - */ @Slf4j(topic = "net") @Component public class FetchBlockService { @@ -123,7 +119,7 @@ private void fetchBlockProcess(FetchBlockInfo fetchBlock) { optionalPeerConnection.ifPresent(firstPeer -> { if (shouldFetchBlock(firstPeer, fetchBlock)) { firstPeer.getAdvInvRequest().put(item, System.currentTimeMillis()); - firstPeer.fastSend(new FetchInvDataMessage(Collections.singletonList(item.getHash()), + firstPeer.sendMessage(new FetchInvDataMessage(Collections.singletonList(item.getHash()), item.getType())); this.fetchBlockInfo = null; } @@ -152,7 +148,7 @@ private boolean shouldFetchBlock(PeerConnection newPeer, FetchBlockInfo fetchBlo private double getPeerTop75(PeerConnection peerConnection) { return MetricsUtil.getHistogram(MetricsKey.NET_LATENCY_FETCH_BLOCK - + peerConnection.getNode().getHost()).getSnapshot().get75thPercentile(); + + peerConnection.getInetAddress()).getSnapshot().get75thPercentile(); } private static class FetchBlockInfo { diff --git a/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java b/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java new file mode 100644 index 00000000000..22ba096fb20 --- /dev/null +++ b/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java @@ -0,0 +1,119 @@ +package org.tron.core.net.service.handshake; + +import java.util.Arrays; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.tron.common.utils.ByteArray; +import org.tron.core.ChainBaseManager; +import org.tron.core.config.args.Args; +import org.tron.core.net.TronNetService; +import org.tron.core.net.message.handshake.HelloMessage; +import org.tron.core.net.peer.PeerConnection; +import org.tron.core.net.peer.PeerManager; +import org.tron.core.net.service.relay.RelayService; +import org.tron.p2p.discover.Node; +import org.tron.protos.Protocol.ReasonCode; + +@Slf4j(topic = "net") +@Component +public class HandshakeService { + + @Autowired + private RelayService relayService; + + @Autowired + private ChainBaseManager chainBaseManager; + + public void startHandshake(PeerConnection peer) { + sendHelloMessage(peer, peer.getChannel().getStartTime()); + } + + public void processHelloMessage(PeerConnection peer, HelloMessage msg) { + if (peer.getHelloMessageReceive() != null) { + logger.warn("Peer {} receive dup hello message", peer.getInetSocketAddress()); + peer.disconnect(ReasonCode.BAD_PROTOCOL); + return; + } + + TronNetService.getP2pService().updateNodeId(peer.getChannel(), msg.getFrom().getHexId()); + if (peer.isDisconnect()) { + logger.info("Duplicate Peer {}", peer.getInetSocketAddress()); + peer.disconnect(ReasonCode.DUPLICATE_PEER); + return; + } + + if (!msg.valid()) { + logger.warn("Peer {} invalid hello message parameters, " + + "GenesisBlockId: {}, SolidBlockId: {}, HeadBlockId: {}", + peer.getInetSocketAddress(), + ByteArray.toHexString(msg.getInstance().getGenesisBlockId().getHash().toByteArray()), + ByteArray.toHexString(msg.getInstance().getSolidBlockId().getHash().toByteArray()), + ByteArray.toHexString(msg.getInstance().getHeadBlockId().getHash().toByteArray())); + peer.disconnect(ReasonCode.UNEXPECTED_IDENTITY); + return; + } + + peer.setAddress(msg.getHelloMessage().getAddress()); + + if (!relayService.checkHelloMessage(msg, peer.getChannel())) { + peer.disconnect(ReasonCode.UNEXPECTED_IDENTITY); + return; + } + + long headBlockNum = chainBaseManager.getHeadBlockNum(); + long lowestBlockNum = msg.getLowestBlockNum(); + if (lowestBlockNum > headBlockNum) { + logger.info("Peer {} miss block, lowestBlockNum:{}, headBlockNum:{}", + peer.getInetSocketAddress(), lowestBlockNum, headBlockNum); + peer.disconnect(ReasonCode.LIGHT_NODE_SYNC_FAIL); + return; + } + + if (msg.getVersion() != Args.getInstance().getNodeP2pVersion()) { + logger.info("Peer {} different p2p version, peer->{}, me->{}", + peer.getInetSocketAddress(), msg.getVersion(), + Args.getInstance().getNodeP2pVersion()); + peer.disconnect(ReasonCode.INCOMPATIBLE_VERSION); + return; + } + + if (!Arrays.equals(chainBaseManager.getGenesisBlockId().getBytes(), + msg.getGenesisBlockId().getBytes())) { + logger.info("Peer {} different genesis block, peer->{}, me->{}", + peer.getInetSocketAddress(), + msg.getGenesisBlockId().getString(), + chainBaseManager.getGenesisBlockId().getString()); + peer.disconnect(ReasonCode.INCOMPATIBLE_CHAIN); + return; + } + + if (chainBaseManager.getSolidBlockId().getNum() >= msg.getSolidBlockId().getNum() + && !chainBaseManager.containBlockInMainChain(msg.getSolidBlockId())) { + logger.info("Peer {} different solid block, peer->{}, me->{}", + peer.getInetSocketAddress(), + msg.getSolidBlockId().getString(), + chainBaseManager.getSolidBlockId().getString()); + peer.disconnect(ReasonCode.FORKED); + return; + } + + peer.setHelloMessageReceive(msg); + + peer.getChannel().updateLatency( + System.currentTimeMillis() - peer.getChannel().getStartTime()); + PeerManager.sortPeers(); + peer.onConnect(); + return; + } + + private void sendHelloMessage(PeerConnection peer, long time) { + Node node = new Node(TronNetService.getP2pConfig().getNodeID(), + TronNetService.getP2pConfig().getIp(), + TronNetService.getP2pConfig().getPort()); + HelloMessage message = new HelloMessage(node, time, ChainBaseManager.getChainBaseManager()); + peer.sendMessage(message); + peer.setHelloMessageSend(message); + } + +} diff --git a/framework/src/main/java/org/tron/core/net/service/keepalive/KeepAliveService.java b/framework/src/main/java/org/tron/core/net/service/keepalive/KeepAliveService.java new file mode 100644 index 00000000000..35c2843c046 --- /dev/null +++ b/framework/src/main/java/org/tron/core/net/service/keepalive/KeepAliveService.java @@ -0,0 +1,66 @@ +package org.tron.core.net.service.keepalive; + +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.tron.core.net.TronNetService; +import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.TronMessage; +import org.tron.core.net.message.keepalive.PingMessage; +import org.tron.core.net.message.keepalive.PongMessage; +import org.tron.core.net.peer.PeerConnection; +import org.tron.protos.Protocol; + +@Slf4j(topic = "net") +@Component +public class KeepAliveService { + + private long KEEP_ALIVE_TIMEOUT = 10_000; + + private long PING_TIMEOUT = 20_000; + + public long PING_PERIOD = 60_000; + + private final ScheduledExecutorService executor = + Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "KeepAlive")); + + public void init() { + executor.scheduleWithFixedDelay(() -> { + try { + long now = System.currentTimeMillis(); + TronNetService.getPeers().forEach(p -> { + long pingSent = p.getChannel().pingSent; + long lastSendTime = p.getChannel().getLastSendTime(); + if (p.getChannel().waitForPong) { + if (now - pingSent > PING_TIMEOUT) { + p.disconnect(Protocol.ReasonCode.TIME_OUT); + } + } else { + if (now - lastSendTime > KEEP_ALIVE_TIMEOUT || now - pingSent > PING_PERIOD) { + p.sendMessage(new PingMessage()); + p.getChannel().waitForPong = true; + p.getChannel().pingSent = now; + } + } + }); + } catch (Throwable t) { + logger.error("Exception in keep alive task.", t); + } + }, 2, 2, TimeUnit.SECONDS); + } + + public void close() { + executor.shutdown(); + } + + public void processMessage(PeerConnection peer, TronMessage message) { + if (message.getType().equals(MessageTypes.P2P_PING)) { + peer.sendMessage(new PongMessage()); + } else { + peer.getChannel().updateLatency(System.currentTimeMillis() - peer.getChannel().pingSent); + peer.getChannel().waitForPong = false; + } + } +} diff --git a/framework/src/main/java/org/tron/core/net/service/nodepersist/DBNode.java b/framework/src/main/java/org/tron/core/net/service/nodepersist/DBNode.java new file mode 100644 index 00000000000..74b4902dd77 --- /dev/null +++ b/framework/src/main/java/org/tron/core/net/service/nodepersist/DBNode.java @@ -0,0 +1,24 @@ +package org.tron.core.net.service.nodepersist; + +import lombok.Getter; +import lombok.Setter; + +public class DBNode { + + @Getter + @Setter + private String host; + + @Getter + @Setter + private int port; + + public DBNode() { + } + + public DBNode(String host, int port) { + this.host = host; + this.port = port; + } + +} diff --git a/framework/src/main/java/org/tron/core/net/service/nodepersist/DBNodes.java b/framework/src/main/java/org/tron/core/net/service/nodepersist/DBNodes.java new file mode 100644 index 00000000000..f56616d81e7 --- /dev/null +++ b/framework/src/main/java/org/tron/core/net/service/nodepersist/DBNodes.java @@ -0,0 +1,13 @@ +package org.tron.core.net.service.nodepersist; + +import java.util.ArrayList; +import java.util.List; +import lombok.Getter; +import lombok.Setter; + +public class DBNodes { + + @Getter + @Setter + private List nodes = new ArrayList<>(); +} \ No newline at end of file diff --git a/framework/src/main/java/org/tron/core/net/service/nodepersist/NodePersistService.java b/framework/src/main/java/org/tron/core/net/service/nodepersist/NodePersistService.java new file mode 100644 index 00000000000..02ff1e7a1aa --- /dev/null +++ b/framework/src/main/java/org/tron/core/net/service/nodepersist/NodePersistService.java @@ -0,0 +1,91 @@ +package org.tron.core.net.service.nodepersist; + +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Timer; +import java.util.TimerTask; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.JsonUtil; +import org.tron.core.ChainBaseManager; +import org.tron.core.capsule.BytesCapsule; +import org.tron.core.config.args.Args; +import org.tron.core.net.TronNetService; +import org.tron.p2p.discover.Node; + +@Slf4j(topic = "net") +@Component +public class NodePersistService { + private static final byte[] DB_KEY_PEERS = "peers".getBytes(); + private static final long DB_COMMIT_RATE = 1 * 60 * 1000L; + private static final int MAX_NODES_WRITE_TO_DB = 30; + + private boolean isNodePersist = Args.getInstance().isNodeDiscoveryPersist(); + + private ChainBaseManager chainBaseManager = ChainBaseManager.getInstance(); + + private Timer nodePersistTaskTimer = new Timer("NodePersistTaskTimer"); + + public void init() { + if (isNodePersist) { + nodePersistTaskTimer.scheduleAtFixedRate(new TimerTask() { + @Override + public void run() { + dbWrite(); + } + }, DB_COMMIT_RATE, DB_COMMIT_RATE); + } + } + + public void close() { + try { + nodePersistTaskTimer.cancel(); + } catch (Exception e) { + logger.error("Close nodePersistTaskTimer failed", e); + } + } + + public List dbRead() { + List nodes = new ArrayList<>(); + try { + byte[] nodeBytes = chainBaseManager.getCommonStore().get(DB_KEY_PEERS).getData(); + if (ByteArray.isEmpty(nodeBytes)) { + return nodes; + } + DBNodes dbNodes = JsonUtil.json2Obj(new String(nodeBytes), DBNodes.class); + logger.info("Read node from store: {} nodes", dbNodes.getNodes().size()); + dbNodes.getNodes().forEach(n -> nodes.add(new InetSocketAddress(n.getHost(), n.getPort()))); + } catch (Exception e) { + logger.warn("DB read nodes failed, {}", e.getMessage()); + } + return nodes; + } + + private void dbWrite() { + try { + List batch = new ArrayList<>(); + List tableNodes = TronNetService.getP2pService().getTableNodes(); + tableNodes.sort(Comparator.comparingLong(value -> -value.getUpdateTime())); + for (Node n : tableNodes) { + batch.add(new DBNode(n.getHost(), n.getPort())); + } + + if (batch.size() > MAX_NODES_WRITE_TO_DB) { + batch = batch.subList(0, MAX_NODES_WRITE_TO_DB); + } + + DBNodes dbNodes = new DBNodes(); + dbNodes.setNodes(batch); + + logger.info("Write nodes to store: {}/{} nodes", batch.size(), tableNodes.size()); + + chainBaseManager.getCommonStore() + .put(DB_KEY_PEERS, new BytesCapsule(JsonUtil.obj2Json(dbNodes).getBytes())); + } catch (Exception e) { + logger.warn("DB write nodes failed, {}", e.getMessage()); + } + } +} diff --git a/framework/src/main/java/org/tron/common/overlay/server/FastForward.java b/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java similarity index 67% rename from framework/src/main/java/org/tron/common/overlay/server/FastForward.java rename to framework/src/main/java/org/tron/core/net/service/relay/RelayService.java index bc19fa206d9..46f3b5c7374 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/FastForward.java +++ b/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java @@ -1,15 +1,16 @@ -package org.tron.common.overlay.server; +package org.tron.core.net.service.relay; import com.google.protobuf.ByteString; -import java.net.InetAddress; import java.net.InetSocketAddress; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.binary.Hex; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; @@ -17,28 +18,40 @@ import org.tron.common.backup.BackupManager.BackupStatusEnum; import org.tron.common.crypto.SignInterface; import org.tron.common.crypto.SignUtils; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.overlay.message.HelloMessage; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.ByteArray; import org.tron.common.utils.Sha256Hash; +import org.tron.core.ChainBaseManager; import org.tron.core.capsule.TransactionCapsule; import org.tron.core.config.args.Args; import org.tron.core.db.Manager; +import org.tron.core.net.TronNetDelegate; +import org.tron.core.net.TronNetService; +import org.tron.core.net.message.adv.BlockMessage; +import org.tron.core.net.message.handshake.HelloMessage; +import org.tron.core.net.peer.Item; +import org.tron.core.net.peer.PeerConnection; import org.tron.core.store.WitnessScheduleStore; +import org.tron.p2p.P2pConfig; +import org.tron.p2p.connection.Channel; import org.tron.protos.Protocol; -import org.tron.protos.Protocol.ReasonCode; @Slf4j(topic = "net") @Component -public class FastForward { +public class RelayService { + + @Autowired + private ChainBaseManager chainBaseManager; + + @Autowired + private TronNetDelegate tronNetDelegate; @Autowired private ApplicationContext ctx; - private Manager manager; + private P2pConfig p2pConfig; - private ChannelManager channelManager; + private Manager manager; private WitnessScheduleStore witnessScheduleStore; @@ -47,15 +60,19 @@ public class FastForward { private ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); private CommonParameter parameter = Args.getInstance(); - private List fastForwardNodes = parameter.getFastForwardNodes(); + + private List fastForwardNodes = parameter.getFastForwardNodes(); + private ByteString witnessAddress = ByteString .copyFrom(Args.getLocalWitnesses().getWitnessAccountAddress(CommonParameter.getInstance() .isECKeyCryptoEngine())); + private int keySize = Args.getLocalWitnesses().getPrivateKeys().size(); + private int maxFastForwardNum = Args.getInstance().getMaxFastForwardNum(); + public void init() { manager = ctx.getBean(Manager.class); - channelManager = ctx.getBean(ChannelManager.class); witnessScheduleStore = ctx.getBean(WitnessScheduleStore.class); backupManager = ctx.getBean(BackupManager.class); @@ -82,8 +99,7 @@ public void init() { public void fillHelloMessage(HelloMessage message, Channel channel) { if (isActiveWitness()) { - fastForwardNodes.forEach(node -> { - InetAddress address = new InetSocketAddress(node.getHost(), node.getPort()).getAddress(); + fastForwardNodes.forEach(address -> { if (address.equals(channel.getInetAddress())) { SignInterface cryptoEngine = SignUtils .fromPrivate(ByteArray.fromHexString(Args.getLocalWitnesses().getPrivateKey()), @@ -135,7 +151,7 @@ public boolean checkHelloMessage(HelloMessage message, Channel channel) { flag = Arrays.equals(sigAddress, witnessPermissionAddress); } if (flag) { - channelManager.getTrustNodes().put(channel.getInetAddress(), channel.getNode()); + p2pConfig.getTrustNodes().add(channel.getInetAddress()); } return flag; } catch (Exception e) { @@ -153,21 +169,48 @@ private boolean isActiveWitness() { } private void connect() { - fastForwardNodes.forEach(node -> { - InetAddress address = new InetSocketAddress(node.getHost(), node.getPort()).getAddress(); - channelManager.getActiveNodes().put(address, node); - }); + fastForwardNodes.forEach(address -> p2pConfig.getActiveNodes().add(address)); } private void disconnect() { - fastForwardNodes.forEach(node -> { - InetAddress address = new InetSocketAddress(node.getHost(), node.getPort()).getAddress(); - channelManager.getActiveNodes().remove(address); - channelManager.getActivePeers().forEach(channel -> { - if (channel.getInetAddress().equals(address)) { - channel.disconnect(ReasonCode.RESET); + fastForwardNodes.forEach(address -> { + p2pConfig.getActiveNodes().remove(address); + TronNetService.getPeers().forEach(peer -> { + if (peer.getInetAddress().equals(address.getAddress())) { + peer.getChannel().close(); } }); }); } + + private Set getNextWitnesses(ByteString key, Integer count) { + List list = chainBaseManager.getWitnessScheduleStore().getActiveWitnesses(); + int index = list.indexOf(key); + if (index < 0) { + return new HashSet<>(list); + } + Set set = new HashSet<>(); + for (; count > 0; count--) { + set.add(list.get(++index % list.size())); + } + return set; + } + + public void broadcast(BlockMessage msg) { + Set witnesses = getNextWitnesses( + msg.getBlockCapsule().getWitnessAddress(), maxFastForwardNum); + Item item = new Item(msg.getBlockId(), Protocol.Inventory.InventoryType.BLOCK); + List peers = tronNetDelegate.getActivePeer().stream() + .filter(peer -> !peer.isNeedSyncFromPeer() && !peer.isNeedSyncFromUs()) + .filter(peer -> peer.getAdvInvReceive().getIfPresent(item) == null + && peer.getAdvInvSpread().getIfPresent(item) == null) + .filter(peer -> peer.getAddress() != null && witnesses.contains(peer.getAddress())) + .collect(Collectors.toList()); + + peers.forEach(peer -> { + peer.sendMessage(msg); + peer.getAdvInvSpread().put(item, System.currentTimeMillis()); + peer.setFastForwardBlock(msg.getBlockId()); + }); + } } diff --git a/framework/src/main/java/org/tron/common/overlay/discover/node/statistics/MessageCount.java b/framework/src/main/java/org/tron/core/net/service/statistics/MessageCount.java similarity index 93% rename from framework/src/main/java/org/tron/common/overlay/discover/node/statistics/MessageCount.java rename to framework/src/main/java/org/tron/core/net/service/statistics/MessageCount.java index 28f0acfbf9e..750f4c52286 100644 --- a/framework/src/main/java/org/tron/common/overlay/discover/node/statistics/MessageCount.java +++ b/framework/src/main/java/org/tron/core/net/service/statistics/MessageCount.java @@ -1,8 +1,8 @@ -package org.tron.common.overlay.discover.node.statistics; +package org.tron.core.net.service.statistics; import lombok.extern.slf4j.Slf4j; -@Slf4j(topic = "discover") +@Slf4j(topic = "net") public class MessageCount { private static int SIZE = 60; diff --git a/framework/src/main/java/org/tron/common/overlay/discover/node/statistics/MessageStatistics.java b/framework/src/main/java/org/tron/core/net/service/statistics/MessageStatistics.java similarity index 78% rename from framework/src/main/java/org/tron/common/overlay/discover/node/statistics/MessageStatistics.java rename to framework/src/main/java/org/tron/core/net/service/statistics/MessageStatistics.java index fa285929639..6ed67026ada 100644 --- a/framework/src/main/java/org/tron/common/overlay/discover/node/statistics/MessageStatistics.java +++ b/framework/src/main/java/org/tron/core/net/service/statistics/MessageStatistics.java @@ -1,27 +1,13 @@ -package org.tron.common.overlay.discover.node.statistics; +package org.tron.core.net.service.statistics; -import lombok.extern.slf4j.Slf4j; -import org.tron.common.net.udp.message.UdpMessageTypeEnum; import org.tron.common.overlay.message.Message; -import org.tron.core.net.message.FetchInvDataMessage; -import org.tron.core.net.message.InventoryMessage; import org.tron.core.net.message.MessageTypes; -import org.tron.core.net.message.TransactionsMessage; +import org.tron.core.net.message.adv.FetchInvDataMessage; +import org.tron.core.net.message.adv.InventoryMessage; +import org.tron.core.net.message.adv.TransactionsMessage; -@Slf4j public class MessageStatistics { - //udp discovery - public final MessageCount discoverInPing = new MessageCount(); - public final MessageCount discoverOutPing = new MessageCount(); - public final MessageCount discoverInPong = new MessageCount(); - public final MessageCount discoverOutPong = new MessageCount(); - public final MessageCount discoverInFindNode = new MessageCount(); - public final MessageCount discoverOutFindNode = new MessageCount(); - public final MessageCount discoverInNeighbours = new MessageCount(); - public final MessageCount discoverOutNeighbours = new MessageCount(); - - //tcp p2p public final MessageCount p2pInHello = new MessageCount(); public final MessageCount p2pOutHello = new MessageCount(); public final MessageCount p2pInPing = new MessageCount(); @@ -31,7 +17,6 @@ public class MessageStatistics { public final MessageCount p2pInDisconnect = new MessageCount(); public final MessageCount p2pOutDisconnect = new MessageCount(); - //tcp tron public final MessageCount tronInMessage = new MessageCount(); public final MessageCount tronOutMessage = new MessageCount(); @@ -69,14 +54,6 @@ public class MessageStatistics { public final MessageCount tronOutBlock = new MessageCount(); public final MessageCount tronOutAdvBlock = new MessageCount(); - public void addUdpInMessage(UdpMessageTypeEnum type) { - addUdpMessage(type, true); - } - - public void addUdpOutMessage(UdpMessageTypeEnum type) { - addUdpMessage(type, false); - } - public void addTcpInMessage(Message msg) { addTcpMessage(msg, true); } @@ -85,41 +62,6 @@ public void addTcpOutMessage(Message msg) { addTcpMessage(msg, false); } - private void addUdpMessage(UdpMessageTypeEnum type, boolean flag) { - switch (type) { - case DISCOVER_PING: - if (flag) { - discoverInPing.add(); - } else { - discoverOutPing.add(); - } - break; - case DISCOVER_PONG: - if (flag) { - discoverInPong.add(); - } else { - discoverOutPong.add(); - } - break; - case DISCOVER_FIND_NODE: - if (flag) { - discoverInFindNode.add(); - } else { - discoverOutFindNode.add(); - } - break; - case DISCOVER_NEIGHBORS: - if (flag) { - discoverInNeighbours.add(); - } else { - discoverOutNeighbours.add(); - } - break; - default: - break; - } - } - private void addTcpMessage(Message msg, boolean flag) { if (flag) { @@ -216,8 +158,8 @@ private void addTcpMessage(Message msg, boolean flag) { break; } } - - + + private void messageProcess(MessageTypes messageType, MessageCount inTrx, MessageCount inTrxEle, diff --git a/framework/src/main/java/org/tron/core/net/service/statistics/NodeStatistics.java b/framework/src/main/java/org/tron/core/net/service/statistics/NodeStatistics.java new file mode 100644 index 00000000000..1bfd7a41738 --- /dev/null +++ b/framework/src/main/java/org/tron/core/net/service/statistics/NodeStatistics.java @@ -0,0 +1,54 @@ +package org.tron.core.net.service.statistics; + +import lombok.Getter; +import org.tron.protos.Protocol; + +public class NodeStatistics { + @Getter + private Protocol.ReasonCode remoteDisconnectReason = null; + @Getter + private Protocol.ReasonCode localDisconnectReason = null; + @Getter + private int disconnectTimes = 0; + private long lastDisconnectedTime = 0; + private long firstDisconnectedTime = 0; + private long start = System.currentTimeMillis(); + + public Protocol.ReasonCode getDisconnectReason() { + if (localDisconnectReason != null) { + return localDisconnectReason; + } + if (remoteDisconnectReason != null) { + return remoteDisconnectReason; + } + return Protocol.ReasonCode.UNKNOWN; + } + + public void nodeDisconnectedRemote(Protocol.ReasonCode reason) { + remoteDisconnectReason = reason; + notifyDisconnect(); + } + + public void nodeDisconnectedLocal(Protocol.ReasonCode reason) { + localDisconnectReason = reason; + notifyDisconnect(); + } + + private void notifyDisconnect() { + lastDisconnectedTime = System.currentTimeMillis(); + if (firstDisconnectedTime == 0) { + firstDisconnectedTime = lastDisconnectedTime; + } + disconnectTimes++; + } + + @Override + public String toString() { + return new StringBuilder() + .append("time:").append(System.currentTimeMillis() - start) + .append(", disconnectTimes:").append(disconnectTimes) + .append(", localReason:").append(localDisconnectReason) + .append(", remoteReason:").append(remoteDisconnectReason).toString(); + } + +} diff --git a/framework/src/main/java/org/tron/core/net/service/statistics/PeerStatistics.java b/framework/src/main/java/org/tron/core/net/service/statistics/PeerStatistics.java new file mode 100644 index 00000000000..8a73d8f21f1 --- /dev/null +++ b/framework/src/main/java/org/tron/core/net/service/statistics/PeerStatistics.java @@ -0,0 +1,5 @@ +package org.tron.core.net.service.statistics; + +public class PeerStatistics { + public final MessageStatistics messageStatistics = new MessageStatistics(); +} diff --git a/framework/src/main/java/org/tron/core/net/service/statistics/TronStatsManager.java b/framework/src/main/java/org/tron/core/net/service/statistics/TronStatsManager.java new file mode 100644 index 00000000000..cf241102dbf --- /dev/null +++ b/framework/src/main/java/org/tron/core/net/service/statistics/TronStatsManager.java @@ -0,0 +1,83 @@ +package org.tron.core.net.service.statistics; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import java.net.InetAddress; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; +import org.tron.common.prometheus.MetricKeys; +import org.tron.common.prometheus.MetricLabels; +import org.tron.common.prometheus.Metrics; +import org.tron.core.metrics.MetricsKey; +import org.tron.core.metrics.MetricsUtil; +import org.tron.core.net.TronNetService; +import org.tron.p2p.stats.P2pStats; + +@Slf4j(topic = "net") +@Component +public class TronStatsManager { + private static volatile long TCP_TRAFFIC_IN = 0; + private static volatile long TCP_TRAFFIC_OUT = 0; + private static volatile long UDP_TRAFFIC_IN = 0; + private static volatile long UDP_TRAFFIC_OUT = 0; + + private static Cache cache = CacheBuilder.newBuilder() + .maximumSize(3000).recordStats().build(); + + private ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + + public static NodeStatistics getNodeStatistics(InetAddress inetAddress) { + NodeStatistics nodeStatistics = cache.getIfPresent(inetAddress); + if (nodeStatistics == null) { + nodeStatistics = new NodeStatistics(); + cache.put(inetAddress, nodeStatistics); + } + return nodeStatistics; + } + + public void init() { + executor.scheduleWithFixedDelay(() -> { + try { + work(); + } catch (Throwable t) { + logger.error("Exception in traffic stats worker, {}", t.getMessage()); + } + }, 1, 1, TimeUnit.SECONDS); + } + + public void close() { + try { + executor.shutdownNow(); + } catch (Exception e) { + logger.error("Exception in shutdown traffic stats worker, {}", e.getMessage()); + } + } + + private void work() { + P2pStats stats = TronNetService.getP2pService().getP2pStats(); + + MetricsUtil.meterMark(MetricsKey.NET_TCP_IN_TRAFFIC, + stats.getTcpInSize() - TCP_TRAFFIC_IN); + Metrics.histogramObserve(MetricKeys.Histogram.TCP_BYTES, + stats.getTcpInSize() - TCP_TRAFFIC_IN, + MetricLabels.Histogram.TRAFFIC_IN); + MetricsUtil.meterMark(MetricsKey.NET_TCP_OUT_TRAFFIC, + stats.getTcpOutSize() - TCP_TRAFFIC_OUT); + Metrics.histogramObserve(MetricKeys.Histogram.UDP_BYTES, + stats.getUdpInSize() - UDP_TRAFFIC_IN, + MetricLabels.Histogram.TRAFFIC_IN); + MetricsUtil.meterMark(MetricsKey.NET_UDP_OUT_TRAFFIC, + stats.getUdpOutSize() - UDP_TRAFFIC_OUT); + Metrics.histogramObserve(MetricKeys.Histogram.UDP_BYTES, + stats.getUdpOutSize() - UDP_TRAFFIC_OUT, + MetricLabels.Histogram.TRAFFIC_OUT); + + TCP_TRAFFIC_IN = stats.getTcpInSize(); + TCP_TRAFFIC_OUT = stats.getTcpOutSize(); + UDP_TRAFFIC_IN = stats.getUdpInSize(); + UDP_TRAFFIC_OUT = stats.getUdpOutSize(); + } +} diff --git a/framework/src/main/java/org/tron/core/net/service/SyncService.java b/framework/src/main/java/org/tron/core/net/service/sync/SyncService.java similarity index 96% rename from framework/src/main/java/org/tron/core/net/service/SyncService.java rename to framework/src/main/java/org/tron/core/net/service/sync/SyncService.java index 79697d7459f..f9351199b3e 100644 --- a/framework/src/main/java/org/tron/core/net/service/SyncService.java +++ b/framework/src/main/java/org/tron/core/net/service/sync/SyncService.java @@ -1,4 +1,4 @@ -package org.tron.core.net.service; +package org.tron.core.net.service.sync; import static org.tron.core.config.Parameter.NetConstants.MAX_BLOCK_FETCH_PER_PEER; @@ -18,7 +18,6 @@ import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; -import org.tron.common.overlay.server.Channel.TronState; import org.tron.common.utils.Pair; import org.tron.core.capsule.BlockCapsule; import org.tron.core.capsule.BlockCapsule.BlockId; @@ -27,11 +26,12 @@ import org.tron.core.exception.P2pException; import org.tron.core.exception.P2pException.TypeEnum; import org.tron.core.net.TronNetDelegate; -import org.tron.core.net.message.BlockMessage; -import org.tron.core.net.message.FetchInvDataMessage; -import org.tron.core.net.message.SyncBlockChainMessage; +import org.tron.core.net.message.adv.BlockMessage; +import org.tron.core.net.message.adv.FetchInvDataMessage; +import org.tron.core.net.message.sync.SyncBlockChainMessage; import org.tron.core.net.messagehandler.PbftDataSyncHandler; import org.tron.core.net.peer.PeerConnection; +import org.tron.core.net.peer.TronState; import org.tron.protos.Protocol.Inventory.InventoryType; import org.tron.protos.Protocol.ReasonCode; @@ -95,7 +95,7 @@ public void close() { public void startSync(PeerConnection peer) { if (peer.getTronState().equals(TronState.SYNCING)) { - logger.warn("Start sync failed, peer {} is in sync", peer.getNode().getHost()); + logger.warn("Start sync failed, peer {} is in sync", peer.getInetSocketAddress()); return; } peer.setTronState(TronState.SYNCING); @@ -109,7 +109,7 @@ public void startSync(PeerConnection peer) { public void syncNext(PeerConnection peer) { try { if (peer.getSyncChainRequested() != null) { - logger.warn("Peer {} is in sync", peer.getNode().getHost()); + logger.warn("Peer {} is in sync", peer.getInetSocketAddress()); return; } LinkedList chainSummary = getBlockChainSummary(peer); @@ -201,7 +201,6 @@ private LinkedList getBlockChainSummary(PeerConnection peer) throws P2p private void startFetchSyncBlock() { HashMap> send = new HashMap<>(); - tronNetDelegate.getActivePeer().stream() .filter(peer -> peer.isNeedSyncFromPeer() && peer.isIdle()) .forEach(peer -> { diff --git a/framework/src/main/java/org/tron/core/services/NodeInfoService.java b/framework/src/main/java/org/tron/core/services/NodeInfoService.java index 79aff5b8a71..c67f522f1bc 100644 --- a/framework/src/main/java/org/tron/core/services/NodeInfoService.java +++ b/framework/src/main/java/org/tron/core/services/NodeInfoService.java @@ -11,6 +11,8 @@ import java.util.Arrays; import java.util.List; import java.util.Map.Entry; + +import org.apache.commons.codec.binary.Hex; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.ArrayUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -21,14 +23,20 @@ import org.tron.common.entity.NodeInfo.MachineInfo.DeadLockThreadInfo; import org.tron.common.entity.NodeInfo.MachineInfo.MemoryDescInfo; import org.tron.common.entity.PeerInfo; -import org.tron.common.overlay.discover.node.NodeManager; -import org.tron.common.overlay.server.SyncPool; import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.MetricTime; import org.tron.core.ChainBaseManager; import org.tron.core.db.Manager; +import org.tron.core.net.P2pEventHandlerImpl; +import org.tron.core.net.TronNetService; import org.tron.core.net.peer.PeerConnection; +import org.tron.core.net.peer.PeerManager; +import org.tron.core.net.service.statistics.NodeStatistics; +import org.tron.core.net.service.statistics.PeerStatistics; import org.tron.core.services.WitnessProductBlockService.CheatWitnessInfo; +import org.tron.p2p.P2pConfig; +import org.tron.p2p.P2pService; +import org.tron.p2p.connection.Channel; import org.tron.program.Version; import org.tron.protos.Protocol.ReasonCode; @@ -42,12 +50,6 @@ public class NodeInfoService { .getOperatingSystemMXBean(); private CommonParameter parameter = CommonParameter.getInstance(); - @Autowired - private SyncPool syncPool; - - @Autowired - private NodeManager nodeManager; - @Autowired private Manager dbManager; @@ -124,48 +126,49 @@ private void setMachineInfo(NodeInfo nodeInfo) { } private void setConnectInfo(NodeInfo nodeInfo) { - nodeInfo.setCurrentConnectCount(syncPool.getActivePeers().size()); - nodeInfo.setActiveConnectCount(syncPool.getActivePeersCount().get()); - nodeInfo.setPassiveConnectCount(syncPool.getPassivePeersCount().get()); - long totalFlow = 0; + int activeCnt = PeerManager.getActivePeersCount().get(); + int passiveCnt = PeerManager.getPassivePeersCount().get(); + nodeInfo.setCurrentConnectCount(activeCnt + passiveCnt); + nodeInfo.setActiveConnectCount(activeCnt); + nodeInfo.setPassiveConnectCount(passiveCnt); List peerInfoList = new ArrayList<>(); - for (PeerConnection peerConnection : syncPool.getActivePeers()) { + for (PeerConnection peerConnection : PeerManager.getPeers()) { + Channel channel = peerConnection.getChannel(); + NodeStatistics nodeStatistics = peerConnection.getNodeStatistics(); + P2pService P2pService = TronNetService.getP2pService(); + P2pConfig p2pConfig = TronNetService.getP2pConfig(); PeerInfo peerInfo = new PeerInfo(); peerInfo.setHeadBlockWeBothHave(peerConnection.getBlockBothHave().getString()); - peerInfo.setActive(peerConnection.isActive()); - peerInfo.setAvgLatency(peerConnection.getNodeStatistics().pingMessageLatency.getAvg()); + peerInfo.setActive(peerConnection.getChannel().isActive()); + peerInfo.setAvgLatency(peerConnection.getChannel().getLatency()); peerInfo.setBlockInPorcSize(peerConnection.getSyncBlockInProcess().size()); - peerInfo.setConnectTime(peerConnection.getStartTime()); - peerInfo.setDisconnectTimes(peerConnection.getNodeStatistics().getDisconnectTimes()); + peerInfo.setConnectTime(channel.getStartTime()); + peerInfo.setDisconnectTimes(nodeStatistics.getDisconnectTimes()); //peerInfo.setHeadBlockTimeWeBothHave(peerConnection.getHeadBlockTimeWeBothHave()); - peerInfo.setHost(peerConnection.getNode().getHost()); - peerInfo.setInFlow(peerConnection.getNodeStatistics().tcpFlow.getTotalCount()); + peerInfo.setHost(channel.getInetAddress().toString()); peerInfo.setLastBlockUpdateTime(peerConnection.getBlockBothHaveUpdateTime()); peerInfo.setLastSyncBlock(peerConnection.getLastSyncBlockId() == null ? "" : peerConnection.getLastSyncBlockId().getString()); - ReasonCode reasonCode = peerConnection.getNodeStatistics() - .getTronLastLocalDisconnectReason(); + ReasonCode reasonCode = nodeStatistics.getLocalDisconnectReason(); peerInfo.setLocalDisconnectReason(reasonCode == null ? "" : reasonCode.toString()); - reasonCode = peerConnection.getNodeStatistics().getTronLastRemoteDisconnectReason(); + reasonCode = nodeStatistics.getRemoteDisconnectReason(); peerInfo.setRemoteDisconnectReason(reasonCode == null ? "" : reasonCode.toString()); peerInfo.setNeedSyncFromPeer(peerConnection.isNeedSyncFromPeer()); peerInfo.setNeedSyncFromUs(peerConnection.isNeedSyncFromUs()); - peerInfo.setNodeCount(nodeManager.getTable().getAllNodes().size()); - peerInfo.setNodeId(peerConnection.getNode().getHexId()); - peerInfo.setPort(peerConnection.getNode().getPort()); + int tableNodesSize = P2pService.getTableNodes().size(); + peerInfo.setNodeCount(tableNodesSize); + peerInfo.setNodeId(Hex.encodeHexString(p2pConfig.getNodeID())); + peerInfo.setPort(p2pConfig.getPort()); peerInfo.setRemainNum(peerConnection.getRemainNum()); - peerInfo.setScore(peerConnection.getNodeStatistics().getReputation()); peerInfo.setSyncBlockRequestedSize(peerConnection.getSyncBlockRequested().size()); peerInfo.setSyncFlag(peerConnection.isDisconnect()); peerInfo.setSyncToFetchSize(peerConnection.getSyncBlockToFetch().size()); peerInfo.setSyncToFetchSizePeekNum(peerConnection.getSyncBlockToFetch().size() > 0 ? peerConnection.getSyncBlockToFetch().peek().getNum() : -1); peerInfo.setUnFetchSynNum(peerConnection.getRemainNum()); - totalFlow += peerConnection.getNodeStatistics().tcpFlow.getTotalCount(); peerInfoList.add(peerInfo); } nodeInfo.setPeerList(peerInfoList); - nodeInfo.setTotalFlow(totalFlow); } private void setConfigNodeInfo(NodeInfo nodeInfo) { @@ -177,7 +180,7 @@ private void setConfigNodeInfo(NodeInfo nodeInfo) { configNodeInfo.setDiscoverEnable(parameter.isNodeDiscoveryEnable()); configNodeInfo.setActiveNodeSize(parameter.getActiveNodes().size()); configNodeInfo.setPassiveNodeSize(parameter.getPassiveNodes().size()); - configNodeInfo.setSendNodeSize(parameter.getSeedNodes().size()); + configNodeInfo.setSendNodeSize(parameter.getSeedNode().getIpList().size()); configNodeInfo.setMaxConnectCount(parameter.getMaxConnections()); configNodeInfo.setSameIpMaxConnectCount(parameter.getMaxConnectionsWithSameIp()); configNodeInfo.setBackupListenPort(parameter.getBackupPort()); diff --git a/framework/src/main/java/org/tron/core/services/RpcApiService.java b/framework/src/main/java/org/tron/core/services/RpcApiService.java index c964536aa33..487f7d7076c 100755 --- a/framework/src/main/java/org/tron/core/services/RpcApiService.java +++ b/framework/src/main/java/org/tron/core/services/RpcApiService.java @@ -88,8 +88,6 @@ import org.tron.common.application.Service; import org.tron.common.crypto.SignInterface; import org.tron.common.crypto.SignUtils; -import org.tron.common.overlay.discover.node.NodeHandler; -import org.tron.common.overlay.discover.node.NodeManager; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.ByteArray; import org.tron.common.utils.Sha256Hash; @@ -111,6 +109,7 @@ import org.tron.core.exception.VMIllegalException; import org.tron.core.exception.ZksnarkException; import org.tron.core.metrics.MetricsApiService; +import org.tron.core.net.TronNetService; import org.tron.core.services.filter.LiteFnQueryGrpcInterceptor; import org.tron.core.services.ratelimiter.RateLimiterInterceptor; import org.tron.core.services.ratelimiter.RpcApiAccessInterceptor; @@ -188,8 +187,6 @@ public class RpcApiService implements Service { @Autowired private ChainBaseManager chainBaseManager; - @Autowired - private NodeManager nodeManager; @Autowired private Wallet wallet; @@ -1603,24 +1600,13 @@ public void getTransactionCountByBlockNum(NumberMessage request, @Override public void listNodes(EmptyMessage request, StreamObserver responseObserver) { - List handlerList = nodeManager.dumpActiveNodes(); - - Map nodeHandlerMap = new HashMap<>(); - for (NodeHandler handler : handlerList) { - String key = handler.getNode().getHexId() + handler.getNode().getHost(); - nodeHandlerMap.put(key, handler); - } - NodeList.Builder nodeListBuilder = NodeList.newBuilder(); - - nodeHandlerMap.entrySet().stream() - .forEach(v -> { - org.tron.common.overlay.discover.node.Node node = v.getValue().getNode(); - nodeListBuilder.addNodes(Node.newBuilder().setAddress( + TronNetService.getP2pService().getConnectableNodes().forEach(node -> { + nodeListBuilder.addNodes(Node.newBuilder().setAddress( Address.newBuilder() - .setHost(ByteString.copyFrom(ByteArray.fromString(node.getHost()))) - .setPort(node.getPort()))); - }); + .setHost(ByteString.copyFrom(ByteArray.fromString(node.getHost()))) + .setPort(node.getPort()))); + }); responseObserver.onNext(nodeListBuilder.build()); responseObserver.onCompleted(); } diff --git a/framework/src/main/java/org/tron/program/FullNode.java b/framework/src/main/java/org/tron/program/FullNode.java index 63c1edaaecf..a056609a665 100644 --- a/framework/src/main/java/org/tron/program/FullNode.java +++ b/framework/src/main/java/org/tron/program/FullNode.java @@ -15,6 +15,7 @@ import org.tron.core.Constant; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; +import org.tron.core.net.P2pEventHandlerImpl; import org.tron.core.services.RpcApiService; import org.tron.core.services.http.FullNodeHttpApiService; import org.tron.core.services.interfaceJsonRpcOnPBFT.JsonRpcServiceOnPBFT; @@ -79,7 +80,6 @@ public static void main(String[] args) { TronApplicationContext context = new TronApplicationContext(beanFactory); context.register(DefaultConfig.class); - context.refresh(); Application appT = ApplicationFactory.create(context); shutdown(appT); diff --git a/framework/src/main/java/org/tron/program/SolidityNode.java b/framework/src/main/java/org/tron/program/SolidityNode.java index e0c1fcaa231..9884a14a62b 100644 --- a/framework/src/main/java/org/tron/program/SolidityNode.java +++ b/framework/src/main/java/org/tron/program/SolidityNode.java @@ -11,9 +11,7 @@ import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; -import org.tron.common.overlay.client.DatabaseGrpcClient; -import org.tron.common.overlay.discover.DiscoverServer; -import org.tron.common.overlay.discover.node.NodeManager; +import org.tron.common.client.DatabaseGrpcClient; import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.Metrics; import org.tron.core.ChainBaseManager; @@ -22,7 +20,6 @@ import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; import org.tron.core.db.Manager; -import org.tron.core.net.TronNetService; import org.tron.core.services.RpcApiService; import org.tron.core.services.http.solidity.SolidityNodeHttpApiService; import org.tron.protos.Protocol.Block; @@ -97,14 +94,6 @@ public static void main(String[] args) { appT.startServices(); appT.startup(); - //Disable peer discovery for solidity node - DiscoverServer discoverServer = context.getBean(DiscoverServer.class); - discoverServer.close(); - NodeManager nodeManager = context.getBean(NodeManager.class); - nodeManager.close(); - TronNetService tronNetService = context.getBean(TronNetService.class); - tronNetService.stop(); - SolidityNode node = new SolidityNode(appT.getDbManager()); node.start(); diff --git a/framework/src/test/java/org/tron/common/overlay/discover/node/NodeHandlerTest.java b/framework/src/test/java/org/tron/common/overlay/discover/node/NodeHandlerTest.java deleted file mode 100644 index dfe24f8e644..00000000000 --- a/framework/src/test/java/org/tron/common/overlay/discover/node/NodeHandlerTest.java +++ /dev/null @@ -1,100 +0,0 @@ -package org.tron.common.overlay.discover.node; - -import java.io.File; -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.tron.common.application.TronApplicationContext; -import org.tron.common.utils.FileUtil; -import org.tron.core.ChainBaseManager; -import org.tron.core.Constant; -import org.tron.core.config.DefaultConfig; -import org.tron.core.config.args.Args; - - -public class NodeHandlerTest { - - private static final Logger logger = LoggerFactory.getLogger("Test"); - // private static Manager dbManager; - private static TronApplicationContext context; - // private Application appTest; - // private CommonParameter argsTest; - private static Node currNode; - private static Node oldNode; - private static Node replaceNode; - private static NodeHandler currHandler; - private static NodeHandler oldHandler; - private static NodeHandler replaceHandler; - private static NodeManager nodeManager; - private static String dbPath = "NodeHandlerTest"; - - static { - Args.setParam(new String[]{"-d", dbPath}, Constant.TEST_CONF); - context = new TronApplicationContext(DefaultConfig.class); - } - - /** - * init the application. - */ - @BeforeClass - public static void init() { - initNodes(); - } - - /** - * destroy the context. - */ - @AfterClass - public static void destroy() { - Args.clearParam(); - context.destroy(); - if (FileUtil.deleteDir(new File(dbPath))) { - logger.info("Release resources successful."); - } else { - logger.info("Release resources failure."); - } - } - - /** - * init nodes. - */ - public static void initNodes() { - // dbManager = context.getBean(Manager.class); - nodeManager = new NodeManager(context.getBean(ChainBaseManager.class)); - String currNodeId = "74c11ffad1d59d7b1a56691a0b84a53f0791c92361357364f1d2537" - + "898407ef0249bbbf5a4ce8cff9e34e2fdf8bac883540e026d1e5d6ebf536414bdde81198e"; - String oldNodeId = "74c11ffad1d59d7b2c56691a0b84a53f0791c92361357364f1d2537898407e" - + "f0249bbbf5a4ce8cff9e34e2fdf8bac883540e026d1e5d6ebf536414bdde81198e"; - String replaceNodeId = "74c11ffad1d59d7b1a56691a0b84a53f0791c92361357364f1d2537" - + "837407ef0249bbbf5a4ce8cff9e34e2fdf8bac883540e026d1e5d6ebf536414bdde81198e"; - currNode = new Node(currNodeId.getBytes(), "47.95.206.44", 18885, 18888); - oldNode = new Node(oldNodeId.getBytes(), "36.95.165.44", 18885, 18888); - replaceNode = new Node(replaceNodeId.getBytes(), "47.29.177.44", 18885, 18888); - currHandler = new NodeHandler(currNode, nodeManager); - oldHandler = new NodeHandler(oldNode, nodeManager); - replaceHandler = new NodeHandler(replaceNode, nodeManager); - } - - @Test - public void stateNonActiveTest() throws Exception { - Class clazz = NodeHandler.class; - Constructor cn = clazz.getDeclaredConstructor(Node.class, NodeManager.class); - NodeHandler nh = cn.newInstance(oldNode, nodeManager); - Field declaredField = clazz.getDeclaredField("replaceCandidate"); - declaredField.setAccessible(true); - declaredField.set(nh, replaceHandler); - - nodeManager.getTable().addNode(oldNode); - nh.changeState(NodeHandler.State.EVICTCANDIDATE); - nh.changeState(NodeHandler.State.NONACTIVE); - replaceHandler.changeState(NodeHandler.State.ALIVE); - - Assert.assertFalse(nodeManager.getTable().contains(oldNode)); - Assert.assertTrue(nodeManager.getTable().contains(replaceNode)); - } -} diff --git a/framework/src/test/java/org/tron/common/overlay/discover/node/NodeManagerTest.java b/framework/src/test/java/org/tron/common/overlay/discover/node/NodeManagerTest.java deleted file mode 100644 index f5b870bacdf..00000000000 --- a/framework/src/test/java/org/tron/common/overlay/discover/node/NodeManagerTest.java +++ /dev/null @@ -1,226 +0,0 @@ -package org.tron.common.overlay.discover.node; - -import java.io.File; -import java.lang.reflect.Constructor; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.tron.common.application.Application; -import org.tron.common.application.TronApplicationContext; -import org.tron.common.parameter.CommonParameter; -import org.tron.common.utils.FileUtil; -import org.tron.core.ChainBaseManager; -import org.tron.core.Constant; -import org.tron.core.config.DefaultConfig; -import org.tron.core.config.args.Args; -import org.tron.core.db.Manager; - - -public class NodeManagerTest { - - private static final Logger logger = LoggerFactory.getLogger("Test"); - private static Manager manager; - private static NodeManager nodeManager; - private static TronApplicationContext context; - private static CommonParameter argsTest; - private static Application appTest; - private static Class nodeManagerClazz; - private static String dbPath = "NodeManagerTest"; - - static { - Args.setParam(new String[]{"-d", dbPath}, Constant.TEST_CONF); - context = new TronApplicationContext(DefaultConfig.class); - } - - /** - * start the application. - */ - @BeforeClass - public static void init() { - // argsTest = Args.getInstance(); - // Args.setParam(new String[]{"--output-directory", dbPath}, - // Constant.TEST_CONF); - // context = new TronApplicationContext(DefaultConfig.class); - // appTest = ApplicationFactory.create(context); - // appTest.initServices(argsTest); - // appTest.startServices(); - // appTest.startup(); - try { - initManager(); - } catch (Exception e) { - logger.error("init failed {}", e.getMessage()); - } - } - - /** - * destroy the context. - */ - @AfterClass - public static void destroy() { - Args.clearParam(); - context.destroy(); - if (FileUtil.deleteDir(new File(dbPath))) { - logger.info("Release resources successful."); - } else { - logger.info("Release resources failure."); - } - } - - /** - * init the managers. - */ - // @Before - public static void initManager() throws Exception { - nodeManagerClazz = NodeManager.class; - // Constructor handlerConstructor - // = nodeManagerClazz.getConstructor(ChainBaseManager.class); - manager = context.getBean(Manager.class); - // nodeManager = handlerConstructor.newInstance(context.getBean(ChainBaseManager.class)); - nodeManager = new NodeManager(context.getBean(ChainBaseManager.class)); - } - - @Test - public void isNodeAliveTest() { - Node node = new Node(new byte[64], "128.0.0.1", 18889, 18889); - nodeManager.getTable().addNode(node); - NodeHandler nodeHandler = new NodeHandler(node, nodeManager); - nodeHandler.changeState(NodeHandler.State.ACTIVE); - Assert.assertTrue(nodeManager.isNodeAlive(nodeHandler)); - nodeHandler.changeState(NodeHandler.State.ALIVE); - Assert.assertTrue(nodeManager.isNodeAlive(nodeHandler)); - nodeHandler.changeState(NodeHandler.State.EVICTCANDIDATE); - Assert.assertTrue(nodeManager.isNodeAlive(nodeHandler)); - } - - @Test - public void trimTableTest_removeByReputation() throws Exception { - //insert 3001 nodes(isConnectible = true) with threshold = 3000 - final int totalNodes = insertValues(3002); - Assert.assertEquals(calculateTrimNodes(totalNodes, 0), getHandlerMapSize()); - - clearNodeManager(); - } - - @Test - public void trimTableTest_removeNotConnectibleNodes() throws Exception { - final int totalNodes = insertValues(3000); - insertNotConnectibleNodes(); - Method method = nodeManagerClazz.getDeclaredMethod("trimTable"); - method.setAccessible(true); - method.invoke(nodeManager); - Assert.assertEquals(calculateTrimNodes(totalNodes, 2), getHandlerMapSize()); - - clearNodeManager(); - } - - private void clearNodeManager() { - nodeManager.clearNodeHandlerMap(); - } - - /** - * calculate nodes number after table trim. - * - * @param totalNodes total nodes inserted - * @param wrongNodes isConnectable = false - * @return nodes count after trimTable() - */ - public int calculateTrimNodes(int totalNodes, int wrongNodes) { - if (totalNodes + wrongNodes > 3000) { - if (totalNodes <= 3000) { - return totalNodes; - } else { - int result = 2000 + ((totalNodes + wrongNodes) % 2000 - 1001); - return result; - } - } - return totalNodes + wrongNodes; - } - - /** - * insert valid nodes in map. - * - * @param totalNodes total nodes to be inserted. - * @return total nodes inserted. - */ - public int insertValues(int totalNodes) throws Exception { - //put 3001 nodes in nodeHandlerMap - int ipPart3 = 1; - int ipPart4 = 1; - for (int i = 0; i < totalNodes; i++) { - StringBuilder stringBuilder = new StringBuilder("128.0."); - byte[] bytes = new byte[64]; - bytes[0] = (byte) (i + 1); - stringBuilder.append(ipPart3); - stringBuilder.append("."); - stringBuilder.append(ipPart4); - ipPart4++; - if (ipPart4 == 256) { - ipPart3++; - ipPart4 = 1; - } - Class nodeClazz = Node.class; - Constructor nodeConstructor - = nodeClazz.getConstructor(byte[].class, String.class, int.class, int.class); - Node node = nodeConstructor.newInstance(bytes, stringBuilder.toString(), 18889, 18889); - Field isConnectableField = nodeClazz.getDeclaredField("p2pVersion"); - isConnectableField.setAccessible(true); - isConnectableField.set(node, Args.getInstance().getNodeP2pVersion()); - nodeManager.getNodeHandler(node); - } - return totalNodes; - } - - /** - * insert nodes with illegal p2p version. - */ - public void insertNotConnectibleNodes() throws Exception { - Class nodeClazz = Node.class; - Constructor nodeConstructor - = nodeClazz.getConstructor(byte[].class, String.class, int.class, int.class); - Node wrongNode1 = nodeConstructor.newInstance(new byte[64], "128.0.0.1", 1111, 18889); - byte[] id = new byte[64]; - id[63] = 1; - Node wrongNode2 = nodeConstructor.newInstance(id, "128.0.0.2", 1111, 18889); - Field isConnectableField = nodeClazz.getDeclaredField("p2pVersion"); - isConnectableField.setAccessible(true); - isConnectableField.set(wrongNode1, 999); - isConnectableField.set(wrongNode2, 999); - nodeManager.getNodeHandler(wrongNode1); - nodeManager.getNodeHandler(wrongNode2); - } - - - /** - * get the size of nodeHandlerMap. - * - * @return NodeManager.nodeHandlerMap - */ - public int getHandlerMapSize() throws Exception { - Field mapField = nodeManagerClazz.getDeclaredField("nodeHandlerMap"); - mapField.setAccessible(true); - Map nodeHandlerMap = (ConcurrentHashMap) mapField.get(nodeManager); - return nodeHandlerMap.size(); - } - - @Test - public void dumpActiveNodesTest() { - Node node1 = new Node(new byte[64], "128.0.0.1", 18889, 18889); - Node node2 = new Node(new byte[64], "128.0.0.2", 18889, 18889); - Node node3 = new Node(new byte[64], "128.0.0.3", 18889, 18889); - NodeHandler nodeHandler1 = nodeManager.getNodeHandler(node1); - NodeHandler nodeHandler2 = nodeManager.getNodeHandler(node2); - NodeHandler nodeHandler3 = nodeManager.getNodeHandler(node3); - nodeHandler1.changeState(NodeHandler.State.ALIVE); - nodeHandler2.changeState(NodeHandler.State.ACTIVE); - nodeHandler3.changeState(NodeHandler.State.NONACTIVE); - int activeNodes = nodeManager.dumpActiveNodes().size(); - Assert.assertEquals(2, activeNodes); - } -} diff --git a/framework/src/test/java/org/tron/common/overlay/discover/node/NodeStatisticsTest.java b/framework/src/test/java/org/tron/common/overlay/discover/node/NodeStatisticsTest.java deleted file mode 100644 index 157c31e26a3..00000000000 --- a/framework/src/test/java/org/tron/common/overlay/discover/node/NodeStatisticsTest.java +++ /dev/null @@ -1,150 +0,0 @@ -package org.tron.common.overlay.discover.node; - -import java.lang.reflect.Field; -import java.util.ArrayList; -import java.util.LinkedList; -import org.bouncycastle.util.encoders.Hex; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.tron.common.net.udp.message.UdpMessageTypeEnum; -import org.tron.common.overlay.discover.node.statistics.MessageStatistics; -import org.tron.common.overlay.discover.node.statistics.NodeStatistics; -import org.tron.common.overlay.message.DisconnectMessage; -import org.tron.common.overlay.message.PongMessage; -import org.tron.common.utils.Sha256Hash; -import org.tron.core.capsule.BlockCapsule; -import org.tron.core.net.message.BlockMessage; -import org.tron.core.net.message.ChainInventoryMessage; -import org.tron.core.net.message.FetchInvDataMessage; -import org.tron.core.net.message.InventoryMessage; -import org.tron.core.net.message.MessageTypes; -import org.tron.core.net.message.SyncBlockChainMessage; -import org.tron.core.net.message.TransactionsMessage; -import org.tron.protos.Protocol; - -public class NodeStatisticsTest { - - private NodeStatistics nodeStatistics; - - @Before - public void init() { - this.nodeStatistics = new NodeStatistics(); - } - - @Test - public void testNode() throws NoSuchFieldException, IllegalAccessException { - Protocol.ReasonCode reasonCode = this.nodeStatistics.getDisconnectReason(); - Assert.assertEquals(Protocol.ReasonCode.UNKNOWN, reasonCode); - - boolean isReputationPenalized = this.nodeStatistics.isReputationPenalized(); - Assert.assertFalse(isReputationPenalized); - - this.nodeStatistics.setPredefined(true); - Assert.assertTrue(this.nodeStatistics.isPredefined()); - - this.nodeStatistics.setPersistedReputation(10000); - this.nodeStatistics.nodeDisconnectedRemote(Protocol.ReasonCode.INCOMPATIBLE_VERSION); - isReputationPenalized = this.nodeStatistics.isReputationPenalized(); - Assert.assertTrue(isReputationPenalized); - - Field field = this.nodeStatistics.getClass().getDeclaredField("firstDisconnectedTime"); - field.setAccessible(true); - field.set(this.nodeStatistics, System.currentTimeMillis() - 60 * 60 * 1000L - 1); - isReputationPenalized = this.nodeStatistics.isReputationPenalized(); - Assert.assertFalse(isReputationPenalized); - reasonCode = this.nodeStatistics.getDisconnectReason(); - Assert.assertEquals(Protocol.ReasonCode.UNKNOWN, reasonCode); - - String str = this.nodeStatistics.toString(); - //System.out.println(str); - Assert.assertNotNull(str); - - this.nodeStatistics.nodeIsHaveDataTransfer(); - this.nodeStatistics.resetTcpFlow(); - this.nodeStatistics.discoverMessageLatency.add(10); - this.nodeStatistics.discoverMessageLatency.add(20); - long avg = this.nodeStatistics.discoverMessageLatency.getAvg(); - Assert.assertEquals(15, avg); - - } - - @Test - public void testMessage() { - MessageStatistics statistics = this.nodeStatistics.messageStatistics; - statistics.addUdpInMessage(UdpMessageTypeEnum.DISCOVER_FIND_NODE); - statistics.addUdpOutMessage(UdpMessageTypeEnum.DISCOVER_NEIGHBORS); - statistics.addUdpInMessage(UdpMessageTypeEnum.DISCOVER_NEIGHBORS); - statistics.addUdpOutMessage(UdpMessageTypeEnum.DISCOVER_FIND_NODE); - Assert.assertEquals(1, statistics.discoverInFindNode.getTotalCount()); - long inFindNodeCount = statistics.discoverInFindNode.getTotalCount(); - long outNeighbours = statistics.discoverOutNeighbours.getTotalCount(); - Assert.assertEquals(inFindNodeCount, outNeighbours); - - PongMessage pongMessage = new PongMessage(MessageTypes.P2P_PONG.asByte(), Hex.decode("C0")); - pongMessage.getData(); - String pongStr = pongMessage.toString(); - Assert.assertNotNull(pongStr); - statistics.addTcpInMessage(pongMessage); - statistics.addTcpOutMessage(pongMessage); - Assert.assertEquals(1, statistics.p2pInPong.getTotalCount()); - - DisconnectMessage disconnectMessage = new DisconnectMessage(Protocol.ReasonCode.TOO_MANY_PEERS); - Assert.assertEquals(Protocol.ReasonCode.TOO_MANY_PEERS, disconnectMessage.getReasonCode()); - statistics.addTcpInMessage(disconnectMessage); - statistics.addTcpOutMessage(disconnectMessage); - Assert.assertEquals(1, statistics.p2pOutDisconnect.getTotalCount()); - - SyncBlockChainMessage syncBlockChainMessage = new SyncBlockChainMessage(new ArrayList<>()); - String syncBlockChainStr = syncBlockChainMessage.toString(); - Assert.assertNotNull(syncBlockChainStr); - statistics.addTcpInMessage(syncBlockChainMessage); - statistics.addTcpOutMessage(syncBlockChainMessage); - Assert.assertEquals(1, statistics.tronInSyncBlockChain.getTotalCount()); - - ChainInventoryMessage chainInventoryMessage = new ChainInventoryMessage(new ArrayList<>(), 0L); - String chainInventoryMessageStr = chainInventoryMessage.toString(); - Assert.assertNotNull(chainInventoryMessageStr); - statistics.addTcpInMessage(chainInventoryMessage); - statistics.addTcpOutMessage(chainInventoryMessage); - Assert.assertEquals(1, statistics.tronOutBlockChainInventory.getTotalCount()); - - InventoryMessage invMsgTrx = - new InventoryMessage(new ArrayList<>(), Protocol.Inventory.InventoryType.TRX); - String inventoryMessageStr = invMsgTrx.toString(); - Assert.assertNotNull(inventoryMessageStr); - statistics.addTcpInMessage(invMsgTrx); - statistics.addTcpOutMessage(invMsgTrx); - InventoryMessage invMsgBlock = - new InventoryMessage(new ArrayList<>(), Protocol.Inventory.InventoryType.BLOCK); - MessageTypes invType = invMsgBlock.getInvMessageType(); - Assert.assertEquals(MessageTypes.BLOCK, invType); - statistics.addTcpInMessage(invMsgBlock); - statistics.addTcpOutMessage(invMsgBlock); - Assert.assertEquals(1, statistics.tronInBlockInventory.getTotalCount()); - - FetchInvDataMessage fetchInvDataTrx = - new FetchInvDataMessage(new ArrayList<>(), Protocol.Inventory.InventoryType.TRX); - statistics.addTcpInMessage(fetchInvDataTrx); - statistics.addTcpOutMessage(fetchInvDataTrx); - FetchInvDataMessage fetchInvDataBlock = - new FetchInvDataMessage(new ArrayList<>(), Protocol.Inventory.InventoryType.BLOCK); - statistics.addTcpInMessage(fetchInvDataBlock); - statistics.addTcpOutMessage(fetchInvDataBlock); - Assert.assertEquals(1, statistics.tronInTrxFetchInvData.getTotalCount()); - - TransactionsMessage transactionsMessage = - new TransactionsMessage(new LinkedList<>()); - statistics.addTcpInMessage(transactionsMessage); - statistics.addTcpOutMessage(transactionsMessage); - Assert.assertEquals(1, statistics.tronInTrxs.getTotalCount()); - - BlockCapsule blockCapsule = new BlockCapsule(1, Sha256Hash.ZERO_HASH, - System.currentTimeMillis(), Sha256Hash.ZERO_HASH.getByteString()); - BlockMessage blockMessage = new BlockMessage(blockCapsule); - statistics.addTcpInMessage(blockMessage); - statistics.addTcpOutMessage(blockMessage); - long inBlockCount = statistics.tronInBlock.getTotalCount(); - Assert.assertEquals(1, inBlockCount); - } -} diff --git a/framework/src/test/java/org/tron/common/overlay/discover/node/statistics/ReputationTest.java b/framework/src/test/java/org/tron/common/overlay/discover/node/statistics/ReputationTest.java deleted file mode 100644 index cee5d1d76dd..00000000000 --- a/framework/src/test/java/org/tron/common/overlay/discover/node/statistics/ReputationTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.tron.common.overlay.discover.node.statistics; - -import org.junit.Assert; -import org.junit.Test; - -public class ReputationTest { - - NodeStatistics nodeStatistics = new NodeStatistics(); - Reputation reputation = new Reputation(nodeStatistics); - - @Test - public void testGetScore() { - Assert.assertEquals(0, reputation.getScore()); - - nodeStatistics.messageStatistics.discoverInPong.add(3); - Assert.assertEquals(100, reputation.getScore()); - - nodeStatistics.messageStatistics.discoverOutPing.add(3); - Assert.assertEquals(200, reputation.getScore()); - - nodeStatistics.messageStatistics.discoverOutPing.add(1); - Assert.assertEquals(150, reputation.getScore()); - - nodeStatistics.tcpFlow.add(10240 * 5); - Assert.assertEquals(155, reputation.getScore()); - - nodeStatistics.discoverMessageLatency.add(100); - Assert.assertEquals(165, reputation.getScore()); - - nodeStatistics.notifyDisconnect(); - Assert.assertEquals(155, reputation.getScore()); - } -} diff --git a/framework/src/test/java/org/tron/common/overlay/discover/table/NodeEntryTest.java b/framework/src/test/java/org/tron/common/overlay/discover/table/NodeEntryTest.java deleted file mode 100644 index f30b02d3953..00000000000 --- a/framework/src/test/java/org/tron/common/overlay/discover/table/NodeEntryTest.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.tron.common.overlay.discover.table; - -import org.junit.Assert; -import org.junit.Test; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.utils.ByteArray; - -public class NodeEntryTest { - - @Test - public void test() throws InterruptedException { - Node node1 = Node.instanceOf("127.0.0.1:10001"); - NodeEntry nodeEntry = new NodeEntry(Node.getNodeId(), node1); - - long lastModified = nodeEntry.getModified(); - Thread.sleep(1); - nodeEntry.touch(); - long nowModified = nodeEntry.getModified(); - Assert.assertNotEquals(lastModified, nowModified); - - Node node2 = Node.instanceOf("127.0.0.1:10002"); - NodeEntry nodeEntry2 = new NodeEntry(Node.getNodeId(), node2); - boolean isDif = nodeEntry.equals(nodeEntry2); - Assert.assertTrue(isDif); - } - - @Test - public void testDistance() { - byte[] randomId = Node.getNodeId(); - String hexRandomIdStr = ByteArray.toHexString(randomId); - Assert.assertEquals(128, hexRandomIdStr.length()); - - byte[] nodeId1 = ByteArray.fromHexString( - "0000000000000000000000000000000000000000000000000000000000000000" - + "0000000000000000000000000000000000000000000000000000000000000000"); - byte[] nodeId2 = ByteArray.fromHexString( - "a000000000000000000000000000000000000000000000000000000000000000" - + "0000000000000000000000000000000000000000000000000000000000000000"); - Assert.assertEquals(256, NodeEntry.distance(nodeId1, nodeId2)); - - byte[] nodeId3 = ByteArray.fromHexString( - "0000000000000000000000000000000000000000000000000000000000000001" - + "0000000000000000000000000000000000000000000000000000000000000000"); - Assert.assertEquals(1, NodeEntry.distance(nodeId1, nodeId3)); - - byte[] nodeId4 = ByteArray.fromHexString( - "0000000000000000000000000000000000000000000000000000000000000000" - + "8000000000000000000000000000000000000000000000000000000000000000"); - Assert.assertEquals(0, NodeEntry.distance(nodeId1, nodeId4)); // => 0 - - byte[] nodeId5 = ByteArray.fromHexString( - "0000000000000000000000000000000000000000000000000000000000000000" - + "4000000000000000000000000000000000000000000000000000000000000000"); - Assert.assertEquals(-1, NodeEntry.distance(nodeId1, nodeId5)); // => 0 - - byte[] nodeId6 = ByteArray.fromHexString( - "0000000000000000000000000000000000000000000000000000000000000000" - + "2000000000000000000000000000000000000000000000000000000000000000"); - Assert.assertEquals(-2, NodeEntry.distance(nodeId1, nodeId6)); // => 0 - - byte[] nodeId7 = ByteArray.fromHexString( - "0000000000000000000000000000000000000000000000000000000000000000" - + "0000000000000000000000000000000000000000000000000000000000000001"); - Assert.assertEquals(-255, NodeEntry.distance(nodeId1, nodeId7)); // => 0 - - Assert.assertEquals(-256, NodeEntry.distance(nodeId1, nodeId1)); // => 0 - } - -} diff --git a/framework/src/test/java/org/tron/common/overlay/discover/table/NodeTableTest.java b/framework/src/test/java/org/tron/common/overlay/discover/table/NodeTableTest.java deleted file mode 100644 index ed43aea43a2..00000000000 --- a/framework/src/test/java/org/tron/common/overlay/discover/table/NodeTableTest.java +++ /dev/null @@ -1,206 +0,0 @@ -package org.tron.common.overlay.discover.table; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.testng.Assert; -import org.tron.common.overlay.discover.node.Node; - - -public class NodeTableTest { - - private static final Logger logger = LoggerFactory.getLogger("Test"); - private Node homeNode; - private NodeTable nodeTable; - private String[] ips; - private List ids; - - @Test - public void test() { - Node node1 = Node.instanceOf("127.0.0.5:10002"); - - NodeTable table = new NodeTable(node1); - Node nodeTemp = table.getNode(); - Assert.assertEquals(10002, nodeTemp.getPort()); - Assert.assertEquals(0, table.getNodesCount()); - Assert.assertEquals(0, table.getBucketsCount()); - - Node node2 = Node.instanceOf("127.0.0.1:10003"); - Node node3 = Node.instanceOf("127.0.0.2:10004"); - table.addNode(node2); - table.addNode(node3); - int bucketsCount = table.getBucketsCount(); - int nodeCount = table.getNodesCount(); - Assert.assertEquals(2, nodeCount); - Assert.assertTrue(bucketsCount > 0); - - boolean isExist = table.contains(node2); - table.touchNode(node2); - Assert.assertTrue(isExist); - - byte[] targetId = Node.getNodeId(); - List nodeList = table.getClosestNodes(targetId); - Assert.assertTrue(nodeList.isEmpty()); - //Assert.assertTrue(true); - } - - /** - * init nodes for test. - */ - @Before - public void init() { - ids = new ArrayList(); - for (int i = 0; i < KademliaOptions.BUCKET_SIZE + 1; i++) { - byte[] id = new byte[64]; - id[0] = 17; - id[1] = 16; - if (i < 10) { - id[63] = (byte) i; - } else { - id[62] = 1; - id[63] = (byte) (i - 10); - } - ids.add(id); - } - - ips = new String[KademliaOptions.BUCKET_SIZE + 1]; - byte[] homeId = new byte[64]; - homeNode = new Node(homeId, "127.0.0.1", 18888, 18888); - nodeTable = new NodeTable(homeNode); - ips[0] = "127.0.0.2"; - ips[1] = "127.0.0.3"; - ips[2] = "127.0.0.4"; - ips[3] = "127.0.0.5"; - ips[4] = "127.0.0.6"; - ips[5] = "127.0.0.7"; - ips[6] = "127.0.0.8"; - ips[7] = "127.0.0.9"; - ips[8] = "127.0.0.10"; - ips[9] = "127.0.0.11"; - ips[10] = "127.0.0.12"; - ips[11] = "127.0.0.13"; - ips[12] = "127.0.0.14"; - ips[13] = "127.0.0.15"; - ips[14] = "127.0.0.16"; - ips[15] = "127.0.0.17"; - ips[16] = "127.0.0.18"; - } - - @Test - public void addNodeTest() { - Node node = new Node(ids.get(0), ips[0], 18888, 18888); - Assert.assertEquals(0, nodeTable.getNodesCount()); - nodeTable.addNode(node); - Assert.assertEquals(1, nodeTable.getNodesCount()); - Assert.assertTrue(nodeTable.contains(node)); - } - - @Test - public void addDupNodeTest() throws Exception { - Node node = new Node(ids.get(0), ips[0], 18888, 18888); - nodeTable.addNode(node); - long firstTouchTime = nodeTable.getAllNodes().get(0).getModified(); - TimeUnit.MILLISECONDS.sleep(20); - nodeTable.addNode(node); - long lastTouchTime = nodeTable.getAllNodes().get(0).getModified(); - Assert.assertTrue(lastTouchTime > firstTouchTime); - Assert.assertEquals(1, nodeTable.getNodesCount()); - } - - @Test - public void addNode_bucketFullTest() throws Exception { - for (int i = 0; i < KademliaOptions.BUCKET_SIZE; i++) { - TimeUnit.MILLISECONDS.sleep(10); - addNode(new Node(ids.get(i), ips[i], 18888, 18888)); - } - Node lastSeen = nodeTable.addNode(new Node(ids.get(16), ips[16], 18888, 18888)); - Assert.assertTrue(null != lastSeen); - Assert.assertEquals(ips[15], lastSeen.getHost()); - } - - public void addNode(Node n) { - nodeTable.addNode(n); - } - - @Test - public void dropNodeTest() { - Node node = new Node(ids.get(0), ips[0], 18888, 18888); - nodeTable.addNode(node); - Assert.assertTrue(nodeTable.contains(node)); - nodeTable.dropNode(node); - Assert.assertTrue(!nodeTable.contains(node)); - nodeTable.addNode(node); - nodeTable.dropNode(new Node(ids.get(1), ips[0], 10000, 10000)); - Assert.assertTrue(!nodeTable.contains(node)); - } - - @Test - public void getBucketsCountTest() { - Assert.assertEquals(0, nodeTable.getBucketsCount()); - Node node = new Node(ids.get(0), ips[0], 18888, 18888); - nodeTable.addNode(node); - Assert.assertEquals(1, nodeTable.getBucketsCount()); - } - - @Test - public void touchNodeTest() throws Exception { - Node node = new Node(ids.get(0), ips[0], 18888, 18888); - nodeTable.addNode(node); - long firstTouchTime = nodeTable.getAllNodes().get(0).getModified(); - TimeUnit.MILLISECONDS.sleep(10); - nodeTable.touchNode(node); - long lastTouchTime = nodeTable.getAllNodes().get(0).getModified(); - Assert.assertTrue(firstTouchTime < lastTouchTime); - } - - @Test - public void containsTest() { - Node node = new Node(ids.get(0), ips[0], 18888, 18888); - Assert.assertTrue(!nodeTable.contains(node)); - nodeTable.addNode(node); - Assert.assertTrue(nodeTable.contains(node)); - } - - @Test - public void getBuckIdTest() { - Node node = new Node(ids.get(0), ips[0], 18888, 18888); //id: 11100...000 - nodeTable.addNode(node); - NodeEntry nodeEntry = new NodeEntry(homeNode.getId(), node); - Assert.assertEquals(252, nodeTable.getBucketId(nodeEntry)); - } - - @Test - public void getClosestNodes_nodesMoreThanBucketCapacity() throws Exception { - byte[] bytes = new byte[64]; - bytes[0] = 15; - Node nearNode = new Node(bytes, "127.0.0.19", 18888, 18888); - bytes[0] = 70; - Node farNode = new Node(bytes, "127.0.0.20", 18888, 18888); - nodeTable.addNode(nearNode); - nodeTable.addNode(farNode); - for (int i = 0; i < KademliaOptions.BUCKET_SIZE - 1; i++) { - //To control totally 17 nodes, however closest's capacity is 16 - nodeTable.addNode(new Node(ids.get(i), ips[i], 18888, 18888)); - TimeUnit.MILLISECONDS.sleep(10); - } - Assert.assertTrue(nodeTable.getBucketsCount() > 1); - //3 buckets, nearnode's distance is 252, far's is 255, others' are 253 - List closest = nodeTable.getClosestNodes(homeNode.getId()); - Assert.assertTrue(closest.contains(nearNode)); - //the farest node should be excluded - } - - @Test - public void getClosestNodes_isDiscoverNode() { - Node node = new Node(ids.get(0), ips[0], 18888); - //This constructor builds a node with isFakeNodeId = true - nodeTable.addNode(node); - List closest = nodeTable.getClosestNodes(homeNode.getId()); - Assert.assertTrue(closest.isEmpty()); - } - -} diff --git a/framework/src/test/java/org/tron/common/overlay/discover/table/TimeComparatorTest.java b/framework/src/test/java/org/tron/common/overlay/discover/table/TimeComparatorTest.java deleted file mode 100644 index b1c3a82f50e..00000000000 --- a/framework/src/test/java/org/tron/common/overlay/discover/table/TimeComparatorTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package org.tron.common.overlay.discover.table; - -import org.junit.Assert; -import org.junit.Test; -import org.tron.common.overlay.discover.node.Node; - -public class TimeComparatorTest { - - @Test - public void test() throws InterruptedException { - Node node1 = Node.instanceOf("127.0.0.1:10001"); - NodeEntry ne1 = new NodeEntry(Node.getNodeId(), node1); - Thread.sleep(1); - Node node2 = Node.instanceOf("127.0.0.1:10002"); - NodeEntry ne2 = new NodeEntry(Node.getNodeId(), node2); - TimeComparator tc = new TimeComparator(); - int result = tc.compare(ne1, ne2); - Assert.assertEquals(1, result); - - } -} diff --git a/framework/src/test/java/org/tron/common/utils/JsonUtilTest.java b/framework/src/test/java/org/tron/common/utils/JsonUtilTest.java index ae006f21521..9a4efb722e3 100644 --- a/framework/src/test/java/org/tron/common/utils/JsonUtilTest.java +++ b/framework/src/test/java/org/tron/common/utils/JsonUtilTest.java @@ -1,31 +1,35 @@ package org.tron.common.utils; -import java.util.ArrayList; -import java.util.List; +import lombok.Data; import org.junit.Assert; import org.junit.Test; -import org.tron.common.overlay.discover.node.DBNode; -import org.tron.common.overlay.discover.node.DBNodeStats; -import org.tron.common.overlay.discover.node.Node; public class JsonUtilTest { + @Data + public static class A { + private String key; + private int value; + + public A() {} + + public A(String key, int value) { + this.key = key; + this.value = value; + } + } + @Test public void test() { - DBNode dbNode = new DBNode(); - DBNodeStats dbNodeStats = new DBNodeStats(Node.getNodeId(), "1.0.0.1", 1000, 100); - List nodes = new ArrayList(); - nodes.add(dbNodeStats); - dbNode.setNodes(nodes); - - String jsonString = JsonUtil.obj2Json(dbNode); + A a1 = new A(); + a1.setKey("abc"); + a1.setValue(100); - DBNode dbNode2 = JsonUtil.json2Obj(jsonString, DBNode.class); + String jsonString = JsonUtil.obj2Json(a1); - dbNodeStats = dbNode2.getNodes().get(0); + A a2 = JsonUtil.json2Obj(jsonString, A.class); - Assert.assertEquals(dbNodeStats.getHost(), "1.0.0.1"); - Assert.assertEquals(dbNodeStats.getPort(), 1000); - Assert.assertEquals(dbNodeStats.getReputation(), 100); + Assert.assertEquals(a2.getKey(), "abc"); + Assert.assertEquals(a2.getValue(), 100); } } diff --git a/framework/src/test/java/org/tron/core/net/BaseNet.java b/framework/src/test/java/org/tron/core/net/BaseNet.java index cfd71080e4a..bb7ca85ef7f 100644 --- a/framework/src/test/java/org/tron/core/net/BaseNet.java +++ b/framework/src/test/java/org/tron/core/net/BaseNet.java @@ -113,7 +113,7 @@ public void destroy() { Collection peerConnections = ReflectUtils .invokeMethod(tronNetDelegate, "getActivePeer"); for (PeerConnection peer : peerConnections) { - peer.close(); + peer.getChannel().close(); } context.destroy(); diff --git a/framework/src/test/java/org/tron/core/net/BaseNetTest.java b/framework/src/test/java/org/tron/core/net/BaseNetTest.java index fee2695a80a..fd0847c1f08 100644 --- a/framework/src/test/java/org/tron/core/net/BaseNetTest.java +++ b/framework/src/test/java/org/tron/core/net/BaseNetTest.java @@ -9,10 +9,8 @@ public class BaseNetTest extends BaseNet { @Test - public void test() throws Exception { + public void test() { new NodeInfoServiceTest(context).test(); - new UdpTest(context).test(); - new TcpTest(context).test(); new DelegationServiceTest(context).test(); } } diff --git a/framework/src/test/java/org/tron/core/net/DisconnectMessageTest.java b/framework/src/test/java/org/tron/core/net/DisconnectMessageTest.java index 0155e5e5356..8ba255ccf08 100644 --- a/framework/src/test/java/org/tron/core/net/DisconnectMessageTest.java +++ b/framework/src/test/java/org/tron/core/net/DisconnectMessageTest.java @@ -4,13 +4,11 @@ import org.tron.protos.Protocol.DisconnectMessageOrBuilder; public class DisconnectMessageTest extends com.google.protobuf.GeneratedMessageV3 implements - // @@protoc_insertion_point(message_implements:protocol.DisconnectMessage) DisconnectMessageOrBuilder { public static final int REASON_FIELD_NUMBER = 1; public static final int NAME_FIELD_NUMBER = 2; private static final long serialVersionUID = 0L; - // @@protoc_insertion_point(class_scope:protocol.DisconnectMessage) private static final DisconnectMessageTest DEFAULT_INSTANCE; private static final com.google.protobuf.Parser PARSER = new com.google.protobuf.AbstractParser() { diff --git a/framework/src/test/java/org/tron/core/net/MessageTest.java b/framework/src/test/java/org/tron/core/net/MessageTest.java index 6d344dd61bc..0400d16b669 100644 --- a/framework/src/test/java/org/tron/core/net/MessageTest.java +++ b/framework/src/test/java/org/tron/core/net/MessageTest.java @@ -2,9 +2,9 @@ import org.junit.Assert; import org.junit.Test; -import org.tron.common.overlay.message.DisconnectMessage; import org.tron.core.exception.P2pException; import org.tron.core.net.message.MessageTypes; +import org.tron.core.net.message.base.DisconnectMessage; import org.tron.protos.Protocol.ReasonCode; public class MessageTest { diff --git a/framework/src/test/java/org/tron/core/net/TcpTest.java b/framework/src/test/java/org/tron/core/net/TcpTest.java deleted file mode 100644 index 5884efffd91..00000000000 --- a/framework/src/test/java/org/tron/core/net/TcpTest.java +++ /dev/null @@ -1,294 +0,0 @@ -package org.tron.core.net; - -import static org.tron.core.net.message.MessageTypes.P2P_DISCONNECT; -import static org.tron.core.net.message.MessageTypes.P2P_HELLO; -import static org.tron.protos.Protocol.ReasonCode.DUPLICATE_PEER; -import static org.tron.protos.Protocol.ReasonCode.FORKED; -import static org.tron.protos.Protocol.ReasonCode.INCOMPATIBLE_CHAIN; -import static org.tron.protos.Protocol.ReasonCode.INCOMPATIBLE_VERSION; -import static org.tron.protos.Protocol.ReasonCode.LIGHT_NODE_SYNC_FAIL; - -import com.google.common.cache.CacheBuilder; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.channel.Channel; -import io.netty.channel.ChannelFutureListener; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ByteToMessageDecoder; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.TimeUnit; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.ArrayUtils; -import org.junit.Assert; -import org.tron.common.application.TronApplicationContext; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.overlay.discover.node.NodeManager; -import org.tron.common.overlay.message.DisconnectMessage; -import org.tron.common.overlay.message.HelloMessage; -import org.tron.common.overlay.message.Message; -import org.tron.common.overlay.message.P2pMessage; -import org.tron.common.overlay.message.P2pMessageFactory; -import org.tron.common.overlay.server.ChannelManager; -import org.tron.common.overlay.server.SyncPool; -import org.tron.common.utils.ByteArray; -import org.tron.common.utils.ReflectUtils; -import org.tron.core.ChainBaseManager; -import org.tron.core.capsule.BlockCapsule; -import org.tron.core.capsule.BlockCapsule.BlockId; -import org.tron.core.capsule.BytesCapsule; -import org.tron.core.config.args.Args; -import org.tron.core.db.Manager; -import org.tron.core.net.message.BlockMessage; -import org.tron.core.net.peer.PeerConnection; -import org.tron.protos.Protocol; -import org.tron.protos.Protocol.Block; - -@Slf4j -public class TcpTest { - - Node node = Node.instanceOf("127.0.0.1:" + Args.getInstance().getNodeListenPort()); - private ChannelManager channelManager; - private Manager manager; - private ChainBaseManager chainBaseManager; - private SyncPool pool; - private TronNetDelegate tronNetDelegate; - private int tryTimes = 10; - private int sleepTime = 1000; - private boolean finish = false; - - public TcpTest(TronApplicationContext context) { - channelManager = context.getBean(ChannelManager.class); - manager = context.getBean(Manager.class); - chainBaseManager = context.getBean(ChainBaseManager.class); - pool = context.getBean(SyncPool.class); - tronNetDelegate = context.getBean(TronNetDelegate.class); - } - - public void normalTest() throws InterruptedException { - Channel channel = BaseNet.connect(new HandshakeHandler(TestType.normal)); - HelloMessage message = new HelloMessage(node, System.currentTimeMillis(), chainBaseManager); - sendMessage(channel, message); - validResultCloseConnect(channel); - } - - public void errorGenesisBlockIdTest() throws InterruptedException { - Channel channel = BaseNet.connect(new HandshakeHandler(TestType.errorGenesisBlock)); - HelloMessage message = new HelloMessage(node, System.currentTimeMillis(), chainBaseManager); - Protocol.HelloMessage.BlockId genesisBlockId = Protocol.HelloMessage.BlockId.newBuilder() - .setHash(new BlockId().getByteString()) - .setNumber(new BlockId().getNum()) - .build(); - message.setHelloMessage( - message.getHelloMessage().toBuilder().setGenesisBlockId(genesisBlockId).build()); - sendMessage(channel, message); - validResultCloseConnect(channel); - } - - public void errorVersionTest() throws InterruptedException { - Channel channel = BaseNet.connect(new HandshakeHandler(TestType.errorVersion)); - Args.getInstance().setNodeP2pVersion(1); - HelloMessage message = new HelloMessage(node, System.currentTimeMillis(), chainBaseManager); - Args.getInstance().setNodeP2pVersion(2); - sendMessage(channel, message); - validResultCloseConnect(channel); - } - - public void errorSolidBlockIdTest() throws InterruptedException { - Channel channel = BaseNet.connect(new HandshakeHandler(TestType.errorSolid)); - HelloMessage message = new HelloMessage(node, System.currentTimeMillis(), chainBaseManager); - Protocol.HelloMessage.BlockId sBlockId = Protocol.HelloMessage.BlockId.newBuilder() - .setHash(new BlockId().getByteString()) - .setNumber(new BlockId().getNum()) - .build(); - message.setHelloMessage( - message.getHelloMessage().toBuilder().setSolidBlockId(sBlockId).build()); - sendMessage(channel, message); - validResultCloseConnect(channel); - } - - public void repeatConnectTest() throws InterruptedException { - Channel channel = BaseNet.connect(new HandshakeHandler(TestType.normal)); - HelloMessage message = new HelloMessage(node, System.currentTimeMillis(), chainBaseManager); - sendMessage(channel, message); - validResultUnCloseConnect(); - Channel repeatChannel = BaseNet.connect(new HandshakeHandler(TestType.repeatConnect)); - sendMessage(repeatChannel, message); - validResultCloseConnect(repeatChannel); - clearConnect(channel); - } - - public void unHandshakeTest() throws InterruptedException { - List beforeActivePeers = - ReflectUtils.getFieldValue(pool, "activePeers"); - int beforeSize = beforeActivePeers.size(); - Channel channel = BaseNet.connect(new HandshakeHandler(TestType.normal)); - BlockMessage message = new BlockMessage(new BlockCapsule(Block.getDefaultInstance())); - sendMessage(channel, message); - List afterActivePeers = - ReflectUtils.getFieldValue(pool, "activePeers"); - int afterSize = afterActivePeers.size(); - Assert.assertEquals(beforeSize, afterSize); - clearConnect(channel); - } - - public void errorMsgTest() throws InterruptedException { - Channel channel = BaseNet.connect(new HandshakeHandler(TestType.normal)); - HelloMessage message = new HelloMessage(node, System.currentTimeMillis(), chainBaseManager); - sendMessage(channel, message); - validResultUnCloseConnect(); - List beforeActivePeers = - ReflectUtils.getFieldValue(pool, "activePeers"); - int beforeSize = beforeActivePeers.size(); - logger.info("beforeSize : {}", beforeSize); - channel.writeAndFlush( - Unpooled.wrappedBuffer(ArrayUtils.add("nihao".getBytes(), 0, (byte) 1))) - .addListener((ChannelFutureListener) future -> { - if (future.isSuccess()) { - logger.info("send msg success"); - } else { - logger.error("send msg fail", future.cause()); - } - }); - Thread.sleep(2000); - List afterActivePeers = - ReflectUtils.getFieldValue(pool, "activePeers"); - int afterSize = afterActivePeers.size(); - logger.info("afterSize : {}", afterSize); - Assert.assertEquals(beforeSize, afterSize + 1); - clearConnect(channel); - } - - public void errorLowestBlockNumTest() throws InterruptedException { - Channel channel = BaseNet.connect(new HandshakeHandler(TestType.errorLowestBlockNum)); - HelloMessage message = new HelloMessage(node, System.currentTimeMillis(), chainBaseManager); - message.setHelloMessage( - message.getHelloMessage().toBuilder().setNodeType(1).setLowestBlockNum(100).build()); - sendMessage(channel, message); - validResultCloseConnect(channel); - } - - private void sendMessage(Channel channel, Message message) { - channel.writeAndFlush(message.getSendData()) - .addListener((ChannelFutureListener) future -> { - if (future.isSuccess()) { - logger.info("send msg success"); - } else { - logger.error("send msg fail", future.cause()); - } - }); - } - - private void validResultCloseConnect(Channel channel) throws InterruptedException { - int trys = 0; - while (!finish && ++trys < tryTimes) { - Thread.sleep(sleepTime); - } - Assert.assertEquals(finish, true); - finish = false; - channel.close(); - Thread.sleep(sleepTime); - Collection peerConnections = ReflectUtils - .invokeMethod(tronNetDelegate, "getActivePeer"); - for (PeerConnection peer : peerConnections) { - peer.close(); - } - ReflectUtils.setFieldValue(channelManager, "recentlyDisconnected", - CacheBuilder.newBuilder().maximumSize(1000) - .expireAfterWrite(30, TimeUnit.SECONDS).recordStats().build()); - } - - private void validResultUnCloseConnect() throws InterruptedException { - int n = 0; - while (!finish && ++n < tryTimes) { - Thread.sleep(sleepTime); - } - Assert.assertEquals(finish, true); - finish = false; - } - - private void clearConnect(Channel channel) throws InterruptedException { - channel.close(); - Thread.sleep(sleepTime); - Collection peerConnections = ReflectUtils - .invokeMethod(tronNetDelegate, "getActivePeer"); - for (PeerConnection peer : peerConnections) { - peer.close(); - } - ReflectUtils.setFieldValue(channelManager, "recentlyDisconnected", - CacheBuilder.newBuilder().maximumSize(1000) - .expireAfterWrite(30, TimeUnit.SECONDS).recordStats().build()); - } - - public void test() throws InterruptedException { - logger.info("begin normal test "); - normalTest(); - logger.info("begin errorGenesisBlockId test "); - errorGenesisBlockIdTest(); - logger.info("begin errorVersion test "); - errorVersionTest(); - logger.info("begin errorSolidBlockId test "); - errorSolidBlockIdTest(); - logger.info("begin repeatConnect test"); - repeatConnectTest(); - logger.info("begin unHandshake test"); - unHandshakeTest(); - logger.info("begin errorMsg test"); - errorLowestBlockNumTest(); - logger.info("begin errorLowestBlockNum test"); - errorMsgTest(); - } - - private enum TestType { - normal, errorGenesisBlock, errorVersion, errorSolid, - repeatConnect, errorLowestBlockNum - } - - private class HandshakeHandler extends ByteToMessageDecoder { - - private P2pMessageFactory messageFactory = new P2pMessageFactory(); - - private TestType testType; - - public HandshakeHandler(TestType testType) { - this.testType = testType; - } - - @Override - protected void decode(ChannelHandlerContext ctx, ByteBuf buffer, List out) - throws Exception { - byte[] encoded = new byte[buffer.readableBytes()]; - buffer.readBytes(encoded); - P2pMessage msg = messageFactory.create(encoded); - switch (testType) { - case normal: - Assert.assertEquals(msg.getType(), P2P_HELLO); - break; - case errorGenesisBlock: - Assert.assertEquals(msg.getType(), P2P_DISCONNECT); - Assert.assertEquals(((DisconnectMessage) msg).getReasonCode(), INCOMPATIBLE_CHAIN); - break; - case errorVersion: - Assert.assertEquals(msg.getType(), P2P_DISCONNECT); - Assert.assertEquals(((DisconnectMessage) msg).getReasonCode(), INCOMPATIBLE_VERSION); - break; - case errorSolid: - Assert.assertEquals(msg.getType(), P2P_DISCONNECT); - Assert.assertEquals(((DisconnectMessage) msg).getReasonCode(), FORKED); - break; - case repeatConnect: - Assert.assertEquals(msg.getType(), P2P_DISCONNECT); - Assert.assertEquals(((DisconnectMessage) msg).getReasonCode(), DUPLICATE_PEER); - break; - case errorLowestBlockNum: - Assert.assertEquals(msg.getType(), P2P_DISCONNECT); - Assert.assertEquals(((DisconnectMessage) msg).getReasonCode(), LIGHT_NODE_SYNC_FAIL); - break; - default: - break; - } - - finish = true; - } - } -} diff --git a/framework/src/test/java/org/tron/core/net/UdpTest.java b/framework/src/test/java/org/tron/core/net/UdpTest.java deleted file mode 100644 index 6d90e4e748a..00000000000 --- a/framework/src/test/java/org/tron/core/net/UdpTest.java +++ /dev/null @@ -1,127 +0,0 @@ -package org.tron.core.net; - -import java.net.DatagramPacket; -import java.net.DatagramSocket; -import java.net.InetAddress; -import java.util.Arrays; -import java.util.List; -import lombok.extern.slf4j.Slf4j; -import org.junit.Assert; -import org.testng.collections.Lists; -import org.tron.common.application.TronApplicationContext; -import org.tron.common.net.udp.message.Message; -import org.tron.common.net.udp.message.discover.FindNodeMessage; -import org.tron.common.net.udp.message.discover.NeighborsMessage; -import org.tron.common.net.udp.message.discover.PingMessage; -import org.tron.common.net.udp.message.discover.PongMessage; -import org.tron.common.overlay.discover.node.Node; -import org.tron.common.overlay.discover.node.NodeManager; -import org.tron.core.config.args.Args; - -@Slf4j -public class UdpTest { - - private NodeManager nodeManager; - private int port = Args.getInstance().getNodeListenPort(); - //private volatile boolean finishFlag = false; - //private long timeOut = 30_000; - - public UdpTest(TronApplicationContext context) { - nodeManager = context.getBean(NodeManager.class); - } - - public void test() throws Exception { - /* - Thread thread = new Thread(() -> { - try { - discover(); - } catch (Exception e) { - logger.info("Discover test failed.", e); - } - }); - thread.start(); - - long time = System.currentTimeMillis(); - while (!finishFlag && System.currentTimeMillis() - time < timeOut) { - Thread.sleep(1000); - } - if (!finishFlag) { - thread.interrupt(); - Assert.assertTrue(false); - } - */ - } - - public void discover() throws Exception { - - InetAddress server = InetAddress.getByName("127.0.0.1"); - - Node from = Node.instanceOf("127.0.0.1:10002"); - Node peer1 = Node.instanceOf("127.0.0.1:10003"); - Node peer2 = Node.instanceOf("127.0.0.1:10004"); - - Assert.assertTrue(!nodeManager.hasNodeHandler(peer1)); - Assert.assertTrue(!nodeManager.hasNodeHandler(peer2)); - Assert.assertTrue(nodeManager.getTable().getAllNodes().isEmpty()); - - PingMessage pingMessage = new PingMessage(from, nodeManager.getPublicHomeNode()); - DatagramPacket pingPacket = new DatagramPacket(pingMessage.getSendData(), - pingMessage.getSendData().length, server, port); - - FindNodeMessage findNodeMessage = new FindNodeMessage(from, Node.getNodeId()); - DatagramPacket findNodePacket = new DatagramPacket(findNodeMessage.getSendData(), - findNodeMessage.getSendData().length, server, port); - - DatagramSocket socket = new DatagramSocket(); - - // send ping msg - socket.send(pingPacket); - byte[] data = new byte[1024]; - DatagramPacket packet = new DatagramPacket(data, data.length); - - boolean pingFlag = false; - boolean pongFlag = false; - boolean findNodeFlag = false; - boolean neighborsFlag = false; - while (true) { - socket.receive(packet); - byte[] bytes = Arrays.copyOfRange(data, 0, packet.getLength()); - Message msg = Message.parse(bytes); - Assert.assertTrue( - Arrays.equals(msg.getFrom().getId(), nodeManager.getPublicHomeNode().getId())); - if (!pingFlag) { - pingFlag = true; - Assert.assertTrue(msg instanceof PingMessage); - Assert.assertTrue(Arrays.equals(((PingMessage) msg).getTo().getId(), from.getId())); - PongMessage pongMessage = new PongMessage(from); - DatagramPacket pongPacket = new DatagramPacket(pongMessage.getSendData(), - pongMessage.getSendData().length, server, port); - socket.send(pongPacket); - } else if (!pongFlag) { - pongFlag = true; - Assert.assertTrue(msg instanceof PongMessage); - } else if (!findNodeFlag) { - findNodeFlag = true; - Assert.assertTrue(msg instanceof FindNodeMessage); - List peers = Lists.newArrayList(peer1, peer2); - NeighborsMessage neighborsMessage = new NeighborsMessage(from, peers, msg.getTimestamp()); - DatagramPacket neighborsPacket = new DatagramPacket(neighborsMessage.getSendData(), - neighborsMessage.getSendData().length, server, port); - socket.send(neighborsPacket); - socket.send(findNodePacket); - } else if (!neighborsFlag) { - Assert.assertTrue(msg instanceof NeighborsMessage); - break; - } - } - - Assert.assertTrue(nodeManager.hasNodeHandler(peer1)); - Assert.assertTrue(nodeManager.hasNodeHandler(peer2)); - Assert.assertTrue(nodeManager.getTable().getAllNodes().size() == 1); - - socket.close(); - - //finishFlag = true; - } -} - diff --git a/framework/src/test/java/org/tron/core/net/messagehandler/BlockMsgHandlerTest.java b/framework/src/test/java/org/tron/core/net/messagehandler/BlockMsgHandlerTest.java index f5edf091a13..5b2abfd0705 100644 --- a/framework/src/test/java/org/tron/core/net/messagehandler/BlockMsgHandlerTest.java +++ b/framework/src/test/java/org/tron/core/net/messagehandler/BlockMsgHandlerTest.java @@ -7,7 +7,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.testng.collections.Lists; import org.tron.common.application.TronApplicationContext; import org.tron.common.utils.Sha256Hash; import org.tron.core.Constant; @@ -16,7 +15,7 @@ import org.tron.core.config.Parameter; import org.tron.core.config.args.Args; import org.tron.core.exception.P2pException; -import org.tron.core.net.message.BlockMessage; +import org.tron.core.net.message.adv.BlockMessage; import org.tron.core.net.peer.Item; import org.tron.core.net.peer.PeerConnection; import org.tron.protos.Protocol.Inventory.InventoryType; diff --git a/framework/src/test/java/org/tron/core/net/messagehandler/ChainInventoryMsgHandlerTest.java b/framework/src/test/java/org/tron/core/net/messagehandler/ChainInventoryMsgHandlerTest.java index ee99a7dde6f..95cb9d0597f 100644 --- a/framework/src/test/java/org/tron/core/net/messagehandler/ChainInventoryMsgHandlerTest.java +++ b/framework/src/test/java/org/tron/core/net/messagehandler/ChainInventoryMsgHandlerTest.java @@ -9,7 +9,7 @@ import org.tron.core.capsule.BlockCapsule.BlockId; import org.tron.core.config.Parameter.NetConstants; import org.tron.core.exception.P2pException; -import org.tron.core.net.message.ChainInventoryMessage; +import org.tron.core.net.message.sync.ChainInventoryMessage; import org.tron.core.net.peer.PeerConnection; public class ChainInventoryMsgHandlerTest { diff --git a/framework/src/test/java/org/tron/core/net/messagehandler/InventoryMsgHandlerTest.java b/framework/src/test/java/org/tron/core/net/messagehandler/InventoryMsgHandlerTest.java index 23a44ac3444..97db6207b2a 100644 --- a/framework/src/test/java/org/tron/core/net/messagehandler/InventoryMsgHandlerTest.java +++ b/framework/src/test/java/org/tron/core/net/messagehandler/InventoryMsgHandlerTest.java @@ -1,20 +1,24 @@ package org.tron.core.net.messagehandler; +import java.lang.reflect.Field; +import java.net.InetAddress; +import java.net.InetSocketAddress; import java.util.ArrayList; import org.junit.Test; -import org.tron.core.net.message.InventoryMessage; +import org.tron.core.net.message.adv.InventoryMessage; import org.tron.core.net.peer.PeerConnection; +import org.tron.p2p.connection.Channel; import org.tron.protos.Protocol.Inventory.InventoryType; public class InventoryMsgHandlerTest { private InventoryMsgHandler handler = new InventoryMsgHandler(); - private PeerConnection peer = new PeerConnection(); @Test - public void testProcessMessage() { + public void testProcessMessage() throws Exception { InventoryMessage msg = new InventoryMessage(new ArrayList<>(), InventoryType.TRX); - + PeerConnection peer = new PeerConnection(); + peer.setChannel(getChannel("1.0.0.3", 1000)); peer.setNeedSyncFromPeer(true); peer.setNeedSyncFromUs(true); handler.processMessage(peer, msg); @@ -28,4 +32,20 @@ public void testProcessMessage() { handler.processMessage(peer, msg); } + + private Channel getChannel(String host, int port) throws Exception { + Channel channel = new Channel(); + InetSocketAddress inetSocketAddress = new InetSocketAddress(host, port); + + Field field = channel.getClass().getDeclaredField("inetSocketAddress"); + field.setAccessible(true); + field.set(channel, inetSocketAddress); + + InetAddress inetAddress = inetSocketAddress.getAddress(); + field = channel.getClass().getDeclaredField("inetAddress"); + field.setAccessible(true); + field.set(channel, inetAddress); + + return channel; + } } diff --git a/framework/src/test/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandlerTest.java b/framework/src/test/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandlerTest.java index 9749c59d911..a0c37d246da 100644 --- a/framework/src/test/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandlerTest.java +++ b/framework/src/test/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandlerTest.java @@ -4,7 +4,7 @@ import org.junit.Assert; import org.junit.Test; import org.tron.core.exception.P2pException; -import org.tron.core.net.message.SyncBlockChainMessage; +import org.tron.core.net.message.sync.SyncBlockChainMessage; import org.tron.core.net.peer.PeerConnection; public class SyncBlockChainMsgHandlerTest { diff --git a/framework/src/test/java/org/tron/core/net/services/AdvServiceTest.java b/framework/src/test/java/org/tron/core/net/services/AdvServiceTest.java index d850ab66958..a6189346ae5 100644 --- a/framework/src/test/java/org/tron/core/net/services/AdvServiceTest.java +++ b/framework/src/test/java/org/tron/core/net/services/AdvServiceTest.java @@ -9,7 +9,6 @@ import org.junit.Before; import org.junit.Test; import org.tron.common.application.TronApplicationContext; -import org.tron.common.overlay.server.SyncPool; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.FileUtil; import org.tron.common.utils.ReflectUtils; @@ -18,21 +17,22 @@ import org.tron.core.capsule.BlockCapsule; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; -import org.tron.core.net.message.BlockMessage; -import org.tron.core.net.message.TransactionMessage; +import org.tron.core.net.P2pEventHandlerImpl; +import org.tron.core.net.message.adv.BlockMessage; +import org.tron.core.net.message.adv.TransactionMessage; import org.tron.core.net.peer.Item; import org.tron.core.net.peer.PeerConnection; -import org.tron.core.net.service.AdvService; +import org.tron.core.net.service.adv.AdvService; +import org.tron.p2p.P2pEventHandler; import org.tron.protos.Protocol; import org.tron.protos.Protocol.Inventory.InventoryType; -//@Ignore public class AdvServiceTest { protected TronApplicationContext context; private AdvService service; private PeerConnection peer; - private SyncPool syncPool; + private P2pEventHandlerImpl p2pEventHandler; private String dbPath = "output-adv-service-test"; /** @@ -52,7 +52,6 @@ public void init() { @After public void destroy() { Args.clearParam(); - context.destroy(); FileUtil.deleteDir(new File(dbPath)); } @@ -60,7 +59,6 @@ public void destroy() { public void test() { testAddInv(); testBroadcast(); - //testFastSend(); testTrxBroadcast(); } @@ -87,51 +85,22 @@ private void testBroadcast() { try { peer = context.getBean(PeerConnection.class); - syncPool = context.getBean(SyncPool.class); + p2pEventHandler = context.getBean(P2pEventHandlerImpl.class); List peers = Lists.newArrayList(); peers.add(peer); - ReflectUtils.setFieldValue(syncPool, "activePeers", peers); + ReflectUtils.setFieldValue(P2pEventHandler.class, "peers", peers); BlockCapsule blockCapsule = new BlockCapsule(1, Sha256Hash.ZERO_HASH, System.currentTimeMillis(), Sha256Hash.ZERO_HASH.getByteString()); BlockMessage msg = new BlockMessage(blockCapsule); service.broadcast(msg); Item item = new Item(blockCapsule.getBlockId(), InventoryType.BLOCK); Assert.assertNotNull(service.getMessage(item)); - - peer.close(); - syncPool.close(); } catch (NullPointerException e) { System.out.println(e); } } - /* - private void testFastSend() { - - try { - peer = context.getBean(PeerConnection.class); - syncPool = context.getBean(SyncPool.class); - - List peers = Lists.newArrayList(); - peers.add(peer); - ReflectUtils.setFieldValue(syncPool, "activePeers", peers); - BlockCapsule blockCapsule = new BlockCapsule(1, Sha256Hash.ZERO_HASH, - System.currentTimeMillis(), Sha256Hash.ZERO_HASH.getByteString()); - BlockMessage msg = new BlockMessage(blockCapsule); - service.fastForward(msg); - Item item = new Item(blockCapsule.getBlockId(), InventoryType.BLOCK); - //Assert.assertNull(service.getMessage(item)); - - peer.getAdvInvRequest().put(item, System.currentTimeMillis()); - service.onDisconnect(peer); - peer.close(); - syncPool.close(); - } catch (NullPointerException e) { - System.out.println(e); - } - } - */ private void testTrxBroadcast() { Protocol.Transaction trx = Protocol.Transaction.newBuilder().build(); diff --git a/framework/src/test/java/org/tron/core/net/services/RelayServiceTest.java b/framework/src/test/java/org/tron/core/net/services/RelayServiceTest.java index 69dffe7ab29..b4deb6aff4b 100644 --- a/framework/src/test/java/org/tron/core/net/services/RelayServiceTest.java +++ b/framework/src/test/java/org/tron/core/net/services/RelayServiceTest.java @@ -2,21 +2,18 @@ import com.google.common.collect.Lists; import com.google.protobuf.ByteString; - import java.io.File; import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.Set; - import org.bouncycastle.util.encoders.Hex; import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.tron.common.application.TronApplicationContext; -import org.tron.common.overlay.server.SyncPool; import org.tron.common.utils.FileUtil; import org.tron.common.utils.ReflectUtils; import org.tron.core.ChainBaseManager; @@ -25,10 +22,11 @@ import org.tron.core.capsule.WitnessCapsule; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; -import org.tron.core.net.message.BlockMessage; +import org.tron.core.net.P2pEventHandlerImpl; +import org.tron.core.net.message.adv.BlockMessage; import org.tron.core.net.peer.Item; import org.tron.core.net.peer.PeerConnection; -import org.tron.core.net.service.RelayService; +import org.tron.core.net.service.relay.RelayService; import org.tron.protos.Protocol; public class RelayServiceTest { @@ -37,7 +35,7 @@ public class RelayServiceTest { private RelayService service; private ChainBaseManager chainBaseManager; private PeerConnection peer; - private SyncPool syncPool; + private P2pEventHandlerImpl p2pEventHandler; private String dbPath = "output-relay-service-test"; /** @@ -50,6 +48,7 @@ public void init() { context = new TronApplicationContext(DefaultConfig.class); service = context.getBean(RelayService.class); chainBaseManager = context.getBean(ChainBaseManager.class); + p2pEventHandler = context.getBean(P2pEventHandlerImpl.class); } @After @@ -108,11 +107,11 @@ private void testBroadcast() { peer.setAddress(getFromHexString("A0299F3DB80A24B20A254B89CE639D59132F157F13")); peer.setNeedSyncFromPeer(false); peer.setNeedSyncFromUs(false); - syncPool = context.getBean(SyncPool.class); + p2pEventHandler = context.getBean(P2pEventHandlerImpl.class); List peers = Lists.newArrayList(); peers.add(peer); - ReflectUtils.setFieldValue(syncPool, "activePeers", peers); + ReflectUtils.setFieldValue(p2pEventHandler, "activePeers", peers); BlockCapsule blockCapsule = new BlockCapsule(chainBaseManager.getHeadBlockNum() + 1, chainBaseManager.getHeadBlockId(), 0, getFromHexString("A04711BF7AFBDF44557DEFBDF4C4E7AA6138C6331F")); @@ -121,8 +120,7 @@ private void testBroadcast() { Item item = new Item(blockCapsule.getBlockId(), Protocol.Inventory.InventoryType.BLOCK); Assert.assertEquals(1, peer.getAdvInvSpread().size()); Assert.assertNotNull(peer.getAdvInvSpread().getIfPresent(item)); - peer.close(); - syncPool.close(); + peer.getChannel().close(); } catch (NullPointerException e) { System.out.println(e); } diff --git a/framework/src/test/java/org/tron/program/SolidityNodeTest.java b/framework/src/test/java/org/tron/program/SolidityNodeTest.java index 307b44d0b9c..0bd2dccfe85 100755 --- a/framework/src/test/java/org/tron/program/SolidityNodeTest.java +++ b/framework/src/test/java/org/tron/program/SolidityNodeTest.java @@ -9,7 +9,7 @@ import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; -import org.tron.common.overlay.client.DatabaseGrpcClient; +import org.tron.common.client.DatabaseGrpcClient; import org.tron.core.Constant; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; From 27ca2453fe3a7fb2d0153fce65bcab2ae44c5f86 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Tue, 15 Nov 2022 15:36:00 +0800 Subject: [PATCH 02/13] feat(net): optimize p2p service shutdown logic --- .../src/main/java/org/tron/core/net/TronNetService.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/framework/src/main/java/org/tron/core/net/TronNetService.java b/framework/src/main/java/org/tron/core/net/TronNetService.java index c46c4688071..5f8bbbe8ad2 100644 --- a/framework/src/main/java/org/tron/core/net/TronNetService.java +++ b/framework/src/main/java/org/tron/core/net/TronNetService.java @@ -32,7 +32,7 @@ public class TronNetService { private static P2pConfig p2pConfig; @Getter - private static P2pService p2pService; + private static P2pService p2pService = new P2pService(); @Autowired private AdvService advService; @@ -63,10 +63,12 @@ public class TronNetService { @Autowired private TronStatsManager tronStatsManager; + private volatile boolean init; + public void start() { try { + init = true; p2pConfig = getConfig(); - p2pService = new P2pService(); p2pService.start(p2pConfig); p2pService.register(p2pEventHandler); advService.init(); @@ -85,6 +87,9 @@ public void start() { } public void close() { + if (!init) { + return; + } PeerManager.close(); tronStatsManager.close(); nodePersistService.close(); From ce4f290eb505cf826e7771a401e73b1fb1f4eda8 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Tue, 15 Nov 2022 17:08:44 +0800 Subject: [PATCH 03/13] feat(net): update p2p version to v0.1.4 --- framework/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/framework/build.gradle b/framework/build.gradle index 9e1e57413ea..bbbeaee8a63 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -44,10 +44,10 @@ dependencies { testCompile group: 'org.testng', name: 'testng', version: '6.14.3' - compile group: 'com.github.tronprotocol', name: 'libp2p', version: 'test-v0.1.3' - compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' + compile group: 'com.github.tronprotocol', name: 'libp2p', version: 'test-v0.1.4' + compile group: 'com.typesafe', name: 'config', version: '1.3.2' compile "com.cedarsoftware:java-util:1.8.0" From 68254125bcd33aa2783a56dff559d63138f40ba7 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Tue, 15 Nov 2022 18:35:43 +0800 Subject: [PATCH 04/13] feat(net): solve sonar problems --- .../java/org/tron/core/net/P2pEventHandlerImpl.java | 12 ++++++++---- .../main/java/org/tron/core/net/TronNetService.java | 5 ++++- .../java/org/tron/core/net/peer/PeerConnection.java | 4 ++-- .../core/net/service/handshake/HandshakeService.java | 1 - .../core/net/service/keepalive/KeepAliveService.java | 2 +- .../core/net/service/statistics/NodeStatistics.java | 12 ++++++------ .../net/service/statistics/TronStatsManager.java | 8 ++++---- 7 files changed, 25 insertions(+), 19 deletions(-) diff --git a/framework/src/main/java/org/tron/core/net/P2pEventHandlerImpl.java b/framework/src/main/java/org/tron/core/net/P2pEventHandlerImpl.java index 6a50ccb0a2e..d19134b62ae 100644 --- a/framework/src/main/java/org/tron/core/net/P2pEventHandlerImpl.java +++ b/framework/src/main/java/org/tron/core/net/P2pEventHandlerImpl.java @@ -131,12 +131,14 @@ public void onMessage(Channel c, byte[] data) { private void processMessage(PeerConnection peer, byte[] data) { long startTime = System.currentTimeMillis(); TronMessage msg = null; + MessageTypes type = null; try { msg = TronMessageFactory.create(data); + type = msg.getType(); peer.getPeerStatistics().messageStatistics.addTcpInMessage(msg); logger.info("Receive message from peer: {}, {}", peer.getInetSocketAddress(), msg); - switch (msg.getType()) { + switch (type) { case P2P_PING: case P2P_PONG: keepAliveService.processMessage(peer, msg); @@ -179,9 +181,11 @@ private void processMessage(PeerConnection peer, byte[] data) { long costs = System.currentTimeMillis() - startTime; if (costs > 50) { logger.info("Message processing costs {} ms, peer: {}, type: {}, time tag: {}", - costs, peer.getInetSocketAddress(), msg.getType(), getTimeTag(costs)); - Metrics.histogramObserve(MetricKeys.Histogram.MESSAGE_PROCESS_LATENCY, - costs / Metrics.MILLISECONDS_PER_SECOND, msg.getType().name()); + costs, peer.getInetSocketAddress(), type, getTimeTag(costs)); + if (type != null) { + Metrics.histogramObserve(MetricKeys.Histogram.MESSAGE_PROCESS_LATENCY, + costs / Metrics.MILLISECONDS_PER_SECOND, type.name()); + } } } } diff --git a/framework/src/main/java/org/tron/core/net/TronNetService.java b/framework/src/main/java/org/tron/core/net/TronNetService.java index 5f8bbbe8ad2..8b930dafade 100644 --- a/framework/src/main/java/org/tron/core/net/TronNetService.java +++ b/framework/src/main/java/org/tron/core/net/TronNetService.java @@ -65,10 +65,13 @@ public class TronNetService { private volatile boolean init; + private static void setP2pConfig(P2pConfig config) { + TronNetService.p2pConfig = config; + } public void start() { try { init = true; - p2pConfig = getConfig(); + setP2pConfig(getConfig()); p2pService.start(p2pConfig); p2pService.register(p2pEventHandler); advService.init(); diff --git a/framework/src/main/java/org/tron/core/net/peer/PeerConnection.java b/framework/src/main/java/org/tron/core/net/peer/PeerConnection.java index 3c35e08dd0a..260b82747f4 100644 --- a/framework/src/main/java/org/tron/core/net/peer/PeerConnection.java +++ b/framework/src/main/java/org/tron/core/net/peer/PeerConnection.java @@ -90,7 +90,7 @@ public class PeerConnection { private int invCacheSize = 20_000; - private long BAD_PEER_BAN_TIME = 3600 * 1000; + private long BAD_PEER_BAN_TIME = 3_600_000; @Setter @Getter @@ -282,7 +282,7 @@ private boolean needToLog(Message msg) { @Override public boolean equals(Object o) { - if (o == null || !(o instanceof PeerConnection)) { + if (!(o instanceof PeerConnection)) { return false; } return this.channel.equals(((PeerConnection) o).getChannel()); diff --git a/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java b/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java index 22ba096fb20..d0ece8734d6 100644 --- a/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java +++ b/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java @@ -104,7 +104,6 @@ public void processHelloMessage(PeerConnection peer, HelloMessage msg) { System.currentTimeMillis() - peer.getChannel().getStartTime()); PeerManager.sortPeers(); peer.onConnect(); - return; } private void sendHelloMessage(PeerConnection peer, long time) { diff --git a/framework/src/main/java/org/tron/core/net/service/keepalive/KeepAliveService.java b/framework/src/main/java/org/tron/core/net/service/keepalive/KeepAliveService.java index 35c2843c046..49ae692b6f1 100644 --- a/framework/src/main/java/org/tron/core/net/service/keepalive/KeepAliveService.java +++ b/framework/src/main/java/org/tron/core/net/service/keepalive/KeepAliveService.java @@ -21,7 +21,7 @@ public class KeepAliveService { private long PING_TIMEOUT = 20_000; - public long PING_PERIOD = 60_000; + private long PING_PERIOD = 60_000; private final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "KeepAlive")); diff --git a/framework/src/main/java/org/tron/core/net/service/statistics/NodeStatistics.java b/framework/src/main/java/org/tron/core/net/service/statistics/NodeStatistics.java index 1bfd7a41738..aef293d493c 100644 --- a/framework/src/main/java/org/tron/core/net/service/statistics/NodeStatistics.java +++ b/framework/src/main/java/org/tron/core/net/service/statistics/NodeStatistics.java @@ -10,8 +10,8 @@ public class NodeStatistics { private Protocol.ReasonCode localDisconnectReason = null; @Getter private int disconnectTimes = 0; - private long lastDisconnectedTime = 0; - private long firstDisconnectedTime = 0; +// private long lastDisconnectedTime = 0; +// private long firstDisconnectedTime = 0; private long start = System.currentTimeMillis(); public Protocol.ReasonCode getDisconnectReason() { @@ -35,10 +35,10 @@ public void nodeDisconnectedLocal(Protocol.ReasonCode reason) { } private void notifyDisconnect() { - lastDisconnectedTime = System.currentTimeMillis(); - if (firstDisconnectedTime == 0) { - firstDisconnectedTime = lastDisconnectedTime; - } +// lastDisconnectedTime = System.currentTimeMillis(); +// if (firstDisconnectedTime == 0) { +// firstDisconnectedTime = lastDisconnectedTime; +// } disconnectTimes++; } diff --git a/framework/src/main/java/org/tron/core/net/service/statistics/TronStatsManager.java b/framework/src/main/java/org/tron/core/net/service/statistics/TronStatsManager.java index cf241102dbf..da9b4b0f2d3 100644 --- a/framework/src/main/java/org/tron/core/net/service/statistics/TronStatsManager.java +++ b/framework/src/main/java/org/tron/core/net/service/statistics/TronStatsManager.java @@ -19,10 +19,10 @@ @Slf4j(topic = "net") @Component public class TronStatsManager { - private static volatile long TCP_TRAFFIC_IN = 0; - private static volatile long TCP_TRAFFIC_OUT = 0; - private static volatile long UDP_TRAFFIC_IN = 0; - private static volatile long UDP_TRAFFIC_OUT = 0; + private volatile long TCP_TRAFFIC_IN = 0; + private volatile long TCP_TRAFFIC_OUT = 0; + private volatile long UDP_TRAFFIC_IN = 0; + private volatile long UDP_TRAFFIC_OUT = 0; private static Cache cache = CacheBuilder.newBuilder() .maximumSize(3000).recordStats().build(); From 736a518d614bd5e1dc6a8477f406fb2d48ba1e8b Mon Sep 17 00:00:00 2001 From: wubin01 Date: Tue, 15 Nov 2022 18:51:39 +0800 Subject: [PATCH 05/13] feat(net): remove unused code --- .../src/main/java/org/tron/core/net/TronNetService.java | 1 + .../tron/core/net/service/statistics/NodeStatistics.java | 6 ------ 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/framework/src/main/java/org/tron/core/net/TronNetService.java b/framework/src/main/java/org/tron/core/net/TronNetService.java index 8b930dafade..e036dd062d1 100644 --- a/framework/src/main/java/org/tron/core/net/TronNetService.java +++ b/framework/src/main/java/org/tron/core/net/TronNetService.java @@ -68,6 +68,7 @@ public class TronNetService { private static void setP2pConfig(P2pConfig config) { TronNetService.p2pConfig = config; } + public void start() { try { init = true; diff --git a/framework/src/main/java/org/tron/core/net/service/statistics/NodeStatistics.java b/framework/src/main/java/org/tron/core/net/service/statistics/NodeStatistics.java index aef293d493c..116b9a5c7e5 100644 --- a/framework/src/main/java/org/tron/core/net/service/statistics/NodeStatistics.java +++ b/framework/src/main/java/org/tron/core/net/service/statistics/NodeStatistics.java @@ -10,8 +10,6 @@ public class NodeStatistics { private Protocol.ReasonCode localDisconnectReason = null; @Getter private int disconnectTimes = 0; -// private long lastDisconnectedTime = 0; -// private long firstDisconnectedTime = 0; private long start = System.currentTimeMillis(); public Protocol.ReasonCode getDisconnectReason() { @@ -35,10 +33,6 @@ public void nodeDisconnectedLocal(Protocol.ReasonCode reason) { } private void notifyDisconnect() { -// lastDisconnectedTime = System.currentTimeMillis(); -// if (firstDisconnectedTime == 0) { -// firstDisconnectedTime = lastDisconnectedTime; -// } disconnectTimes++; } From 4353450af09aaaa4b6d8141532de1c6325fe9f19 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Wed, 16 Nov 2022 15:29:45 +0800 Subject: [PATCH 06/13] modify the network parameter initialization value --- .../main/java/org/tron/core/config/args/Args.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index a773c467481..92441675e7d 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -123,9 +123,9 @@ public static void clearParam() { PARAMETER.nodeDiscoveryEnable = false; PARAMETER.nodeDiscoveryPersist = false; PARAMETER.nodeConnectionTimeout = 2000; - PARAMETER.activeNodes = Collections.emptyList(); - PARAMETER.passiveNodes = Collections.emptyList(); - PARAMETER.fastForwardNodes = Collections.emptyList(); + PARAMETER.activeNodes = new ArrayList<>(); + PARAMETER.passiveNodes = new ArrayList<>(); + PARAMETER.fastForwardNodes = new ArrayList<>(); PARAMETER.maxFastForwardNum = 3; PARAMETER.nodeChannelReadTimeout = 0; PARAMETER.maxConnections = 30; @@ -1116,10 +1116,10 @@ private static RateLimiterInitialization getRateLimiterFromConfig( private static List getInetSocketAddress( final com.typesafe.config.Config config, String path) { + List ret = new ArrayList<>(); if (!config.hasPath(path)) { - return Collections.emptyList(); + return ret; } - List ret = new ArrayList<>(); List list = config.getStringList(path); for (String configString : list) { String[] sz = configString.split(":"); @@ -1137,10 +1137,10 @@ private static List getInetSocketAddress( private static List getInetAddress( final com.typesafe.config.Config config, String path) { + List ret = new ArrayList<>(); if (!config.hasPath(path)) { - return Collections.emptyList(); + return ret; } - List ret = new ArrayList<>(); List list = config.getStringList(path); for (String configString : list) { try { From 9f9780d2345f99a3d5633e406540dc9d184bf315 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Mon, 21 Nov 2022 12:30:12 +0800 Subject: [PATCH 07/13] feat(net): optimize the logic of generating hello message --- .../org/tron/core/net/service/handshake/HandshakeService.java | 1 + .../main/java/org/tron/core/net/service/relay/RelayService.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java b/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java index d0ece8734d6..7b19c9dd3d5 100644 --- a/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java +++ b/framework/src/main/java/org/tron/core/net/service/handshake/HandshakeService.java @@ -111,6 +111,7 @@ private void sendHelloMessage(PeerConnection peer, long time) { TronNetService.getP2pConfig().getIp(), TronNetService.getP2pConfig().getPort()); HelloMessage message = new HelloMessage(node, time, ChainBaseManager.getChainBaseManager()); + relayService.fillHelloMessage(message, peer.getChannel()); peer.sendMessage(message); peer.setHelloMessageSend(message); } diff --git a/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java b/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java index 46f3b5c7374..ec231809ea5 100644 --- a/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java +++ b/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java @@ -100,7 +100,7 @@ public void init() { public void fillHelloMessage(HelloMessage message, Channel channel) { if (isActiveWitness()) { fastForwardNodes.forEach(address -> { - if (address.equals(channel.getInetAddress())) { + if (address.getAddress().equals(channel.getInetAddress())) { SignInterface cryptoEngine = SignUtils .fromPrivate(ByteArray.fromHexString(Args.getLocalWitnesses().getPrivateKey()), Args.getInstance().isECKeyCryptoEngine()); From 4ad5f0ebb11cdf5bc034b93c7bef3e6ee5e27ad1 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Mon, 21 Nov 2022 20:12:52 +0800 Subject: [PATCH 08/13] feat(net): modify hello message toString function --- .../tron/core/net/P2pEventHandlerImpl.java | 2 +- .../net/message/handshake/HelloMessage.java | 22 ++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/framework/src/main/java/org/tron/core/net/P2pEventHandlerImpl.java b/framework/src/main/java/org/tron/core/net/P2pEventHandlerImpl.java index d19134b62ae..6a164d774da 100644 --- a/framework/src/main/java/org/tron/core/net/P2pEventHandlerImpl.java +++ b/framework/src/main/java/org/tron/core/net/P2pEventHandlerImpl.java @@ -119,7 +119,7 @@ public void onMessage(Channel c, byte[] data) { pbftMsgHandler.processMessage(peerConnection, message); } catch (Exception e) { logger.warn("PBFT Message from {} process failed, {}", - peerConnection.getInetSocketAddress(), message, e); + peerConnection.getInetSocketAddress(), message, e.getMessage()); peerConnection.disconnect(Protocol.ReasonCode.BAD_PROTOCOL); } return; diff --git a/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java b/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java index 01cf19b2a0b..d4ed7282386 100755 --- a/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/handshake/HelloMessage.java @@ -3,6 +3,7 @@ import com.google.protobuf.ByteString; import lombok.Getter; import org.tron.common.utils.ByteArray; +import org.tron.common.utils.StringUtil; import org.tron.core.ChainBaseManager; import org.tron.core.Constant; import org.tron.core.capsule.BlockCapsule; @@ -128,13 +129,28 @@ public Class getAnswerMessage() { @Override public String toString() { - return new StringBuilder().append(super.toString()) + StringBuilder builder = new StringBuilder(); + + builder.append(super.toString()) .append("from: ").append(getFrom().getInetSocketAddress()).append("\n") .append("timestamp: ").append(getTimestamp()).append("\n") .append("headBlockId: ").append(getHeadBlockId().getString()).append("\n") .append("nodeType: ").append(helloMessage.getNodeType()).append("\n") - .append("lowestBlockNum: ").append(helloMessage.getLowestBlockNum()).append("\n") - .toString(); + .append("lowestBlockNum: ").append(helloMessage.getLowestBlockNum()).append("\n"); + + ByteString address = helloMessage.getAddress(); + if (address != null && !address.isEmpty()) { + builder.append("address:") + .append(StringUtil.encode58Check(address.toByteArray())).append("\n"); + } + + ByteString signature = helloMessage.getSignature(); + if (signature != null && !signature.isEmpty()) { + builder.append("signature:") + .append(signature.toByteArray().length).append("\n"); + } + + return builder.toString(); } public Protocol.HelloMessage getInstance() { From 8ebf5930f71510dbad00710dbe4aedac00d2347d Mon Sep 17 00:00:00 2001 From: wubin01 Date: Tue, 22 Nov 2022 11:10:52 +0800 Subject: [PATCH 09/13] feat(net): enable the message forwarding function --- .../src/main/java/org/tron/core/net/TronNetService.java | 6 ++++++ .../java/org/tron/core/net/service/relay/RelayService.java | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/framework/src/main/java/org/tron/core/net/TronNetService.java b/framework/src/main/java/org/tron/core/net/TronNetService.java index e036dd062d1..3c9554d79fa 100644 --- a/framework/src/main/java/org/tron/core/net/TronNetService.java +++ b/framework/src/main/java/org/tron/core/net/TronNetService.java @@ -19,6 +19,7 @@ import org.tron.core.net.service.fetchblock.FetchBlockService; import org.tron.core.net.service.keepalive.KeepAliveService; import org.tron.core.net.service.nodepersist.NodePersistService; +import org.tron.core.net.service.relay.RelayService; import org.tron.core.net.service.statistics.TronStatsManager; import org.tron.core.net.service.sync.SyncService; import org.tron.p2p.P2pConfig; @@ -63,6 +64,9 @@ public class TronNetService { @Autowired private TronStatsManager tronStatsManager; + @Autowired + private RelayService relayService; + private volatile boolean init; private static void setP2pConfig(P2pConfig config) { @@ -84,6 +88,7 @@ public void start() { nodePersistService.init(); tronStatsManager.init(); PeerManager.init(); + relayService.init(); logger.info("Net service start successfully"); } catch (Exception e) { logger.error("Net service start failed", e); @@ -104,6 +109,7 @@ public void close() { transactionsMsgHandler.close(); fetchBlockService.close(); p2pService.close(); + relayService.close(); logger.info("Net service closed successfully"); } diff --git a/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java b/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java index ec231809ea5..380f097361b 100644 --- a/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java +++ b/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java @@ -97,6 +97,10 @@ public void init() { }, 30, 100, TimeUnit.SECONDS); } + public void close() { + executorService.shutdown(); + } + public void fillHelloMessage(HelloMessage message, Channel channel) { if (isActiveWitness()) { fastForwardNodes.forEach(address -> { From d3d53d906a614541e7c5bd8adc00d1d218ed6f05 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Tue, 22 Nov 2022 11:51:33 +0800 Subject: [PATCH 10/13] fix(net): solve p2p configuration initialization problem --- .../main/java/org/tron/core/net/service/relay/RelayService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java b/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java index 380f097361b..7dc75ec89a0 100644 --- a/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java +++ b/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java @@ -49,7 +49,7 @@ public class RelayService { @Autowired private ApplicationContext ctx; - private P2pConfig p2pConfig; + private P2pConfig p2pConfig = TronNetService.getP2pConfig(); private Manager manager; From 7c8fc3b86d8455eed9ac8b3c188d816fc49e1d30 Mon Sep 17 00:00:00 2001 From: wubin01 Date: Tue, 22 Nov 2022 12:22:18 +0800 Subject: [PATCH 11/13] fix(net): solve p2p configuration initialization problem --- .../org/tron/core/net/service/relay/RelayService.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java b/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java index 7dc75ec89a0..478f0714875 100644 --- a/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java +++ b/framework/src/main/java/org/tron/core/net/service/relay/RelayService.java @@ -49,8 +49,6 @@ public class RelayService { @Autowired private ApplicationContext ctx; - private P2pConfig p2pConfig = TronNetService.getP2pConfig(); - private Manager manager; private WitnessScheduleStore witnessScheduleStore; @@ -155,7 +153,7 @@ public boolean checkHelloMessage(HelloMessage message, Channel channel) { flag = Arrays.equals(sigAddress, witnessPermissionAddress); } if (flag) { - p2pConfig.getTrustNodes().add(channel.getInetAddress()); + TronNetService.getP2pConfig().getTrustNodes().add(channel.getInetAddress()); } return flag; } catch (Exception e) { @@ -173,12 +171,13 @@ private boolean isActiveWitness() { } private void connect() { - fastForwardNodes.forEach(address -> p2pConfig.getActiveNodes().add(address)); + fastForwardNodes.forEach(address -> + TronNetService.getP2pConfig().getActiveNodes().add(address)); } private void disconnect() { fastForwardNodes.forEach(address -> { - p2pConfig.getActiveNodes().remove(address); + TronNetService.getP2pConfig().getActiveNodes().remove(address); TronNetService.getPeers().forEach(peer -> { if (peer.getInetAddress().equals(address.getAddress())) { peer.getChannel().close(); From b3f5fa991343925a8cf62866ff95a84881cf123c Mon Sep 17 00:00:00 2001 From: chengtx01 Date: Wed, 30 Nov 2022 17:01:10 +0800 Subject: [PATCH 12/13] fix(net): introduce a new libp2p dependency to fix the logging issue --- framework/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/framework/build.gradle b/framework/build.gradle index bbbeaee8a63..2cb1a61e245 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -46,7 +46,7 @@ dependencies { compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' - compile group: 'com.github.tronprotocol', name: 'libp2p', version: 'test-v0.1.4' + compile group: 'com.github.tronprotocol', name: 'libp2p', version: 'test-v0.1.5' compile group: 'com.typesafe', name: 'config', version: '1.3.2' From 4159b3a7c70fef5b2fe91ea34ff9704e869fe05a Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Wed, 23 Nov 2022 13:42:31 +0800 Subject: [PATCH 13/13] feat(toolkit): add db conversion and data splitting extensions 1. convert leveldb to rocksdb 2. split lite full node data 3. optimize gradle build --- framework/build.gradle | 30 +- .../java/org/tron/core/config/args/Args.java | 2 +- .../org/tron/tool/litefullnode/DbTool.java | 2 +- .../tool/litefullnode/LiteFullNodeTool.java | 12 + .../java/org/tron/tool/litefullnode/Util.java | 2 +- plugins/README.md | 115 +++ plugins/build.gradle | 27 +- .../org/tron/plugins/ArchiveManifest.java | 1 - .../src/main/java/org/tron/plugins/Db.java | 43 +- .../main/java/org/tron/plugins/DbArchive.java | 58 +- .../main/java/org/tron/plugins/DbConvert.java | 360 +++++++++ .../main/java/org/tron/plugins/DbLite.java | 740 ++++++++++++++++++ .../main/java/org/tron/plugins/DbMove.java | 66 +- .../MarketOrderPriceComparatorForLevelDB.java | 28 + .../MarketOrderPriceComparatorForRockDB.java | 38 + .../org/tron/plugins/utils/ByteArray.java | 101 +++ .../org/tron/plugins/utils/CryptoUitls.java | 6 + .../java/org/tron/plugins/utils/DBUtils.java | 153 ++++ .../org/tron/plugins/utils/FileUtils.java | 189 +++++ .../org/tron/plugins/utils/MarketUtils.java | 148 ++++ .../org/tron/plugins/utils/Sha256Hash.java | 202 +++++ .../tron/plugins/utils/db/DBInterface.java | 21 + .../org/tron/plugins/utils/db/DBIterator.java | 59 ++ .../org/tron/plugins/utils/db/DbTool.java | 187 +++++ .../tron/plugins/utils/db/LevelDBImpl.java | 46 ++ .../plugins/utils/db/LevelDBIterator.java | 58 ++ .../tron/plugins/utils/db/RockDBIterator.java | 77 ++ .../tron/plugins/utils/db/RocksDBImpl.java | 64 ++ plugins/src/main/resources/logback.xml | 7 +- .../java/org/tron/plugins/DbConvertTest.java | 110 +++ .../java/org/tron/plugins/DbMoveTest.java | 38 +- 31 files changed, 2813 insertions(+), 177 deletions(-) create mode 100644 plugins/README.md create mode 100644 plugins/src/main/java/org/tron/plugins/DbConvert.java create mode 100644 plugins/src/main/java/org/tron/plugins/DbLite.java create mode 100644 plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java create mode 100644 plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/ByteArray.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/CryptoUitls.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/DBUtils.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/FileUtils.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/MarketUtils.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/Sha256Hash.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/db/DBInterface.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/db/DBIterator.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/db/DbTool.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/db/LevelDBIterator.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java create mode 100644 plugins/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java create mode 100644 plugins/src/test/java/org/tron/plugins/DbConvertTest.java diff --git a/framework/build.gradle b/framework/build.gradle index 329f0e6d265..66acc047c80 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -267,14 +267,34 @@ createScript(project, 'org.tron.program.DBConvert', 'DBConvert') createScript(project, 'org.tron.tool.litefullnode.LiteFullNodeTool', 'LiteFullNodeTool') def releaseBinary = hasProperty('binaryRelease') ? getProperty('binaryRelease') : 'true' +def skipSolidity = hasProperty('skipSolidity') ? true : false +def skipKeystore = hasProperty('skipKeystore') ? true : false +def skipConvert = hasProperty('skipConvert') ? true : false +def skipLite = hasProperty('skipLite') ? true : false +def skipAll = hasProperty('skipAll') ? true : false if (releaseBinary == 'true') { artifacts { - archives(binaryRelease('buildSolidityNodeJar', 'SolidityNode', 'org.tron.program.SolidityNode'), - binaryRelease('buildFullNodeJar', 'FullNode', 'org.tron.program.FullNode'), - binaryRelease('buildKeystoreFactoryJar', 'KeystoreFactory', 'org.tron.program.KeystoreFactory'), - binaryRelease('buildDBConvertJar', 'DBConvert', 'org.tron.program.DBConvert'), - binaryRelease('buildLiteFullNodeToolJar', 'LiteFullNodeTool', 'org.tron.tool.litefullnode.LiteFullNodeTool')) + archives(binaryRelease('buildFullNodeJar', 'FullNode', 'org.tron.program.FullNode')) } + if (!skipAll) { + if (!skipSolidity) { + artifacts { + archives(binaryRelease('buildSolidityNodeJar', 'SolidityNode', 'org.tron.program.SolidityNode'))} + } + if (!skipKeystore) { + artifacts { + archives(binaryRelease('buildKeystoreFactoryJar', 'KeystoreFactory', 'org.tron.program.KeystoreFactory'))} + } + if (!skipConvert) { + artifacts { + archives(binaryRelease('buildDBConvertJar', 'DBConvert', 'org.tron.program.DBConvert'))} + } + if (!skipLite) { + artifacts { + archives(binaryRelease('buildLiteFullNodeToolJar', 'LiteFullNodeTool', 'org.tron.tool.litefullnode.LiteFullNodeTool'))} + } + } + } task copyToParent(type: Copy) { diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index a0b428df59f..4386f592ea2 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -1437,7 +1437,7 @@ public static boolean checkIsLiteFullNode() { PARAMETER.storage.getDbDirectory(), Constant.INFO_FILE_NAME).toString(); if (FileUtil.isExists(infoFile)) { String value = PropUtil.readProperty(infoFile, Constant.SPLIT_BLOCK_NUM); - return !"".equals(value) && Long.parseLong(value) > 0; + return !"".equals(value) && Long.parseLong(value) > 1; } return false; } diff --git a/framework/src/main/java/org/tron/tool/litefullnode/DbTool.java b/framework/src/main/java/org/tron/tool/litefullnode/DbTool.java index 49d4fac8384..5f43361eddf 100644 --- a/framework/src/main/java/org/tron/tool/litefullnode/DbTool.java +++ b/framework/src/main/java/org/tron/tool/litefullnode/DbTool.java @@ -27,7 +27,7 @@ public class DbTool { private static final String KEY_ENGINE = "ENGINE"; - private static final String ENGINE_FILE = "engine.properties"; + public static final String ENGINE_FILE = "engine.properties"; private static final String FILE_SEPARATOR = File.separator; private static final String ROCKSDB = "ROCKSDB"; diff --git a/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java b/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java index bec69227c52..bc72388e156 100644 --- a/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java +++ b/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java @@ -14,6 +14,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; import java.util.List; @@ -331,6 +332,17 @@ private void fillSnapshotBlockAndTransDb(String sourceDir, String snapshotDir) DBInterface destCommonDb = DbTool.getDB(snapshotDir, COMMON_DB_NAME); destCommonDb.put(DB_KEY_NODE_TYPE, ByteArray.fromInt(Constant.NODE_TYPE_LIGHT_NODE)); destCommonDb.put(DB_KEY_LOWEST_BLOCK_NUM, ByteArray.fromLong(startIndex)); + // copy engine.properties for block、block-index、trans from source if exist + copyEngineIfExist(sourceDir, snapshotDir, BLOCK_DB_NAME, BLOCK_INDEX_DB_NAME, TRANS_DB_NAME); + } + + private void copyEngineIfExist(String source, String dest, String... dbNames) { + for (String dbName : dbNames) { + Path ori = Paths.get(source, dbName, DbTool.ENGINE_FILE); + if (ori.toFile().exists()) { + Util.copy(ori, Paths.get(dest, dbName, DbTool.ENGINE_FILE)); + } + } } private byte[] getGenesisBlockHash(String parentDir) throws IOException, RocksDBException { diff --git a/framework/src/main/java/org/tron/tool/litefullnode/Util.java b/framework/src/main/java/org/tron/tool/litefullnode/Util.java index 227ce5c5de4..0e4898b8031 100644 --- a/framework/src/main/java/org/tron/tool/litefullnode/Util.java +++ b/framework/src/main/java/org/tron/tool/litefullnode/Util.java @@ -43,7 +43,7 @@ public static void copyDatabases(Path src, Path dest, List subDirs) }); } - private static void copy(Path source, Path dest) { + public static void copy(Path source, Path dest) { try { // create hard link when file is .sst if (source.toString().endsWith(".sst")) { diff --git a/plugins/README.md b/plugins/README.md new file mode 100644 index 00000000000..ff1fea4d17b --- /dev/null +++ b/plugins/README.md @@ -0,0 +1,115 @@ +# Toolkit Manual + +This package contains a set of tools for Tron, the following is the documentation for each tool. + +## DB Archive + +DB archive provides the ability to reformat the manifest according to the current `database`, +parameters are compatible with previous `ArchiveManifest`. + +Parameter explanation: + +- `-b | --batch-size`: [int] specify the batch manifest size,default：80000. +- `-d | --database-directory`: [string] specify the database directory to be processed,default：output-directory/database. +- `-m | --manifest-size`: [int] specify the minimum required manifest file size ，unit:M，default：0. +- `-h | --help`: provide the help info + +Demo: + +```shell script +# full command + java -jar Toolkit.jar db archive [-h] [-b=] [-d=] [-m=] +# examples + java -jar Toolkit.jar db archive #1. use default settings + java -jar Toolkit.jar db archive -d /tmp/db/database #2. Specify the database directory as /tmp/db/database + java -jar Toolkit.jar db archive -b 64000 #3. Specify the batch size to 64000 when optimizing Manifest + java -jar Toolkit.jar db archive -m 128 #4. Specify optimization only when Manifest exceeds 128M +``` + + +## DB Convert + +DB convert provides a helper which can convert LevelDB data to RocksDB data, parameters are compatible with previous `DBConvert`. + +Parameter explanation: + +- ``: Input path for leveldb. Default: output-directory/database +- ``: Output path for rocksdb. Default: output-directory-dst/database +- `--safe`: In safe mode, read data from leveldb then put rocksdb, it's a very time-consuming procedure. If not, just change engine.properties from leveldb to rocksdb,rocksdb + is compatible with leveldb for current version.This may not be the case in the future.Default: false +- `-h | --help`: provide the help info + +Demo: + +```shell script +# full command + java -jar Toolkit.jar db convert [-h] [--safe] +# examples + java -jar Toolkit.jar db convert output-directory/database /tmp/databse +``` + + +## DB Lite + +DB Lite provides lite database, parameters are compatible with previous `LiteFullNodeTool`. + +Parameter explanation: + +- `-o | --operate`: [split,merge]. Default: split. +- `-t | --type`: only used with operate=split: [snapshot,history]. Default: snapshot. +- `-fn | --fn-data-path`: the database path to be split or merged. +- `-ds | --dataset-path`: when operation is `split`,`dataset-path` is the path that store the `snapshot` or `history`,when + operation is `split`,`dataset-path` is the `history` data path. +- `-h | --help`: provide the help info + +Demo: + +```shell script +# full command + java -jar Toolkit.jar db lite [-h] -ds= -fn= [-o=] [-t=] +# examples + #split and get a snapshot dataset + java -jar Toolkit.jar db lite -o split -t snapshot --fn-data-path output-directory/database --dataset-path /tmp + #split and get a history dataset + java -jar Toolkit.jar db lite -o split -t history --fn-data-path output-directory/database --dataset-path /tmp + #merge history dataset and snapshot dataset + java -jar Toolkit.jar db lite -o split -t history --fn-data-path /tmp/snapshot --dataset-path /tmp/history +``` + +## DB Move + +DB Move provides a helper to move some dbs to pre-set new path. For example move `block`, `transactionRetStore` or `transactionHistoryStore` to HDD,reduce storage expenses + +Parameter explanation: + +- `-c | --config`: config file. Default: config.conf. +- `-d | --database-directory`: database directory path. Default: output-directory. +- `-h | --help`: provide the help info + +Demo: + +Take the example of moving `block` and `trans` + +```conf +storage { + ...... + properties = [ + { + name = "block", + path = "/data1/tron", + }, + { + name = "trans", + path = "/data1/tron", + } + ] + ...... +} +``` + +```shell script +# full command + java -jar Toolkit.jar db mv [-h] [-c=] [-d=] +# examples + java -jar Toolkit.jar db mv -c main_net_config.conf -d /data/tron/output-directory +``` \ No newline at end of file diff --git a/plugins/build.gradle b/plugins/build.gradle index f6815e606b3..7dff7295869 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -33,10 +33,11 @@ dependencies { compile group: 'info.picocli', name: 'picocli', version: '4.6.3' compile group: 'com.typesafe', name: 'config', version: '1.3.2' compile group: 'me.tongfei', name: 'progressbar', version: '0.9.3' - - - compile 'com.github.halibobo1205.leveldb-java:leveldb:v0.12.5' - compile 'com.github.halibobo1205.leveldb-java:leveldb-api:v0.12.5' + compile group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' + compile group: 'org.rocksdb', name: 'rocksdbjni', version: '5.15.10' + compile 'com.halibobor:leveldbjni-all:1.18.2' + compile 'com.halibobor:leveldb:1.18.2' + compile project(":protocol") } check.dependsOn 'lint' @@ -104,6 +105,11 @@ def binaryRelease(taskName, jarName, mainClass) { } } + // exclude these files for bouncycastle + exclude "META-INF/*.SF" + exclude "META-INF/*.DSA" + exclude "META-INF/*.RSA" + manifest { attributes "Main-Class": "${mainClass}" } @@ -132,11 +138,18 @@ createScript(project, 'org.tron.plugins.ArchiveManifest', 'ArchiveManifest') createScript(project, 'org.tron.plugins.Toolkit', 'Toolkit') def releaseBinary = hasProperty('binaryRelease') ? getProperty('binaryRelease') : 'true' +def skipArchive = hasProperty('skipArchive') ? true : false +def skipAll = hasProperty('skipAll') ? true : false if (releaseBinary == 'true') { artifacts { - archives( - binaryRelease('buildArchiveManifestJar', 'ArchiveManifest', 'org.tron.plugins.ArchiveManifest'), - binaryRelease('buildToolkitJar', 'Toolkit', 'org.tron.plugins.Toolkit')) + archives(binaryRelease('buildToolkitJar', 'Toolkit', 'org.tron.plugins.Toolkit')) + } + if (!skipAll) { + if (!skipArchive) { + artifacts { + archives(binaryRelease('buildArchiveManifestJar', 'ArchiveManifest', 'org.tron.plugins.ArchiveManifest')) + } + } } } diff --git a/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java b/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java index 793404ddc2c..4d54df6d299 100644 --- a/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java +++ b/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java @@ -82,7 +82,6 @@ public static org.iq80.leveldb.Options newDefaultLevelDbOptions() { dbOptions.maxOpenFiles(1000); dbOptions.maxBatchSize(64_000); dbOptions.maxManifestSize(128); - dbOptions.fast(false); return dbOptions; } diff --git a/plugins/src/main/java/org/tron/plugins/Db.java b/plugins/src/main/java/org/tron/plugins/Db.java index 702f505b147..e7296a2cbc5 100644 --- a/plugins/src/main/java/org/tron/plugins/Db.java +++ b/plugins/src/main/java/org/tron/plugins/Db.java @@ -1,47 +1,18 @@ package org.tron.plugins; -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; -import java.io.File; -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; import picocli.CommandLine; @CommandLine.Command(name = "db", mixinStandardHelpOptions = true, version = "db command 1.0", description = "An rich command set that provides high-level operations for dbs.", - subcommands = {CommandLine.HelpCommand.class, DbMove.class, DbArchive.class}, - commandListHeading = "%nCommands:%n%nThe most commonly used git commands are:%n" + subcommands = {CommandLine.HelpCommand.class, + DbMove.class, + DbArchive.class, + DbConvert.class, + DbLite.class + }, + commandListHeading = "%nCommands:%n%nThe most commonly used db commands are:%n" ) public class Db { - - static class ConfigConverter implements CommandLine.ITypeConverter { - ConfigConverter() { - } - - public Config convert(String value) throws IOException { - File file = Paths.get(value).toFile(); - if (file.exists() && file.isFile()) { - return ConfigFactory.parseFile(Paths.get(value).toFile()); - } else { - throw new IOException("DB config [" + value + "] not exist!"); - } - } - } - - static class PathConverter implements CommandLine.ITypeConverter { - PathConverter() { - } - - public Path convert(String value) throws IOException { - File file = Paths.get(value).toFile(); - if (file.exists() && file.isDirectory()) { - return file.toPath(); - } else { - throw new IOException("DB path [" + value + "] not exist!"); - } - } - } } diff --git a/plugins/src/main/java/org/tron/plugins/DbArchive.java b/plugins/src/main/java/org/tron/plugins/DbArchive.java index 2339fc9590b..e3032731ede 100644 --- a/plugins/src/main/java/org/tron/plugins/DbArchive.java +++ b/plugins/src/main/java/org/tron/plugins/DbArchive.java @@ -3,25 +3,14 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.iq80.leveldb.impl.Iq80DBFactory.factory; -import java.io.BufferedInputStream; -import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Objects; -import java.util.Properties; import java.util.concurrent.Callable; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; @@ -30,11 +19,12 @@ import org.iq80.leveldb.DB; import org.iq80.leveldb.Options; import org.iq80.leveldb.impl.Filename; +import org.tron.plugins.utils.FileUtils; import picocli.CommandLine; import picocli.CommandLine.Option; @Slf4j(topic = "archive") -@CommandLine.Command(name = "archive", description = "a helper to rewrite leveldb manifest.") +@CommandLine.Command(name = "archive", description = "A helper to rewrite leveldb manifest.") public class DbArchive implements Callable { @CommandLine.Spec @@ -55,7 +45,7 @@ public class DbArchive implements Callable { description = "manifest min size(M) to archive. Default: ${DEFAULT-VALUE}") private int maxManifestSize; - @Option(names = {"-h", "--help"}, help = true) + @Option(names = {"-h", "--help"}) private boolean help; @@ -189,7 +179,7 @@ public void doArchive() { } try { if (checkManifest(levelDbFile.toString())) { - if (!checkEngine()) { + if (!FileUtils.isLevelDBEngine(srcDbPath)) { logger.info("Db {},not leveldb, ignored.", this.name); return; } @@ -203,46 +193,6 @@ public void doArchive() { throw new RuntimeException("Db " + this.name + " archive failed.", e); } } - - public boolean checkEngine() { - String dir = this.srcDbPath.toString(); - String enginePath = dir + File.separator + "engine.properties"; - if (!new File(enginePath).exists() && !writeProperty(enginePath, KEY_ENGINE, LEVELDB)) { - return false; - } - String engine = readProperty(enginePath, KEY_ENGINE); - return LEVELDB.equals(engine); - } - - public static String readProperty(String file, String key) { - try (FileInputStream fileInputStream = new FileInputStream(file); - InputStream inputStream = new BufferedInputStream(fileInputStream)) { - Properties prop = new Properties(); - prop.load(inputStream); - return new String(prop.getProperty(key, "").getBytes(StandardCharsets.ISO_8859_1), - UTF_8); - } catch (Exception e) { - logger.error("readProperty", e); - return ""; - } - } - - public static boolean writeProperty(String file, String key, String value) { - try (OutputStream out = new FileOutputStream(file); - FileInputStream fis = new FileInputStream(file); - BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, UTF_8))) { - BufferedReader bf = new BufferedReader(new InputStreamReader(fis, UTF_8)); - Properties properties = new Properties(); - properties.load(bf); - properties.setProperty(key, value); - properties.store(bw, "Generated by the application. PLEASE DO NOT EDIT! "); - } catch (Exception e) { - logger.warn("writeProperty", e); - return false; - } - return true; - } - } } \ No newline at end of file diff --git a/plugins/src/main/java/org/tron/plugins/DbConvert.java b/plugins/src/main/java/org/tron/plugins/DbConvert.java new file mode 100644 index 00000000000..a75b235bbcf --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/DbConvert.java @@ -0,0 +1,360 @@ +package org.tron.plugins; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import me.tongfei.progressbar.ProgressBar; +import org.fusesource.leveldbjni.JniDBFactory; +import org.iq80.leveldb.DB; +import org.iq80.leveldb.DBIterator; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; +import org.rocksdb.Status; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.FileUtils; +import picocli.CommandLine; + + +@Slf4j(topic = "convert") +@CommandLine.Command(name = "convert", + description = "Covert leveldb to rocksdb.", + exitCodeListHeading = "Exit Codes:%n", + exitCodeList = { + "0:Successful", + "n:Internal error: exception occurred,please check toolkit.log"}) +public class DbConvert implements Callable { + + static { + RocksDB.loadLibrary(); + } + + private static final int BATCH = 256; + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + @CommandLine.Parameters(index = "0", defaultValue = "output-directory/database", + description = " Input path for leveldb. Default: ${DEFAULT-VALUE}") + private File src; + @CommandLine.Parameters(index = "1", defaultValue = "output-directory-dst/database", + description = "Output path for rocksdb. Default: ${DEFAULT-VALUE}") + private File dest; + + @CommandLine.Option(names = {"--safe"}, + description = "In safe mode, read data from leveldb then put rocksdb." + + "If not, just change engine.properties from leveldb to rocksdb," + + "rocksdb is compatible with leveldb for current version." + + "This may not be the case in the future." + + "Default: ${DEFAULT-VALUE}") + private boolean safe; + + @CommandLine.Option(names = {"-h", "--help"}) + private boolean help; + + + @Override + public Integer call() throws Exception { + if (help) { + spec.commandLine().usage(System.out); + return 0; + } + if (!src.exists()) { + logger.info(" {} does not exist.", src); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .errorText(String.format("%s does not exist.", src))); + return 404; + } + List files = Arrays.stream(Objects.requireNonNull(src.listFiles())) + .filter(File::isDirectory) + .filter(e -> !DBUtils.CHECKPOINT_DB_V2.equals(e.getName())) + .collect(Collectors.toList()); + + // add checkpoint v2 convert + File cpV2Dir = new File(Paths.get(src.toString(), DBUtils.CHECKPOINT_DB_V2).toString()); + List cpList = new ArrayList<>(); + if (cpV2Dir.exists()) { + cpList = Arrays.stream(Objects.requireNonNull(cpV2Dir.listFiles())) + .filter(File::isDirectory) + .collect(Collectors.toList()); + } + + if (files.isEmpty()) { + logger.info("{} does not contain any database.", src); + spec.commandLine().getOut().format("%s does not contain any database.", src).println(); + return 0; + } + final long time = System.currentTimeMillis(); + List services = new ArrayList<>(); + files.forEach(f -> services.add( + new DbConverter(src.getPath(), dest.getPath(), f.getName(), safe))); + cpList.forEach(f -> services.add( + new DbConverter( + Paths.get(src.getPath(), DBUtils.CHECKPOINT_DB_V2).toString(), + Paths.get(dest.getPath(), DBUtils.CHECKPOINT_DB_V2).toString(), + f.getName(), safe))); + List fails = ProgressBar.wrap(services.stream(), "convert task").parallel().map( + dbConverter -> { + try { + return dbConverter.doConvert() ? null : dbConverter.name(); + } catch (Exception e) { + logger.error("{}", e); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .errorText(e.getMessage())); + return dbConverter.name(); + } + }).filter(Objects::nonNull).collect(Collectors.toList()); + long during = (System.currentTimeMillis() - time) / 1000; + spec.commandLine().getOut().format("convert db done, fails: %s, take %d s.", + fails, during).println(); + logger.info("database convert use {} seconds total, fails: {}.", during, fails); + return fails.size(); + } + + interface Converter { + + boolean doConvert() throws Exception; + + String name(); + } + + static class DbConverter implements Converter { + private final String srcDir; + private final String dstDir; + private final String dbName; + private final Path srcDbPath; + private final Path dstDbPath; + + private long srcDbKeyCount = 0L; + private long dstDbKeyCount = 0L; + private long srcDbKeySum = 0L; + private long dstDbKeySum = 0L; + private long srcDbValueSum = 0L; + private long dstDbValueSum = 0L; + + private boolean safe; + + public DbConverter(String srcDir, String dstDir, String name, boolean safe) { + this.srcDir = srcDir; + this.dstDir = dstDir; + this.dbName = name; + this.srcDbPath = Paths.get(this.srcDir, name); + this.dstDbPath = Paths.get(this.dstDir, name); + this.safe = safe; + } + + @Override + public boolean doConvert() throws Exception { + + if (checkDone(this.dstDbPath.toString())) { + logger.info(" {} is done, skip it.", this.dbName); + return true; + } + + File levelDbFile = srcDbPath.toFile(); + if (!levelDbFile.exists()) { + logger.info(" {} does not exist.", srcDbPath); + return true; + } + if (!FileUtils.isLevelDBEngine(srcDbPath)) { + logger.info("Db {},not leveldb, ignored.", this.dbName); + return true; + } + long startTime = System.currentTimeMillis(); + if (this.dstDbPath.toFile().exists()) { + logger.info(" {} begin to clear exist database directory", this.dbName); + FileUtils.deleteDir(this.dstDbPath.toFile()); + logger.info(" {} clear exist database directory done.", this.dbName); + } + + FileUtils.createDirIfNotExists(dstDir); + + logger.info("Convert database {} start", this.dbName); + if (safe) { + convertLevelToRocks(); + compact(); + } else { + FileUtils.copyDir(Paths.get(srcDir), Paths.get(dstDir), dbName); + } + boolean result = check() && createEngine(dstDbPath.toString()); + long etime = System.currentTimeMillis(); + + if (result) { + if (safe) { + logger.info("Convert database {} successful end with {} key-value {} minutes", + this.dbName, this.srcDbKeyCount, (etime - startTime) / 1000.0 / 60); + } else { + logger.info("Convert database {} successful end {} minutes", + this.dbName, (etime - startTime) / 1000.0 / 60); + } + + } else { + logger.info("Convert database {} failure", this.dbName); + if (this.dstDbPath.toFile().exists()) { + logger.info(" {} begin to clear exist database directory", this.dbName); + FileUtils.deleteDir(this.dstDbPath.toFile()); + logger.info(" {} clear exist database directory done.", this.dbName); + } + } + return result; + } + + @Override + public String name() { + return dbName; + } + + private void batchInsert(RocksDB rocks, List keys, List values) + throws Exception { + try (org.rocksdb.WriteBatch batch = new org.rocksdb.WriteBatch()) { + for (int i = 0; i < keys.size(); i++) { + byte[] k = keys.get(i); + byte[] v = values.get(i); + batch.put(k, v); + } + write(rocks, batch); + } + keys.clear(); + values.clear(); + } + + /** + * https://github.com/facebook/rocksdb/issues/6625. + * + * @param rocks db + * @param batch write batch + * @throws Exception RocksDBException + */ + private void write(RocksDB rocks, org.rocksdb.WriteBatch batch) throws Exception { + try { + rocks.write(new org.rocksdb.WriteOptions(), batch); + } catch (RocksDBException e) { + // retry + if (maybeRetry(e)) { + TimeUnit.MILLISECONDS.sleep(1); + write(rocks, batch); + } else { + throw e; + } + } + } + + private boolean maybeRetry(RocksDBException e) { + boolean retry = false; + if (e.getStatus() != null) { + retry = e.getStatus().getCode() == Status.Code.TryAgain + || e.getStatus().getCode() == Status.Code.Busy + || e.getStatus().getCode() == Status.Code.Incomplete; + } + return retry || (e.getMessage() != null && ("Write stall".equalsIgnoreCase(e.getMessage()) + || ("Incomplete").equalsIgnoreCase(e.getMessage()))); + } + + /** + * https://github.com/facebook/rocksdb/wiki/RocksDB-FAQ . + * What's the fastest way to load data into RocksDB? + * + * @return if ok + */ + public void convertLevelToRocks() throws Exception { + List keys = new ArrayList<>(BATCH); + List values = new ArrayList<>(BATCH); + JniDBFactory.pushMemoryPool(1024 * 1024); + try ( + DB level = DBUtils.newLevelDb(srcDbPath); + RocksDB rocks = DBUtils.newRocksDbForBulkLoad(dstDbPath); + DBIterator levelIterator = level.iterator( + new org.iq80.leveldb.ReadOptions().fillCache(false))) { + + levelIterator.seekToFirst(); + + while (levelIterator.hasNext()) { + Map.Entry entry = levelIterator.next(); + byte[] key = entry.getKey(); + byte[] value = entry.getValue(); + srcDbKeyCount++; + srcDbKeySum = byteArrayToIntWithOne(srcDbKeySum, key); + srcDbValueSum = byteArrayToIntWithOne(srcDbValueSum, value); + keys.add(key); + values.add(value); + if (keys.size() >= BATCH) { + batchInsert(rocks, keys, values); + } + } + // clear + if (!keys.isEmpty()) { + batchInsert(rocks, keys, values); + } + } finally { + JniDBFactory.popMemoryPool(); + } + } + + private void compact() throws RocksDBException { + if (DBUtils.MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(this.dbName)) { + return; + } + try (RocksDB rocks = DBUtils.newRocksDb(this.dstDbPath)) { + logger.info("compact database {} start", this.dbName); + rocks.compactRange(); + logger.info("compact database {} end", this.dbName); + } + } + + private boolean check() throws RocksDBException { + if (!safe) { + return true; + } + try ( + RocksDB rocks = DBUtils.newRocksDbReadOnly(this.dstDbPath); + org.rocksdb.ReadOptions r = new org.rocksdb.ReadOptions().setFillCache(false); + RocksIterator rocksIterator = rocks.newIterator(r)) { + + // check + logger.info("check database {} start", this.dbName); + for (rocksIterator.seekToFirst(); rocksIterator.isValid(); rocksIterator.next()) { + byte[] key = rocksIterator.key(); + byte[] value = rocksIterator.value(); + dstDbKeyCount++; + dstDbKeySum = byteArrayToIntWithOne(dstDbKeySum, key); + dstDbValueSum = byteArrayToIntWithOne(dstDbValueSum, value); + } + logger.info("Check database {} end,dstDbKeyCount {}, dstDbKeySum {}, dstDbValueSum {}," + + "srcDbKeyCount {}, srcDbKeySum {}, srcDbValueSum {}", + dbName, dstDbKeyCount, dstDbKeySum, dstDbValueSum, + srcDbKeyCount, srcDbKeySum, srcDbValueSum); + return dstDbKeyCount == srcDbKeyCount && dstDbKeySum == srcDbKeySum + && dstDbValueSum == srcDbValueSum; + } + } + } + + private static boolean createEngine(String dir) { + String enginePath = dir + File.separator + DBUtils.FILE_ENGINE; + if (!FileUtils.createFileIfNotExists(enginePath)) { + return false; + } + return FileUtils.writeProperty(enginePath, DBUtils.KEY_ENGINE, DBUtils.ROCKSDB); + } + + private static boolean checkDone(String dir) { + String enginePath = dir + File.separator + DBUtils.FILE_ENGINE; + return FileUtils.isExists(enginePath); + } + + private static long byteArrayToIntWithOne(long sum, byte[] b) { + for (byte oneByte : b) { + sum += oneByte; + } + return sum; + } + +} diff --git a/plugins/src/main/java/org/tron/plugins/DbLite.java b/plugins/src/main/java/org/tron/plugins/DbLite.java new file mode 100644 index 00000000000..51b588bf9b5 --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/DbLite.java @@ -0,0 +1,740 @@ +package org.tron.plugins; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.primitives.Bytes; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; +import java.util.stream.LongStream; +import lombok.extern.slf4j.Slf4j; +import me.tongfei.progressbar.ProgressBar; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.ByteArray; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.FileUtils; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DBIterator; +import org.tron.plugins.utils.db.DbTool; +import org.tron.protos.Protocol; +import picocli.CommandLine; + +@Slf4j(topic = "lite") +@CommandLine.Command(name = "lite", + description = "Split lite data for java-tron.", + exitCodeListHeading = "Exit Codes:%n", + exitCodeList = { + "0:Successful", + "1:Internal error: exception occurred,please check toolkit.log"}) +public class DbLite implements Callable { + + private static final byte[] DB_KEY_LOWEST_BLOCK_NUM = "lowest_block_num".getBytes(); + private static final byte[] DB_KEY_NODE_TYPE = "node_type".getBytes(); + + private static final long START_TIME = System.currentTimeMillis() / 1000; + + private static long RECENT_BLKS = 65536; + + private static final String SNAPSHOT_DIR_NAME = "snapshot"; + private static final String HISTORY_DIR_NAME = "history"; + private static final String INFO_FILE_NAME = "info.properties"; + private static final String BACKUP_DIR_PREFIX = ".bak_"; + private static final String CHECKPOINT_DB = "tmp"; + private static final String BLOCK_DB_NAME = "block"; + private static final String BLOCK_INDEX_DB_NAME = "block-index"; + private static final String TRANS_DB_NAME = "trans"; + private static final String COMMON_DB_NAME = "common"; + private static final String TRANSACTION_RET_DB_NAME = "transactionRetStore"; + private static final String TRANSACTION_HISTORY_DB_NAME = "transactionHistoryStore"; + private static final String PROPERTIES_DB_NAME = "properties"; + private static final String TRANS_CACHE_DB_NAME = "trans-cache"; + + private static final List archiveDbs = Arrays.asList( + BLOCK_DB_NAME, + BLOCK_INDEX_DB_NAME, + TRANS_DB_NAME, + TRANSACTION_RET_DB_NAME, + TRANSACTION_HISTORY_DB_NAME); + + enum Operate { split, merge } + + enum Type { snapshot, history } + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + + @CommandLine.Option( + names = {"--operate", "-o"}, + defaultValue = "split", + description = "operate: [ ${COMPLETION-CANDIDATES} ]. Default: ${DEFAULT-VALUE}", + order = 1) + private Operate operate; + + @CommandLine.Option( + names = {"--type", "-t"}, + defaultValue = "snapshot", + description = "only used with operate=split: [ ${COMPLETION-CANDIDATES} ]." + + " Default: ${DEFAULT-VALUE}", + order = 2) + private Type type; + + @CommandLine.Option( + names = {"--fn-data-path", "-fn"}, + required = true, + description = "the database path to be split or merged.", + order = 3) + private String fnDataPath; + + @CommandLine.Option( + names = {"--dataset-path", "-ds"}, + required = true, + description = "when operation is `split`," + + "`dataset-path` is the path that store the `snapshot` or `history`," + + "when operation is `split`," + + "`dataset-path` is the `history` data path.", + order = 4) + private String datasetPath; + + @CommandLine.Option( + names = {"--help", "-h"}, + order = 5) + private boolean help; + + + @Override + public Integer call() { + if (help) { + spec.commandLine().usage(System.out); + return 0; + } + try { + switch (this.operate) { + case split: + if (Type.snapshot == this.type) { + generateSnapshot(fnDataPath, datasetPath); + } else if (Type.history == type) { + generateHistory(fnDataPath, datasetPath); + } + break; + case merge: + completeHistoryData(datasetPath, fnDataPath); + break; + default: + } + return 0; + } catch (Exception e) { + logger.error("{}", e); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .errorText(e.getMessage())); + spec.commandLine().usage(System.out); + return 1; + } finally { + DbTool.close(); + } + } + + /** + * Create the snapshot dataset. + * + * @param sourceDir the original fullnode database dir, + * same with {storage.db.directory} in conf file. + * @param snapshotDir the path that stores the snapshot dataset + */ + public void generateSnapshot(String sourceDir, String snapshotDir) { + logger.info("Start create snapshot."); + spec.commandLine().getOut().println("Start create snapshot."); + long start = System.currentTimeMillis(); + snapshotDir = Paths.get(snapshotDir, SNAPSHOT_DIR_NAME).toString(); + try { + hasEnoughBlock(sourceDir); + List snapshotDbs = getSnapshotDbs(sourceDir); + split(sourceDir, snapshotDir, snapshotDbs); + mergeCheckpoint2Snapshot(sourceDir, snapshotDir); + // write genesisBlock , latest recent blocks and trans + fillSnapshotBlockAndTransDb(sourceDir, snapshotDir); + // save min block to info + generateInfoProperties(Paths.get(snapshotDir, INFO_FILE_NAME).toString(), + getSecondBlock(snapshotDir)); + } catch (IOException | RocksDBException e) { + logger.error("Create snapshot failed, {}.", e.getMessage()); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .stackTraceText(e)); + return; + } + long during = (System.currentTimeMillis() - start) / 1000; + logger.info("Create snapshot finished, take {} s.", during); + spec.commandLine().getOut().format("Create snapshot finished, take %d s.", during).println(); + } + + /** + * Create the history dataset. + * + * @param sourceDir the original fullnode database dir, + * same with {storage.db.directory} in conf file. + * @param historyDir the path that stores the history dataset + */ + public void generateHistory(String sourceDir, String historyDir) { + logger.info("Start create history."); + spec.commandLine().getOut().println("Start create history."); + long start = System.currentTimeMillis(); + historyDir = Paths.get(historyDir, HISTORY_DIR_NAME).toString(); + try { + if (isLite(sourceDir)) { + throw new IllegalStateException( + String.format("Unavailable sourceDir: %s is not fullNode data.", sourceDir)); + } + hasEnoughBlock(sourceDir); + split(sourceDir, historyDir, archiveDbs); + mergeCheckpoint2History(sourceDir, historyDir); + // save max block to info + generateInfoProperties(Paths.get(historyDir, INFO_FILE_NAME).toString(), + getLatestBlockHeaderNum(sourceDir)); + } catch (IOException | RocksDBException e) { + logger.error("Create history failed, {}.", e.getMessage()); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .stackTraceText(e)); + return; + } + long during = (System.currentTimeMillis() - start) / 1000; + logger.info("Create history finished, take {} s.", during); + spec.commandLine().getOut().format("Create history finished, take %d s.", during).println(); + } + + /** + * Merge the history dataset into database. + * + * @param historyDir the path that stores the history dataset + * + * @param liteDir lite fullnode database path + */ + public void completeHistoryData(String historyDir, String liteDir) { + logger.info("Start merge history to lite node."); + spec.commandLine().getOut().println("Start merge history to lite node."); + long start = System.currentTimeMillis(); + try { + // check historyDir is from lite data + if (isLite(historyDir)) { + throw new IllegalStateException( + String.format("Unavailable history: %s is not generated by fullNode data.", + historyDir)); + } + // 1. check block number and genesis block are compatible, + // and return the block numbers of snapshot and history + BlockNumInfo blockNumInfo = checkAndGetBlockNumInfo(historyDir, liteDir); + // 2. move archive dbs to bak + backupArchiveDbs(liteDir); + // 3. copy history data to liteDir + copyHistory2Database(historyDir, liteDir); + // 4. delete the extra block data in history data + trimExtraHistory(liteDir, blockNumInfo); + // 5. merge bak to database + mergeBak2Database(liteDir, blockNumInfo); + // 6. delete bak dir + deleteBackupArchiveDbs(liteDir); + // 7. delete snapshot flag + deleteSnapshotFlag(liteDir); + } catch (IOException | RocksDBException e) { + logger.error("Merge history data to database failed, {}.", e.getMessage()); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .stackTraceText(e)); + return; + } + long during = (System.currentTimeMillis() - start) / 1000; + logger.info("Merge history finished, take {} s.", during); + spec.commandLine().getOut().format("Merge history finished, take %d s.", during).println(); + } + + private List getSnapshotDbs(String sourceDir) { + List snapshotDbs = Lists.newArrayList(); + File basePath = new File(sourceDir); + Arrays.stream(Objects.requireNonNull(basePath.listFiles())) + .filter(File::isDirectory) + .filter(dir -> !archiveDbs.contains(dir.getName())) + .forEach(dir -> snapshotDbs.add(dir.getName())); + return snapshotDbs; + } + + private void mergeCheckpoint2Snapshot(String sourceDir, String historyDir) { + List snapshotDbs = getSnapshotDbs(sourceDir); + mergeCheckpoint(sourceDir, historyDir, snapshotDbs); + } + + private void mergeCheckpoint2History(String sourceDir, String destDir) { + mergeCheckpoint(sourceDir, destDir, archiveDbs); + } + + private void split(String sourceDir, String destDir, List dbs) throws IOException { + logger.info("Begin to split the dbs."); + spec.commandLine().getOut().println("Begin to split the dbs."); + if (!new File(sourceDir).isDirectory()) { + throw new RuntimeException(String.format("sourceDir: %s must be a directory ", sourceDir)); + } + File destPath = new File(destDir); + if (new File(destDir).exists()) { + throw new RuntimeException(String.format( + "destDir: %s is already exist, please remove it first", destDir)); + } + if (!destPath.mkdirs()) { + throw new RuntimeException(String.format("destDir: %s create failed, please check", destDir)); + } + FileUtils.copyDatabases(Paths.get(sourceDir), Paths.get(destDir), dbs); + } + + private void mergeCheckpoint(String sourceDir, String destDir, List destDbs) { + logger.info("Begin to merge checkpoint to dataset."); + spec.commandLine().getOut().println("Begin to merge checkpoint to dataset."); + try { + List cpList = getCheckpointV2List(sourceDir); + if (cpList.size() > 0) { + for (String cp : cpList) { + DBInterface checkpointDb = DbTool.getDB( + sourceDir + "/" + DBUtils.CHECKPOINT_DB_V2, cp); + recover(checkpointDb, destDir, destDbs); + } + } else if (Paths.get(sourceDir, CHECKPOINT_DB).toFile().exists()) { + DBInterface tmpDb = DbTool.getDB(sourceDir, CHECKPOINT_DB); + recover(tmpDb, destDir, destDbs); + } + } catch (IOException | RocksDBException e) { + throw new RuntimeException(e); + } + } + + private void recover(DBInterface db, String destDir, List destDbs) + throws IOException, RocksDBException { + try (DBIterator iterator = db.iterator()) { + for (iterator.seekToFirst(); iterator.hasNext(); iterator.next()) { + byte[] key = iterator.getKey(); + byte[] value = iterator.getValue(); + String dbName = DBUtils.simpleDecode(key); + // skip trans-cache db + if (TRANS_CACHE_DB_NAME.equalsIgnoreCase(dbName)) { + continue; + } + byte[] realKey = Arrays.copyOfRange(key, dbName.getBytes().length + 4, key.length); + byte[] realValue = + value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); + if (destDbs != null && destDbs.contains(dbName)) { + DBInterface destDb = DbTool.getDB(destDir, dbName); + if (realValue != null) { + destDb.put(realKey, realValue); + } else { + byte op = value[0]; + if (DBUtils.Operator.DELETE.getValue() == op) { + destDb.delete(realKey); + } else { + destDb.put(realKey, new byte[0]); + } + } + } + } + } + } + + private void generateInfoProperties(String propertyfile, long num) + throws IOException, RocksDBException { + logger.info("Create {} for dataset.", INFO_FILE_NAME); + spec.commandLine().getOut().format("Create %s for dataset.", INFO_FILE_NAME).println(); + if (!FileUtils.createFileIfNotExists(propertyfile)) { + throw new RuntimeException("Create properties file failed."); + } + if (!FileUtils.writeProperty(propertyfile, DBUtils.SPLIT_BLOCK_NUM, Long.toString(num))) { + throw new RuntimeException("Write properties file failed."); + } + } + + private long getLatestBlockHeaderNum(String databaseDir) throws IOException, RocksDBException { + // query latest_block_header_number from checkpoint first + final String latestBlockHeaderNumber = "latest_block_header_number"; + List cpList = getCheckpointV2List(databaseDir); + DBInterface checkpointDb; + if (cpList.size() > 0) { + String lastestCp = cpList.get(cpList.size() - 1); + checkpointDb = DbTool.getDB( + databaseDir + "/" + DBUtils.CHECKPOINT_DB_V2, lastestCp); + } else { + checkpointDb = DbTool.getDB(databaseDir, CHECKPOINT_DB); + } + Long blockNumber = getLatestBlockHeaderNumFromCP(checkpointDb, + latestBlockHeaderNumber.getBytes()); + if (blockNumber != null) { + return blockNumber; + } + // query from propertiesDb if checkpoint not contains latest_block_header_number + DBInterface propertiesDb = DbTool.getDB(databaseDir, PROPERTIES_DB_NAME); + return Optional.ofNullable(propertiesDb.get(ByteArray.fromString(latestBlockHeaderNumber))) + .map(ByteArray::toLong) + .orElseThrow( + () -> new IllegalArgumentException("not found latest block header number")); + } + + private Long getLatestBlockHeaderNumFromCP(DBInterface db, byte[] key) { + byte[] value = db.get(Bytes.concat(simpleEncode(PROPERTIES_DB_NAME), key)); + if (value != null && value.length > 1) { + return ByteArray.toLong(Arrays.copyOfRange(value, 1, value.length)); + } + return null; + } + + /** + * recent blocks, trans and genesis block. + */ + private void fillSnapshotBlockAndTransDb(String sourceDir, String snapshotDir) + throws IOException, RocksDBException { + logger.info("Begin to fill {} block, genesis block and trans to snapshot.", RECENT_BLKS); + spec.commandLine().getOut().format( + "Begin to fill %d block, genesis block and trans to snapshot.", RECENT_BLKS).println(); + DBInterface sourceBlockIndexDb = DbTool.getDB(sourceDir, BLOCK_INDEX_DB_NAME); + DBInterface sourceBlockDb = DbTool.getDB(sourceDir, BLOCK_DB_NAME); + // init snapshot db ,keep engine same as source + DBInterface destBlockDb = DbTool.getDB(sourceDir, snapshotDir, BLOCK_DB_NAME); + DBInterface destBlockIndexDb = DbTool.getDB(sourceDir, snapshotDir, BLOCK_INDEX_DB_NAME); + DBInterface destTransDb = DbTool.getDB(sourceDir, snapshotDir, TRANS_DB_NAME); + // put genesis block and block-index into snapshot + long genesisBlockNum = 0L; + byte[] genesisBlockID = sourceBlockIndexDb.get(ByteArray.fromLong(genesisBlockNum)); + destBlockIndexDb.put(ByteArray.fromLong(genesisBlockNum), genesisBlockID); + destBlockDb.put(genesisBlockID, sourceBlockDb.get(genesisBlockID)); + + long latestBlockNum = getLatestBlockHeaderNum(sourceDir); + long startIndex = latestBlockNum - RECENT_BLKS + 1; + // put the recent blocks and trans in snapshot + ProgressBar.wrap(LongStream.rangeClosed(startIndex, latestBlockNum), "fillBlockAndTrans") + .forEach(blockNum -> { + try { + byte[] blockId = getDataFromSourceDB(sourceDir, BLOCK_INDEX_DB_NAME, + Longs.toByteArray(blockNum)); + byte[] block = getDataFromSourceDB(sourceDir, BLOCK_DB_NAME, blockId); + // put block + destBlockDb.put(blockId, block); + // put block index + destBlockIndexDb.put(ByteArray.fromLong(blockNum), blockId); + // put trans + long finalBlockNum = blockNum; + Protocol.Block.parseFrom(block).getTransactionsList().stream().map( + tc -> DBUtils.getTransactionId(tc).getBytes()) + .map(bytes -> Maps.immutableEntry(bytes, Longs.toByteArray(finalBlockNum))) + .forEach(e -> destTransDb.put(e.getKey(), e.getValue())); + } catch (IOException | RocksDBException e) { + throw new RuntimeException(e.getMessage()); + } + }); + + DBInterface destCommonDb = DbTool.getDB(snapshotDir, COMMON_DB_NAME); + destCommonDb.put(DB_KEY_NODE_TYPE, ByteArray.fromInt(DBUtils.NODE_TYPE_LIGHT_NODE)); + destCommonDb.put(DB_KEY_LOWEST_BLOCK_NUM, ByteArray.fromLong(startIndex)); + // copy engine.properties for block、block-index、trans from source if exist + copyEngineIfExist(sourceDir, snapshotDir, BLOCK_DB_NAME, BLOCK_INDEX_DB_NAME, TRANS_DB_NAME); + } + + private void copyEngineIfExist(String source, String dest, String... dbNames) { + for (String dbName : dbNames) { + Path ori = Paths.get(source, dbName, DBUtils.FILE_ENGINE); + if (ori.toFile().exists()) { + FileUtils.copy(ori, Paths.get(dest, dbName, DBUtils.FILE_ENGINE)); + } + } + } + + private byte[] getGenesisBlockHash(String parentDir) throws IOException, RocksDBException { + long genesisBlockNum = 0L; + DBInterface blockIndexDb = DbTool.getDB(parentDir, BLOCK_INDEX_DB_NAME); + byte[] result = blockIndexDb.get(ByteArray.fromLong(genesisBlockNum)); + // when merge history, block-index db will be moved to bak dir and replaced by history + // so should close this db and reopen it. + DbTool.closeDB(parentDir, BLOCK_INDEX_DB_NAME); + return result; + } + + private static byte[] simpleEncode(String s) { + byte[] bytes = s.getBytes(); + byte[] length = Ints.toByteArray(bytes.length); + byte[] r = new byte[4 + bytes.length]; + System.arraycopy(length, 0, r, 0, 4); + System.arraycopy(bytes, 0, r, 4, bytes.length); + return r; + } + + private BlockNumInfo checkAndGetBlockNumInfo(String historyDir, String liteDir) + throws IOException, RocksDBException { + logger.info("Check the compatibility of this history."); + spec.commandLine().getOut().println("Check the compatibility of this history."); + String snapshotInfo = Paths.get(liteDir, INFO_FILE_NAME).toString(); + String historyInfo = Paths.get(historyDir, INFO_FILE_NAME).toString(); + if (!FileUtils.isExists(snapshotInfo)) { + throw new FileNotFoundException( + "Snapshot property file is not found. maybe this is a complete fullnode?"); + } + if (!FileUtils.isExists(historyInfo)) { + throw new FileNotFoundException("history property file is not found."); + } + // min block for snapshot + long snapshotMinNum = getSecondBlock(liteDir); + // max block for snapshot + long snapshotMaxNum = getLatestBlockHeaderNum(liteDir); + // max block for history + long historyMaxNum = Long.parseLong(FileUtils.readProperty(historyInfo, DBUtils + .SPLIT_BLOCK_NUM)); + if (historyMaxNum < snapshotMinNum) { + throw new RuntimeException( + String.format( + "History max block is lower than snapshot min number, history: %d, snapshot: %d", + historyMaxNum, snapshotMinNum)); + } + // check genesis block is equal + if (!Arrays.equals(getGenesisBlockHash(liteDir), getGenesisBlockHash(historyDir))) { + throw new RuntimeException(String.format( + "Genesis block hash is not equal, history: %s, database: %s", + Arrays.toString(getGenesisBlockHash(historyDir)), + Arrays.toString(getGenesisBlockHash(liteDir)))); + } + return new BlockNumInfo(snapshotMinNum, historyMaxNum, snapshotMaxNum); + } + + private void backupArchiveDbs(String databaseDir) throws IOException { + Path bakDir = Paths.get(databaseDir, BACKUP_DIR_PREFIX + START_TIME); + logger.info("Backup the archive dbs to {}.", bakDir); + spec.commandLine().getOut().format("Backup the archive dbs to %s.", bakDir).println(); + if (!FileUtils.createDirIfNotExists(bakDir.toString())) { + throw new RuntimeException(String.format("create bak dir %s failed", bakDir)); + } + FileUtils.copyDatabases(Paths.get(databaseDir), bakDir, archiveDbs); + archiveDbs.forEach(db -> FileUtils.deleteDir(new File(databaseDir, db))); + } + + private void copyHistory2Database(String historyDir, String databaseDir) throws IOException { + logger.info("Begin to copy history to database."); + spec.commandLine().getOut().println("Begin to copy history to database."); + FileUtils.copyDatabases(Paths.get(historyDir), Paths.get(databaseDir), archiveDbs); + } + + private void trimExtraHistory(String liteDir, BlockNumInfo blockNumInfo) + throws IOException, RocksDBException { + long start = blockNumInfo.getSnapshotMaxNum() + 1; + long end = blockNumInfo.getHistoryMaxNum(); + if (start > end) { + logger.info("Ignore trimming the history data, from {} to {}.", start, end); + spec.commandLine().getOut() + .format("Ignore trimming the history data, from %d to %d.", start, end).println(); + return; + } + logger.info("Begin to trim the history data, from {} to {}.", start, end); + spec.commandLine().getOut() + .format("Begin to trim the history data, from %d to %d.", start, end).println(); + DBInterface blockIndexDb = DbTool.getDB(liteDir, BLOCK_INDEX_DB_NAME); + DBInterface blockDb = DbTool.getDB(liteDir, BLOCK_DB_NAME); + DBInterface transDb = DbTool.getDB(liteDir, TRANS_DB_NAME); + DBInterface tranRetDb = DbTool.getDB(liteDir, TRANSACTION_RET_DB_NAME); + + + ProgressBar.wrap(LongStream.rangeClosed(start, end) + .boxed() + .sorted((a, b) -> Long.compare(b, a)), "trimHistory").forEach(n -> { + try { + byte[] blockIdHash = blockIndexDb.get(ByteArray.fromLong(n)); + Protocol.Block block = Protocol.Block.parseFrom(blockDb.get(blockIdHash)); + // delete transactions + for (Protocol.Transaction e : block.getTransactionsList()) { + transDb.delete(DBUtils.getTransactionId(e).getBytes()); + } + // delete transaction result + tranRetDb.delete(ByteArray.fromLong(n)); + // delete block + blockDb.delete(blockIdHash); + // delete block index + blockIndexDb.delete(ByteArray.fromLong(n)); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + private void mergeBak2Database(String liteDir, BlockNumInfo blockNumInfo) throws + IOException, RocksDBException { + long start = blockNumInfo.getHistoryMaxNum() + 1; + long end = blockNumInfo.getSnapshotMaxNum(); + + if (start > end) { + logger.info("Ignore merging the bak data, start {} end {}.", start, end); + spec.commandLine().getOut() + .format("Ignore merging the bak data, start %d end %d.", start, end).println(); + return; + } + + + Path bakDir = Paths.get(liteDir, BACKUP_DIR_PREFIX + START_TIME); + logger.info("Begin to merge {} to database, start {} end {}.", bakDir, start, end); + spec.commandLine().getOut() + .format("Begin to merge %s to database, start %d end %d.", bakDir, start, end).println(); + byte[] head = ByteArray.fromLong(start); + // use seek to skip + archiveDbs.stream().parallel().forEach(dbName -> { + try { + DBInterface bakDb = DbTool.getDB(bakDir.toString(), dbName); + DBInterface destDb = DbTool.getDB(liteDir, dbName); + try (DBIterator iterator = bakDb.iterator()) { + if (TRANS_DB_NAME.equals(dbName) || TRANSACTION_HISTORY_DB_NAME.equals(dbName)) { + iterator.seekToFirst(); + } else { + iterator.seek(head); + } + ProgressBar.wrap(iterator, dbName) + .forEachRemaining(e -> destDb.put(e.getKey(), e.getValue())); + } + } catch (IOException | RocksDBException e) { + throw new RuntimeException(e); + } + }); + } + + private byte[] getDataFromSourceDB(String sourceDir, String dbName, byte[] key) + throws IOException, RocksDBException { + DBInterface sourceDb = DbTool.getDB(sourceDir, dbName); + DBInterface checkpointDb = DbTool.getDB(sourceDir, CHECKPOINT_DB); + // get data from tmp first. + byte[] valueFromTmp = checkpointDb.get(Bytes.concat(simpleEncode(dbName), key)); + byte[] value; + if (isEmptyBytes(valueFromTmp)) { + value = sourceDb.get(key); + } else { + value = valueFromTmp.length == 1 + ? null : Arrays.copyOfRange(valueFromTmp, 1, valueFromTmp.length); + } + if (isEmptyBytes(value)) { + throw new RuntimeException(String.format("data not found in store, dbName: %s, key: %s", + dbName, Arrays.toString(key))); + } + return value; + } + + /** + * return true if byte array is null or length is 0. + * + * @param b bytes + * @return true or false + */ + private static boolean isEmptyBytes(byte[] b) { + if (b != null) { + return b.length == 0; + } + return true; + } + + private void deleteSnapshotFlag(String databaseDir) throws IOException, RocksDBException { + logger.info("Delete the info file from {}.", databaseDir); + spec.commandLine().getOut().format("Delete the info file from %s.", databaseDir).println(); + Files.delete(Paths.get(databaseDir, INFO_FILE_NAME)); + if (!isLite(databaseDir)) { + DBInterface destCommonDb = DbTool.getDB(databaseDir, COMMON_DB_NAME); + destCommonDb.delete(DB_KEY_NODE_TYPE); + destCommonDb.delete(DB_KEY_LOWEST_BLOCK_NUM); + logger.info("Deleted {} and {} from {} to identify this node is a real fullnode.", + "node_type", "lowest_block_num", COMMON_DB_NAME); + spec.commandLine().getOut().format( + "Deleted %s and %s from %s to identify this node is a real fullnode.", + "node_type", "lowest_block_num", COMMON_DB_NAME).println(); + } + + } + + private void deleteBackupArchiveDbs(String liteDir) throws IOException, RocksDBException { + + Path bakDir = Paths.get(liteDir, BACKUP_DIR_PREFIX + START_TIME); + logger.info("Begin to delete bak dir {}.", bakDir); + spec.commandLine().getOut().format("Begin to delete bak dir %s.", bakDir).println(); + if (FileUtils.deleteDir(bakDir.toFile())) { + logger.info("End to delete bak dir {}.", bakDir); + spec.commandLine().getOut().format("End to delete bak dir %s.", bakDir).println(); + } else { + logger.info("Fail to delete bak dir {}, please remove manually.", bakDir); + spec.commandLine().getOut().format("Fail to delete bak dir %s, please remove manually.", + bakDir).println(); + } + } + + private void hasEnoughBlock(String sourceDir) throws RocksDBException, IOException { + // check latest + long latest = getLatestBlockHeaderNum(sourceDir); + // check second ,skip 0; + long second = getSecondBlock(sourceDir); + if (latest - second + 1 < RECENT_BLKS) { + throw new NoSuchElementException( + String.format("At least %d blocks in block store, actual latestBlock:%d, firstBlock:%d.", + RECENT_BLKS, latest, second)); + } + } + + private boolean isLite(String databaseDir) throws RocksDBException, IOException { + return getSecondBlock(databaseDir) > 1; + } + + private long getSecondBlock(String databaseDir) throws RocksDBException, IOException { + long num = 0; + DBInterface sourceBlockIndexDb = DbTool.getDB(databaseDir, BLOCK_INDEX_DB_NAME); + DBIterator iterator = sourceBlockIndexDb.iterator(); + iterator.seek(ByteArray.fromLong(1)); + if (iterator.hasNext()) { + num = Longs.fromByteArray(iterator.getKey()); + } + return num; + } + + @VisibleForTesting + public static void setRecentBlks(long recentBlks) { + RECENT_BLKS = recentBlks; + } + + @VisibleForTesting + public static void reSetRecentBlks() { + RECENT_BLKS = 65536; + } + + private List getCheckpointV2List(String sourceDir) { + File file = new File(Paths.get(sourceDir, DBUtils.CHECKPOINT_DB_V2).toString()); + if (file.exists() && file.isDirectory() && file.list() != null) { + return Arrays.stream(Objects.requireNonNull(file.list())).sorted() + .collect(Collectors.toList()); + } + return Lists.newArrayList(); + } + + static class BlockNumInfo { + private final long snapshotMinNum; + private final long snapshotMaxNum; + private final long historyMaxNum; + + public BlockNumInfo(long snapshotMinNum, long historyMaxNum, long snapshotMaxNum) { + this.snapshotMinNum = snapshotMinNum; + this.historyMaxNum = historyMaxNum; + this.snapshotMaxNum = snapshotMaxNum; + } + + public long getSnapshotMinNum() { + return snapshotMinNum; + } + + public long getHistoryMaxNum() { + return historyMaxNum; + } + + public long getSnapshotMaxNum() { + return snapshotMaxNum; + } + } +} + + + diff --git a/plugins/src/main/java/org/tron/plugins/DbMove.java b/plugins/src/main/java/org/tron/plugins/DbMove.java index 77c3dca26f8..a5619d2d7ed 100644 --- a/plugins/src/main/java/org/tron/plugins/DbMove.java +++ b/plugins/src/main/java/org/tron/plugins/DbMove.java @@ -15,12 +15,15 @@ import java.util.Set; import java.util.concurrent.Callable; import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; import me.tongfei.progressbar.ProgressBar; +import org.tron.plugins.utils.FileUtils; import picocli.CommandLine; import picocli.CommandLine.Command; +@Slf4j(topic = "move") @Command(name = "mv", aliases = "move", - description = "mv db to pre-set new path . For example HDD,reduce storage expenses.") + description = "Move db to pre-set new path . For example HDD,reduce storage expenses.") public class DbMove implements Callable { private static final String PROPERTIES_CONFIG_KEY = "storage.properties"; @@ -35,24 +38,24 @@ public class DbMove implements Callable { @CommandLine.Option(names = {"-d", "--database-directory"}, defaultValue = "output-directory", - converter = Db.PathConverter.class, + converter = PathConverter.class, description = "database directory path. Default: ${DEFAULT-VALUE}") static Path database; @CommandLine.Option(names = {"-c", "--config"}, defaultValue = "config.conf", converter = ConfigConverter.class, - order = Integer.MAX_VALUE, description = " config file. Default: ${DEFAULT-VALUE}") Config config; - @CommandLine.Option(names = {"-h", "--help"}, help = true, description = "display a help message") - boolean help; + @CommandLine.Option(names = {"-h", "--help"}) + static boolean help; @Override public Integer call() throws Exception { if (help) { spec.commandLine().usage(System.out); + help = false; return 0; } @@ -129,7 +132,7 @@ private void run(Property p) { } }); try { - if (deleteDir(p.original.toFile())) { + if (FileUtils.deleteDir(p.original.toFile())) { Files.createSymbolicLink(p.original, p.destination); } } catch (IOException | UnsupportedOperationException x) { @@ -144,20 +147,6 @@ private void printNotExist() { spec.commandLine().getErr().println(NOT_FIND); } - /** - * delete directory. - */ - public static boolean deleteDir(File dir) { - if (dir.isDirectory()) { - String[] children = dir.list(); - if (children != null) { - for (String child : children) { - deleteDir(new File(dir, child)); - } - } - } - return dir.delete(); - } static class Property { @@ -174,7 +163,7 @@ public Property(String name, Path original, Path destination) throws IOException if (this.original.toFile().isFile()) { throw new IOException(this.original + " is a file!"); } - if (isSymbolicLink(original.toFile())) { + if (FileUtils.isSymbolicLink(original.toFile())) { throw new IOException(original + " is symbolicLink!"); } this.destination = destination.toFile().getCanonicalFile().toPath(); @@ -185,21 +174,6 @@ public Property(String name, Path original, Path destination) throws IOException throw new IOException("destination and original can not be same:[" + this.original + "]!"); } } - - public boolean isSymbolicLink(File file) throws IOException { - if (file == null) { - throw new NullPointerException("File must not be null"); - } - - File canon; - if (file.getParent() == null) { - canon = file; - } else { - File canonDir = file.getParentFile().getCanonicalFile(); - canon = new File(canonDir, file.getName()); - } - return !canon.getCanonicalFile().equals(canon.getAbsoluteFile()); - } } static class ConfigConverter implements CommandLine.ITypeConverter { @@ -210,6 +184,9 @@ static class ConfigConverter implements CommandLine.ITypeConverter { } public Config convert(String value) throws Exception { + if (help) { + return null; + } File file = Paths.get(value).toFile(); if (file.exists() && file.isFile()) { Config config = ConfigFactory.parseFile(Paths.get(value).toFile()); @@ -248,4 +225,21 @@ public Config convert(String value) throws Exception { } } } + + static class PathConverter implements CommandLine.ITypeConverter { + PathConverter() { + } + + public Path convert(String value) throws IOException { + if (help) { + return null; + } + File file = Paths.get(value).toFile(); + if (file.exists() && file.isDirectory()) { + return file.toPath(); + } else { + throw new IOException("DB path [" + value + "] not exist!"); + } + } + } } diff --git a/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java b/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java new file mode 100644 index 00000000000..0879f770e1f --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java @@ -0,0 +1,28 @@ +package org.tron.plugins.comparator; + +import org.iq80.leveldb.DBComparator; +import org.tron.plugins.utils.MarketUtils; + +public class MarketOrderPriceComparatorForLevelDB implements DBComparator { + + @Override + public String name() { + return "MarketOrderPriceComparator"; + } + + @Override + public byte[] findShortestSeparator(byte[] start, byte[] limit) { + return new byte[0]; + } + + @Override + public byte[] findShortSuccessor(byte[] key) { + return new byte[0]; + } + + @Override + public int compare(byte[] o1, byte[] o2) { + return MarketUtils.comparePriceKey(o1, o2); + } + +} \ No newline at end of file diff --git a/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java b/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java new file mode 100644 index 00000000000..cd718bdd2d7 --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java @@ -0,0 +1,38 @@ +package org.tron.plugins.comparator; + +import org.rocksdb.ComparatorOptions; +import org.rocksdb.DirectSlice; +import org.rocksdb.util.DirectBytewiseComparator; +import org.tron.plugins.utils.MarketUtils; + +public class MarketOrderPriceComparatorForRockDB extends DirectBytewiseComparator { + + public MarketOrderPriceComparatorForRockDB(final ComparatorOptions copt) { + super(copt); + } + + @Override + public String name() { + return "MarketOrderPriceComparator"; + } + + @Override + public int compare(final DirectSlice a, final DirectSlice b) { + return MarketUtils.comparePriceKey(convertDataToBytes(a), convertDataToBytes(b)); + } + + /** + * DirectSlice.data().array will throw UnsupportedOperationException. + * */ + public byte[] convertDataToBytes(DirectSlice directSlice) { + int capacity = directSlice.data().capacity(); + byte[] bytes = new byte[capacity]; + + for (int i = 0; i < capacity; i++) { + bytes[i] = directSlice.get(i); + } + + return bytes; + } + +} \ No newline at end of file diff --git a/plugins/src/main/java/org/tron/plugins/utils/ByteArray.java b/plugins/src/main/java/org/tron/plugins/utils/ByteArray.java new file mode 100644 index 00000000000..3422c36ca9d --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/ByteArray.java @@ -0,0 +1,101 @@ +package org.tron.plugins.utils; + +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import java.math.BigInteger; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; +import org.bouncycastle.util.encoders.Hex; + +public class ByteArray { + + public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; + public static final byte[] ZERO_BYTE_ARRAY = new byte[]{0}; + public static final int WORD_SIZE = 32; + + /** + * get bytes data from string data. + */ + public static byte[] fromString(String s) { + return StringUtils.isBlank(s) ? null : s.getBytes(); + } + + /** + * get string data from bytes data. + */ + public static String toStr(byte[] b) { + return ArrayUtils.isEmpty(b) ? null : new String(b); + } + + public static byte[] fromLong(long val) { + return Longs.toByteArray(val); + } + + /** + * get long data from bytes data. + */ + public static long toLong(byte[] b) { + return ArrayUtils.isEmpty(b) ? 0 : new BigInteger(1, b).longValue(); + } + + public static byte[] fromInt(int val) { + return Ints.toByteArray(val); + } + + /** + * get int data from bytes data. + */ + public static int toInt(byte[] b) { + return ArrayUtils.isEmpty(b) ? 0 : new BigInteger(1, b).intValue(); + } + + public static int compareUnsigned(byte[] a, byte[] b) { + if (a == b) { + return 0; + } + if (a == null) { + return -1; + } + if (b == null) { + return 1; + } + int minLen = Math.min(a.length, b.length); + for (int i = 0; i < minLen; ++i) { + int aVal = a[i] & 0xFF; + int bVal = b[i] & 0xFF; + if (aVal < bVal) { + return -1; + } + if (aVal > bVal) { + return 1; + } + } + if (a.length < b.length) { + return -1; + } + if (a.length > b.length) { + return 1; + } + return 0; + } + + public static String toHexString(byte[] data) { + return data == null ? "" : Hex.toHexString(data); + } + + /** + * get bytes data from hex string data. + */ + public static byte[] fromHexString(String data) { + if (data == null) { + return EMPTY_BYTE_ARRAY; + } + if (data.startsWith("0x")) { + data = data.substring(2); + } + if (data.length() % 2 != 0) { + data = "0" + data; + } + return Hex.decode(data); + } +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/CryptoUitls.java b/plugins/src/main/java/org/tron/plugins/utils/CryptoUitls.java new file mode 100644 index 00000000000..6d3e4ccb548 --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/CryptoUitls.java @@ -0,0 +1,6 @@ +package org.tron.plugins.utils; + +public class CryptoUitls { + + public static final String ECKey_ENGINE = "ECKey"; +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/DBUtils.java b/plugins/src/main/java/org/tron/plugins/utils/DBUtils.java new file mode 100644 index 00000000000..19547e2b5a5 --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/DBUtils.java @@ -0,0 +1,153 @@ +package org.tron.plugins.utils; + +import static org.fusesource.leveldbjni.JniDBFactory.factory; + +import com.google.common.primitives.Ints; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Arrays; +import lombok.Getter; +import org.iq80.leveldb.CompressionType; +import org.iq80.leveldb.DB; +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.BloomFilter; +import org.rocksdb.ComparatorOptions; +import org.rocksdb.Options; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.tron.plugins.comparator.MarketOrderPriceComparatorForLevelDB; +import org.tron.plugins.comparator.MarketOrderPriceComparatorForRockDB; +import org.tron.protos.Protocol; + +public class DBUtils { + + + public enum Operator { + CREATE((byte) 0), + MODIFY((byte) 1), + DELETE((byte) 2), + PUT((byte) 3); + + @Getter + private byte value; + + Operator(byte value) { + this.value = value; + } + + static Operator valueOf(byte b) { + switch (b) { + case 0: + return Operator.CREATE; + case 1: + return Operator.MODIFY; + case 2: + return Operator.DELETE; + case 3: + return Operator.PUT; + default: + return null; + } + } + } + + public static final String SPLIT_BLOCK_NUM = "split_block_num"; + public static final String MARKET_PAIR_PRICE_TO_ORDER = "market_pair_price_to_order"; + public static final String CHECKPOINT_DB_V2 = "checkpoint"; + public static final String TMP = "tmp"; + + public static final int NODE_TYPE_LIGHT_NODE = 1; + + public static final String KEY_ENGINE = "ENGINE"; + public static final String FILE_ENGINE = "engine.properties"; + public static final String LEVELDB = "LEVELDB"; + public static final String ROCKSDB = "ROCKSDB"; + + public static DB newLevelDb(Path db) throws IOException { + File file = db.toFile(); + org.iq80.leveldb.Options dbOptions = newDefaultLevelDbOptions(); + if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(file.getName())) { + dbOptions.comparator(new MarketOrderPriceComparatorForLevelDB()); + } + return factory.open(file, dbOptions); + } + + public static org.iq80.leveldb.Options newDefaultLevelDbOptions() { + org.iq80.leveldb.Options dbOptions = new org.iq80.leveldb.Options(); + dbOptions.createIfMissing(true); + dbOptions.paranoidChecks(true); + dbOptions.verifyChecksums(true); + dbOptions.compressionType(CompressionType.SNAPPY); + dbOptions.blockSize(4 * 1024); + dbOptions.writeBufferSize(10 * 1024 * 1024); + dbOptions.cacheSize(10 * 1024 * 1024L); + dbOptions.maxOpenFiles(1000); + return dbOptions; + } + + private static Options newDefaultRocksDbOptions(boolean forBulkLoad) { + Options options = new Options(); + options.setCreateIfMissing(true); + options.setIncreaseParallelism(1); + options.setNumLevels(7); + options.setMaxOpenFiles(5000); + options.setTargetFileSizeBase(64 * 1024 * 1024); + options.setTargetFileSizeMultiplier(1); + options.setMaxBytesForLevelBase(512 * 1024 * 1024); + options.setMaxBackgroundCompactions(Math.max(1, Runtime.getRuntime().availableProcessors())); + options.setLevel0FileNumCompactionTrigger(4); + options.setLevelCompactionDynamicLevelBytes(true); + final BlockBasedTableConfig tableCfg; + options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig()); + tableCfg.setBlockSize(64 * 1024); + tableCfg.setBlockCacheSize(32 * 1024 * 1024); + tableCfg.setCacheIndexAndFilterBlocks(true); + tableCfg.setPinL0FilterAndIndexBlocksInCache(true); + tableCfg.setFilter(new BloomFilter(10, false)); + if (forBulkLoad) { + options.prepareForBulkLoad(); + } + return options; + } + + public static RocksDB newRocksDb(Path db) throws RocksDBException { + try (Options options = newDefaultRocksDbOptions(false)) { + if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { + options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); + } + return RocksDB.open(options, db.toString()); + } + } + + public static RocksDB newRocksDbForBulkLoad(Path db) throws RocksDBException { + try (Options options = newDefaultRocksDbOptions(true)) { + if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { + options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); + } + return RocksDB.open(options, db.toString()); + } + } + + + public static RocksDB newRocksDbReadOnly(Path db) throws RocksDBException { + try (Options options = newDefaultRocksDbOptions(false)) { + if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { + options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); + } + return RocksDB.openReadOnly(options, db.toString()); + } + } + + public static String simpleDecode(byte[] bytes) { + byte[] lengthBytes = Arrays.copyOf(bytes, 4); + int length = Ints.fromByteArray(lengthBytes); + byte[] value = Arrays.copyOfRange(bytes, 4, 4 + length); + return new String(value); + } + + public static Sha256Hash getTransactionId(Protocol.Transaction transaction) { + return Sha256Hash.of(true, + transaction.getRawData().toByteArray()); + } +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/FileUtils.java b/plugins/src/main/java/org/tron/plugins/utils/FileUtils.java new file mode 100644 index 00000000000..b07b4469dc3 --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/FileUtils.java @@ -0,0 +1,189 @@ +package org.tron.plugins.utils; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.FileSystemException; +import java.nio.file.FileVisitOption; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.List; +import java.util.Properties; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class FileUtils { + + public static boolean isLevelDBEngine(Path path) { + String dir = path.toString(); + String enginePath = dir + File.separator + "engine.properties"; + if (!new File(enginePath).exists() + && !writeProperty(enginePath, DBUtils.KEY_ENGINE, DBUtils.LEVELDB)) { + return false; + } + String engine = readProperty(enginePath, DBUtils.KEY_ENGINE); + return DBUtils.LEVELDB.equals(engine); + } + + public static String readProperty(String file, String key) { + try (FileInputStream fileInputStream = new FileInputStream(file); + InputStream inputStream = new BufferedInputStream(fileInputStream)) { + Properties prop = new Properties(); + prop.load(inputStream); + return new String(prop.getProperty(key, "").getBytes(StandardCharsets.ISO_8859_1), + StandardCharsets.UTF_8); + } catch (Exception e) { + logger.error("readProperty", e); + return ""; + } + } + + public static boolean writeProperty(String file, String key, String value) { + try (OutputStream o = new FileOutputStream(file); + FileInputStream f = new FileInputStream(file); + BufferedWriter w = new BufferedWriter(new OutputStreamWriter(o, StandardCharsets.UTF_8)); + BufferedReader r = new BufferedReader(new InputStreamReader(f, StandardCharsets.UTF_8)) + ) { + Properties properties = new Properties(); + properties.load(r); + properties.setProperty(key, value); + properties.store(w, "Generated by the application. PLEASE DO NOT EDIT! "); + } catch (Exception e) { + logger.warn("writeProperty", e); + return false; + } + return true; + } + + + /** + * delete directory. + */ + public static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + String[] children = dir.list(); + for (int i = 0; i < children.length; i++) { + boolean success = deleteDir(new File(dir, children[i])); + if (!success) { + return false; + } + } + } + return dir.delete(); + } + + public static boolean createFileIfNotExists(String filepath) { + File file = new File(filepath); + if (!file.exists()) { + try { + file.createNewFile(); + } catch (Exception e) { + return false; + } + } + return true; + } + + public static boolean createDirIfNotExists(String dirPath) { + File dir = new File(dirPath); + if (!dir.exists()) { + return dir.mkdirs(); + } + return true; + } + + public static boolean isExists(String path) { + File file = new File(path); + return file.exists(); + } + + public static boolean isSymbolicLink(File file) throws IOException { + if (file == null) { + throw new NullPointerException("File must not be null"); + } + + File canon; + if (file.getParent() == null) { + canon = file; + } else { + File canonDir = file.getParentFile().getCanonicalFile(); + canon = new File(canonDir, file.getName()); + } + return !canon.getCanonicalFile().equals(canon.getAbsoluteFile()); + } + + /** + * Copy src to dest, if dest is a directory and already exists, throw Exception. + * + *

Note: This method is not rigorous, because all the dirs that its FileName + * is contained in List(subDirs) will be filtered, this may result in unpredictable result. + * just used in LiteFullNodeTool. + * + * @param src Path or File + * @param dest Path or File + * @param subDirs only the subDirs in {@code src} will be copied + * @throws IOException IOException + */ + public static void copyDatabases(Path src, Path dest, List subDirs) + throws IOException { + // create subdirs, as using parallel() to run, so should create dirs first. + subDirs.forEach(dir -> { + if (isExists(Paths.get(src.toString(), dir).toString())) { + try { + Files.walk(Paths.get(src.toString(), dir), FileVisitOption.FOLLOW_LINKS) + .forEach(source -> copy(source, dest.resolve(src.relativize(source)))); + } catch (IOException e) { + logger.error("copy database failed, src: {}, dest: {}, error: {}", + Paths.get(src.toString(), dir), Paths.get(dest.toString(), dir), e.getMessage()); + throw new RuntimeException(e); + } + } + }); + } + + public static void copyDir(Path src, Path dest, String dir) { + if (isExists(Paths.get(src.toString(), dir).toString())) { + try { + if (createDirIfNotExists(Paths.get(dest.toString(), dir).toString())) { + Files.walk(Paths.get(src.toString(), dir), FileVisitOption.FOLLOW_LINKS) + .forEach(source -> copy(source, dest.resolve(src.relativize(source)))); + } else { + throw new IOException(String.format("dest %s create fail ", + Paths.get(dest.toString(), dir))); + } + } catch (IOException e) { + logger.error("copy dir failed, src: {}, dest: {}, error: {}", + Paths.get(src.toString(), dir), Paths.get(dest.toString(), dir), e.getMessage()); + throw new RuntimeException(e); + } + } + } + + public static void copy(Path source, Path dest) { + try { + // create hard link when file is .sst + if (source.toString().endsWith(".sst")) { + try { + java.nio.file.Files.createLink(dest, source); + } catch (FileSystemException e) { + java.nio.file.Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); + } + } else { + java.nio.file.Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING); + } + } catch (Exception e) { + throw new RuntimeException(e.getMessage(), e); + } + } +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/MarketUtils.java b/plugins/src/main/java/org/tron/plugins/utils/MarketUtils.java new file mode 100644 index 00000000000..a36fc8ad57c --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/MarketUtils.java @@ -0,0 +1,148 @@ +package org.tron.plugins.utils; + +import java.math.BigInteger; +import org.tron.plugins.utils.ByteArray; + +public class MarketUtils { + + public static final int TOKEN_ID_LENGTH = ByteArray + .fromString(Long.toString(Long.MAX_VALUE)).length; // 19 + + + + /** + * In order to avoid the difference between the data of same key stored and fetched by hashMap and + * levelDB, when creating the price key, we will find the GCD (Greatest Common Divisor) of + * sellTokenQuantity and buyTokenQuantity. + */ + public static byte[] createPairPriceKey(byte[] sellTokenId, byte[] buyTokenId, + long sellTokenQuantity, long buyTokenQuantity) { + + byte[] sellTokenQuantityBytes; + byte[] buyTokenQuantityBytes; + + // cal the GCD + long gcd = findGCD(sellTokenQuantity, buyTokenQuantity); + if (gcd == 0) { + sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity); + buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity); + } else { + sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity / gcd); + buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity / gcd); + } + + return doCreatePairPriceKey(sellTokenId, buyTokenId, + sellTokenQuantityBytes, buyTokenQuantityBytes); + } + + public static long findGCD(long number1, long number2) { + if (number1 == 0 || number2 == 0) { + return 0; + } + return calGCD(number1, number2); + } + + private static long calGCD(long number1, long number2) { + if (number2 == 0) { + return number1; + } + return calGCD(number2, number1 % number2); + } + + + private static byte[] doCreatePairPriceKey(byte[] sellTokenId, byte[] buyTokenId, + byte[] sellTokenQuantity, byte[] buyTokenQuantity) { + byte[] result = new byte[TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + + sellTokenQuantity.length + buyTokenQuantity.length]; + + System.arraycopy(sellTokenId, 0, result, 0, sellTokenId.length); + System.arraycopy(buyTokenId, 0, result, TOKEN_ID_LENGTH, buyTokenId.length); + System.arraycopy(sellTokenQuantity, 0, result, + TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, + sellTokenQuantity.length); + System.arraycopy(buyTokenQuantity, 0, result, + TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + buyTokenQuantity.length, + buyTokenQuantity.length); + + return result; + } + + + public static int comparePriceKey(byte[] o1, byte[] o2) { + //compare pair + byte[] pair1 = new byte[TOKEN_ID_LENGTH * 2]; + byte[] pair2 = new byte[TOKEN_ID_LENGTH * 2]; + + System.arraycopy(o1, 0, pair1, 0, TOKEN_ID_LENGTH * 2); + System.arraycopy(o2, 0, pair2, 0, TOKEN_ID_LENGTH * 2); + + int pairResult = ByteArray.compareUnsigned(pair1, pair2); + if (pairResult != 0) { + return pairResult; + } + + //compare price + byte[] getSellTokenQuantity1 = new byte[8]; + byte[] getBuyTokenQuantity1 = new byte[8]; + + byte[] getSellTokenQuantity2 = new byte[8]; + byte[] getBuyTokenQuantity2 = new byte[8]; + + int longByteNum = 8; + + System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, + getSellTokenQuantity1, 0, longByteNum); + System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum, + getBuyTokenQuantity1, 0, longByteNum); + + System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, + getSellTokenQuantity2, 0, longByteNum); + System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum, + getBuyTokenQuantity2, 0, longByteNum); + + long sellTokenQuantity1 = ByteArray.toLong(getSellTokenQuantity1); + long buyTokenQuantity1 = ByteArray.toLong(getBuyTokenQuantity1); + long sellTokenQuantity2 = ByteArray.toLong(getSellTokenQuantity2); + long buyTokenQuantity2 = ByteArray.toLong(getBuyTokenQuantity2); + + if ((sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) + && (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0)) { + return 0; + } + + if (sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) { + return -1; + } + + if (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0) { + return 1; + } + + return comparePrice(sellTokenQuantity1, buyTokenQuantity1, + sellTokenQuantity2, buyTokenQuantity2); + + } + + /** + * Note: the params should be the same token pair, or you should change the order. + * All the quantity should be bigger than 0. + * */ + public static int comparePrice(long price1SellQuantity, long price1BuyQuantity, + long price2SellQuantity, long price2BuyQuantity) { + try { + return Long.compare(Math.multiplyExact(price1BuyQuantity, price2SellQuantity), + Math.multiplyExact(price2BuyQuantity, price1SellQuantity)); + + } catch (ArithmeticException ex) { + // do nothing here, because we will use BigInteger to compute again + } + + BigInteger price1BuyQuantityBI = BigInteger.valueOf(price1BuyQuantity); + BigInteger price1SellQuantityBI = BigInteger.valueOf(price1SellQuantity); + BigInteger price2BuyQuantityBI = BigInteger.valueOf(price2BuyQuantity); + BigInteger price2SellQuantityBI = BigInteger.valueOf(price2SellQuantity); + + return price1BuyQuantityBI.multiply(price2SellQuantityBI) + .compareTo(price2BuyQuantityBI.multiply(price1SellQuantityBI)); + } +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/Sha256Hash.java b/plugins/src/main/java/org/tron/plugins/utils/Sha256Hash.java new file mode 100644 index 00000000000..5fe80601b66 --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/Sha256Hash.java @@ -0,0 +1,202 @@ +package org.tron.plugins.utils; + +/* + * Copyright 2011 Google Inc. + * Copyright 2014 Andreas Schildbach + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.io.ByteStreams; +import com.google.common.primitives.Ints; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.Serializable; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import org.bouncycastle.crypto.digests.SM3Digest; + + +/** + * A Sha256Hash just wraps a byte[] so that equals and hashcode work correctly, allowing it to be + * used as keys in a map. It also checks that the length is correct and provides a bit more type + * safety. + */ +public class Sha256Hash implements Serializable, Comparable { + + public static final int LENGTH = 32; // bytes + + private final byte[] bytes; + + /** + * Use {@link #wrap(byte[])} instead. + */ + @Deprecated + public Sha256Hash(byte[] rawHashBytes) { + checkArgument(rawHashBytes.length == LENGTH); + this.bytes = rawHashBytes; + } + + /** + * Creates a new instance that wraps the given hash value. + * + * @param rawHashBytes the raw hash bytes to wrap + * @return a new instance + * @throws IllegalArgumentException if the given array length is not exactly 32 + */ + @SuppressWarnings("deprecation") // the constructor will be made private in the future + public static Sha256Hash wrap(byte[] rawHashBytes) { + return new Sha256Hash(rawHashBytes); + } + + + /** + * Creates a new instance containing the calculated (one-time) hash of the given bytes. + * + * @param contents the bytes on which the hash value is calculated + * @return a new instance containing the calculated (one-time) hash + */ + public static Sha256Hash of(boolean isSha256, byte[] contents) { + return wrap(hash(isSha256, contents)); + } + + /** + * Creates a new instance containing the calculated (one-time) hash of the given file's contents. + * The file contents are read fully into memory, so this method should only be used with small + * files. + * + * @param file the file on which the hash value is calculated + * @return a new instance containing the calculated (one-time) hash + * @throws IOException if an error occurs while reading the file + */ + public static Sha256Hash of(boolean isSha256, File file) throws IOException { + + try (FileInputStream in = new FileInputStream(file)) { + return of(isSha256, ByteStreams.toByteArray(in)); + } + } + + + /** + * Returns a new SHA-256 MessageDigest instance. This is a convenience method which wraps the + * checked exception that can never occur with a RuntimeException. + * + * @return a new SHA-256 MessageDigest instance + */ + public static MessageDigest newDigest() { + try { + return MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); // Can't happen. + } + } + + /** + * Returns a new SM3 MessageDigest instance. This is a convenience method which wraps the checked + * exception that can never occur with a RuntimeException. + * + * @return a new SM3 MessageDigest instance + */ + public static SM3Digest newSM3Digest() { + return new SM3Digest(); + } + + /** + * Calculates the SHA-256 hash of the given bytes. + * + * @param input the bytes to hash + * @return the hash (in big-endian order) + */ + public static byte[] hash(boolean isSha256, byte[] input) { + return hash(isSha256, input, 0, input.length); + } + + /** + * Calculates the SHA-256 hash of the given byte range. + * + * @param input the array containing the bytes to hash + * @param offset the offset within the array of the bytes to hash + * @param length the number of bytes to hash + * @return the hash (in big-endian order) + */ + public static byte[] hash(boolean isSha256, byte[] input, int offset, int length) { + if (isSha256) { + MessageDigest digest = newDigest(); + digest.update(input, offset, length); + return digest.digest(); + } else { + SM3Digest digest = newSM3Digest(); + digest.update(input, offset, length); + byte[] eHash = new byte[digest.getDigestSize()]; + digest.doFinal(eHash, 0); + return eHash; + } + + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof Sha256Hash)) { + return false; + } + return Arrays.equals(bytes, ((Sha256Hash) o).bytes); + } + + @Override + public String toString() { + return ByteArray.toHexString(bytes); + } + + /** + * Returns the last four bytes of the wrapped hash. This should be unique enough to be a suitable + * hash code even for blocks, where the goal is to try and get the first bytes to be zeros (i.e. + * the value as a big integer lower than the target value). + */ + @Override + public int hashCode() { + // Use the last 4 bytes, not the first 4 which are often zeros in Bitcoin. + return Ints + .fromBytes(bytes[LENGTH - 4], bytes[LENGTH - 3], bytes[LENGTH - 2], bytes[LENGTH - 1]); + } + + /** + * Returns the internal byte array, without defensively copying. Therefore do NOT modify the + * returned array. + */ + public byte[] getBytes() { + return bytes; + } + + + @Override + public int compareTo(final Sha256Hash other) { + for (int i = LENGTH - 1; i >= 0; i--) { + final int thisByte = this.bytes[i] & 0xff; + final int otherByte = other.bytes[i] & 0xff; + if (thisByte > otherByte) { + return 1; + } + if (thisByte < otherByte) { + return -1; + } + } + return 0; + } +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/DBInterface.java b/plugins/src/main/java/org/tron/plugins/utils/db/DBInterface.java new file mode 100644 index 00000000000..b0f7c58c587 --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/db/DBInterface.java @@ -0,0 +1,21 @@ +package org.tron.plugins.utils.db; + +import java.io.Closeable; +import java.io.IOException; + + +public interface DBInterface extends Closeable { + + byte[] get(byte[] key); + + void put(byte[] key, byte[] value); + + void delete(byte[] key); + + DBIterator iterator(); + + long size(); + + void close() throws IOException; + +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/DBIterator.java b/plugins/src/main/java/org/tron/plugins/utils/db/DBIterator.java new file mode 100644 index 00000000000..b5cbd705331 --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/db/DBIterator.java @@ -0,0 +1,59 @@ +package org.tron.plugins.utils.db; + +import java.io.Closeable; +import java.util.Iterator; +import java.util.Map; + +public interface DBIterator extends Iterator>, Closeable { + + /** + * An iterator is either positioned at a key/value pair, or + * not valid. This method returns true iff the iterator is valid. + * + * @return an iterator is either positioned at a key/value pair + */ + boolean valid(); + + /** + * Position at the first key in the source that is at or past target. + * The iterator is valid() after this call iff the source contains + * an entry that comes at or past target. + * + * @param key target + */ + void seek(byte[] key); + + /** + * Position at the first key in the source. The iterator is valid() + * after this call iff the source is not empty. + */ + void seekToFirst(); + + /** + * Position at the last key in the source. The iterator is + * valid() after this call iff the source is not empty. + */ + void seekToLast(); + + boolean hasNext(); + + /** + * The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator. + * REQUIRES: valid() + * + * @return the key for the current entry + */ + byte[] getKey(); + + /** + * The underlying storage for + * the returned slice is valid only until the next modification of + * the iterator. + * REQUIRES: valid() + * + * @return the value for the current entry + */ + byte[] getValue(); +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/DbTool.java b/plugins/src/main/java/org/tron/plugins/utils/db/DbTool.java new file mode 100644 index 00000000000..6e8df4c59fb --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/db/DbTool.java @@ -0,0 +1,187 @@ +package org.tron.plugins.utils.db; + +import com.google.common.collect.Maps; +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Iterator; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.FileUtils; + + +@Slf4j(topic = "tool") +public class DbTool { + + private static final String KEY_ENGINE = "ENGINE"; + private static final String ENGINE_FILE = "engine.properties"; + private static final String FILE_SEPARATOR = File.separator; + private static final String ROCKSDB = "ROCKSDB"; + + private static final Map dbMap = Maps.newConcurrentMap(); + + enum DbType { + LevelDB, + RocksDB + } + + /** + * Get the DB object according to the specified path, + * create db object when not exists, otherwise get it from the dbMap. + * + * @param sourceDir the parent path of db + * @param dbName db dir name + * + * @return db object + * + * @throws IOException leveldb error + * @throws RocksDBException rocksdb error + */ + public static DBInterface getDB(String sourceDir, String dbName) + throws IOException, RocksDBException { + DbType type = getDbType(sourceDir, dbName); + return getDB(sourceDir, dbName, type); + } + + /** + * Get the DB object according to the specified path, keep engine same with source. + * + * @param sourceDir read engine + * @param destDir to be open parent path + * @param dbName database name + * + * @return db object + * + * @throws IOException leveldb error + * @throws RocksDBException rocksdb error + */ + public static DBInterface getDB(String sourceDir, String destDir, String dbName) + throws IOException, RocksDBException { + DbType type = getDbType(sourceDir, dbName); + return getDB(destDir, dbName, type); + } + + /** + * Get the DB object according to the specified path and engine. + * + * @param sourceDir to be open parent path + * @param dbName database name + * @param type engine + * @return db object + * @throws IOException leveldb error + * @throws RocksDBException rocksdb error + */ + public static DBInterface getDB(String sourceDir, String dbName, DbType type) + throws IOException, RocksDBException { + Path path = Paths.get(sourceDir, dbName); + if (dbMap.containsKey(path.toString())) { + return dbMap.get(path.toString()); + } + DBInterface db; + switch (type) { + case LevelDB: + db = openLevelDb(path); + dbMap.put(path.toString(), db); + break; + case RocksDB: + db = openRocksDb(path); + dbMap.put(path.toString(), db); + break; + default: + throw new IllegalStateException("Unexpected value: " + type); + } + return db; + } + + /** + * Get the DB object according to the specified path, + * not managed by dbMap. + * + * @param sourceDir the parent path of db + * @param dbName db dir name + * + * @return db object + * + * @throws IOException leveldb error + * @throws RocksDBException rocksdb error + */ + public static DBInterface getDB(Path sourceDir, String dbName) + throws IOException, RocksDBException { + Path path = Paths.get(sourceDir.toString(), dbName); + DbType type = getDbType(sourceDir.toString(), dbName); + switch (type) { + case LevelDB: + return openLevelDb(path); + case RocksDB: + return openRocksDb(path); + default: + throw new IllegalStateException("Unexpected value: " + type); + } + } + + /** + * Close db. + * + * @param sourceDir db parentPath + * @param dbName db dirname + * + * @throws IOException IOException + */ + public static void closeDB(String sourceDir, String dbName) + throws IOException { + Path path = Paths.get(sourceDir, dbName); + DBInterface db = dbMap.get(path.toString()); + if (db != null) { + try { + dbMap.remove(path.toString()); + db.close(); + } catch (IOException e) { + logger.error("close db {} error: {}", path, e); + throw e; + } + } + } + + /** + * Close all dbs. + */ + public static void close() { + Iterator> iterator = dbMap.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry next = iterator.next(); + try { + next.getValue().close(); + } catch (IOException e) { + logger.error("close db failed, db: {}", next.getKey(), e); + } + iterator.remove(); + } + } + + private static DbType getDbType(String sourceDir, String dbName) { + String engineFile = String.format("%s%s%s%s%s", sourceDir, FILE_SEPARATOR, + dbName, FILE_SEPARATOR, ENGINE_FILE); + if (!new File(engineFile).exists()) { + return DbType.LevelDB; + } + String engine = FileUtils.readProperty(engineFile, KEY_ENGINE); + if (engine.equalsIgnoreCase(ROCKSDB)) { + return DbType.RocksDB; + } else { + return DbType.LevelDB; + } + } + + private static LevelDBImpl openLevelDb(Path db) throws IOException { + return new LevelDBImpl(DBUtils.newLevelDb(db)); + } + + private static RocksDBImpl openRocksDb(Path db) throws RocksDBException { + return new RocksDBImpl(DBUtils.newRocksDb(db)); + } + + +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java b/plugins/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java new file mode 100644 index 00000000000..3a89e77af40 --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java @@ -0,0 +1,46 @@ +package org.tron.plugins.utils.db; + +import com.google.common.collect.Streams; +import java.io.IOException; +import org.iq80.leveldb.DB; +import org.iq80.leveldb.ReadOptions; + + +public class LevelDBImpl implements DBInterface { + + private DB leveldb; + + public LevelDBImpl(DB leveldb) { + this.leveldb = leveldb; + } + + @Override + public byte[] get(byte[] key) { + return leveldb.get(key); + } + + @Override + public void put(byte[] key, byte[] value) { + leveldb.put(key, value); + } + + @Override + public void delete(byte[] key) { + leveldb.delete(key); + } + + @Override + public DBIterator iterator() { + return new LevelDBIterator(leveldb.iterator(new ReadOptions().fillCache(false))); + } + + @Override + public long size() { + return Streams.stream(leveldb.iterator()).count(); + } + + @Override + public void close() throws IOException { + leveldb.close(); + } +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/LevelDBIterator.java b/plugins/src/main/java/org/tron/plugins/utils/db/LevelDBIterator.java new file mode 100644 index 00000000000..b5f0028dd6f --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/db/LevelDBIterator.java @@ -0,0 +1,58 @@ +package org.tron.plugins.utils.db; + +import java.io.IOException; +import java.util.Map; + +public class LevelDBIterator implements DBIterator { + + private final org.iq80.leveldb.DBIterator iterator; + + public LevelDBIterator(org.iq80.leveldb.DBIterator iterator) { + this.iterator = iterator; + } + + @Override + public boolean valid() { + return iterator.hasNext(); + } + + @Override + public void seek(byte[] key) { + iterator.seek(key); + } + + @Override + public void seekToFirst() { + iterator.seekToFirst(); + } + + @Override + public void seekToLast() { + iterator.seekToLast(); + } + + @Override + public boolean hasNext() { + return iterator.hasNext(); + } + + @Override + public byte[] getKey() { + return iterator.peekNext().getKey(); + } + + @Override + public byte[] getValue() { + return iterator.peekNext().getValue(); + } + + @Override + public Map.Entry next() { + return iterator.next(); + } + + @Override + public void close() throws IOException { + iterator.close(); + } +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java b/plugins/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java new file mode 100644 index 00000000000..d3e17d9173f --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java @@ -0,0 +1,77 @@ +package org.tron.plugins.utils.db; + +import java.io.IOException; +import java.util.Map; +import org.rocksdb.RocksIterator; + +public class RockDBIterator implements DBIterator { + + private final RocksIterator iterator; + + public RockDBIterator(RocksIterator iterator) { + this.iterator = iterator; + } + + @Override + public boolean valid() { + return iterator.isValid(); + } + + @Override + public void seek(byte[] key) { + iterator.seek(key); + } + + @Override + public void seekToFirst() { + iterator.seekToFirst(); + } + + @Override + public void seekToLast() { + iterator.seekToLast(); + } + + @Override + public boolean hasNext() { + return iterator.isValid(); + } + + @Override + public byte[] getKey() { + return iterator.key(); + } + + @Override + public byte[] getValue() { + return iterator.value(); + } + + @Override + public Map.Entry next() { + byte[] key = iterator.key(); + byte[] value = iterator.value(); + iterator.next(); + return new Map.Entry() { + @Override + public byte[] getKey() { + return key; + } + + @Override + public byte[] getValue() { + return value; + } + + @Override + public byte[] setValue(byte[] value) { + throw new UnsupportedOperationException(); + } + }; + } + + @Override + public void close() throws IOException { + iterator.close(); + } +} diff --git a/plugins/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java b/plugins/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java new file mode 100644 index 00000000000..88b25b0413f --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java @@ -0,0 +1,64 @@ +package org.tron.plugins.utils.db; + +import java.io.IOException; +import org.rocksdb.RocksDBException; +import org.rocksdb.RocksIterator; + +public class RocksDBImpl implements DBInterface { + + private org.rocksdb.RocksDB rocksDB; + + public RocksDBImpl(org.rocksdb.RocksDB rocksDB) { + this.rocksDB = rocksDB; + } + + @Override + public byte[] get(byte[] key) { + try { + return rocksDB.get(key); + } catch (RocksDBException e) { + e.printStackTrace(); + } + return null; + } + + @Override + public void put(byte[] key, byte[] value) { + try { + rocksDB.put(key, value); + } catch (RocksDBException e) { + e.printStackTrace(); + } + } + + @Override + public void delete(byte[] key) { + try { + rocksDB.delete(key); + } catch (RocksDBException e) { + e.printStackTrace(); + } + } + + @Override + public DBIterator iterator() { + return new RockDBIterator(rocksDB.newIterator( + new org.rocksdb.ReadOptions().setFillCache(false))); + } + + @Override + public long size() { + RocksIterator iterator = rocksDB.newIterator(); + long size = 0; + for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { + size++; + } + iterator.close(); + return size; + } + + @Override + public void close() throws IOException { + rocksDB.close(); + } +} diff --git a/plugins/src/main/resources/logback.xml b/plugins/src/main/resources/logback.xml index 5e632f0d424..6c415042e38 100644 --- a/plugins/src/main/resources/logback.xml +++ b/plugins/src/main/resources/logback.xml @@ -16,11 +16,11 @@ - ./logs/archive.log + ./logs/toolkit.log - ./logs/archive-%d{yyyy-MM-dd}.%i.log.gz + ./logs/toolkit-%d{yyyy-MM-dd}.%i.log.gz 500MB 10 @@ -49,6 +49,9 @@ + + + diff --git a/plugins/src/test/java/org/tron/plugins/DbConvertTest.java b/plugins/src/test/java/org/tron/plugins/DbConvertTest.java new file mode 100644 index 00000000000..91996815c01 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbConvertTest.java @@ -0,0 +1,110 @@ +package org.tron.plugins; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import org.iq80.leveldb.DB; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import org.tron.plugins.utils.ByteArray; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.FileUtils; +import org.tron.plugins.utils.MarketUtils; +import picocli.CommandLine; + +public class DbConvertTest { + + + private static final String INPUT_DIRECTORY = "output-directory/convert-database/"; + private static final String OUTPUT_DIRECTORY = "output-directory/convert-database-dest/"; + private static final String ACCOUNT = "account"; + private static final String MARKET = DBUtils.MARKET_PAIR_PRICE_TO_ORDER; + CommandLine cli = new CommandLine(new Toolkit()); + + + @BeforeClass + public static void init() throws IOException { + if (new File(INPUT_DIRECTORY).mkdirs()) { + initDB(new File(INPUT_DIRECTORY,ACCOUNT)); + initDB(new File(INPUT_DIRECTORY,MARKET)); + } + } + + private static void initDB(File file) throws IOException { + try (DB db = DBUtils.newLevelDb(file.toPath())) { + if (MARKET.equalsIgnoreCase(file.getName())) { + byte[] sellTokenID1 = ByteArray.fromString("100"); + byte[] buyTokenID1 = ByteArray.fromString("200"); + byte[] pairPriceKey1 = MarketUtils.createPairPriceKey( + sellTokenID1, + buyTokenID1, + 1000L, + 2001L + ); + byte[] pairPriceKey2 = MarketUtils.createPairPriceKey( + sellTokenID1, + buyTokenID1, + 1000L, + 2002L + ); + byte[] pairPriceKey3 = MarketUtils.createPairPriceKey( + sellTokenID1, + buyTokenID1, + 1000L, + 2003L + ); + + + //Use out-of-order insertion，key in store should be 1,2,3 + db.put(pairPriceKey1, "1".getBytes(StandardCharsets.UTF_8)); + db.put(pairPriceKey2, "2".getBytes(StandardCharsets.UTF_8)); + db.put(pairPriceKey3, "3".getBytes(StandardCharsets.UTF_8)); + } else { + for (int i = 0; i < 100; i++) { + byte[] bytes = UUID.randomUUID().toString().getBytes(StandardCharsets.UTF_8); + db.put(bytes, bytes); + } + } + } + } + + @AfterClass + public static void destroy() { + FileUtils.deleteDir(new File(INPUT_DIRECTORY)); + FileUtils.deleteDir(new File(OUTPUT_DIRECTORY)); + } + + @Test + public void testRun() { + String[] args = new String[] { "db", "convert", INPUT_DIRECTORY, OUTPUT_DIRECTORY }; + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testHelp() { + String[] args = new String[] {"db", "convert", "-h"}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testNotExist() { + String[] args = new String[] {"db", "convert", + OUTPUT_DIRECTORY + File.separator + UUID.randomUUID(), + OUTPUT_DIRECTORY}; + Assert.assertEquals(404, cli.execute(args)); + } + + @Test + public void testEmpty() { + File file = new File(OUTPUT_DIRECTORY + File.separator + UUID.randomUUID()); + file.mkdirs(); + file.deleteOnExit(); + String[] args = new String[] {"db", "convert", file.toString(), OUTPUT_DIRECTORY}; + Assert.assertEquals(0, cli.execute(args)); + } + +} diff --git a/plugins/src/test/java/org/tron/plugins/DbMoveTest.java b/plugins/src/test/java/org/tron/plugins/DbMoveTest.java index 6bbac663c2c..c1bc6b470fc 100644 --- a/plugins/src/test/java/org/tron/plugins/DbMoveTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbMoveTest.java @@ -2,25 +2,17 @@ import static org.iq80.leveldb.impl.Iq80DBFactory.factory; -import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.OutputStreamWriter; import java.net.URL; -import java.nio.charset.StandardCharsets; import java.nio.file.Paths; -import java.util.Properties; import java.util.UUID; import lombok.extern.slf4j.Slf4j; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.tron.plugins.utils.FileUtils; import picocli.CommandLine; @Slf4j @@ -41,15 +33,15 @@ public class DbMoveTest { public static void init() throws IOException { File file = new File(OUTPUT_DIRECTORY_DATABASE, ACCOUNT); factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); file = new File(OUTPUT_DIRECTORY_DATABASE, MARKET); factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); file = new File(OUTPUT_DIRECTORY_DATABASE, TRANS); factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); } @@ -58,26 +50,6 @@ public static void destroy() { deleteDir(new File(OUTPUT_DIRECTORY)); } - private static void writeProperty(String filename, String key, String value) throws IOException { - File file = new File(filename); - if (!file.exists()) { - file.createNewFile(); - } - - try (FileInputStream fis = new FileInputStream(file); - OutputStream out = new FileOutputStream(file); - BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, - StandardCharsets.UTF_8))) { - BufferedReader bf = new BufferedReader(new InputStreamReader(fis, StandardCharsets.UTF_8)); - Properties properties = new Properties(); - properties.load(bf); - properties.setProperty(key, value); - properties.store(bw, "Generated by the application. PLEASE DO NOT EDIT! "); - } catch (Exception e) { - logger.warn("{}", e); - } - } - /** * delete directory. */ @@ -122,7 +94,7 @@ public void testDuplicate() { @Test public void testHelp() { - String[] args = new String[] {}; + String[] args = new String[] {"db", "mv", "-h"}; CommandLine cli = new CommandLine(new Toolkit()); Assert.assertEquals(0, cli.execute(args)); }