diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index 051bed60aa5..00000000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: Docker - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -on: - push: - tags: - - "GreatVoyage-v*" - -env: - # Use docker.io for Docker Hub if empty - REGISTRY: docker.io - # github.repository as / - IMAGE_NAME: ${{ github.repository }} - - -jobs: - build: - - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - # This is used to complete the identity challenge - # with sigstore/fulcio when running outside of PRs. - id-token: write - - steps: - - name: Checkout repository - uses: actions/checkout@v2 - - # Install the cosign tool except on PR - # https://github.com/sigstore/cosign-installer - - name: Install cosign - if: github.event_name != 'pull_request' - uses: sigstore/cosign-installer@1e95c1de343b5b0c23352d6417ee3e48d5bcd422 - with: - cosign-release: 'v1.4.0' - - - # Workaround: https://github.com/docker/build-push-action/issues/461 - - name: Setup Docker buildx - uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf - - # Login against a Docker registry except on PR - # https://github.com/docker/login-action - - name: Log into registry ${{ env.REGISTRY }} - if: github.event_name != 'pull_request' - uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c - with: - registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - # Extract metadata (tags, labels) for Docker - # https://github.com/docker/metadata-action - - name: Extract Docker metadata - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - # Build and push Docker image with Buildx (don't push on PR) - # https://github.com/docker/build-push-action - - name: Build and push Docker image - id: build-and-push - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: . - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - - name: Sign the published Docker image - if: ${{ github.event_name != 'pull_request' }} - env: - COSIGN_EXPERIMENTAL: "true" - # This step uses the identity token to provision an ephemeral certificate - # against the sigstore community Fulcio instance. - run: cosign sign ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@${{ steps.build-and-push.outputs.digest }} diff --git a/actuator/src/main/java/org/tron/core/actuator/TransferActuator.java b/actuator/src/main/java/org/tron/core/actuator/TransferActuator.java index 94fef0b5c12..5e3d605aed7 100755 --- a/actuator/src/main/java/org/tron/core/actuator/TransferActuator.java +++ b/actuator/src/main/java/org/tron/core/actuator/TransferActuator.java @@ -9,6 +9,7 @@ import lombok.extern.slf4j.Slf4j; import org.tron.common.utils.Commons; import org.tron.common.utils.DecodeUtil; +import org.tron.common.utils.StringUtil; import org.tron.core.capsule.AccountCapsule; import org.tron.core.capsule.ContractCapsule; import org.tron.core.capsule.TransactionResultCapsule; @@ -156,6 +157,8 @@ public boolean validate() throws ContractValidateException { } if (balance < Math.addExact(amount, fee)) { + logger.warn("Balance is not sufficient. Account: {}, balance: {}, amount: {}, fee: {}.", + StringUtil.encode58Check(ownerAddress), balance, amount, fee); throw new ContractValidateException( "Validate TransferContract error, balance is not sufficient."); } diff --git a/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java b/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java index 20ad57d4929..0ae85146eda 100644 --- a/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java +++ b/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java @@ -102,22 +102,24 @@ public LevelDbDataSourceImpl(String parentPath, String dataBaseName) { public void initDB() { resetDbLock.writeLock().lock(); try { - logger.debug("~> LevelDbDataSourceImpl.initDB(): " + dataBaseName); + logger.debug("Init DB: {}.", dataBaseName); if (isAlive()) { return; } if (dataBaseName == null) { - throw new NullPointerException("no name set to the dbStore"); + throw new IllegalArgumentException("No name set to the dbStore"); } try { openDatabase(options); alive = true; } catch (IOException ioe) { - throw new RuntimeException("Can't initialize database", ioe); + throw new RuntimeException(String.format("Can't initialize database, %s", dataBaseName), + ioe); } + logger.debug("Init DB {} done.", dataBaseName); } finally { resetDbLock.writeLock().unlock(); } @@ -133,12 +135,16 @@ private void openDatabase(Options dbOptions) throws IOException { } try { database = factory.open(dbPath.toFile(), dbOptions); - logger.info("DB {} open success with : writeBufferSize {}M,cacheSize {}M,maxOpenFiles {}.", - this.getDBName(), dbOptions.writeBufferSize() / 1024 / 1024, - dbOptions.cacheSize() / 1024 / 1024, dbOptions.maxOpenFiles()); + if (!this.getDBName().startsWith("checkpoint")) { + logger.info("DB {} open success with writeBufferSize {} M, cacheSize {} M, maxOpenFiles {}.", + this.getDBName(), dbOptions.writeBufferSize() / 1024 / 1024, + dbOptions.cacheSize() / 1024 / 1024, dbOptions.maxOpenFiles()); + } } catch (IOException e) { if (e.getMessage().contains("Corruption:")) { + logger.warn("DB {} corruption detected, try to repair it.", this.getDBName()); factory.repair(dbPath.toFile(), dbOptions); + logger.warn("DB {} corruption detected, repair done.", this.getDBName()); database = factory.open(dbPath.toFile(), dbOptions); } else { throw e; @@ -460,7 +466,7 @@ public void closeDB() { database.close(); alive = false; } catch (IOException e) { - logger.error("Failed to find the dbStore file on the closeDB: {} ", dataBaseName); + logger.error("Failed to find the dbStore file on the closeDB: {}.", dataBaseName, e); } finally { resetDbLock.writeLock().unlock(); } diff --git a/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java b/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java index 0733094312f..35248619933 100644 --- a/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java +++ b/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java @@ -43,13 +43,12 @@ import org.tron.core.db2.common.WrappedByteArray; -@Slf4j +@Slf4j(topic = "DB") @NoArgsConstructor public class RocksDbDataSourceImpl extends DbStat implements DbSourceInter, Iterable>, Instance { ReadOptions readOpts; - private static final String FAIL_TO_INIT_DATABASE = "Failed to initialize database"; private String dataBaseName; private RocksDB database; private boolean alive; @@ -102,6 +101,7 @@ public void closeDB() { database.close(); alive = false; } catch (Exception e) { + logger.error("Failed to find the dbStore file on the closeDB: {}.", dataBaseName, e); } finally { resetDbLock.writeLock().unlock(); } @@ -116,7 +116,7 @@ public void resetDb() { private boolean quitIfNotAlive() { if (!isAlive()) { - logger.warn("db is not alive"); + logger.warn("DB {} is not alive.", dataBaseName); } return !isAlive(); } @@ -181,8 +181,8 @@ public boolean checkOrInitEngine() { public void initDB() { if (!checkOrInitEngine()) { - logger.error("database engine do not match"); - throw new RuntimeException(FAIL_TO_INIT_DATABASE); + throw new RuntimeException( + String.format("failed to check database: %s, engine do not match", dataBaseName)); } initDB(RocksDbSettings.getSettings()); } @@ -194,7 +194,7 @@ public void initDB(RocksDbSettings settings) { return; } if (dataBaseName == null) { - throw new NullPointerException("no name set to the dbStore"); + throw new IllegalArgumentException("No name set to the dbStore"); } try (Options options = new Options()) { @@ -238,7 +238,7 @@ public void initDB(RocksDbSettings settings) { .setVerifyChecksums(false); try { - logger.debug("Opening database"); + logger.debug("Opening database {}.", dataBaseName); final Path dbPath = getDbPath(); if (!Files.isSymbolicLink(dbPath.getParent())) { @@ -248,17 +248,17 @@ public void initDB(RocksDbSettings settings) { try { database = RocksDB.open(options, dbPath.toString()); } catch (RocksDBException e) { - logger.error(e.getMessage(), e); - throw new RuntimeException(FAIL_TO_INIT_DATABASE, e); + throw new RuntimeException( + String.format("failed to open database: %s", dataBaseName), e); } alive = true; } catch (IOException ioe) { - logger.error(ioe.getMessage(), ioe); - throw new RuntimeException(FAIL_TO_INIT_DATABASE, ioe); + throw new RuntimeException( + String.format("failed to init database: %s", dataBaseName), ioe); } - logger.debug("<~ RocksDbDataSource.initDB(): " + dataBaseName); + logger.debug("Init DB {} done.", dataBaseName); } } finally { resetDbLock.writeLock().unlock(); @@ -274,7 +274,7 @@ public void putData(byte[] key, byte[] value) { try { database.put(key, value); } catch (RocksDBException e) { - throw new RuntimeException(e); + throw new RuntimeException(dataBaseName, e); } finally { resetDbLock.readLock().unlock(); } @@ -289,7 +289,7 @@ public byte[] getData(byte[] key) { try { return database.get(key); } catch (RocksDBException e) { - throw new RuntimeException(e); + throw new RuntimeException(dataBaseName, e); } finally { resetDbLock.readLock().unlock(); } @@ -304,7 +304,7 @@ public void deleteData(byte[] key) { try { database.delete(key); } catch (RocksDBException e) { - throw new RuntimeException(e); + throw new RuntimeException(dataBaseName, e); } finally { resetDbLock.readLock().unlock(); } @@ -365,7 +365,7 @@ public void updateByBatch(Map rows, WriteOptionsWrapper optionsW try { updateByBatchInner(rows); } catch (Exception e1) { - throw new RuntimeException(e); + throw new RuntimeException(dataBaseName, e1); } } finally { resetDbLock.readLock().unlock(); @@ -384,7 +384,7 @@ public void updateByBatch(Map rows) { try { updateByBatchInner(rows); } catch (Exception e1) { - throw new RuntimeException(e); + throw new RuntimeException(dataBaseName, e1); } } finally { resetDbLock.readLock().unlock(); diff --git a/chainbase/src/main/java/org/tron/common/utils/Commons.java b/chainbase/src/main/java/org/tron/common/utils/Commons.java index f2f7578d200..55542d494b4 100644 --- a/chainbase/src/main/java/org/tron/common/utils/Commons.java +++ b/chainbase/src/main/java/org/tron/common/utils/Commons.java @@ -74,7 +74,8 @@ public static void adjustBalance(AccountStore accountStore, AccountCapsule accou if (amount < 0 && balance < -amount) { throw new BalanceInsufficientException( - StringUtil.createReadableString(account.createDbKey()) + " insufficient balance"); + String.format("%s insufficient balance, balance: %d, amount: %d", + StringUtil.createReadableString(account.createDbKey()), balance, -amount)); } account.setBalance(Math.addExact(balance, amount)); accountStore.put(account.getAddress().toByteArray(), account); @@ -120,12 +121,16 @@ public static void adjustAssetBalanceV2(AccountCapsule account, String AssetID, if (amount < 0) { if (!account.reduceAssetAmountV2(AssetID.getBytes(), -amount, dynamicPropertiesStore, assetIssueStore)) { - throw new BalanceInsufficientException("reduceAssetAmount failed !"); + throw new BalanceInsufficientException( + String.format("reduceAssetAmount failed! account: %s", + StringUtil.encode58Check(account.createDbKey()))); } } else if (amount > 0 && !account.addAssetAmountV2(AssetID.getBytes(), amount, dynamicPropertiesStore, assetIssueStore)) { - throw new BalanceInsufficientException("addAssetAmount failed !"); + throw new BalanceInsufficientException( + String.format("addAssetAmount failed! account: %s", + StringUtil.encode58Check(account.createDbKey()))); } accountStore.put(account.getAddress().toByteArray(), account); } @@ -135,7 +140,8 @@ public static void adjustTotalShieldedPoolValue(long valueBalance, long totalShieldedPoolValue = Math .subtractExact(dynamicPropertiesStore.getTotalShieldedPoolValue(), valueBalance); if (totalShieldedPoolValue < 0) { - throw new BalanceInsufficientException("Total shielded pool value can not below 0"); + throw new BalanceInsufficientException(String.format( + "total shielded pool value can not below 0, actual: %d", totalShieldedPoolValue)); } dynamicPropertiesStore.saveTotalShieldedPoolValue(totalShieldedPoolValue); } diff --git a/chainbase/src/main/java/org/tron/common/utils/ForkController.java b/chainbase/src/main/java/org/tron/common/utils/ForkController.java index b2cba4929dd..db4d77fd809 100644 --- a/chainbase/src/main/java/org/tron/common/utils/ForkController.java +++ b/chainbase/src/main/java/org/tron/common/utils/ForkController.java @@ -63,7 +63,7 @@ private boolean passOld(int version) { private boolean passNew(int version) { ForkBlockVersionEnum versionEnum = ForkBlockVersionEnum.getForkBlockVersionEnum(version); if (versionEnum == null) { - logger.error("not exist block version: {}", version); + logger.error("Not exist block version: {}.", version); return false; } long latestBlockTime = manager.getDynamicPropertiesStore().getLatestBlockHeaderTimestamp(); @@ -167,7 +167,7 @@ public synchronized void update(BlockCapsule blockCapsule) { stats[slot] = VERSION_UPGRADE; manager.getDynamicPropertiesStore().statsByVersion(version, stats); logger.info( - "*******update hard fork:{}, witness size:{}, solt:{}, witness:{}, version:{}", + "Update hard fork: {}, witness size: {}, solt: {}, witness: {}, version: {}.", Streams.zip(witnesses.stream(), Stream.of(ArrayUtils.toObject(stats)), Maps::immutableEntry) .map(e -> Maps .immutableEntry(encode58Check(e.getKey().toByteArray()), e.getValue())) diff --git a/chainbase/src/main/java/org/tron/common/utils/LocalWitnesses.java b/chainbase/src/main/java/org/tron/common/utils/LocalWitnesses.java index 00cd396143c..940a107a2ac 100644 --- a/chainbase/src/main/java/org/tron/common/utils/LocalWitnesses.java +++ b/chainbase/src/main/java/org/tron/common/utils/LocalWitnesses.java @@ -88,8 +88,8 @@ private void validate(String privateKey) { if (StringUtils.isNotBlank(privateKey) && privateKey.length() != ChainConstant.PRIVATE_KEY_LENGTH) { throw new IllegalArgumentException( - "Private key(" + privateKey + ") must be " + ChainConstant.PRIVATE_KEY_LENGTH - + "-bits hex string."); + String.format("private key must be %d-bits hex string, actual: %d", + ChainConstant.PRIVATE_KEY_LENGTH, privateKey.length())); } } @@ -101,7 +101,7 @@ public void addPrivateKeys(String privateKey) { //get the first one recently public String getPrivateKey() { if (CollectionUtils.isEmpty(privateKeys)) { - logger.warn("privateKey is null"); + logger.warn("PrivateKey is null."); return null; } return privateKeys.get(0); @@ -109,7 +109,7 @@ public String getPrivateKey() { public byte[] getPublicKey() { if (CollectionUtils.isEmpty(privateKeys)) { - logger.warn("privateKey is null"); + logger.warn("PrivateKey is null."); return null; } byte[] privateKey = ByteArray.fromHexString(getPrivateKey()); diff --git a/chainbase/src/main/java/org/tron/common/utils/WalletUtil.java b/chainbase/src/main/java/org/tron/common/utils/WalletUtil.java index 36f09ba6cc7..330ce35b6e9 100644 --- a/chainbase/src/main/java/org/tron/common/utils/WalletUtil.java +++ b/chainbase/src/main/java/org/tron/common/utils/WalletUtil.java @@ -28,7 +28,8 @@ public static boolean checkPermissionOperations(Permission permission, Contract throws PermissionException { ByteString operations = permission.getOperations(); if (operations.size() != 32) { - throw new PermissionException("operations size must 32"); + throw new PermissionException(String.format("operations size must 32, actual: %d", + operations.size())); } int contractType = contract.getTypeValue(); boolean b = (operations.byteAt(contractType / 8) & (1 << (contractType % 8))) != 0; diff --git a/chainbase/src/main/java/org/tron/core/ChainBaseManager.java b/chainbase/src/main/java/org/tron/core/ChainBaseManager.java index b2a9f643a42..a2f3cd67b8e 100644 --- a/chainbase/src/main/java/org/tron/core/ChainBaseManager.java +++ b/chainbase/src/main/java/org/tron/core/ChainBaseManager.java @@ -233,13 +233,13 @@ public class ChainBaseManager { private DbStatService dbStatService; public void closeOneStore(ITronChainBase database) { - logger.info("******** begin to close " + database.getName() + " ********"); + logger.info("******** Begin to close {}. ********", database.getName()); try { database.close(); } catch (Exception e) { - logger.info("failed to close " + database.getName() + ". " + e); + logger.info("Failed to close {}.", database.getName(), e); } finally { - logger.info("******** end to close " + database.getName() + " ********"); + logger.info("******** End to close {}. ********", database.getName()); } } @@ -297,8 +297,7 @@ public BlockCapsule getHead() throws HeaderNotFound { if (CollectionUtils.isNotEmpty(blocks)) { return blocks.get(0); } else { - logger.info("Header block Not Found"); - throw new HeaderNotFound("Header block Not Found"); + throw new HeaderNotFound("header block not found"); } } diff --git a/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java b/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java index cd35bf4a253..f3c2af334cb 100755 --- a/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java +++ b/chainbase/src/main/java/org/tron/core/capsule/TransactionCapsule.java @@ -105,6 +105,15 @@ public class TransactionCapsule implements ProtoCapsule { @Getter @Setter private long order; + private byte[] ownerAddress; + + public byte[] getOwnerAddress() { + if (this.ownerAddress == null) { + this.ownerAddress = getOwner(this.transaction.getRawData().getContract(0)); + } + return this.ownerAddress; + } + /** * constructor TransactionCapsule. @@ -544,7 +553,7 @@ public void addSign(byte[] privateKey, AccountStore accountStore) throws PermissionException, SignatureException, SignatureFormatException { Transaction.Contract contract = this.transaction.getRawData().getContract(0); int permissionId = contract.getPermissionId(); - byte[] owner = getOwner(contract); + byte[] owner = getOwnerAddress(); AccountCapsule account = accountStore.get(owner); if (account == null) { throw new PermissionException("Account is not exist!"); @@ -632,7 +641,7 @@ public boolean validateSignature(AccountStore accountStore, if (contract.getType() != ContractType.ShieldedTransferContract) { validatePubSignature(accountStore, dynamicPropertiesStore); } else { //ShieldedTransfer - byte[] owner = getOwner(contract); + byte[] owner = getOwnerAddress(); if (!ArrayUtils.isEmpty(owner)) { //transfer from transparent address validatePubSignature(accountStore, dynamicPropertiesStore); } else { //transfer from shielded address diff --git a/chainbase/src/main/java/org/tron/core/db/BandwidthProcessor.java b/chainbase/src/main/java/org/tron/core/db/BandwidthProcessor.java index 324e3bbf5ac..c2aba118405 100644 --- a/chainbase/src/main/java/org/tron/core/db/BandwidthProcessor.java +++ b/chainbase/src/main/java/org/tron/core/db/BandwidthProcessor.java @@ -11,6 +11,7 @@ import lombok.extern.slf4j.Slf4j; import org.tron.common.utils.ByteArray; import org.tron.common.utils.Commons; +import org.tron.common.utils.StringUtil; import org.tron.core.ChainBaseManager; import org.tron.core.Constant; import org.tron.core.capsule.AccountCapsule; @@ -115,12 +116,13 @@ public void consume(TransactionCapsule trx, TransactionTrace trace) bytesSize += Constant.MAX_RESULT_SIZE_IN_TX; } - logger.debug("trxId {}, bandwidth cost: {}", trx.getTransactionId(), bytesSize); + logger.debug("TxId {}, bandwidth cost: {}.", trx.getTransactionId(), bytesSize); trace.setNetBill(bytesSize, 0); byte[] address = TransactionCapsule.getOwner(contract); AccountCapsule accountCapsule = chainBaseManager.getAccountStore().get(address); if (accountCapsule == null) { - throw new ContractValidateException("account does not exist"); + throw new ContractValidateException(String.format("account [%s] does not exist", + StringUtil.encode58Check(address))); } long now = chainBaseManager.getHeadSlot(); if (contractCreateNewAccount(contract)) { @@ -147,8 +149,9 @@ public void consume(TransactionCapsule trx, TransactionTrace trace) long fee = chainBaseManager.getDynamicPropertiesStore().getTransactionFee() * bytesSize; throw new AccountResourceInsufficientException( - "Account has insufficient bandwidth[" + bytesSize + "] and balance[" - + fee + "] to create new account"); + String.format( + "account [%s] has insufficient bandwidth[%d] and balance[%d] to create new account", + StringUtil.encode58Check(address), bytesSize, fee)); } } @@ -172,7 +175,10 @@ private void consumeForCreateNewAccount(AccountCapsule accountCapsule, long byte if (!ret) { ret = consumeFeeForCreateNewAccount(accountCapsule, trace); if (!ret) { - throw new AccountResourceInsufficientException(); + throw new AccountResourceInsufficientException(String.format( + "account [%s] has insufficient bandwidth[%d] and balance[%d] to create new account", + StringUtil.encode58Check(accountCapsule.createDbKey()), bytes, + chainBaseManager.getDynamicPropertiesStore().getCreateAccountFee())); } } } @@ -267,7 +273,7 @@ private boolean useAssetAccountNet(Contract contract, AccountCapsule accountCaps chainBaseManager.getAssetIssueStore(), chainBaseManager.getAssetIssueV2Store()) .get(assetName.toByteArray()); if (assetIssueCapsule == null) { - throw new ContractValidateException("asset does not exist"); + throw new ContractValidateException(String.format("asset [%s] does not exist", assetName)); } String tokenName = ByteArray.toStr(assetName.toByteArray()); @@ -284,7 +290,9 @@ private boolean useAssetAccountNet(Contract contract, AccountCapsule accountCaps publicLatestFreeNetTime, now); if (bytes > (publicFreeAssetNetLimit - newPublicFreeAssetNetUsage)) { - logger.debug("The " + tokenID + " public free bandwidth is not enough"); + logger.debug("The {} public free bandwidth is not enough." + + " Bytes: {}, publicFreeAssetNetLimit: {}, newPublicFreeAssetNetUsage: {}.", + tokenID, bytes, publicFreeAssetNetLimit, newPublicFreeAssetNetUsage); return false; } @@ -306,7 +314,9 @@ private boolean useAssetAccountNet(Contract contract, AccountCapsule accountCaps latestAssetOperationTime, now); if (bytes > (freeAssetNetLimit - newFreeAssetNetUsage)) { - logger.debug("The " + tokenID + " free bandwidth is not enough"); + logger.debug("The {} free bandwidth is not enough." + + " Bytes: {}, freeAssetNetLimit: {}, newFreeAssetNetUsage:{}.", + tokenID, bytes, freeAssetNetLimit, newFreeAssetNetUsage); return false; } @@ -321,7 +331,9 @@ private boolean useAssetAccountNet(Contract contract, AccountCapsule accountCaps issuerNetUsage, 0, latestConsumeTime, now); if (bytes > (issuerNetLimit - newIssuerNetUsage)) { - logger.debug("The " + tokenID + " issuer's bandwidth is not enough"); + logger.debug("The {} issuer's bandwidth is not enough." + + " Bytes: {}, issuerNetLimit: {}, newIssuerNetUsage:{}.", + tokenID, bytes, issuerNetLimit, newIssuerNetUsage); return false; } @@ -399,7 +411,9 @@ private boolean useAccountNet(AccountCapsule accountCapsule, long bytes, long no long newNetUsage = increase(accountCapsule, BANDWIDTH, netUsage, 0, latestConsumeTime, now); if (bytes > (netLimit - newNetUsage)) { - logger.debug("net usage is running out, now use free net usage"); + logger.debug("Net usage is running out, now use free net usage." + + " Bytes: {}, netLimit: {}, newNetUsage: {}.", + bytes, netLimit, newNetUsage); return false; } @@ -422,7 +436,9 @@ private boolean useFreeNet(AccountCapsule accountCapsule, long bytes, long now) long newFreeNetUsage = increase(freeNetUsage, 0, latestConsumeFreeTime, now); if (bytes > (freeNetLimit - newFreeNetUsage)) { - logger.debug("free net usage is running out"); + logger.debug("Free net usage is running out." + + " Bytes: {}, freeNetLimit: {}, newFreeNetUsage: {}.", + bytes, freeNetLimit, newFreeNetUsage); return false; } @@ -433,7 +449,9 @@ private boolean useFreeNet(AccountCapsule accountCapsule, long bytes, long now) long newPublicNetUsage = increase(publicNetUsage, 0, publicNetTime, now); if (bytes > (publicNetLimit - newPublicNetUsage)) { - logger.debug("free public net usage is running out"); + logger.debug("Free public net usage is running out." + + " Bytes: {}, publicNetLimit: {}, newPublicNetUsage: {}.", + bytes, publicNetLimit, newPublicNetUsage); return false; } diff --git a/chainbase/src/main/java/org/tron/core/db/BlockIndexStore.java b/chainbase/src/main/java/org/tron/core/db/BlockIndexStore.java index 2cfd3d7ae28..50402002f88 100644 --- a/chainbase/src/main/java/org/tron/core/db/BlockIndexStore.java +++ b/chainbase/src/main/java/org/tron/core/db/BlockIndexStore.java @@ -29,7 +29,7 @@ public BlockId get(Long num) throws ItemNotFoundException { BytesCapsule value = getUnchecked(ByteArray.fromLong(num)); if (value == null || value.getData() == null) { - throw new ItemNotFoundException("number: " + num + " is not found!"); + throw new ItemNotFoundException(String.format("number: %d is not found!", num)); } return new BlockId(Sha256Hash.wrap(value.getData()), num); } @@ -39,7 +39,8 @@ public BytesCapsule get(byte[] key) throws ItemNotFoundException { byte[] value = revokingDB.getUnchecked(key); if (ArrayUtils.isEmpty(value)) { - throw new ItemNotFoundException("number: " + Arrays.toString(key) + " is not found!"); + throw new ItemNotFoundException(String.format("number: %d is not found!", + ByteArray.toLong(key))); } return new BytesCapsule(value); } diff --git a/chainbase/src/main/java/org/tron/core/db/BlockStore.java b/chainbase/src/main/java/org/tron/core/db/BlockStore.java index e4efe388043..9198b1c5198 100644 --- a/chainbase/src/main/java/org/tron/core/db/BlockStore.java +++ b/chainbase/src/main/java/org/tron/core/db/BlockStore.java @@ -15,14 +15,15 @@ package org.tron.core.db; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; +import org.tron.common.error.TronDBException; import org.tron.common.utils.Sha256Hash; import org.tron.core.capsule.BlockCapsule; import org.tron.core.capsule.BlockCapsule.BlockId; @@ -39,31 +40,24 @@ private BlockStore(@Value("block") String dbName) { public List getLimitNumber(long startNumber, long limit) { BlockId startBlockId = new BlockId(Sha256Hash.ZERO_HASH, startNumber); - return revokingDB.getValuesNext(startBlockId.getBytes(), limit).stream() - .map(bytes -> { - try { - return new BlockCapsule(bytes); - } catch (BadItemException ignored) { - } - return null; - }) - .filter(Objects::nonNull) - .sorted(Comparator.comparing(BlockCapsule::getNum)) - .collect(Collectors.toList()); + return pack(revokingDB.getValuesNext(startBlockId.getBytes(), limit)); } public List getBlockByLatestNum(long getNum) { + return pack(revokingDB.getlatestValues(getNum)); + } - return revokingDB.getlatestValues(getNum).stream() - .map(bytes -> { - try { - return new BlockCapsule(bytes); - } catch (BadItemException ignored) { - } - return null; - }) - .filter(Objects::nonNull) - .sorted(Comparator.comparing(BlockCapsule::getNum)) - .collect(Collectors.toList()); + private List pack(Set values) { + List blocks = new ArrayList<>(); + for (byte[] bytes : values) { + try { + blocks.add(new BlockCapsule(bytes)); + } catch (BadItemException e) { + logger.error("Find bad item: {}", e.getMessage()); + // throw new TronDBException(e); + } + } + blocks.sort(Comparator.comparing(BlockCapsule::getNum)); + return blocks; } } diff --git a/chainbase/src/main/java/org/tron/core/db/CommonDataBase.java b/chainbase/src/main/java/org/tron/core/db/CommonDataBase.java index a607bb1e44f..a72b581a97c 100644 --- a/chainbase/src/main/java/org/tron/core/db/CommonDataBase.java +++ b/chainbase/src/main/java/org/tron/core/db/CommonDataBase.java @@ -5,7 +5,7 @@ import org.springframework.stereotype.Component; import org.tron.common.utils.ByteArray; -@Slf4j +@Slf4j(topic = "DB") @Component public class CommonDataBase extends TronDatabase { @@ -37,7 +37,7 @@ public boolean has(byte[] key) { public void saveLatestPbftBlockNum(long number) { if (number <= getLatestPbftBlockNum()) { - logger.warn("pbft number {} <= latest number {}", number, getLatestPbftBlockNum()); + logger.warn("PBFT number {} <= latest number {}.", number, getLatestPbftBlockNum()); return; } this.put(LATEST_PBFT_BLOCK_NUM, ByteArray.fromLong(number)); diff --git a/chainbase/src/main/java/org/tron/core/db/EnergyProcessor.java b/chainbase/src/main/java/org/tron/core/db/EnergyProcessor.java index 20e2c94b2d2..fe36575c512 100644 --- a/chainbase/src/main/java/org/tron/core/db/EnergyProcessor.java +++ b/chainbase/src/main/java/org/tron/core/db/EnergyProcessor.java @@ -86,9 +86,8 @@ public void updateAdaptiveTotalEnergyLimit() { ); dynamicPropertiesStore.saveTotalEnergyCurrentLimit(result); - logger.debug( - "adjust totalEnergyCurrentLimit, old[" + totalEnergyCurrentLimit + "], new[" + result - + "]"); + logger.debug("Adjust totalEnergyCurrentLimit, old: {}, new: {}.", + totalEnergyCurrentLimit, result); } @Override diff --git a/chainbase/src/main/java/org/tron/core/db/KhaosDatabase.java b/chainbase/src/main/java/org/tron/core/db/KhaosDatabase.java index 9c9c968aa73..c1d3bceeb4b 100644 --- a/chainbase/src/main/java/org/tron/core/db/KhaosDatabase.java +++ b/chainbase/src/main/java/org/tron/core/db/KhaosDatabase.java @@ -110,12 +110,13 @@ public BlockCapsule push(BlockCapsule blk) if (kblock != null) { if (blk.getNum() != kblock.num + 1) { throw new BadNumberBlockException( - "parent number :" + kblock.num + ",block number :" + blk.getNum()); + String.format("parent number: %d , block number: %d", kblock.num, blk.getNum()) + ); } block.setParent(kblock); } else { miniUnlinkedStore.insert(block); - logger.error("blk:{}, head:{}, miniStore:{}, miniUnlinkedStore:{}", + logger.error("Block: {}, head: {}, miniStore: {}, miniUnlinkedStore: {}.", blk, head, miniStore, @@ -327,7 +328,7 @@ protected boolean removeEldestEntry(Map.Entry> entry numKblkMap.remove(k); v.forEach(b -> { hashKblkMap.remove(b.id); - logger.info("remove from khaosDatabase:{}", b.id); + logger.info("Remove from khaosDatabase: {}.", b.id); }); }); diff --git a/chainbase/src/main/java/org/tron/core/db/PbftSignDataStore.java b/chainbase/src/main/java/org/tron/core/db/PbftSignDataStore.java index 784abca70aa..d65e24ce6de 100644 --- a/chainbase/src/main/java/org/tron/core/db/PbftSignDataStore.java +++ b/chainbase/src/main/java/org/tron/core/db/PbftSignDataStore.java @@ -6,7 +6,7 @@ import org.tron.core.capsule.PbftSignCapsule; import org.tron.protos.Protocol.PBFTMessage.DataType; -@Slf4j +@Slf4j(topic = "DB") @Component public class PbftSignDataStore extends TronDatabase { diff --git a/chainbase/src/main/java/org/tron/core/db/StorageMarket.java b/chainbase/src/main/java/org/tron/core/db/StorageMarket.java index 66c0904b622..10a3b656565 100644 --- a/chainbase/src/main/java/org/tron/core/db/StorageMarket.java +++ b/chainbase/src/main/java/org/tron/core/db/StorageMarket.java @@ -7,11 +7,8 @@ @Slf4j(topic = "DB") public class StorageMarket { - + private static final String LOG_MSG = "NewTotalPool: {}, newTotalReserved: {}."; private static final long MS_PER_YEAR = 365 * 24 * 3600 * 1000L; - private static final String NEW_TOTAL_RESERVED = " newTotalReserved: "; - private static final String NEW_STORAGE_LIMIT = " newStorageLimit: "; - private static final String NEW_TOTAL_POOL = "newTotalPool: "; private AccountStore accountStore; private DynamicPropertiesStore dynamicPropertiesStore; private long supply = 1_000_000_000_000_000L; @@ -23,12 +20,12 @@ public StorageMarket(AccountStore accountStore, DynamicPropertiesStore dynamicPr } private long exchangeToSupply(boolean isTRX, long quant) { - logger.info("isTRX: " + isTRX); + logger.info("IsTRX: {}.", isTRX); long balance = isTRX ? dynamicPropertiesStore.getTotalStoragePool() : dynamicPropertiesStore.getTotalStorageReserved(); - logger.info("balance: " + balance); + logger.info("Balance: {}.", balance); long newBalance = balance + quant; - logger.info("balance + quant: " + (balance + quant)); + logger.info("Balance + quant: {}.", balance + quant); // if (isTRX) { // dbManager.getDynamicPropertiesStore().saveTotalStoragePool(newBalance); @@ -37,7 +34,7 @@ private long exchangeToSupply(boolean isTRX, long quant) { // } double issuedSupply = -supply * (1.0 - Math.pow(1.0 + (double) quant / newBalance, 0.0005)); - logger.info("issuedSupply: " + issuedSupply); + logger.info("IssuedSupply: {}.", issuedSupply); long out = (long) issuedSupply; supply += out; @@ -45,12 +42,12 @@ private long exchangeToSupply(boolean isTRX, long quant) { } private long exchangeToSupply2(boolean isTRX, long quant) { - logger.info("isTRX: " + isTRX); + logger.info("IsTRX: {}.", isTRX); long balance = isTRX ? dynamicPropertiesStore.getTotalStoragePool() : dynamicPropertiesStore.getTotalStorageReserved(); - logger.info("balance: " + balance); + logger.info("Balance: {}.", balance); long newBalance = balance - quant; - logger.info("balance - quant: " + (balance - quant)); + logger.info("Balance - quant: {}.", balance - quant); // if (isTRX) { // dbManager.getDynamicPropertiesStore().saveTotalStoragePool(newBalance); @@ -59,7 +56,7 @@ private long exchangeToSupply2(boolean isTRX, long quant) { // } double issuedSupply = -supply * (1.0 - Math.pow(1.0 + (double) quant / newBalance, 0.0005)); - logger.info("issuedSupply: " + issuedSupply); + logger.info("IssuedSupply: {}.", issuedSupply); long out = (long) issuedSupply; supply += out; @@ -73,12 +70,12 @@ private long exchange_from_supply(boolean isTRX, long supplyQuant) { double exchangeBalance = balance * (Math.pow(1.0 + (double) supplyQuant / supply, 2000.0) - 1.0); - logger.info("exchangeBalance: " + exchangeBalance); + logger.info("ExchangeBalance: {}.", exchangeBalance); long out = (long) exchangeBalance; if (isTRX) { out = Math.round(exchangeBalance / 100000) * 100000; - logger.info("---out: " + out); + logger.info("Out: {}.", out); } return out; @@ -95,7 +92,7 @@ public long calculateTax(long duration, long limit) { double millisecondPerYear = MS_PER_YEAR; double feeRate = duration / millisecondPerYear * ratePerYear; long storageTax = (long) (limit * feeRate); - logger.info("storageTax: " + storageTax); + logger.info("StorageTax: {}.", storageTax); return storageTax; } @@ -103,18 +100,18 @@ public long calculateTax(long duration, long limit) { public long tryPayTax(long duration, long limit) { long storageTax = calculateTax(duration, limit); long tax = exchange(storageTax, false); - logger.info("tax: " + tax); + logger.info("Tax: {}.", tax); long newTotalTax = dynamicPropertiesStore.getTotalStorageTax() + tax; long newTotalPool = dynamicPropertiesStore.getTotalStoragePool() - tax; long newTotalReserved = dynamicPropertiesStore.getTotalStorageReserved() + storageTax; - logger.info("reserved: " + dynamicPropertiesStore.getTotalStorageReserved()); + logger.info("Reserved: {}.", dynamicPropertiesStore.getTotalStorageReserved()); boolean eq = dynamicPropertiesStore.getTotalStorageReserved() == 128L * 1024 * 1024 * 1024; - logger.info("reserved == 128GB: " + eq); - logger.info("newTotalTax: " + newTotalTax + " newTotalPool: " + newTotalPool - + NEW_TOTAL_RESERVED + newTotalReserved); + logger.info("Reserved == 128GB: {}.", eq); + logger.info("NewTotalTax: {}, newTotalPool: {}, newTotalReserved: {}.", + newTotalTax, newTotalPool, newTotalReserved); return storageTax; } @@ -122,18 +119,18 @@ public long tryPayTax(long duration, long limit) { public long payTax(long duration, long limit) { long storageTax = calculateTax(duration, limit); long tax = exchange(storageTax, false); - logger.info("tax: " + tax); + logger.info("Tax: {}.", tax); long newTotalTax = dynamicPropertiesStore.getTotalStorageTax() + tax; long newTotalPool = dynamicPropertiesStore.getTotalStoragePool() - tax; long newTotalReserved = dynamicPropertiesStore.getTotalStorageReserved() + storageTax; - logger.info("reserved: " + dynamicPropertiesStore.getTotalStorageReserved()); + logger.info("Reserved: {}.", dynamicPropertiesStore.getTotalStorageReserved()); boolean eq = dynamicPropertiesStore.getTotalStorageReserved() == 128L * 1024 * 1024 * 1024; - logger.info("reserved == 128GB: " + eq); - logger.info("newTotalTax: " + newTotalTax + " newTotalPool: " + newTotalPool - + NEW_TOTAL_RESERVED + newTotalReserved); + logger.info("Reserved == 128GB: {}.", eq); + logger.info("NewTotalTax: {}, newTotalPool: {}, newTotalReserved: {}.", + newTotalTax, newTotalPool, newTotalReserved); dynamicPropertiesStore.saveTotalStorageTax(newTotalTax); dynamicPropertiesStore.saveTotalStoragePool(newTotalPool); dynamicPropertiesStore.saveTotalStorageReserved(newTotalReserved); @@ -162,12 +159,10 @@ public AccountCapsule buyStorageBytes(AccountCapsule accountCapsule, long storag long quant = exchange_from_supply(true, relay); long newBalance = accountCapsule.getBalance() - quant; - logger.info("newBalance: " + newBalance); + logger.info("New balance: {}.", newBalance); long newStorageLimit = currentStorageLimit + storageBought; - logger.info( - "storageBought: " + storageBought + NEW_STORAGE_LIMIT - + newStorageLimit); + logger.info("StorageBought: {}, newStorageLimit: {}.", storageBought, newStorageLimit); accountCapsule.setLatestExchangeStorageTime(now); accountCapsule.setStorageLimit(newStorageLimit); @@ -177,7 +172,7 @@ public AccountCapsule buyStorageBytes(AccountCapsule accountCapsule, long storag long newTotalPool = dynamicPropertiesStore.getTotalStoragePool() + quant; long newTotalReserved = dynamicPropertiesStore.getTotalStorageReserved() - storageBought; - logger.info(NEW_TOTAL_POOL + newTotalPool + NEW_TOTAL_RESERVED + newTotalReserved); + logger.info(LOG_MSG, newTotalPool, newTotalReserved); dynamicPropertiesStore.saveTotalStoragePool(newTotalPool); dynamicPropertiesStore.saveTotalStorageReserved(newTotalReserved); return accountCapsule; @@ -189,13 +184,11 @@ public void buyStorage(AccountCapsule accountCapsule, long quant) { long currentStorageLimit = accountCapsule.getStorageLimit(); long newBalance = accountCapsule.getBalance() - quant; - logger.info("newBalance: " + newBalance); + logger.info("New balance: {}.", newBalance); long storageBought = exchange(quant, true); long newStorageLimit = currentStorageLimit + storageBought; - logger.info( - "storageBought: " + storageBought + NEW_STORAGE_LIMIT - + newStorageLimit); + logger.info("StorageBought: {}, newStorageLimit: {}.", storageBought, newStorageLimit); accountCapsule.setLatestExchangeStorageTime(now); accountCapsule.setStorageLimit(newStorageLimit); @@ -205,7 +198,7 @@ public void buyStorage(AccountCapsule accountCapsule, long quant) { long newTotalPool = dynamicPropertiesStore.getTotalStoragePool() + quant; long newTotalReserved = dynamicPropertiesStore.getTotalStorageReserved() - storageBought; - logger.info(NEW_TOTAL_POOL + newTotalPool + NEW_TOTAL_RESERVED + newTotalReserved); + logger.info(LOG_MSG, newTotalPool, newTotalReserved); dynamicPropertiesStore.saveTotalStoragePool(newTotalPool); dynamicPropertiesStore.saveTotalStorageReserved(newTotalReserved); @@ -219,7 +212,7 @@ public void sellStorage(AccountCapsule accountCapsule, long bytes) { long newBalance = accountCapsule.getBalance() + quant; long newStorageLimit = currentStorageLimit - bytes; - logger.info("quant: " + quant + NEW_STORAGE_LIMIT + newStorageLimit); + logger.info("Quant: {}, newStorageLimit: {}.", quant, newStorageLimit); accountCapsule.setLatestExchangeStorageTime(now); accountCapsule.setStorageLimit(newStorageLimit); @@ -229,7 +222,7 @@ public void sellStorage(AccountCapsule accountCapsule, long bytes) { long newTotalPool = dynamicPropertiesStore.getTotalStoragePool() - quant; long newTotalReserved = dynamicPropertiesStore.getTotalStorageReserved() + bytes; - logger.info(NEW_TOTAL_POOL + newTotalPool + NEW_TOTAL_RESERVED + newTotalReserved); + logger.info(LOG_MSG, newTotalPool, newTotalReserved); dynamicPropertiesStore.saveTotalStoragePool(newTotalPool); dynamicPropertiesStore.saveTotalStorageReserved(newTotalReserved); diff --git a/chainbase/src/main/java/org/tron/core/db/TransactionTrace.java b/chainbase/src/main/java/org/tron/core/db/TransactionTrace.java index 6a74d6d015f..4df135ce310 100644 --- a/chainbase/src/main/java/org/tron/core/db/TransactionTrace.java +++ b/chainbase/src/main/java/org/tron/core/db/TransactionTrace.java @@ -41,7 +41,7 @@ import org.tron.protos.contract.SmartContractOuterClass.SmartContract.ABI; import org.tron.protos.contract.SmartContractOuterClass.TriggerSmartContract; -@Slf4j(topic = "TransactionTrace") +@Slf4j(topic = "DB") public class TransactionTrace { private TransactionCapsule trx; @@ -138,11 +138,10 @@ public void checkIsConstant() throws ContractValidateException, VMIllegalExcepti ContractCapsule contract = contractStore .get(triggerContractFromTransaction.getContractAddress().toByteArray()); if (contract == null) { - logger.info("contract: {} is not in contract store", StringUtil - .encode58Check(triggerContractFromTransaction.getContractAddress().toByteArray())); - throw new ContractValidateException("contract: " + StringUtil - .encode58Check(triggerContractFromTransaction.getContractAddress().toByteArray()) - + " is not in contract store"); + throw new ContractValidateException(String.format("contract: %s is not in contract store", + StringUtil.encode58Check(triggerContractFromTransaction + .getContractAddress().toByteArray()))); + } ABI abi = contract.getInstance().getAbi(); if (WalletUtil.isConstant(abi, triggerContractFromTransaction)) { @@ -224,7 +223,7 @@ public void pay() throws BalanceInsufficientException { long originEnergyLimit = 0; switch (trxType) { case TRX_CONTRACT_CREATION_TYPE: - callerAccount = TransactionCapsule.getOwner(trx.getInstance().getRawData().getContract(0)); + callerAccount = trx.getOwnerAddress(); originAccount = callerAccount; break; case TRX_CONTRACT_CALL_TYPE: @@ -269,14 +268,13 @@ public void check() throws ReceiptCheckErrException { return; } if (Objects.isNull(trx.getContractRet())) { - throw new ReceiptCheckErrException("null resultCode"); + throw new ReceiptCheckErrException( + String.format("null resultCode id: %s", trx.getTransactionId())); } if (!trx.getContractRet().equals(receipt.getResult())) { - logger.info( - "this tx id: {}, the resultCode in received block: {}, the resultCode in self: {}", - Hex.toHexString(trx.getTransactionId().getBytes()), trx.getContractRet(), - receipt.getResult()); - throw new ReceiptCheckErrException("Different resultCode"); + throw new ReceiptCheckErrException(String.format( + "different resultCode txId: %s, expect: %s, actual: %s", + trx.getTransactionId(), trx.getContractRet(), receipt.getResult())); } } diff --git a/chainbase/src/main/java/org/tron/core/db/TronDatabase.java b/chainbase/src/main/java/org/tron/core/db/TronDatabase.java index 9373189941a..04e20af4fd6 100644 --- a/chainbase/src/main/java/org/tron/core/db/TronDatabase.java +++ b/chainbase/src/main/java/org/tron/core/db/TronDatabase.java @@ -2,7 +2,6 @@ import com.google.protobuf.InvalidProtocolBufferException; import java.nio.file.Paths; -import java.util.HashMap; import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; @@ -10,6 +9,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.iq80.leveldb.WriteOptions; +import org.rocksdb.DirectComparator; import org.springframework.beans.factory.annotation.Autowired; import org.tron.common.parameter.CommonParameter; import org.tron.common.storage.WriteOptionsWrapper; @@ -18,11 +18,8 @@ import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl; import org.tron.common.utils.StorageUtils; import org.tron.core.db.common.DbSourceInter; -import org.tron.core.db2.common.LevelDB; -import org.tron.core.db2.common.RocksDB; import org.tron.core.db2.common.WrappedByteArray; import org.tron.core.db2.core.ITronChainBase; -import org.tron.core.db2.core.SnapshotRoot; import org.tron.core.exception.BadItemException; import org.tron.core.exception.ItemNotFoundException; @@ -46,7 +43,7 @@ protected TronDatabase(String dbName) { dbSource = new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(dbName), dbName, - StorageUtils.getOptionsByDbName(dbName), + getOptionsByDbNameForLevelDB(dbName), new WriteOptions().sync(CommonParameter.getInstance() .getStorage().isDbSync())); } else if ("ROCKSDB".equals(CommonParameter.getInstance() @@ -55,7 +52,7 @@ protected TronDatabase(String dbName) { CommonParameter.getInstance().getStorage().getDbDirectory()).toString(); dbSource = new RocksDbDataSourceImpl(parentName, dbName, CommonParameter.getInstance() - .getRocksDBCustomSettings()); + .getRocksDBCustomSettings(), getDirectComparator()); } dbSource.initDB(); @@ -69,6 +66,14 @@ private void init() { protected TronDatabase() { } + protected org.iq80.leveldb.Options getOptionsByDbNameForLevelDB(String dbName) { + return StorageUtils.getOptionsByDbName(dbName); + } + + protected DirectComparator getDirectComparator() { + return null; + } + public DbSourceInter getDbSource() { return dbSource; } diff --git a/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java b/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java index 1b327642212..d1bbed934d5 100644 --- a/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java +++ b/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java @@ -76,12 +76,12 @@ protected TronStoreWithRevoking(String dbName) { dbName, CommonParameter.getInstance() .getRocksDBCustomSettings(), getDirectComparator())); } else { - throw new RuntimeException("dbEngine is error."); + throw new RuntimeException(String.format("db engine %s is error", dbEngine)); } this.revokingDB = new Chainbase(new SnapshotRoot(this.db)); } else { - throw new RuntimeException("db version is error."); + throw new RuntimeException(String.format("db version %d is error", dbVersion)); } } @@ -99,7 +99,7 @@ protected TronStoreWithRevoking(DB db) { this.db = db; this.revokingDB = new Chainbase(new SnapshotRoot(db)); } else { - throw new RuntimeException("db version is only 2.(" + dbVersion + ")"); + throw new RuntimeException(String.format("db version is only 2, actual: %d", dbVersion)); } } @@ -196,7 +196,7 @@ private void closeJniIterator(Iterator iterator) { try { ((DBIterator) iterator).close(); } catch (IOException e) { - logger.error("", e); + logger.error("Close jni iterator.", e); } } } diff --git a/chainbase/src/main/java/org/tron/core/db/accountstate/AccountStateEntity.java b/chainbase/src/main/java/org/tron/core/db/accountstate/AccountStateEntity.java index 858543986fc..d52eb7a0af2 100644 --- a/chainbase/src/main/java/org/tron/core/db/accountstate/AccountStateEntity.java +++ b/chainbase/src/main/java/org/tron/core/db/accountstate/AccountStateEntity.java @@ -1,5 +1,6 @@ package org.tron.core.db.accountstate; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.tron.common.utils.StringUtil; import org.tron.protos.Protocol.Account; @@ -24,8 +25,8 @@ public AccountStateEntity(Account account) { public static AccountStateEntity parse(byte[] data) { try { return new AccountStateEntity().setAccount(Account.parseFrom(data)); - } catch (Exception e) { - logger.error("parse to AccountStateEntity error! reason: {}", e.getMessage()); + } catch (InvalidProtocolBufferException e) { + logger.error("Parse to AccountStateEntity error! reason: {}", e.getMessage()); } return null; } diff --git a/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java b/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java index f095ec5ab5d..a31ebca9409 100644 --- a/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java +++ b/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java @@ -5,9 +5,10 @@ import java.util.NoSuchElementException; import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksIterator; +import org.tron.common.error.TronDBException; -@Slf4j +@Slf4j(topic = "DB") public final class RockStoreIterator implements DBIterator { private RocksIterator dbIterator; @@ -35,11 +36,11 @@ public boolean hasNext() { dbIterator.close(); } } catch (Exception e) { - System.out.println("e:" + e); + logger.error(e.getMessage(), e); try { dbIterator.close(); } catch (Exception e1) { - System.out.println("e1:" + e1); + logger.error(e.getMessage(), e); } } return hasNext; diff --git a/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java b/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java index 997d13dd293..74fe4a70ed0 100755 --- a/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java +++ b/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java @@ -4,6 +4,7 @@ import java.util.Map.Entry; import lombok.extern.slf4j.Slf4j; import org.iq80.leveldb.DBIterator; +import org.tron.common.error.TronDBException; @Slf4j(topic = "DB") public final class StoreIterator implements org.tron.core.db.common.iterator.DBIterator { @@ -34,7 +35,7 @@ public boolean hasNext() { dbIterator.close(); } } catch (Exception e) { - logger.debug(e.getMessage(), e); + logger.error(e.getMessage(), e); } return hasNext; diff --git a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java index 02f8f06fd2e..f574520b222 100644 --- a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java +++ b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java @@ -81,10 +81,10 @@ public TxCacheDB(String name, RecentTransactionStore recentTransactionStore) { name, CommonParameter.getInstance() .getRocksDBCustomSettings())); } else { - throw new RuntimeException("db type is not supported."); + throw new RuntimeException(String.format("db type: %s is not supported", dbEngine)); } } else { - throw new RuntimeException("db version is not supported."); + throw new RuntimeException(String.format("db version: %d is not supported", dbVersion)); } this.bloomFilters[0] = BloomFilter.create(Funnels.byteArrayFunnel(), MAX_BLOCK_SIZE * TRANSACTION_COUNT); @@ -109,8 +109,7 @@ private void initCache() { bloomFilters[1].put(entry.getKey()); persistentSize++; } - logger.info("load transaction cache from persistentStore " - + "db-size:{}, filter-size:{}, filter-fpp:{}, cost:{}ms", + logger.info("Load cache from persistentStore, db: {}, filter: {}, filter-fpp: {}, cost: {} ms.", persistentSize, bloomFilters[1].approximateElementCount(), bloomFilters[1].expectedFpp(), System.currentTimeMillis() - start); @@ -133,8 +132,7 @@ private void init() { trx.getTransactionIds().forEach(tid -> bloomFilters[1].put(Hex.decode(tid))); } - logger.info("load transaction cache from recentTransactionStore" - + " filter-size:{}, filter-fpp:{}, cost:{}ms", + logger.info("Load cache from recentTransactionStore, filter: {}, filter-fpp: {}, cost: {} ms.", bloomFilters[1].approximateElementCount(), bloomFilters[1].expectedFpp(), System.currentTimeMillis() - start); } @@ -159,10 +157,11 @@ public void put(byte[] key, byte[] value) { // init active filter start block filterStartBlock = blockNum; currentFilterIndex = 0; - logger.info("init tx cache bloomFilters at {}", blockNum); + logger.info("Init tx cache bloomFilters at {}.", blockNum); } else if (blockNum - filterStartBlock > MAX_BLOCK_SIZE) { // active filter is full - logger.info("active bloomFilters is full (size={} fpp={}), create a new one (start={})", + logger.info( + "Active bloomFilters is full (size = {} fpp = {}), create a new one (start = {}).", bloomFilters[currentFilterIndex].approximateElementCount(), bloomFilters[currentFilterIndex].expectedFpp(), blockNum); diff --git a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java index 7e2d8f5074a..3810667e465 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java @@ -7,6 +7,9 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; + +import java.io.File; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -15,10 +18,13 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.locks.LockSupport; import java.util.stream.Collectors; import javax.annotation.PostConstruct; + import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -26,7 +32,10 @@ import org.tron.common.error.TronDBException; import org.tron.common.parameter.CommonParameter; import org.tron.common.storage.WriteOptionsWrapper; +import org.tron.common.utils.FileUtil; +import org.tron.common.utils.StorageUtils; import org.tron.core.db.RevokingDatabase; +import org.tron.core.db.TronDatabase; import org.tron.core.db2.ISession; import org.tron.core.db2.common.DB; import org.tron.core.db2.common.IRevokingDB; @@ -34,6 +43,7 @@ import org.tron.core.db2.common.Value; import org.tron.core.db2.common.WrappedByteArray; import org.tron.core.exception.RevokingStoreIllegalStateException; +import org.tron.core.store.CheckPointV2Store; import org.tron.core.store.CheckTmpStore; @Slf4j(topic = "DB") @@ -42,6 +52,8 @@ public class SnapshotManager implements RevokingDatabase { public static final int DEFAULT_MAX_FLUSH_COUNT = 500; public static final int DEFAULT_MIN_FLUSH_COUNT = 1; private static final int DEFAULT_STACK_MAX_SIZE = 256; + private static final long ONE_MINUTE_MILLS = 60*1000L; + private static final String CHECKPOINT_V2_DIR = "checkpoint"; @Getter private List dbs = new ArrayList<>(); @Getter @@ -63,6 +75,8 @@ public class SnapshotManager implements RevokingDatabase { private Map flushServices = new HashMap<>(); + private ScheduledExecutorService pruneCheckpointThread = null; + @Autowired @Setter @Getter @@ -71,11 +85,27 @@ public class SnapshotManager implements RevokingDatabase { @Setter private volatile int maxFlushCount = DEFAULT_MIN_FLUSH_COUNT; + private int checkpointVersion = 1; // default v1 + public SnapshotManager(String checkpointPath) { } @PostConstruct public void init() { + checkpointVersion = CommonParameter.getInstance().getStorage().getCheckpointVersion(); + // prune checkpoint + if (isV2Open()) { + pruneCheckpointThread = Executors.newSingleThreadScheduledExecutor(); + pruneCheckpointThread.scheduleWithFixedDelay(() -> { + try { + if (!unChecked) { + pruneCheckpoint(); + } + } catch (Throwable t) { + logger.error("Exception in prune checkpoint", t); + } + }, 10000, 3600, TimeUnit.MILLISECONDS); + } exitThread = new Thread(() -> { LockSupport.park(); // to Guarantee Some other thread invokes unpark with the current thread as the target @@ -150,7 +180,7 @@ private void retreat() { public void merge() { if (activeSession <= 0) { - throw new RevokingStoreIllegalStateException("activeDialog has to be greater than 0"); + throw new RevokingStoreIllegalStateException(activeSession); } if (size < 2) { @@ -168,7 +198,7 @@ public synchronized void revoke() { } if (activeSession <= 0) { - throw new RevokingStoreIllegalStateException("activeSession has to be greater than 0"); + throw new RevokingStoreIllegalStateException(activeSession); } if (size <= 0) { @@ -187,7 +217,7 @@ public synchronized void revoke() { public synchronized void commit() { if (activeSession <= 0) { - throw new RevokingStoreIllegalStateException("activeSession has to be greater than 0"); + throw new RevokingStoreIllegalStateException(activeSession); } --activeSession; @@ -195,11 +225,13 @@ public synchronized void commit() { public synchronized void pop() { if (activeSession != 0) { - throw new RevokingStoreIllegalStateException("activeSession has to be equal 0"); + throw new RevokingStoreIllegalStateException( + String.format("activeSession has to be equal 0, current %d", activeSession)); } if (size <= 0) { - throw new RevokingStoreIllegalStateException("there is not snapshot to be popped"); + throw new RevokingStoreIllegalStateException( + String.format("there is not snapshot to be popped, current: %d", size)); } disabled = true; @@ -240,10 +272,13 @@ public synchronized void disable() { @Override public void shutdown() { - System.err.println("******** begin to pop revokingDb ********"); - System.err.println("******** before revokingDb size:" + size); + logger.info("******** Begin to pop revokingDb. ********"); + logger.info("******** Before revokingDb size: {}.", size); checkTmpStore.close(); - System.err.println("******** end to pop revokingDb ********"); + logger.info("******** End to pop revokingDb. ********"); + if (pruneCheckpointThread != null) { + pruneCheckpointThread.shutdown(); + } } public void updateSolidity(int hops) { @@ -307,18 +342,21 @@ public void flush() { if (shouldBeRefreshed()) { try { long start = System.currentTimeMillis(); - deleteCheckpoint(); + if (!isV2Open()) { + deleteCheckpoint(); + } createCheckpoint(); + long checkPointEnd = System.currentTimeMillis(); refresh(); flushCount = 0; - logger.info("flush cost:{}, create checkpoint cost:{}, refresh cost:{}", + logger.info("Flush cost: {} ms, create checkpoint cost: {} ms, refresh cost: {} ms.", System.currentTimeMillis() - start, checkPointEnd - start, System.currentTimeMillis() - checkPointEnd ); } catch (TronDBException e) { - logger.error(" Find fatal error , program will be exited soon", e); + logger.error(" Find fatal error, program will be exited soon.", e); hitDown = true; LockSupport.unpark(exitThread); } @@ -326,6 +364,8 @@ public void flush() { } private void createCheckpoint() { + TronDatabase checkPointStore = null; + boolean syncFlag; try { Map batch = new HashMap<>(); for (Chainbase db : dbs) { @@ -348,70 +388,148 @@ private void createCheckpoint() { } } } + if (isV2Open()) { + String dbName = String.valueOf(System.currentTimeMillis()); + checkPointStore = getCheckpointDB(dbName); + syncFlag = CommonParameter.getInstance().getStorage().isCheckpointSync(); + } else { + checkPointStore = checkTmpStore; + syncFlag = CommonParameter.getInstance().getStorage().isDbSync(); + } - checkTmpStore.getDbSource().updateByBatch(batch.entrySet().stream() + checkPointStore.getDbSource().updateByBatch(batch.entrySet().stream() .map(e -> Maps.immutableEntry(e.getKey().getBytes(), e.getValue().getBytes())) .collect(HashMap::new, (m, k) -> m.put(k.getKey(), k.getValue()), HashMap::putAll), - WriteOptionsWrapper.getInstance().sync(CommonParameter - .getInstance().getStorage().isDbSync())); + WriteOptionsWrapper.getInstance().sync(syncFlag)); - } catch ( Exception e) { + } catch (Exception e) { throw new TronDBException(e); + } finally { + if (isV2Open() && checkPointStore != null) { + checkPointStore.close(); + } } } - private void deleteCheckpoint() { - try { - Map hmap = new HashMap<>(); - if (!checkTmpStore.getDbSource().allKeys().isEmpty()) { - for (Map.Entry e : checkTmpStore.getDbSource()) { - hmap.put(e.getKey(), null); - } + private TronDatabase getCheckpointDB(String dbName) { + return new CheckPointV2Store(CHECKPOINT_V2_DIR+"/"+dbName); + } + + private List getCheckpointList() { + String dbPath = Paths.get(StorageUtils.getOutputDirectoryByDbName(CHECKPOINT_V2_DIR), + CommonParameter.getInstance().getStorage().getDbDirectory()).toString(); + File file = new File(Paths.get(dbPath, CHECKPOINT_V2_DIR).toString()); + if (file.exists() && file.isDirectory()) { + String[] subDirs = file.list(); + if (subDirs != null) { + return Arrays.stream(subDirs).sorted().collect(Collectors.toList()); } + } + return null; + } - checkTmpStore.getDbSource().updateByBatch(hmap); - } catch (Exception e) { - throw new TronDBException(e); + private void deleteCheckpoint() { + checkTmpStore.reset(); + } + + private void pruneCheckpoint() { + if (unChecked) { + return; + } + List cpList = getCheckpointList(); + if (cpList == null) { + return; + } + if (cpList.size() < 3) { + return; + } + for (String cp: cpList.subList(0, cpList.size()-3)) { + long timestamp = Long.parseLong(cp); + if (System.currentTimeMillis() - timestamp < ONE_MINUTE_MILLS*2) { + break; + } + String checkpointPath = Paths.get(StorageUtils.getOutputDirectoryByDbName(CHECKPOINT_V2_DIR), + CommonParameter.getInstance().getStorage().getDbDirectory(), CHECKPOINT_V2_DIR).toString(); + if (!FileUtil.recursiveDelete(Paths.get(checkpointPath, cp).toString())) { + logger.error("checkpoint prune failed, timestamp: {}", timestamp); + return; + } + logger.debug("checkpoint prune success, timestamp: {}", timestamp); } } // ensure run this method first after process start. @Override public void check() { - for (Chainbase db : dbs) { + if (!isV2Open()) { + List cpList = getCheckpointList(); + if (cpList != null && cpList.size() != 0) { + logger.error("checkpoint check failed, can't convert checkpoint from v2 to v1"); + System.exit(-1); + } + checkV1(); + } else { + checkV2(); + } + } + + private void checkV1() { + for (Chainbase db: dbs) { if (!Snapshot.isRoot(db.getHead())) { - throw new IllegalStateException("first check."); + throw new IllegalStateException("First check."); } } + recover(checkTmpStore); + unChecked = false; + } - if (!checkTmpStore.getDbSource().allKeys().isEmpty()) { - Map dbMap = dbs.stream() - .map(db -> Maps.immutableEntry(db.getDbName(), db)) - .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - advance(); - for (Map.Entry e : checkTmpStore.getDbSource()) { - byte[] key = e.getKey(); - byte[] value = e.getValue(); - String db = simpleDecode(key); - if (dbMap.get(db) == null) { - continue; - } - byte[] realKey = Arrays.copyOfRange(key, db.getBytes().length + 4, key.length); + private void checkV2() { + logger.info("checkpoint version: {}", CommonParameter.getInstance().getStorage().getCheckpointVersion()); + logger.info("checkpoint sync: {}", CommonParameter.getInstance().getStorage().isCheckpointSync()); + List cpList = getCheckpointList(); + if (cpList == null || cpList.size() == 0) { + logger.info("checkpoint size is 0, using v1 recover"); + checkV1(); + deleteCheckpoint(); + return; + } - byte[] realValue = value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); - if (realValue != null) { - dbMap.get(db).getHead().put(realKey, realValue); - } else { - dbMap.get(db).getHead().remove(realKey); - } + for (String cp: cpList) { + TronDatabase checkPointV2Store = getCheckpointDB(cp); + recover(checkPointV2Store); + checkPointV2Store.close(); + } + logger.info("checkpoint v2 recover success"); + unChecked = false; + } + private void recover(TronDatabase tronDatabase) { + Map dbMap = dbs.stream() + .map(db -> Maps.immutableEntry(db.getDbName(), db)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + advance(); + for (Map.Entry e: tronDatabase.getDbSource()) { + byte[] key = e.getKey(); + byte[] value = e.getValue(); + String db = simpleDecode(key); + if (dbMap.get(db) == null) { + continue; + } + byte[] realKey = Arrays.copyOfRange(key, db.getBytes().length + 4, key.length); + byte[] realValue = value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); + if (realValue != null) { + dbMap.get(db).getHead().put(realKey, realValue); + } else { + dbMap.get(db).getHead().remove(realKey); } - - dbs.forEach(db -> db.getHead().getRoot().merge(db.getHead())); - retreat(); } - unChecked = false; + dbs.forEach(db -> db.getHead().getRoot().merge(db.getHead())); + retreat(); + } + + private boolean isV2Open() { + return checkpointVersion == 2; } private byte[] simpleEncode(String s) { @@ -471,7 +589,7 @@ public void destroy() { snapshotManager.revoke(); } } catch (Exception e) { - logger.error("revoke database error.", e); + logger.error("Revoke database error.", e); } if (disableOnExit) { snapshotManager.disable(); @@ -485,7 +603,7 @@ public void close() { snapshotManager.revoke(); } } catch (Exception e) { - logger.error("revoke database error.", e); + logger.error("Revoke database error.", e); throw new RevokingStoreIllegalStateException(e); } if (disableOnExit) { diff --git a/chainbase/src/main/java/org/tron/core/service/MortgageService.java b/chainbase/src/main/java/org/tron/core/service/MortgageService.java index f11c0693a77..5dd99626930 100644 --- a/chainbase/src/main/java/org/tron/core/service/MortgageService.java +++ b/chainbase/src/main/java/org/tron/core/service/MortgageService.java @@ -74,19 +74,19 @@ public void payStandbyWitness() { for (ByteString b : witnessAddressList) { double eachVotePay = (double) totalPay / voteSum; long pay = (long) (witnessCapsuleMap.get(b).getVoteCount() * eachVotePay); - logger.debug("pay {} stand reward {}", Hex.toHexString(b.toByteArray()), pay); + logger.debug("Pay {} stand reward {}.", Hex.toHexString(b.toByteArray()), pay); payReward(b.toByteArray(), pay); } } } public void payBlockReward(byte[] witnessAddress, long value) { - logger.debug("pay {} block reward {}", Hex.toHexString(witnessAddress), value); + logger.debug("Pay {} block reward {}.", Hex.toHexString(witnessAddress), value); payReward(witnessAddress, value); } public void payTransactionFeeReward(byte[] witnessAddress, long value) { - logger.debug("pay {} transaction fee reward {}", Hex.toHexString(witnessAddress), value); + logger.debug("Pay {} transaction fee reward {}.", Hex.toHexString(witnessAddress), value); payReward(witnessAddress, value); } @@ -125,7 +125,7 @@ public void withdrawReward(byte[] address) { reward = computeReward(beginCycle, endCycle, account); adjustAllowance(address, reward); reward = 0; - logger.info("latest cycle reward {},{}", beginCycle, account.getVotesList()); + logger.info("Latest cycle reward {}, {}.", beginCycle, account.getVotesList()); } beginCycle += 1; } @@ -142,8 +142,8 @@ public void withdrawReward(byte[] address) { delegationStore.setBeginCycle(address, endCycle); delegationStore.setEndCycle(address, endCycle + 1); delegationStore.setAccountVote(endCycle, address, accountCapsule); - logger.info("adjust {} allowance {}, now currentCycle {}, beginCycle {}, endCycle {}, " - + "account vote {},", Hex.toHexString(address), reward, currentCycle, + logger.info("Adjust {} allowance {}, now currentCycle {}, beginCycle {}, endCycle {}, " + + "account vote {}.", Hex.toHexString(address), reward, currentCycle, beginCycle, endCycle, accountCapsule.getVotesList()); } @@ -194,7 +194,7 @@ private long computeReward(long cycle, AccountCapsule accountCapsule) { long userVote = vote.getVoteCount(); double voteRate = (double) userVote / totalVote; reward += voteRate * totalReward; - logger.debug("computeReward {} {} {} {},{},{},{}", cycle, + logger.debug("ComputeReward {}, {}, {}, {}, {}, {}, {}.", cycle, Hex.toHexString(accountCapsule.getAddress().toByteArray()), Hex.toHexString(srAddress), userVote, totalVote, totalReward, reward); } @@ -252,7 +252,7 @@ public void adjustAllowance(byte[] address, long amount) { } adjustAllowance(accountStore, address, amount); } catch (BalanceInsufficientException e) { - logger.error("withdrawReward error: {},{}", Hex.toHexString(address), address, e); + logger.error("WithdrawReward error: {}.", e.getMessage()); } } @@ -266,7 +266,8 @@ public void adjustAllowance(AccountStore accountStore, byte[] accountAddress, lo if (amount < 0 && allowance < -amount) { throw new BalanceInsufficientException( - StringUtil.createReadableString(accountAddress) + " insufficient balance"); + String.format("%s insufficient balance, amount: %d, allowance: %d", + StringUtil.createReadableString(accountAddress), amount, allowance)); } account.setAllowance(allowance + amount); accountStore.put(account.createDbKey(), account); diff --git a/chainbase/src/main/java/org/tron/core/store/CheckPointV2Store.java b/chainbase/src/main/java/org/tron/core/store/CheckPointV2Store.java new file mode 100644 index 00000000000..d5ffaaa9143 --- /dev/null +++ b/chainbase/src/main/java/org/tron/core/store/CheckPointV2Store.java @@ -0,0 +1,48 @@ +package org.tron.core.store; + +import com.google.protobuf.InvalidProtocolBufferException; +import org.springframework.beans.factory.annotation.Autowired; +import org.tron.core.db.TronDatabase; +import org.tron.core.exception.BadItemException; +import org.tron.core.exception.ItemNotFoundException; + +import java.util.Spliterator; +import java.util.function.Consumer; + +public class CheckPointV2Store extends TronDatabase { + + @Autowired + public CheckPointV2Store(String dbPath) { + super(dbPath); + } + + @Override + public void put(byte[] key, byte[] item) { + } + + @Override + public void delete(byte[] key) { + getDbSource().deleteData(key); + } + + @Override + public byte[] get(byte[] key) + throws InvalidProtocolBufferException, ItemNotFoundException, BadItemException { + return null; + } + + @Override + public boolean has(byte[] key) { + return false; + } + + @Override + public void forEach(Consumer action) { + + } + + @Override + public Spliterator spliterator() { + return null; + } +} \ No newline at end of file diff --git a/chainbase/src/main/java/org/tron/core/store/CheckTmpStore.java b/chainbase/src/main/java/org/tron/core/store/CheckTmpStore.java index 870aa9aa1b8..09f60c83898 100644 --- a/chainbase/src/main/java/org/tron/core/store/CheckTmpStore.java +++ b/chainbase/src/main/java/org/tron/core/store/CheckTmpStore.java @@ -24,7 +24,6 @@ public void put(byte[] key, byte[] item) { @Override public void delete(byte[] key) { - } @Override diff --git a/chainbase/src/main/java/org/tron/core/store/DelegationStore.java b/chainbase/src/main/java/org/tron/core/store/DelegationStore.java index 8c799a4250b..4e95e480cfd 100644 --- a/chainbase/src/main/java/org/tron/core/store/DelegationStore.java +++ b/chainbase/src/main/java/org/tron/core/store/DelegationStore.java @@ -13,7 +13,7 @@ import java.math.BigInteger; -@Slf4j +@Slf4j(topic = "DB") @Component public class DelegationStore extends TronStoreWithRevoking { diff --git a/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java b/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java index 1c12ac47767..50329d0c9dc 100644 --- a/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java +++ b/chainbase/src/main/java/org/tron/core/store/DynamicPropertiesStore.java @@ -915,7 +915,7 @@ public long getAbiMoveDone() { } public void saveBlockFilledSlotsIndex(int blockFilledSlotsIndex) { - logger.debug("blockFilledSlotsIndex:" + blockFilledSlotsIndex); + logger.debug("BlockFilledSlotsIndex: {}.", blockFilledSlotsIndex); this.put(BLOCK_FILLED_SLOTS_INDEX, new BytesCapsule(ByteArray.fromInt(blockFilledSlotsIndex))); } @@ -1946,7 +1946,7 @@ public boolean supportShieldedTRC20Transaction() { } public void saveBlockFilledSlots(int[] blockFilledSlots) { - logger.debug("blockFilledSlots:" + intArrayToString(blockFilledSlots)); + logger.debug("BlockFilledSlots: {}.", intArrayToString(blockFilledSlots)); this.put(BLOCK_FILLED_SLOTS, new BytesCapsule(ByteArray.fromString(intArrayToString(blockFilledSlots)))); } @@ -2044,7 +2044,7 @@ public long getLatestBlockHeaderNumberFromDB() { .orElseThrow( () -> new IllegalArgumentException("not found latest block header number")); } catch (ItemNotFoundException | BadItemException e) { - logger.error("{}", e); + logger.error("Get header from DB, {}.", e.getMessage()); } return -1; } @@ -2071,7 +2071,7 @@ public Sha256Hash getLatestBlockHeaderHash() { * save timestamp of creating global latest block. */ public void saveLatestBlockHeaderTimestamp(long t) { - logger.info("update latest block header timestamp = {}", t); + logger.info("Update latest block header timestamp = {}.", t); this.put(LATEST_BLOCK_HEADER_TIMESTAMP, new BytesCapsule(ByteArray.fromLong(t))); } @@ -2079,7 +2079,7 @@ public void saveLatestBlockHeaderTimestamp(long t) { * save number of global latest block. */ public void saveLatestBlockHeaderNumber(long n) { - logger.info("update latest block header number = {}", n); + logger.info("Update latest block header number = {}.", n); this.put(LATEST_BLOCK_HEADER_NUMBER, new BytesCapsule(ByteArray.fromLong(n))); } @@ -2087,12 +2087,12 @@ public void saveLatestBlockHeaderNumber(long n) { * save id of global latest block. */ public void saveLatestBlockHeaderHash(ByteString h) { - logger.info("update latest block header id = {}", ByteArray.toHexString(h.toByteArray())); + logger.info("Update latest block header id = {}.", ByteArray.toHexString(h.toByteArray())); this.put(LATEST_BLOCK_HEADER_HASH, new BytesCapsule(h.toByteArray())); } public void saveStateFlag(int n) { - logger.info("update state flag = {}", n); + logger.info("Update state flag = {}.", n); this.put(STATE_FLAG, new BytesCapsule(ByteArray.fromInt(n))); } @@ -2122,8 +2122,8 @@ public void updateNextMaintenanceTime(long blockTime) { saveNextMaintenanceTime(nextMaintenanceTime); logger.info( - "do update nextMaintenanceTime,currentMaintenanceTime:{}, blockTime:{}," - + "nextMaintenanceTime:{}", + "Do update nextMaintenanceTime, currentMaintenanceTime: {}, blockTime: {}, " + + "nextMaintenanceTime: {}.", new DateTime(currentMaintenanceTime), new DateTime(blockTime), new DateTime(nextMaintenanceTime) ); @@ -2200,7 +2200,7 @@ public long getAllowProtoFilterNum() { * save allow protobuf number. */ public void saveAllowProtoFilterNum(long num) { - logger.info("update allow protobuf number = {}", num); + logger.info("Update allow protobuf number = {}.", num); this.put(ALLOW_PROTO_FILTER_NUM, new BytesCapsule(ByteArray.fromLong(num))); } diff --git a/chainbase/src/main/java/org/tron/core/store/TreeBlockIndexStore.java b/chainbase/src/main/java/org/tron/core/store/TreeBlockIndexStore.java index 95ef3a3373c..fbeaecfda40 100644 --- a/chainbase/src/main/java/org/tron/core/store/TreeBlockIndexStore.java +++ b/chainbase/src/main/java/org/tron/core/store/TreeBlockIndexStore.java @@ -28,7 +28,7 @@ public byte[] get(Long num) throws ItemNotFoundException { BytesCapsule value = getUnchecked(ByteArray.fromLong(num)); if (value == null || value.getData() == null) { - throw new ItemNotFoundException("number: " + num + " is not found!"); + throw new ItemNotFoundException(String.format("number: %d is not found!", num)); } return value.getData(); @@ -39,7 +39,8 @@ public BytesCapsule get(byte[] key) throws ItemNotFoundException { byte[] value = revokingDB.getUnchecked(key); if (ArrayUtils.isEmpty(value)) { - throw new ItemNotFoundException("number: " + Arrays.toString(key) + " is not found!"); + throw new ItemNotFoundException(String.format("number: %d is not found!", + ByteArray.toLong(key))); } return new BytesCapsule(value); } diff --git a/chainbase/src/main/java/org/tron/core/store/WitnessScheduleStore.java b/chainbase/src/main/java/org/tron/core/store/WitnessScheduleStore.java index 227b65b6c98..285638c9f49 100644 --- a/chainbase/src/main/java/org/tron/core/store/WitnessScheduleStore.java +++ b/chainbase/src/main/java/org/tron/core/store/WitnessScheduleStore.java @@ -49,11 +49,11 @@ private List getData(byte[] species) { System.arraycopy(ba, i * ADDRESS_BYTE_ARRAY_LENGTH, b, 0, ADDRESS_BYTE_ARRAY_LENGTH); witnessesAddressList.add(ByteString.copyFrom(b)); } - logger.debug("getWitnesses:" + ByteArray.toStr(species) + witnessesAddressList); + logger.debug("GetWitnesses: {}, {}.", ByteArray.toStr(species), witnessesAddressList); return witnessesAddressList; }).orElseThrow( () -> new IllegalArgumentException( - "not found " + ByteArray.toStr(species) + "Witnesses")); + String.format("not found %s Witnesses", ByteArray.toStr(species)))); } public void saveActiveWitnesses(List witnessesAddressList) { diff --git a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java index a68f379f394..800c5cd3c62 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricKeys.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricKeys.java @@ -58,6 +58,9 @@ public static class Histogram { public static final String TCP_BYTES = "tron:tcp_bytes"; public static final String HTTP_BYTES = "tron:http_bytes"; public static final String INTERNAL_SERVICE_LATENCY = "tron:internal_service_latency_seconds"; + public static final String MESSAGE_PROCESS_LATENCY = "tron:message_process_latency_seconds"; + public static final String BLOCK_FETCH_LATENCY = "tron:block_fetch_latency_seconds"; + public static final String BLOCK_RECEIVE_DELAY = "tron:block_receive_delay_seconds"; private Histogram() { throw new IllegalStateException("Histogram"); diff --git a/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java b/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java index 92be90eb65e..10cbf1d2979 100644 --- a/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java +++ b/common/src/main/java/org/tron/common/prometheus/MetricsHistogram.java @@ -41,6 +41,11 @@ public class MetricsHistogram { "type"); init(MetricKeys.Histogram.HTTP_BYTES, "http_bytes traffic.", "url", "status"); + init(MetricKeys.Histogram.MESSAGE_PROCESS_LATENCY, "process message latency.", + "type"); + init(MetricKeys.Histogram.BLOCK_FETCH_LATENCY, "fetch block latency."); + init(MetricKeys.Histogram.BLOCK_RECEIVE_DELAY, + "receive block delay time, receiveTime - blockTime."); } private MetricsHistogram() { diff --git a/common/src/main/java/org/tron/core/config/args/Storage.java b/common/src/main/java/org/tron/core/config/args/Storage.java index 1ea7e00eb1f..6d911f3b0ac 100644 --- a/common/src/main/java/org/tron/core/config/args/Storage.java +++ b/common/src/main/java/org/tron/core/config/args/Storage.java @@ -69,6 +69,9 @@ public class Storage { private static final String MAX_OPEN_FILES_CONFIG_KEY = "maxOpenFiles"; private static final String EVENT_SUBSCRIBE_CONTRACT_PARSE = "event.subscribe.contractParse"; + private static final String CHECKPOINT_VERSION_KEY = "storage.checkpoint.version"; + private static final String CHECKPOINT_SYNC_KEY = "storage.checkpoint.sync"; + /** * Default values of directory */ @@ -79,6 +82,8 @@ public class Storage { private static final String DEFAULT_DB_DIRECTORY = "database"; private static final String DEFAULT_INDEX_DIRECTORY = "index"; private static final String DEFAULT_INDEX_SWITCH = "on"; + private static final int DEFAULT_CHECKPOINT_VERSION = 1; + private static final boolean DEFAULT_CHECKPOINT_SYNC = true; private static final int DEFAULT_ESTIMATED_TRANSACTIONS = 1000; private Config storage; @@ -120,6 +125,14 @@ public class Storage { @Setter private String transactionHistorySwitch; + @Getter + @Setter + private int checkpointVersion; + + @Getter + @Setter + private boolean checkpointSync; + private Options defaultDbOptions; @Getter @@ -175,6 +188,18 @@ public static String getTransactionHistorySwitchFromConfig(final Config config) : DEFAULT_TRANSACTIONHISTORY_SWITCH; } + public static int getCheckpointVersionFromConfig(final Config config) { + return config.hasPath(CHECKPOINT_VERSION_KEY) + ? config.getInt(CHECKPOINT_VERSION_KEY) + : DEFAULT_CHECKPOINT_VERSION; + } + + public static boolean getCheckpointSyncFromConfig(final Config config) { + return config.hasPath(CHECKPOINT_SYNC_KEY) + ? config.getBoolean(CHECKPOINT_SYNC_KEY) + : DEFAULT_CHECKPOINT_SYNC; + } + public static int getEstimatedTransactionsFromConfig(final Config config) { if (!config.hasPath(ESTIMATED_TRANSACTIONS_CONFIG_KEY)) { return DEFAULT_ESTIMATED_TRANSACTIONS; @@ -205,12 +230,12 @@ private Property createProperty(final ConfigObject conf) { File file = new File(path); if (!file.exists() && !file.mkdirs()) { throw new IllegalArgumentException( - "[storage.properties] can not create storage path: " + path); + String.format("[storage.properties] can not create storage path: %s", path)); } if (!file.canWrite()) { throw new IllegalArgumentException( - "[storage.properties] permission denied to write to: " + path); + String.format("[storage.properties] permission denied to write to: %s ", path)); } property.setPath(path); @@ -251,71 +276,58 @@ private static void setIfNeeded(ConfigObject conf, Options dbOptions) { } if (conf.containsKey(COMPRESSION_TYPE_CONFIG_KEY)) { + String param = conf.get(COMPRESSION_TYPE_CONFIG_KEY).unwrapped().toString(); try { dbOptions.compressionType( - CompressionType.getCompressionTypeByPersistentId( - Integer.parseInt( - conf.get(COMPRESSION_TYPE_CONFIG_KEY).unwrapped().toString() - ) - ) - ); + CompressionType.getCompressionTypeByPersistentId(Integer.parseInt(param))); } catch (NumberFormatException e) { - throw new IllegalArgumentException( - "[storage.properties] compressionType must be Integer type."); + throwIllegalArgumentException(COMPRESSION_TYPE_CONFIG_KEY, Integer.class, param); } } if (conf.containsKey(BLOCK_SIZE_CONFIG_KEY)) { + String param = conf.get(BLOCK_SIZE_CONFIG_KEY).unwrapped().toString(); try { - dbOptions.blockSize( - Integer.parseInt( - conf.get(BLOCK_SIZE_CONFIG_KEY).unwrapped().toString() - ) - ); + dbOptions.blockSize(Integer.parseInt(param)); } catch (NumberFormatException e) { - throw new IllegalArgumentException("[storage.properties] blockSize must be Integer type."); + throwIllegalArgumentException(BLOCK_SIZE_CONFIG_KEY, Integer.class, param); } } if (conf.containsKey(WRITE_BUFFER_SIZE_CONFIG_KEY)) { + String param = conf.get(WRITE_BUFFER_SIZE_CONFIG_KEY).unwrapped().toString(); try { - dbOptions.writeBufferSize( - Integer.parseInt( - conf.get(WRITE_BUFFER_SIZE_CONFIG_KEY).unwrapped().toString() - ) - ); + dbOptions.writeBufferSize(Integer.parseInt(param)); } catch (NumberFormatException e) { - throw new IllegalArgumentException( - "[storage.properties] writeBufferSize must be Integer type."); + throwIllegalArgumentException(WRITE_BUFFER_SIZE_CONFIG_KEY, Integer.class, param); } } if (conf.containsKey(CACHE_SIZE_CONFIG_KEY)) { + String param = conf.get(CACHE_SIZE_CONFIG_KEY).unwrapped().toString(); try { - dbOptions.cacheSize( - Long.parseLong( - conf.get(CACHE_SIZE_CONFIG_KEY).unwrapped().toString() - ) - ); + dbOptions.cacheSize(Long.parseLong(param)); } catch (NumberFormatException e) { - throw new IllegalArgumentException("[storage.properties] cacheSize must be Long type."); + throwIllegalArgumentException(CACHE_SIZE_CONFIG_KEY, Long.class, param); } } if (conf.containsKey(MAX_OPEN_FILES_CONFIG_KEY)) { + String param = conf.get(MAX_OPEN_FILES_CONFIG_KEY).unwrapped().toString(); try { - dbOptions.maxOpenFiles( - Integer.parseInt( - conf.get(MAX_OPEN_FILES_CONFIG_KEY).unwrapped().toString() - ) - ); + dbOptions.maxOpenFiles(Integer.parseInt(param)); } catch (NumberFormatException e) { - throw new IllegalArgumentException( - "[storage.properties] maxOpenFiles must be Integer type."); + throwIllegalArgumentException(MAX_OPEN_FILES_CONFIG_KEY, Integer.class, param); } } } + private static void throwIllegalArgumentException(String param, Class type, String actual) { + throw new IllegalArgumentException( + String.format("[storage.properties] %s must be %s type, actual: %s.", + param, type.getSimpleName(), actual)); + } + /** * Set propertyMap of Storage object from Config * diff --git a/common/src/main/java/org/tron/core/exception/AccountResourceInsufficientException.java b/common/src/main/java/org/tron/core/exception/AccountResourceInsufficientException.java index 1318680d463..ba2807e10ab 100644 --- a/common/src/main/java/org/tron/core/exception/AccountResourceInsufficientException.java +++ b/common/src/main/java/org/tron/core/exception/AccountResourceInsufficientException.java @@ -2,10 +2,6 @@ public class AccountResourceInsufficientException extends TronException { - public AccountResourceInsufficientException() { - super("Insufficient bandwidth and balance to create new account"); - } - public AccountResourceInsufficientException(String message) { super(message); } diff --git a/common/src/main/java/org/tron/core/exception/RevokingStoreIllegalStateException.java b/common/src/main/java/org/tron/core/exception/RevokingStoreIllegalStateException.java index fc0b2d56f23..480f850c111 100755 --- a/common/src/main/java/org/tron/core/exception/RevokingStoreIllegalStateException.java +++ b/common/src/main/java/org/tron/core/exception/RevokingStoreIllegalStateException.java @@ -51,4 +51,9 @@ public RevokingStoreIllegalStateException(String message, Throwable cause) { public RevokingStoreIllegalStateException(Throwable cause) { super("", cause); } + + public RevokingStoreIllegalStateException(int activeSession) { + this(String.format("activeSession %d has to be greater than 0", activeSession)); + } + } diff --git a/consensus/src/main/java/org/tron/consensus/dpos/StateManager.java b/consensus/src/main/java/org/tron/consensus/dpos/StateManager.java index df74b64fec8..19d1f88a243 100644 --- a/consensus/src/main/java/org/tron/consensus/dpos/StateManager.java +++ b/consensus/src/main/java/org/tron/consensus/dpos/StateManager.java @@ -90,12 +90,12 @@ public void receiveBlock(BlockCapsule blockCapsule) { return; } - if (dupBlockCount.get() == 0) { - dupBlockCount.set(new Random().nextInt(10)); - } else { - dupBlockCount.set(10); + if (null != currentBlockId + && currentBlockId.toString().compareTo(blockCapsule.getBlockId().toString()) > 0) { + return; } + dupBlockCount.set(1); dupBlockTime.set(System.currentTimeMillis()); logger.warn("Dup block produced: {}", blockCapsule); diff --git a/framework/src/main/java/org/tron/common/application/ApplicationImpl.java b/framework/src/main/java/org/tron/common/application/ApplicationImpl.java index 1aa158d4be2..1be119f1be3 100644 --- a/framework/src/main/java/org/tron/common/application/ApplicationImpl.java +++ b/framework/src/main/java/org/tron/common/application/ApplicationImpl.java @@ -67,6 +67,7 @@ public void shutdown() { tronNetService.stop(); consensusService.stop(); synchronized (dbManager.getRevokingStore()) { + dbManager.getSession().reset(); closeRevokingStore(); closeAllStore(); } diff --git a/framework/src/main/java/org/tron/common/application/ServiceContainer.java b/framework/src/main/java/org/tron/common/application/ServiceContainer.java index 9692ce2da62..d194360c5ea 100644 --- a/framework/src/main/java/org/tron/common/application/ServiceContainer.java +++ b/framework/src/main/java/org/tron/common/application/ServiceContainer.java @@ -35,29 +35,29 @@ public void add(Service service) { public void init() { for (Service service : this.services) { - logger.debug("Initing " + service.getClass().getSimpleName()); + logger.debug("Initing {}.", service.getClass().getSimpleName()); service.init(); } } public void init(CommonParameter parameter) { for (Service service : this.services) { - logger.debug("Initing " + service.getClass().getSimpleName()); + logger.debug("Initing {}.", service.getClass().getSimpleName()); service.init(parameter); } } public void start() { - logger.debug("Starting services"); + logger.debug("Starting services."); for (Service service : this.services) { - logger.debug("Starting " + service.getClass().getSimpleName()); + logger.debug("Starting {}.", service.getClass().getSimpleName()); service.start(); } } public void stop() { for (Service service : this.services) { - logger.debug("Stopping " + service.getClass().getSimpleName()); + logger.debug("Stopping {}.", service.getClass().getSimpleName()); service.stop(); } } diff --git a/framework/src/main/java/org/tron/common/backup/BackupManager.java b/framework/src/main/java/org/tron/common/backup/BackupManager.java index abbdb6d380b..1d78dd234ad 100644 --- a/framework/src/main/java/org/tron/common/backup/BackupManager.java +++ b/framework/src/main/java/org/tron/common/backup/BackupManager.java @@ -72,7 +72,7 @@ public void init() { try { localIp = InetAddress.getLocalHost().getHostAddress(); } catch (Exception e) { - logger.warn("Failed to get local ip."); + logger.warn("Failed to get local ip"); } for (String member : parameter.getBackupMembers()) { @@ -105,7 +105,7 @@ public void init() { .accept(new UdpEvent(new KeepAliveMessage(status.equals(MASTER), priority), new InetSocketAddress(member, port)))); } catch (Throwable t) { - logger.error("Exception in send keep alive message:{}", t.getMessage()); + logger.error("Exception in send keep alive", t); } }, 1000, keepAliveInterval, TimeUnit.MILLISECONDS); } @@ -120,7 +120,7 @@ public void handleEvent(UdpEvent udpEvent) { return; } if (!members.contains(sender.getHostString())) { - logger.warn("Receive keep alive message from {} is not my member.", sender.getHostString()); + logger.warn("Receive keep alive message from {} is not my member", sender.getHostString()); return; } diff --git a/framework/src/main/java/org/tron/common/net/udp/handler/MessageHandler.java b/framework/src/main/java/org/tron/common/net/udp/handler/MessageHandler.java index 41897e2eebc..656dd9fb667 100644 --- a/framework/src/main/java/org/tron/common/net/udp/handler/MessageHandler.java +++ b/framework/src/main/java/org/tron/common/net/udp/handler/MessageHandler.java @@ -78,7 +78,8 @@ public void channelReadComplete(ChannelHandlerContext ctx) { @Override public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { - logger.info("exception caught, {} {}", ctx.channel().remoteAddress(), cause.getMessage()); + logger.warn("Exception caught in udp message handler, {} {}", + ctx.channel().remoteAddress(), cause.getMessage()); ctx.close(); } } diff --git a/framework/src/main/java/org/tron/common/overlay/client/PeerClient.java b/framework/src/main/java/org/tron/common/overlay/client/PeerClient.java index bdd5eb7606e..60f73eb3fb3 100644 --- a/framework/src/main/java/org/tron/common/overlay/client/PeerClient.java +++ b/framework/src/main/java/org/tron/common/overlay/client/PeerClient.java @@ -45,8 +45,7 @@ public void connect(String host, int port, String remoteId) { ChannelFuture f = connectAsync(host, port, remoteId, false); f.sync().channel().closeFuture().sync(); } catch (Exception e) { - logger - .info("PeerClient: Can't connect to " + host + ":" + port + " (" + e.getMessage() + ")"); + logger.info("Can't connect to {}:{}, cause:{})", host, port, e.getMessage()); } } @@ -55,7 +54,7 @@ public ChannelFuture connectAsync(NodeHandler nodeHandler, boolean discoveryMode return connectAsync(node.getHost(), node.getPort(), node.getHexId(), discoveryMode) .addListener((ChannelFutureListener) future -> { if (!future.isSuccess()) { - logger.warn("connect to {}:{} fail,cause:{}", node.getHost(), node.getPort(), + logger.warn("Connect to {}:{} fail, cause:{}", node.getHost(), node.getPort(), future.cause().getMessage()); nodeHandler.getNodeStatistics().nodeDisconnectedLocal(ReasonCode.CONNECT_FAIL); nodeHandler.getNodeStatistics().notifyDisconnect(); @@ -67,7 +66,7 @@ public ChannelFuture connectAsync(NodeHandler nodeHandler, boolean discoveryMode private ChannelFuture connectAsync(String host, int port, String remoteId, boolean discoveryMode) { - logger.info("connect peer {} {} {}", host, port, remoteId); + logger.info("Connect peer {}:{}, remoteId:{}", host, port, remoteId); TronChannelInitializer tronChannelInitializer = ctx .getBean(TronChannelInitializer.class, remoteId); diff --git a/framework/src/main/java/org/tron/common/overlay/discover/DiscoverServer.java b/framework/src/main/java/org/tron/common/overlay/discover/DiscoverServer.java index 983be8543ef..ce8d577caf2 100755 --- a/framework/src/main/java/org/tron/common/overlay/discover/DiscoverServer.java +++ b/framework/src/main/java/org/tron/common/overlay/discover/DiscoverServer.java @@ -68,7 +68,7 @@ public DiscoverServer(final NodeManager nodeManager) { try { start(); } catch (Exception e) { - logger.error("Discovery server start failed.", e); + logger.error("Discovery server start failed", e); } }, "DiscoverServer").start(); } @@ -111,10 +111,10 @@ public void initChannel(NioDatagramChannel ch) Thread.sleep(5000); } } catch (InterruptedException e) { - logger.warn("Discover server interrupted."); + logger.warn("Discover server interrupted"); Thread.currentThread().interrupt(); } catch (Exception e) { - logger.error("Start discovery server with port {} failed.", port, e); + logger.error("Start discovery server with port {} failed", port, e); } finally { group.shutdownGracefully().sync(); } @@ -127,7 +127,7 @@ public void close() { try { channel.close().await(10, TimeUnit.SECONDS); } catch (Exception e) { - logger.info("Closing discovery server failed.", e); + logger.error("Closing discovery server failed", e); } } @@ -135,7 +135,7 @@ public void close() { try { discoveryExecutor.close(); } catch (Exception e) { - logger.info("Closing discovery executor failed.", e); + logger.error("Closing discovery executor failed", e); } } } diff --git a/framework/src/main/java/org/tron/common/overlay/discover/DiscoverTask.java b/framework/src/main/java/org/tron/common/overlay/discover/DiscoverTask.java index c1fb444e95a..8f105b6ee89 100644 --- a/framework/src/main/java/org/tron/common/overlay/discover/DiscoverTask.java +++ b/framework/src/main/java/org/tron/common/overlay/discover/DiscoverTask.java @@ -40,7 +40,7 @@ public synchronized void discover(byte[] nodeId, int round, List prevTried tried.add(n); wait(50); } catch (Exception ex) { - logger.error("Unexpected Exception " + ex, ex); + logger.error("Unexpected Exception", ex); } } if (tried.size() == KademliaOptions.ALPHA) { diff --git a/framework/src/main/java/org/tron/common/overlay/discover/node/NodeHandler.java b/framework/src/main/java/org/tron/common/overlay/discover/node/NodeHandler.java index 90ecff8cbab..44e8761c29a 100644 --- a/framework/src/main/java/org/tron/common/overlay/discover/node/NodeHandler.java +++ b/framework/src/main/java/org/tron/common/overlay/discover/node/NodeHandler.java @@ -184,7 +184,7 @@ public void handlePong(PongMessage msg) { public void handleNeighbours(NeighborsMessage msg) { if (!waitForNeighbors) { - logger.warn("Receive neighbors from {} without send find nodes.", node.getHost()); + logger.warn("Receive neighbors from {} without send find nodes", node.getHost()); return; } waitForNeighbors = false; @@ -231,7 +231,7 @@ public void sendPing() { handleTimedOut(); } } catch (Exception e) { - logger.error("Unhandled exception", e); + logger.error("Unhandled exception in pong timer schedule", e); } }, pingTimeout, TimeUnit.MILLISECONDS); } diff --git a/framework/src/main/java/org/tron/common/overlay/discover/node/NodeManager.java b/framework/src/main/java/org/tron/common/overlay/discover/node/NodeManager.java index 5efb94a4386..e8f2870fe98 100644 --- a/framework/src/main/java/org/tron/common/overlay/discover/node/NodeManager.java +++ b/framework/src/main/java/org/tron/common/overlay/discover/node/NodeManager.java @@ -78,7 +78,7 @@ public NodeManager(ChainBaseManager chainBaseManager) { bootNodes.add(Node.instanceOf(boot)); } - logger.info("homeNode : {}", homeNode); + logger.info("Home node is {}", homeNode); table = new NodeTable(homeNode); @@ -123,13 +123,13 @@ private void dbRead() { return; } DBNode dbNode = JsonUtil.json2Obj(new String(nodeBytes), DBNode.class); - logger.info("Reading node statistics from store: {} nodes.", dbNode.getNodes().size()); + logger.info("Reading node statistics from store: {} nodes", dbNode.getNodes().size()); dbNode.getNodes().forEach(n -> { Node node = new Node(n.getId(), n.getHost(), n.getPort()); getNodeHandler(node).getNodeStatistics().setPersistedReputation(n.getReputation()); }); } catch (Exception e) { - logger.error("DB read node failed.", e); + logger.error("DB read node failed", e); } } @@ -153,13 +153,13 @@ private void dbWrite() { dbNode.setNodes(batch); - logger.info("Write node statistics to store: m:{}/t:{}/{}/{} nodes.", + logger.info("Write node statistics to store: m:{}/t:{}/{}/{} nodes", nodeHandlerMap.size(), getTable().getAllNodes().size(), size, batch.size()); chainBaseManager.getCommonStore() .put(DB_KEY_PEERS, new BytesCapsule(JsonUtil.obj2Json(dbNode).getBytes())); } catch (Exception e) { - logger.error("DB write node failed.", e); + logger.error("DB write node failed", e); } } @@ -307,7 +307,7 @@ public void close() { nodeManagerTasksTimer.cancel(); pongTimer.shutdownNow(); } catch (Exception e) { - logger.warn("close failed.", e); + logger.error("Close nodeManagerTasksTimer or pongTimer failed", e); } } diff --git a/framework/src/main/java/org/tron/common/overlay/discover/table/NodeEntry.java b/framework/src/main/java/org/tron/common/overlay/discover/table/NodeEntry.java index 2a6c3fb27e1..4d83f58734d 100644 --- a/framework/src/main/java/org/tron/common/overlay/discover/table/NodeEntry.java +++ b/framework/src/main/java/org/tron/common/overlay/discover/table/NodeEntry.java @@ -20,28 +20,14 @@ import org.tron.common.overlay.discover.node.Node; -/** - * Created by kest on 5/25/15. - */ public class NodeEntry { - - private byte[] ownerId; private Node node; private String entryId; private int distance; private long modified; - public NodeEntry(Node n) { - this.node = n; - this.ownerId = n.getId(); - entryId = n.getHost(); - distance = distance(ownerId, n.getId()); - touch(); - } - public NodeEntry(byte[] ownerId, Node n) { this.node = n; - this.ownerId = ownerId; entryId = n.getHost(); distance = distance(ownerId, n.getId()); touch(); @@ -115,6 +101,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return this.node.hashCode(); + return this.entryId.hashCode(); } } diff --git a/framework/src/main/java/org/tron/common/overlay/discover/table/NodeTable.java b/framework/src/main/java/org/tron/common/overlay/discover/table/NodeTable.java index 2ad121fb238..f5ba7615736 100644 --- a/framework/src/main/java/org/tron/common/overlay/discover/table/NodeTable.java +++ b/framework/src/main/java/org/tron/common/overlay/discover/table/NodeTable.java @@ -49,48 +49,37 @@ public final void initialize() { } public synchronized Node addNode(Node n) { - NodeEntry e = new NodeEntry(node.getId(), n); - if (nodes.contains(e)) { - nodes.forEach(nodeEntry -> { - if (nodeEntry.equals(e)) { - nodeEntry.touch(); - } - }); + NodeEntry entry = getNodeEntry(n); + if (entry != null) { + entry.touch(); return null; } + + NodeEntry e = new NodeEntry(node.getId(), n); NodeEntry lastSeen = buckets[getBucketId(e)].addNode(e); if (lastSeen != null) { return lastSeen.getNode(); } - if (!nodes.contains(e)) { - nodes.add(e); - } + nodes.add(e); return null; } public synchronized void dropNode(Node n) { - NodeEntry e = new NodeEntry(node.getId(), n); - buckets[getBucketId(e)].dropNode(e); - nodes.remove(e); + NodeEntry entry = getNodeEntry(n); + if (entry != null) { + nodes.remove(entry); + buckets[getBucketId(entry)].dropNode(entry); + } } public synchronized boolean contains(Node n) { - NodeEntry e = new NodeEntry(node.getId(), n); - for (NodeBucket b : buckets) { - if (b.getNodes().contains(e)) { - return true; - } - } - return false; + return getNodeEntry(n) != null; } public synchronized void touchNode(Node n) { - NodeEntry e = new NodeEntry(node.getId(), n); - for (NodeBucket b : buckets) { - if (b.getNodes().contains(e)) { - b.getNodes().get(b.getNodes().indexOf(e)).touch(); - break; - } + NodeEntry entry = getNodeEntry(n); + if (entry != null) { + entry.touch(); } } @@ -104,10 +93,6 @@ public int getBucketsCount() { return i; } - public synchronized NodeBucket[] getBuckets() { - return buckets; - } - public int getBucketId(NodeEntry e) { int id = e.getDistance() - 1; return id < 0 ? 0 : id; @@ -118,17 +103,9 @@ public synchronized int getNodesCount() { } public synchronized List getAllNodes() { - List nodes = new ArrayList<>(); - - for (NodeBucket b : buckets) { - for (NodeEntry e : b.getNodes()) { - if (!e.getNode().equals(node)) { - nodes.add(e); - } - } - } - - return nodes; + List list = new ArrayList<>(nodes); + list.remove(new NodeEntry(node.getId(), node)); + return list; } public synchronized List getClosestNodes(byte[] targetId) { @@ -145,4 +122,15 @@ public synchronized List getClosestNodes(byte[] targetId) { } return closestNodes; } + + private NodeEntry getNodeEntry(Node n) { + NodeEntry entry = null; + for (NodeEntry e: nodes) { + if (e.getNode().getHost().equals(n.getHost())) { + entry = e; + break; + } + } + return entry; + } } diff --git a/framework/src/main/java/org/tron/common/overlay/message/HelloMessage.java b/framework/src/main/java/org/tron/common/overlay/message/HelloMessage.java index 3fec2284fa2..86c01725aed 100755 --- a/framework/src/main/java/org/tron/common/overlay/message/HelloMessage.java +++ b/framework/src/main/java/org/tron/common/overlay/message/HelloMessage.java @@ -124,7 +124,13 @@ public Class getAnswerMessage() { @Override public String toString() { - return new StringBuilder().append(super.toString()).append(helloMessage.toString()).toString(); + return new StringBuilder().append(super.toString()).append(", ") + .append("from: ").append(getFrom()).append(", ") + .append("timestamp: ").append(getTimestamp()).append(", ") + .append("headBlockId: {").append(getHeadBlockId().getString()).append("}, ") + .append("nodeType: ").append(helloMessage.getNodeType()).append(", ") + .append("lowestBlockNum: ").append(helloMessage.getLowestBlockNum()) + .toString(); } public Protocol.HelloMessage getInstance() { diff --git a/framework/src/main/java/org/tron/common/overlay/server/Channel.java b/framework/src/main/java/org/tron/common/overlay/server/Channel.java index 0a4c92a87a1..f67cd8ba43b 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/Channel.java +++ b/framework/src/main/java/org/tron/common/overlay/server/Channel.java @@ -120,7 +120,7 @@ public void publicHandshakeFinished(ChannelHandlerContext ctx, HelloMessage msg) setStartTime(msg.getTimestamp()); setTronState(TronState.HANDSHAKE_FINISHED); getNodeStatistics().p2pHandShake.add(); - logger.info("Finish handshake with {}.", ctx.channel().remoteAddress()); + logger.info("Finish handshake with {}", ctx.channel().remoteAddress()); } /** @@ -215,7 +215,7 @@ public void setStartTime(long startTime) { public void setTronState(TronState tronState) { this.tronState = tronState; - logger.info("Peer {} status change to {}.", inetSocketAddress, tronState); + logger.info("Peer {} status change to {}", inetSocketAddress, tronState); } public boolean isActive() { diff --git a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java b/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java index d9da821f495..68c5ee78d97 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java +++ b/framework/src/main/java/org/tron/common/overlay/server/ChannelManager.java @@ -93,7 +93,7 @@ public void init() { fastForwardNodes.put(address, node); } - logger.info("Node config, trust {}, active {}, forward {}.", + logger.info("Node config, trust {}, active {}, forward {}", trustNodes.size(), activeNodes.size(), fastForwardNodes.size()); peerConnectionCheckService.init(); @@ -140,7 +140,7 @@ public synchronized boolean processPeer(Channel peer) { if (trustNodes.getIfPresent(peer.getInetAddress()) == null) { if (recentlyDisconnected.getIfPresent(peer) != null) { - logger.info("Peer {} recently disconnected.", peer.getInetAddress()); + logger.info("Peer {} recently disconnected", peer.getInetAddress()); return false; } diff --git a/framework/src/main/java/org/tron/common/overlay/server/FastForward.java b/framework/src/main/java/org/tron/common/overlay/server/FastForward.java index 32b25eed1e1..bc19fa206d9 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/FastForward.java +++ b/framework/src/main/java/org/tron/common/overlay/server/FastForward.java @@ -113,7 +113,7 @@ public boolean checkHelloMessage(HelloMessage message, Channel channel) { } if (!witnessScheduleStore.getActiveWitnesses().contains(msg.getAddress())) { - logger.error("HelloMessage from {}, {} is not a schedule witness.", + logger.warn("HelloMessage from {}, {} is not a schedule witness.", channel.getInetAddress(), ByteArray.toHexString(msg.getAddress().toByteArray())); return false; @@ -139,7 +139,7 @@ public boolean checkHelloMessage(HelloMessage message, Channel channel) { } return flag; } catch (Exception e) { - logger.error("Check hello message failed, msg: {}, {}", message, e); + logger.error("Check hello message failed, msg: {}, {}", message, channel.getInetAddress(), e); return false; } } diff --git a/framework/src/main/java/org/tron/common/overlay/server/HandshakeHandler.java b/framework/src/main/java/org/tron/common/overlay/server/HandshakeHandler.java index 7bb195b2ca8..b36c7f31b5d 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/HandshakeHandler.java +++ b/framework/src/main/java/org/tron/common/overlay/server/HandshakeHandler.java @@ -78,7 +78,7 @@ public class HandshakeHandler extends ByteToMessageDecoder { @Override public void channelActive(ChannelHandlerContext ctx) throws Exception { - logger.info("channel active, {}", ctx.channel().remoteAddress()); + logger.info("Channel active, {}", ctx.channel().remoteAddress()); channel.setChannelHandlerContext(ctx); if (remoteId.length == 64) { channel.initNode(remoteId, ((InetSocketAddress) ctx.channel().remoteAddress()).getPort()); diff --git a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java index 2edf84c8e32..de59cb85736 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java +++ b/framework/src/main/java/org/tron/common/overlay/server/MessageQueue.java @@ -77,15 +77,14 @@ public void activate(ChannelHandlerContext ctx) { } ctx.writeAndFlush(msg.getSendData()).addListener((ChannelFutureListener) future -> { if (!future.isSuccess() && !channel.isDisconnect()) { - logger.warn("Failed to send to {}, {}", ctx.channel().remoteAddress(), msg); + logger.warn("Failed to send to {}, {} ", ctx.channel().remoteAddress(), msg); } }); } catch (InterruptedException e) { - logger.warn("Send message server interrupted."); + logger.warn("Send message server interrupted"); Thread.currentThread().interrupt(); } catch (Exception e) { - logger.error("Failed to send to {}, error info: {}", ctx.channel().remoteAddress(), - e.getMessage()); + logger.error("Failed to send to {} ", ctx.channel().remoteAddress(), e); } } }); @@ -163,7 +162,7 @@ public void close() { sendMsgThread.join(20); sendMsgThread = null; } catch (InterruptedException e) { - logger.warn("Send message join interrupted."); + logger.warn("Send message join interrupted"); Thread.currentThread().interrupt(); } catch (Exception e) { logger.warn("Join send thread failed, peer {}", ctx.channel().remoteAddress()); @@ -197,7 +196,7 @@ private void send() { } if (rt.getRetryTimes() > 0) { channel.getNodeStatistics().nodeDisconnectedLocal(ReasonCode.PING_TIMEOUT); - logger.warn("Wait {} timeout. close channel {}.", + logger.warn("Wait {} timeout. close channel {}", rt.getMsg().getAnswerMessage(), ctx.channel().remoteAddress()); channel.close(); return; diff --git a/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java b/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java index 3aafa4a8e95..e0d9d39b4a7 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java +++ b/framework/src/main/java/org/tron/common/overlay/server/PeerConnectionCheckService.java @@ -30,18 +30,18 @@ public void init() { if (isFastForward || !isOpenFullTcpDisconnect) { return; } - logger.info("Start peer connection check service."); + logger.info("Start peer connection check service"); poolLoopExecutor.scheduleWithFixedDelay(() -> { try { check(); } catch (Throwable t) { - logger.error("Exception in peer connection check.", t); + logger.error("Exception in peer connection check", t); } }, 10, 30, TimeUnit.SECONDS); } public void close() { - logger.info("Close peer connection check service."); + logger.info("Close peer connection check service"); poolLoopExecutor.shutdown(); } diff --git a/framework/src/main/java/org/tron/common/overlay/server/PeerServer.java b/framework/src/main/java/org/tron/common/overlay/server/PeerServer.java index bdff4ccebf7..e00372e9a5d 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/PeerServer.java +++ b/framework/src/main/java/org/tron/common/overlay/server/PeerServer.java @@ -63,7 +63,7 @@ public void start(int port) { logger.info("TCP listener closed"); } catch (Exception e) { - logger.error("Start TCP server failed.", e); + logger.error("Start TCP server failed", e); } finally { workerGroup.shutdownGracefully(); bossGroup.shutdownGracefully(); @@ -77,7 +77,7 @@ public void close() { logger.info("Closing TCP server..."); channelFuture.channel().close().sync(); } catch (Exception e) { - logger.warn("Closing TCP server failed.", e); + logger.error("Closing TCP server failed", e); } } } diff --git a/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java b/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java index d850a387d98..e4b28494404 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java +++ b/framework/src/main/java/org/tron/common/overlay/server/SyncPool.java @@ -225,7 +225,7 @@ public void close() { poolLoopExecutor.shutdownNow(); logExecutor.shutdownNow(); } catch (Exception e) { - logger.warn("Problems shutting down executor", e); + logger.error("Problems shutting down executor", e); } } diff --git a/framework/src/main/java/org/tron/common/overlay/server/TronChannelInitializer.java b/framework/src/main/java/org/tron/common/overlay/server/TronChannelInitializer.java index 4ffefd92ae2..f6732adadd5 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/TronChannelInitializer.java +++ b/framework/src/main/java/org/tron/common/overlay/server/TronChannelInitializer.java @@ -45,7 +45,7 @@ public void initChannel(NioSocketChannel ch) throws Exception { // be aware of channel closing ch.closeFuture().addListener((ChannelFutureListener) future -> { - logger.info("Close channel:" + channel); + logger.info("Close channel: {}", channel); if (!peerDiscoveryMode) { channelManager.notifyDisconnect(channel); } diff --git a/framework/src/main/java/org/tron/common/overlay/server/TrxProtobufVarint32FrameDecoder.java b/framework/src/main/java/org/tron/common/overlay/server/TrxProtobufVarint32FrameDecoder.java index 8be89a178fe..b6696200aff 100644 --- a/framework/src/main/java/org/tron/common/overlay/server/TrxProtobufVarint32FrameDecoder.java +++ b/framework/src/main/java/org/tron/common/overlay/server/TrxProtobufVarint32FrameDecoder.java @@ -76,7 +76,7 @@ protected void decode(ChannelHandlerContext ctx, ByteBuf in, List out) t int preIndex = in.readerIndex(); int length = readRawVarint32(in); if (length >= maxMsgLength) { - logger.warn("recv a big msg, host : {}, msg length is : {}", ctx.channel().remoteAddress(), + logger.warn("Recv a big msg, host : {}, msg length is : {}", ctx.channel().remoteAddress(), length); in.clear(); channel.close(); diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 3fec975d5ba..eae41be2012 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -501,6 +501,11 @@ public static void setParam(final String[] args, final String confFileName) { .filter(StringUtils::isNotEmpty) .orElse(Storage.getTransactionHistorySwitchFromConfig(config))); + PARAMETER.storage + .setCheckpointVersion(Storage.getCheckpointVersionFromConfig(config)); + PARAMETER.storage + .setCheckpointSync(Storage.getCheckpointSyncFromConfig(config)); + PARAMETER.storage.setEstimatedBlockTransactions( Storage.getEstimatedTransactionsFromConfig(config)); diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 457bcda999d..80be86403ba 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -169,7 +169,7 @@ public class Manager { private static final int SHIELDED_TRANS_IN_BLOCK_COUNTS = 1; - private static final String SAVE_BLOCK = "save block: "; + private static final String SAVE_BLOCK = "Save block: {}"; private static final int SLEEP_TIME_OUT = 50; private static final int TX_ID_CACHE_SIZE = 100_000; private static final int SLEEP_FOR_WAIT_LOCK = 10; @@ -266,7 +266,7 @@ public class Manager { if (ex instanceof InterruptedException) { Thread.currentThread().interrupt(); } - logger.error("unknown exception happened in rePush loop", ex); + logger.error("Unknown exception happened in rePush loop.", ex); if (tx != null) { Metrics.counterInc(MetricKeys.Counter.TXS, 1, MetricLabels.Counter.TXS_FAIL, MetricLabels.Counter.TXS_FAIL_ERROR); @@ -291,7 +291,7 @@ public class Manager { logger.info(ex.getMessage()); Thread.currentThread().interrupt(); } catch (Throwable throwable) { - logger.error("unknown throwable happened in process capsule loop", throwable); + logger.error("Unknown throwable happened in process capsule loop.", throwable); } } }; @@ -305,10 +305,11 @@ public class Manager { filterCapsule.processFilterTrigger(); } } catch (InterruptedException e) { - logger.error("filterProcessLoop get InterruptedException, error is {}", e.getMessage()); + logger.error("FilterProcessLoop get InterruptedException, error is {}.", + e.getMessage()); Thread.currentThread().interrupt(); } catch (Throwable throwable) { - logger.error("unknown throwable happened in filterProcessLoop: ", throwable); + logger.error("Unknown throwable happened in filterProcessLoop. ", throwable); } } }; @@ -473,9 +474,9 @@ public void init() { Args.getInstance().getOutputDirectory()); System.exit(1); } catch (BadItemException e) { - logger.error("DB data broken! {}", e); + logger.error("DB data broken {}.", e.getMessage()); logger.error( - "Please delete database directory({}) and restart", + "Please delete database directory({}) and restart.", Args.getInstance().getOutputDirectory()); System.exit(1); } @@ -508,9 +509,9 @@ public void init() { initLiteNode(); long headNum = chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber(); - logger.info("current headNum is: {}", headNum); + logger.info("Current headNum is: {}.", headNum); int nodeType = chainBaseManager.getCommonStore().getNodeType(); - logger.info("node type is: {}", Constant.NODE_TYPE_LIGHT_NODE == nodeType ? "lite" : "full"); + logger.info("Node type is: {}.", Constant.NODE_TYPE_LIGHT_NODE == nodeType ? "lite" : "full"); revokingStore.enable(); validateSignService = Executors .newFixedThreadPool(Args.getInstance().getValidateSignThreadNum()); @@ -548,7 +549,7 @@ public void init() { } if (CommonParameter.getInstance().getShutdownBlockHeight() < headNum) { - logger.info("ShutDownBlockHeight {} is less than headNum {},ignored.", + logger.info("ShutDownBlockHeight {} is less than headNum {}, ignored.", CommonParameter.getInstance().getShutdownBlockHeight(), headNum); CommonParameter.getInstance().setShutdownBlockHeight(-1); } @@ -568,17 +569,17 @@ public void initGenesis() { } else { if (chainBaseManager.hasBlocks()) { logger.error( - "genesis block modify, please delete database directory({}) and restart", + "Genesis block modify, please delete database directory({}) and restart.", Args.getInstance().getOutputDirectory()); System.exit(1); } else { - logger.info("create genesis block"); + logger.info("Create genesis block."); Args.getInstance().setChainId(genesisBlock.getBlockId().toString()); chainBaseManager.getBlockStore().put(genesisBlock.getBlockId().getBytes(), genesisBlock); chainBaseManager.getBlockIndexStore().put(genesisBlock.getBlockId()); - logger.info(SAVE_BLOCK + genesisBlock); + logger.info(SAVE_BLOCK, genesisBlock); // init Dynamic Properties Store chainBaseManager.getDynamicPropertiesStore().saveLatestBlockHeaderNumber(0); chainBaseManager.getDynamicPropertiesStore().saveLatestBlockHeaderHash( @@ -704,8 +705,7 @@ void validateTapos(TransactionCapsule transactionCapsule) throws TaposException ByteArray.toLong(refBlockNumBytes), Hex.toHexString(refBlockHash), Hex.toHexString(blockHash), chainBaseManager.getSolidBlockId().getString(), - chainBaseManager.getHeadBlockId().getString()).toString(); - logger.info(str); + chainBaseManager.getHeadBlockId().getString()); throw new TaposException(str); } } catch (ItemNotFoundException e) { @@ -713,8 +713,7 @@ void validateTapos(TransactionCapsule transactionCapsule) throws TaposException .format("Tapos failed, block not found, ref block %s, %s , solid block %s head block %s", ByteArray.toLong(refBlockNumBytes), Hex.toHexString(refBlockHash), chainBaseManager.getSolidBlockId().getString(), - chainBaseManager.getHeadBlockId().getString()).toString(); - logger.info(str); + chainBaseManager.getHeadBlockId().getString()); throw new TaposException(str); } } @@ -722,23 +721,24 @@ void validateTapos(TransactionCapsule transactionCapsule) throws TaposException void validateCommon(TransactionCapsule transactionCapsule) throws TransactionExpirationException, TooBigTransactionException { if (transactionCapsule.getData().length > Constant.TRANSACTION_MAX_BYTE_SIZE) { - throw new TooBigTransactionException( - "too big transaction, the size is " + transactionCapsule.getData().length + " bytes"); + throw new TooBigTransactionException(String.format( + "Too big transaction, the size is %d bytes", transactionCapsule.getData().length)); } long transactionExpiration = transactionCapsule.getExpiration(); long headBlockTime = chainBaseManager.getHeadBlockTimeStamp(); if (transactionExpiration <= headBlockTime || transactionExpiration > headBlockTime + Constant.MAXIMUM_TIME_UNTIL_EXPIRATION) { throw new TransactionExpirationException( - "transaction expiration, transaction expiration time is " + transactionExpiration - + ", but headBlockTime is " + headBlockTime); + String.format( + "Transaction expiration, transaction expiration time is %d, but headBlockTime is %d", + transactionExpiration, headBlockTime)); } } void validateDup(TransactionCapsule transactionCapsule) throws DupTransactionException { if (containsTransaction(transactionCapsule)) { - logger.debug(ByteArray.toHexString(transactionCapsule.getTransactionId().getBytes())); - throw new DupTransactionException("dup trans"); + throw new DupTransactionException(String.format("dup trans : %s ", + transactionCapsule.getTransactionId())); } } @@ -777,7 +777,8 @@ public boolean pushTransaction(final TransactionCapsule trx) try { if (!trx.validateSignature(chainBaseManager.getAccountStore(), chainBaseManager.getDynamicPropertiesStore())) { - throw new ValidateSignatureException("trans sig validate failed"); + throw new ValidateSignatureException(String.format("trans sig validate failed, id: %s", + trx.getTransactionId())); } synchronized (transactionLock) { @@ -790,7 +791,7 @@ public boolean pushTransaction(final TransactionCapsule trx) } } catch (InterruptedException e) { Thread.currentThread().interrupt(); - logger.debug("the wait has been interrupted"); + logger.debug("The wait has been interrupted."); } } synchronized (this) { @@ -845,7 +846,8 @@ public void consumeMultiSignFee(TransactionCapsule trx, TransactionTrace trace) } } catch (BalanceInsufficientException e) { throw new AccountResourceInsufficientException( - "Account Insufficient balance[" + fee + "] to MultiSign"); + String.format("account %s insufficient balance[%d] to multiSign", + StringUtil.encode58Check(address), fee)); } } @@ -869,10 +871,10 @@ public void eraseBlock() { try { BlockCapsule oldHeadBlock = chainBaseManager.getBlockById( getDynamicPropertiesStore().getLatestBlockHeaderHash()); - logger.info("start to erase block:" + oldHeadBlock); + logger.info("Start to erase block: {}.", oldHeadBlock); khaosDb.pop(); revokingStore.fastPop(); - logger.info("end to erase block:" + oldHeadBlock); + logger.info("End to erase block: {}.", oldHeadBlock); oldHeadBlock.getTransactions().forEach(tc -> poppedTransactions.add(new TransactionCapsule(tc.getInstance()))); Metrics.gaugeInc(MetricKeys.Gauge.MANAGER_QUEUE, oldHeadBlock.getTransactions().size(), @@ -893,7 +895,7 @@ public void pushVerifiedBlock(BlockCapsule block) throws ContractValidateExcepti block.generatedByMyself = true; long start = System.currentTimeMillis(); pushBlock(block); - logger.info("push block cost:{}ms, blockNum:{}, blockHash:{}, trx count:{}", + logger.info("Push block cost: {} ms, blockNum: {}, blockHash: {}, trx count: {}.", System.currentTimeMillis() - start, block.getNum(), block.getBlockId(), @@ -960,7 +962,7 @@ private void switchFork(BlockCapsule newHead) Metrics.counterInc(MetricKeys.Counter.BLOCK_FORK, 1, MetricLabels.FAIL); MetricsUtil.meterMark(MetricsKey.BLOCKCHAIN_FAIL_FORK_COUNT); logger.info( - "this is not the most recent common ancestor, " + "This is not the most recent common ancestor, " + "need to remove all blocks in the fork chain."); BlockCapsule tmp = newHead; while (tmp != null) { @@ -1011,9 +1013,7 @@ private void switchFork(BlockCapsule newHead) if (exception != null) { Metrics.counterInc(MetricKeys.Counter.BLOCK_FORK, 1, MetricLabels.FAIL); MetricsUtil.meterMark(MetricsKey.BLOCKCHAIN_FAIL_FORK_COUNT); - logger.warn("switch back because exception thrown while switching forks. " + exception - .getMessage(), - exception); + logger.warn("Switch back because exception thrown while switching forks.", exception); first.forEach(khaosBlock -> khaosDb.removeBlk(khaosBlock.getBlk().getBlockId())); khaosDb.setHead(binaryTree.getValue().peekFirst()); @@ -1063,8 +1063,7 @@ public List getVerifyTxs(BlockCapsule block) { pendingTransactions.forEach(capsule -> { String txId = Hex.toHexString(capsule.getTransactionId().getBytes()); if (isMultiSignTransaction(capsule.getInstance())) { - Contract contract = capsule.getInstance().getRawData().getContract(0); - String address = Hex.toHexString(TransactionCapsule.getOwner(contract)); + String address = Hex.toHexString(capsule.getOwnerAddress()); multiAddresses.add(address); } else { txIds.add(txId); @@ -1072,8 +1071,7 @@ public List getVerifyTxs(BlockCapsule block) { }); block.getTransactions().forEach(capsule -> { - Contract contract = capsule.getInstance().getRawData().getContract(0); - String address = Hex.toHexString(TransactionCapsule.getOwner(contract)); + String address = Hex.toHexString(capsule.getOwnerAddress()); String txId = Hex.toHexString(capsule.getTransactionId().getBytes()); if (multiAddresses.contains(address) || !txIds.contains(txId)) { txs.add(capsule); @@ -1124,12 +1122,10 @@ public void pushBlock(final BlockCapsule block) if (!block.generatedByMyself) { if (!block.calcMerkleRoot().equals(block.getMerkleRoot())) { - logger.warn( - "The merkle root doesn't match, Calc result is " - + block.calcMerkleRoot() - + " , the headers is " - + block.getMerkleRoot()); - throw new BadBlockException("The merkle hash is not validated"); + logger.warn("Num: {}, the merkle root doesn't match, expect is {} , actual is {}.", + block.getNum(), block.getMerkleRoot(), block.calcMerkleRoot()); + throw new BadBlockException(String.format("The merkle hash is not validated for %d", + block.getNum())); } consensus.receiveBlock(block); } @@ -1138,7 +1134,8 @@ public void pushBlock(final BlockCapsule block) .filter(tran -> isShieldedTransaction(tran.getInstance())) .count() > SHIELDED_TRANS_IN_BLOCK_COUNTS) { throw new BadBlockException( - "shielded transaction count > " + SHIELDED_TRANS_IN_BLOCK_COUNTS); + String.format("num: %d, shielded transaction count > %d", + block.getNum(), SHIELDED_TRANS_IN_BLOCK_COUNTS)); } BlockCapsule newBlock; @@ -1146,8 +1143,8 @@ public void pushBlock(final BlockCapsule block) newBlock = this.khaosDb.push(block); } catch (UnLinkedBlockException e) { logger.error( - "latestBlockHeaderHash:{}, latestBlockHeaderNumber:{}" - + ", latestSolidifiedBlockNum:{}", + "LatestBlockHeaderHash: {}, latestBlockHeaderNumber: {}" + + ", latestSolidifiedBlockNum: {}.", getDynamicPropertiesStore().getLatestBlockHeaderHash(), getDynamicPropertiesStore().getLatestBlockHeaderNumber(), getDynamicPropertiesStore().getLatestSolidifiedBlockNum()); @@ -1168,55 +1165,35 @@ public void pushBlock(final BlockCapsule block) if (!newBlock .getParentHash() .equals(getDynamicPropertiesStore().getLatestBlockHeaderHash())) { - logger.warn( - "switch fork! new head num = {}, block id = {}", - newBlock.getNum(), - newBlock.getBlockId()); + logger.warn("Switch fork! new head num = {}, block id = {}.", + newBlock.getNum(), newBlock.getBlockId()); logger.warn( - "******** before switchFork ******* push block: " - + block.toString() - + ", new block:" - + newBlock.toString() - + ", dynamic head num: " - + chainBaseManager.getDynamicPropertiesStore() - .getLatestBlockHeaderNumber() - + ", dynamic head hash: " - + chainBaseManager.getDynamicPropertiesStore() - .getLatestBlockHeaderHash() - + ", dynamic head timestamp: " - + chainBaseManager.getDynamicPropertiesStore() - .getLatestBlockHeaderTimestamp() - + ", khaosDb head: " - + khaosDb.getHead() - + ", khaosDb miniStore size: " - + khaosDb.getMiniStore().size() - + ", khaosDb unlinkMiniStore size: " - + khaosDb.getMiniUnlinkedStore().size()); + "******** Before switchFork ******* push block: {}, new block: {}, " + + "dynamic head num: {}, dynamic head hash: {}, " + + "dynamic head timestamp: {}, khaosDb head: {}, " + + "khaosDb miniStore size: {}, khaosDb unlinkMiniStore size: {}.", + block, newBlock, + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber(), + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderHash(), + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderTimestamp(), + khaosDb.getHead(), khaosDb.getMiniStore().size(), + khaosDb.getMiniUnlinkedStore().size()); switchFork(newBlock); - logger.info(SAVE_BLOCK + newBlock); + logger.info(SAVE_BLOCK, newBlock); logger.warn( - "******** after switchFork ******* push block: " - + block.toString() - + ", new block:" - + newBlock.toString() - + ", dynamic head num: " - + chainBaseManager.getDynamicPropertiesStore() - .getLatestBlockHeaderNumber() - + ", dynamic head hash: " - + chainBaseManager.getDynamicPropertiesStore() - .getLatestBlockHeaderHash() - + ", dynamic head timestamp: " - + chainBaseManager.getDynamicPropertiesStore() - .getLatestBlockHeaderTimestamp() - + ", khaosDb head: " - + khaosDb.getHead() - + ", khaosDb miniStore size: " - + khaosDb.getMiniStore().size() - + ", khaosDb unlinkMiniStore size: " - + khaosDb.getMiniUnlinkedStore().size()); + "******** After switchFork ******* push block: {}, new block: {}, " + + "dynamic head num: {}, dynamic head hash: {}, " + + "dynamic head timestamp: {}, khaosDb head: {}, " + + "khaosDb miniStore size: {}, khaosDb unlinkMiniStore size: {}.", + block, newBlock, + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber(), + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderHash(), + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderTimestamp(), + khaosDb.getHead(), khaosDb.getMiniStore().size(), + khaosDb.getMiniUnlinkedStore().size()); return; } @@ -1238,7 +1215,7 @@ public void pushBlock(final BlockCapsule block) throw throwable; } } - logger.info(SAVE_BLOCK + newBlock); + logger.info(SAVE_BLOCK, newBlock); } //clear ownerAddressSet if (CollectionUtils.isNotEmpty(ownerAddressSet)) { @@ -1256,7 +1233,7 @@ public void pushBlock(final BlockCapsule block) long cost = System.currentTimeMillis() - start; MetricsUtil.meterMark(MetricsKey.BLOCKCHAIN_BLOCK_PROCESS_TIME, cost); - logger.info("pushBlock block number:{}, cost/txs:{}/{} {}", + logger.info("PushBlock block number: {}, cost/txs: {}/{} {}.", block.getNum(), cost, block.getTransactions().size(), cost > 1000); Metrics.histogramObserve(timer); @@ -1299,7 +1276,7 @@ public LinkedList getBlockChainHashesOnFork(final BlockId forkBlockHash LinkedList blockCapsules = branch.getValue(); if (blockCapsules.isEmpty()) { - logger.info("empty branch {}", forkBlockHash); + logger.info("Empty branch {}.", forkBlockHash); return Lists.newLinkedList(); } @@ -1325,7 +1302,7 @@ public TransactionInfo processTransaction(final TransactionCapsule trxCap, Block return null; } Contract contract = trxCap.getInstance().getRawData().getContract(0); - + Sha256Hash txId = trxCap.getTransactionId(); final Histogram.Timer requestTimer = Metrics.histogramStartTimer( MetricKeys.Histogram.PROCESS_TRANSACTION_LATENCY, Objects.nonNull(blockCap) ? MetricLabels.BLOCK : MetricLabels.TRX, @@ -1342,14 +1319,17 @@ public TransactionInfo processTransaction(final TransactionCapsule trxCap, Block if (trxCap.getInstance().getRawData().getContractList().size() != 1) { throw new ContractSizeNotEqualToOneException( - "act size should be exactly 1, this is extend feature"); + String.format( + "tx %s contract size should be exactly 1, this is extend feature ,actual :%d", + txId, trxCap.getInstance().getRawData().getContractList().size())); } validateDup(trxCap); if (!trxCap.validateSignature(chainBaseManager.getAccountStore(), chainBaseManager.getDynamicPropertiesStore())) { - throw new ValidateSignatureException("transaction signature validate failed"); + throw new ValidateSignatureException( + String.format(" %s transaction signature validate failed", txId)); } TransactionTrace trace = new TransactionTrace(trxCap, StoreFactory.getInstance(), @@ -1366,13 +1346,11 @@ public TransactionInfo processTransaction(final TransactionCapsule trxCap, Block if (Objects.nonNull(blockCap)) { trace.setResult(); if (trace.checkNeedRetry()) { - String txId = Hex.toHexString(trxCap.getTransactionId().getBytes()); - logger.info("Retry for tx id: {}", txId); trace.init(blockCap, eventPluginLoaded); trace.checkIsConstant(); trace.exec(); trace.setResult(); - logger.info("Retry result when push:{}, for tx id: {}, tx resultCode in receipt: {}", + logger.info("Retry result when push: {}, for tx id: {}, tx resultCode in receipt: {}.", blockCap.hasWitnessSignature(), txId, trace.getReceipt().getResult()); } if (blockCap.hasWitnessSignature()) { @@ -1402,7 +1380,7 @@ public TransactionInfo processTransaction(final TransactionCapsule trxCap, Block if (isMultiSignTransaction(trxCap.getInstance())) { - ownerAddressSet.add(ByteArray.toHexString(TransactionCapsule.getOwner(contract))); + ownerAddressSet.add(ByteArray.toHexString(trxCap.getOwnerAddress())); } if (Objects.nonNull(blockCap)) { @@ -1418,8 +1396,12 @@ public TransactionInfo processTransaction(final TransactionCapsule trxCap, Block } long cost = System.currentTimeMillis() - start; if (cost > 100) { - logger.info("Process transaction {} cost {}.", - Hex.toHexString(transactionInfo.getId()), cost); + String type = "broadcast"; + if (Objects.nonNull(blockCap)) { + type = blockCap.hasWitnessSignature() ? "apply" : "pack"; + } + logger.info("Process transaction {} cost {} ms during {}, {}", + Hex.toHexString(transactionInfo.getId()), cost, type, contract.getType().name()); } Metrics.histogramObserve(requestTimer); return transactionInfo.getInstance(); @@ -1432,10 +1414,10 @@ public BlockCapsule generateBlock(Miner miner, long blockTime, long timeout) { String address = StringUtil.encode58Check(miner.getWitnessAddress().toByteArray()); final Histogram.Timer timer = Metrics.histogramStartTimer( MetricKeys.Histogram.BLOCK_GENERATE_LATENCY, address); - Metrics.histogramObserve(MetricKeys.Histogram.MINER_LATENCY, + Metrics.histogramObserve(MetricKeys.Histogram.MINER_DELAY, (System.currentTimeMillis() - blockTime) / Metrics.MILLISECONDS_PER_SECOND, address); long postponedTrxCount = 0; - logger.info("Generate block {} begin", chainBaseManager.getHeadBlockNum() + 1); + logger.info("Generate block {} begin.", chainBaseManager.getHeadBlockNum() + 1); BlockCapsule blockCapsule = new BlockCapsule(chainBaseManager.getHeadBlockNum() + 1, chainBaseManager.getHeadBlockId(), @@ -1451,7 +1433,7 @@ public BlockCapsule generateBlock(Miner miner, long blockTime, long timeout) { AccountCapsule witnessAccount = getAccountStore() .get(miner.getWitnessAddress().toByteArray()); if (!Arrays.equals(privateKeyAddress, witnessAccount.getWitnessPermissionAddress())) { - logger.warn("Witness permission is wrong"); + logger.warn("Witness permission is wrong."); return null; } } @@ -1491,12 +1473,13 @@ public BlockCapsule generateBlock(Miner miner, long blockTime, long timeout) { if (trx == null) { // transaction may be removed by rePushLoop. - logger.warn("Trx is null,fromPending:{},pending:{},repush:{}.", + logger.warn("Trx is null, fromPending: {}, pending: {}, repush: {}.", fromPending, pendingTransactions.size(), rePushTransactions.size()); continue; } if (System.currentTimeMillis() > timeout) { - logger.warn("Processing transaction time exceeds the producing time."); + logger.warn("Processing transaction time exceeds the producing time {}.", + System.currentTimeMillis()); break; } @@ -1512,8 +1495,7 @@ public BlockCapsule generateBlock(Miner miner, long blockTime, long timeout) { continue; } //multi sign transaction - Contract contract = trx.getInstance().getRawData().getContract(0); - byte[] owner = TransactionCapsule.getOwner(contract); + byte[] owner = trx.getOwnerAddress(); String ownerAddress = ByteArray.toHexString(owner); if (accountSet.contains(ownerAddress)) { continue; @@ -1536,8 +1518,8 @@ public BlockCapsule generateBlock(Miner miner, long blockTime, long timeout) { transactionRetCapsule.addTransactionInfo(result); } } catch (Exception e) { - logger.error("Process trx {} failed when generating block: {}", trx.getTransactionId(), - e.getMessage()); + logger.error("Process trx {} failed when generating block {}, {}.", trx.getTransactionId(), + blockCapsule.getNum(), e.getMessage()); } } @@ -1545,8 +1527,8 @@ public BlockCapsule generateBlock(Miner miner, long blockTime, long timeout) { session.reset(); - logger.info("Generate block {} success, trxs:{}, pendingCount: {}, rePushCount: {}," - + " postponedCount: {}", + logger.info("Generate block {} success, trxs: {}, pendingCount: {}, rePushCount: {}," + + " postponedCount: {}.", blockCapsule.getNum(), blockCapsule.getTransactions().size(), pendingTransactions.size(), rePushTransactions.size(), postponedTrxCount); @@ -1560,8 +1542,7 @@ public BlockCapsule generateBlock(Miner miner, long blockTime, long timeout) { } private void filterOwnerAddress(TransactionCapsule transactionCapsule, Set result) { - Contract contract = transactionCapsule.getInstance().getRawData().getContract(0); - byte[] owner = TransactionCapsule.getOwner(contract); + byte[] owner = transactionCapsule.getOwnerAddress(); String ownerAddress = ByteArray.toHexString(owner); if (ownerAddressSet.contains(ownerAddress)) { result.add(ownerAddress); @@ -1631,7 +1612,7 @@ private void processBlock(BlockCapsule block, List txs) try { preValidateTransactionSign(txs); } catch (InterruptedException e) { - logger.error("parallel check sign interrupted exception! block info: {}", block, e); + logger.error("Parallel check sign interrupted exception! block info: {}.", block, e); Thread.currentThread().interrupt(); } } @@ -1748,7 +1729,7 @@ private void postSolidityLogContractTrigger(Long blockNum, Long lastSolidityNum) triggerCapsule.setTriggerName(Trigger.SOLIDITYLOG_TRIGGER_NAME); EventPluginLoader.getInstance().postSolidityLogTrigger(triggerCapsule); } else { - logger.error("postSolidityLogContractTrigger txId={} not contains transaction", + logger.error("PostSolidityLogContractTrigger txId = {} not contains transaction.", triggerCapsule.getTransactionId()); } } @@ -1802,21 +1783,17 @@ public void updateRecentTransaction(BlockCapsule block) { public void updateFork(BlockCapsule block) { int blockVersion = block.getInstance().getBlockHeader().getRawData().getVersion(); if (blockVersion > ChainConstant.BLOCK_VERSION) { - logger.warn("newer block version found: " + blockVersion + ", YOU MUST UPGRADE java-tron!"); + logger.warn("Newer block version found: {}, YOU MUST UPGRADE java-tron!", blockVersion); } - chainBaseManager - .getForkController().update(block); + chainBaseManager.getForkController().update(block); } public long getSyncBeginNumber() { - logger.info("headNumber:" - + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber()); - logger.info( - "syncBeginNumber:" - + (chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() - - revokingStore.size())); - logger.info("solidBlockNumber:" - + chainBaseManager.getDynamicPropertiesStore().getLatestSolidifiedBlockNum()); + logger.info("HeadNumber: {}, syncBeginNumber: {}, solidBlockNumber: {}.", + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber(), + chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() + - revokingStore.size(), + chainBaseManager.getDynamicPropertiesStore().getLatestSolidifiedBlockNum()); return chainBaseManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber() - revokingStore.size(); } @@ -1839,19 +1816,19 @@ public NullifierStore getNullifierStore() { } public void closeAllStore() { - logger.info("******** begin to close db ********"); + logger.info("******** Begin to close db. ********"); chainBaseManager.closeAllStore(); - logger.info("******** end to close db ********"); + logger.info("******** End to close db. ********"); } public void closeOneStore(ITronChainBase database) { - logger.info("******** begin to close " + database.getName() + " ********"); + logger.info("******** Begin to close {}. ********", database.getName()); try { database.close(); } catch (Exception e) { - logger.info("failed to close " + database.getName() + ". " + e); + logger.info("Failed to close {}.", database.getName(), e); } finally { - logger.info("******** end to close " + database.getName() + " ********"); + logger.info("******** End to close {}. ********", database.getName()); } } @@ -1902,17 +1879,17 @@ public void rePush(TransactionCapsule tx) { | AccountResourceInsufficientException | VMIllegalException e) { logger.debug(e.getMessage(), e); } catch (DupTransactionException e) { - logger.debug("pending manager: dup trans", e); + logger.debug("Pending manager: dup trans", e); } catch (TaposException e) { - logger.debug("pending manager: tapos exception", e); + logger.debug("Pending manager: tapos exception", e); } catch (TooBigTransactionException e) { - logger.debug("too big transaction"); + logger.debug("Pending manager: too big transaction", e); } catch (TransactionExpirationException e) { - logger.debug("expiration transaction"); + logger.debug("Pending manager: expiration transaction", e); } catch (ReceiptCheckErrException e) { - logger.debug("outOfSlotTime transaction"); + logger.debug("Pending manager: outOfSlotTime transaction", e); } catch (TooBigTransactionResultException e) { - logger.debug("too big transaction result"); + logger.debug("Pending manager: too big transaction result", e); } } @@ -1941,7 +1918,7 @@ private void startEventSubscribing() { .start(Args.getInstance().getEventPluginConfig()); if (!eventPluginLoaded) { - logger.error("failed to load eventPlugin"); + logger.error("Failed to load eventPlugin."); } FilterQuery eventFilter = Args.getInstance().getEventFilter(); @@ -1956,7 +1933,7 @@ private void startEventSubscribing() { private void postSolidityFilter(final long oldSolidNum, final long latestSolidifiedBlockNumber) { if (oldSolidNum >= latestSolidifiedBlockNumber) { - logger.warn("post solidity filter failed, oldSolidity: {} >= latestSolidity: {}", + logger.warn("Post solidity filter failed, oldSolidity: {} >= latestSolidity: {}.", oldSolidNum, latestSolidifiedBlockNumber); return; } @@ -1965,7 +1942,7 @@ private void postSolidityFilter(final long oldSolidNum, final long latestSolidif try { blockCapsule = chainBaseManager.getBlockByNum(latestSolidifiedBlockNumber); } catch (Exception e) { - logger.error("postSolidityFilter getBlockByNum={} except, {}", + logger.error("PostSolidityFilter getBlockByNum = {} except, {}.", latestSolidifiedBlockNumber, e.getMessage()); return; } @@ -1996,14 +1973,14 @@ private void postSolidityTrigger(final long oldSolidNum, final long latestSolidi blockCapsule = chainBaseManager.getBlockByNum(latestSolidifiedBlockNumber); solidityTriggerCapsule.setTimeStamp(blockCapsule.getTimeStamp()); } catch (Exception e) { - logger.error("postSolidityTrigger getBlockByNum={} except, {}", + logger.error("PostSolidityTrigger getBlockByNum = {} except, {}.", latestSolidifiedBlockNumber, e.getMessage()); } boolean result = triggerCapsuleQueue.offer(solidityTriggerCapsule); if (!result) { - logger.info("too many trigger, lost solidified trigger, " - + "block number: {}", latestSolidifiedBlockNumber); + logger.info("Too many trigger, lost solidified trigger, block number: {}.", + latestSolidifiedBlockNumber); } } @@ -2032,7 +2009,7 @@ private void processTransactionTrigger(BlockCapsule newBlock) { transactionInfoList = transactionInfoListBuilder.build(); } } catch (BadItemException e) { - logger.error("postBlockTrigger getTransactionInfoList blockNum={}, error is {}", + logger.error("PostBlockTrigger getTransactionInfoList blockNum = {}, error is {}.", newBlock.getNum(), e.getMessage()); } @@ -2053,9 +2030,8 @@ private void processTransactionTrigger(BlockCapsule newBlock) { cumulativeLogCount += transactionInfo.getLogCount(); } } else { - logger.error("postBlockTrigger blockNum={} has no transactions or " - + "the sizes of transactionInfoList and transactionCapsuleList are not equal", - newBlock.getNum()); + logger.error("PostBlockTrigger blockNum = {} has no transactions or {}.", newBlock.getNum(), + "the sizes of transactionInfoList and transactionCapsuleList are not equal"); for (TransactionCapsule e : newBlock.getTransactions()) { postTransactionTrigger(e, newBlock); } @@ -2069,14 +2045,14 @@ private void processTransactionTrigger(BlockCapsule newBlock) { private void reOrgLogsFilter() { if (CommonParameter.getInstance().isJsonRpcHttpFullNodeEnable()) { - logger.info("switch fork occurred, post reOrgLogsFilter"); + logger.info("Switch fork occurred, post reOrgLogsFilter."); try { BlockCapsule oldHeadBlock = chainBaseManager.getBlockById( getDynamicPropertiesStore().getLatestBlockHeaderHash()); postLogsFilter(oldHeadBlock, false, true); } catch (BadItemException | ItemNotFoundException e) { - logger.error("block header hash does not exist or is bad: {}", + logger.error("Block header hash does not exist or is bad: {}.", getDynamicPropertiesStore().getLatestBlockHeaderHash()); } } @@ -2085,7 +2061,7 @@ private void reOrgLogsFilter() { private void postBlockFilter(final BlockCapsule blockCapsule, boolean solidified) { BlockFilterCapsule blockFilterCapsule = new BlockFilterCapsule(blockCapsule, solidified); if (!filterCapsuleQueue.offer(blockFilterCapsule)) { - logger.info("too many filters, block filter lost: {}", blockCapsule.getBlockId()); + logger.info("Too many filters, block filter lost: {}.", blockCapsule.getBlockId()); } } @@ -2103,7 +2079,7 @@ private void postLogsFilter(final BlockCapsule blockCapsule, boolean solidified, transactionInfoList.addAll(result.getInstance().getTransactioninfoList()); } } catch (BadItemException e) { - logger.error("processLogsFilter getTransactionInfoList blockNum={}, error is {}", + logger.error("ProcessLogsFilter getTransactionInfoList blockNum = {}, error is {}.", blockNumber, e.getMessage()); return; } @@ -2113,7 +2089,7 @@ private void postLogsFilter(final BlockCapsule blockCapsule, boolean solidified, solidified, removed); if (!filterCapsuleQueue.offer(logsFilterCapsule)) { - logger.info("too many filters, logs filter lost: {}", blockNumber); + logger.info("Too many filters, logs filter lost: {}.", blockNumber); } } } @@ -2135,7 +2111,7 @@ private void postBlockTrigger(final BlockCapsule blockCapsule) { newBlock = chainBaseManager .getBlockByNum(solidityBlkNum); } catch (Exception e) { - logger.error("postBlockTrigger getBlockByNum blkNum={} except, error is {}", + logger.error("PostBlockTrigger getBlockByNum blkNum = {} except, error is {}.", solidityBlkNum, e.getMessage()); } } @@ -2144,7 +2120,7 @@ private void postBlockTrigger(final BlockCapsule blockCapsule) { blockLogTriggerCapsule.setLatestSolidifiedBlockNumber(getDynamicPropertiesStore() .getLatestSolidifiedBlockNum()); if (!triggerCapsuleQueue.offer(blockLogTriggerCapsule)) { - logger.info("too many triggers, block trigger lost: {}", newBlock.getBlockId()); + logger.info("Too many triggers, block trigger lost: {}.", newBlock.getBlockId()); } } @@ -2156,7 +2132,7 @@ private void postBlockTrigger(final BlockCapsule blockCapsule) { try { newBlock = chainBaseManager.getBlockByNum(solidityBlkNum); } catch (Exception e) { - logger.error("postBlockTrigger getBlockByNum blkNum={} except, error is {}", + logger.error("PostBlockTrigger getBlockByNum blkNum = {} except, error is {}.", solidityBlkNum, e.getMessage()); } } else { @@ -2178,7 +2154,7 @@ private long postTransactionTrigger(final TransactionCapsule trxCap, trx.setLatestSolidifiedBlockNumber(getDynamicPropertiesStore() .getLatestSolidifiedBlockNum()); if (!triggerCapsuleQueue.offer(trx)) { - logger.info("too many triggers, transaction trigger lost: {}", trxCap.getTransactionId()); + logger.info("Too many triggers, transaction trigger lost: {}.", trxCap.getTransactionId()); } return trx.getTransactionLogTrigger().getEnergyUsageTotal(); @@ -2191,7 +2167,7 @@ private void postTransactionTrigger(final TransactionCapsule trxCap, trx.setLatestSolidifiedBlockNumber(getDynamicPropertiesStore() .getLatestSolidifiedBlockNum()); if (!triggerCapsuleQueue.offer(trx)) { - logger.info("too many triggers, transaction trigger lost: {}", trxCap.getTransactionId()); + logger.info("Too many triggers, transaction trigger lost: {}.", trxCap.getTransactionId()); } } @@ -2199,7 +2175,7 @@ private void reOrgContractTrigger() { if (eventPluginLoaded && (EventPluginLoader.getInstance().isContractEventTriggerEnable() || EventPluginLoader.getInstance().isContractLogTriggerEnable())) { - logger.info("switchfork occurred, post reOrgContractTrigger"); + logger.info("Switch fork occurred, post reOrgContractTrigger."); try { BlockCapsule oldHeadBlock = chainBaseManager.getBlockById( getDynamicPropertiesStore().getLatestBlockHeaderHash()); @@ -2207,7 +2183,7 @@ private void reOrgContractTrigger() { postContractTrigger(trx.getTrxTrace(), true, oldHeadBlock.getBlockId().toString()); } } catch (BadItemException | ItemNotFoundException e) { - logger.error("block header hash does not exist or is bad: {}", + logger.error("Block header hash does not exist or is bad: {}.", getDynamicPropertiesStore().getLatestBlockHeaderHash()); } } @@ -2231,7 +2207,7 @@ private void postContractTrigger(final TransactionTrace trace, boolean remove, S contractTriggerCapsule.setBlockHash(blockHash); if (!triggerCapsuleQueue.offer(contractTriggerCapsule)) { - logger.info("too many triggers, contract log trigger lost: {}", + logger.info("Too many triggers, contract log trigger lost: {}.", trigger.getTransactionId()); } } @@ -2301,7 +2277,7 @@ private void initLiteNode() { if (needInit) { // copy transaction from recent-transaction to trans - logger.info("load trans for lite node."); + logger.info("Load trans for lite node."); TransactionCapsule item = new TransactionCapsule(Transaction.newBuilder().build()); @@ -2324,7 +2300,7 @@ private void initLiteNode() { trx.getTransactionIds().forEach( tid -> chainBaseManager.getTransactionStore().put(Hex.decode(tid), item)); } - logger.info("load trans complete, trans:{},from={},to={}", + logger.info("Load trans complete, trans: {}, from = {}, to = {}.", transactionCount, minBlock, maxBlock); } } diff --git a/framework/src/main/java/org/tron/core/db/PendingManager.java b/framework/src/main/java/org/tron/core/db/PendingManager.java index 2098ed2178b..0a79d5401e4 100644 --- a/framework/src/main/java/org/tron/core/db/PendingManager.java +++ b/framework/src/main/java/org/tron/core/db/PendingManager.java @@ -34,7 +34,7 @@ public void close() { Metrics.counterInc(MetricKeys.Counter.TXS, 1, MetricLabels.Counter.TXS_FAIL, MetricLabels.Counter.TXS_FAIL_TIMEOUT); if (Args.getInstance().isOpenPrintLog()) { - logger.warn("[timeout] remove tx from repush, txId:{}", tx.getTransactionId()); + logger.warn("Timeout remove tx from repush, txId: {}.", tx.getTransactionId()); } } } @@ -54,7 +54,7 @@ public void close() { Metrics.gaugeSet(MetricKeys.Gauge.MANAGER_QUEUE, 0, MetricLabels.Gauge.QUEUE_POPPED); if (Args.getInstance().isOpenPrintLog()) { - logger.warn("pending tx size:{}", dbManager.getRePushTransactions().size()); + logger.info("Pending tx size: {}.", dbManager.getRePushTransactions().size()); } } @@ -69,7 +69,7 @@ private void txIteration(TransactionCapsule tx) { Metrics.counterInc(MetricKeys.Counter.TXS, 1, MetricLabels.Counter.TXS_FAIL, MetricLabels.Counter.TXS_FAIL_TIMEOUT); if (Args.getInstance().isOpenPrintLog()) { - logger.warn("[timeout] remove tx from pending, txId:{}", tx.getTransactionId()); + logger.warn("Timeout remove tx from pending, txId: {}.", tx.getTransactionId()); } } } catch (InterruptedException e) { diff --git a/framework/src/main/java/org/tron/core/db/accountstate/AccountStateEntity.java b/framework/src/main/java/org/tron/core/db/accountstate/AccountStateEntity.java index 15764bb84d1..6a7307f477d 100644 --- a/framework/src/main/java/org/tron/core/db/accountstate/AccountStateEntity.java +++ b/framework/src/main/java/org/tron/core/db/accountstate/AccountStateEntity.java @@ -1,5 +1,6 @@ package org.tron.core.db.accountstate; +import com.google.protobuf.InvalidProtocolBufferException; import lombok.extern.slf4j.Slf4j; import org.tron.common.utils.StringUtil; import org.tron.protos.Protocol.Account; @@ -23,8 +24,8 @@ public AccountStateEntity(Account account) { public static AccountStateEntity parse(byte[] data) { try { return new AccountStateEntity().setAccount(Account.parseFrom(data)); - } catch (Exception e) { - logger.error("parse to AccountStateEntity error! reason: {}", e.getMessage()); + } catch (InvalidProtocolBufferException e) { + logger.error("Parse to AccountStateEntity error! reason: {}", e.getMessage()); } return null; } diff --git a/framework/src/main/java/org/tron/core/db/accountstate/callback/AccountStateCallBack.java b/framework/src/main/java/org/tron/core/db/accountstate/callback/AccountStateCallBack.java index 37eda516196..dda7fe0726b 100644 --- a/framework/src/main/java/org/tron/core/db/accountstate/callback/AccountStateCallBack.java +++ b/framework/src/main/java/org/tron/core/db/accountstate/callback/AccountStateCallBack.java @@ -84,10 +84,10 @@ public void executePushFinish() throws BadBlockException { newRoot = Hash.EMPTY_TRIE_HASH; } if (!oldRoot.isEmpty() && !Arrays.equals(oldRoot.toByteArray(), newRoot)) { - logger.error("the accountStateRoot hash is error. {}, oldRoot: {}, newRoot: {}", - blockCapsule, ByteArray.toHexString(oldRoot.toByteArray()), - ByteArray.toHexString(newRoot)); - throw new BadBlockException("the accountStateRoot hash is error"); + throw new BadBlockException( + String.format("accountStateRoot hash is error, %s, oldRoot: %s, newRoot: %s.", + blockCapsule, + ByteArray.toHexString(oldRoot.toByteArray()), ByteArray.toHexString(newRoot))); } } diff --git a/framework/src/main/java/org/tron/core/db/backup/BackupDbUtil.java b/framework/src/main/java/org/tron/core/db/backup/BackupDbUtil.java index 2feaa9ae2b7..ecaeb19d004 100644 --- a/framework/src/main/java/org/tron/core/db/backup/BackupDbUtil.java +++ b/framework/src/main/java/org/tron/core/db/backup/BackupDbUtil.java @@ -15,7 +15,7 @@ import org.tron.core.db2.core.SnapshotManager; import org.tron.core.db2.core.SnapshotRoot; -@Slf4j +@Slf4j(topic = "DB") @Component public class BackupDbUtil { @@ -95,16 +95,16 @@ public void doBackup(BlockCapsule block) { deleteBackup(DB_BACKUP_INDEX2); break; default: - logger.warn("invalid backup state"); + logger.warn("invalid backup state {}.", getBackupState()); } } catch (RocksDBException | SecurityException e) { - logger.warn("backup db error:" + e); + logger.warn("Backup db error.", e); } long timeUsed = System.currentTimeMillis() - t1; logger - .info("current block number is {}, backup all store use {} ms!", block.getNum(), timeUsed); + .info("Current block number is {}, backup all store use {} ms!", block.getNum(), timeUsed); if (timeUsed >= 3000) { - logger.warn("backing up db uses too much time."); + logger.warn("Backing up db uses too much time. {} ms.", timeUsed); } } @@ -115,7 +115,7 @@ private void backup(int i) throws RocksDBException { } else if (i == DB_BACKUP_INDEX2) { path = parameter.getDbBackupConfig().getBak2path(); } else { - throw new RuntimeException("Error backup with undefined index"); + throw new RuntimeException(String.format("error backup with undefined index %d", i)); } List stores = ((SnapshotManager) db).getDbs(); for (Chainbase store : stores) { @@ -134,7 +134,7 @@ private void deleteBackup(int i) { } else if (i == DB_BACKUP_INDEX2) { path = parameter.getDbBackupConfig().getBak2path(); } else { - throw new RuntimeException("Error deleteBackup with undefined index"); + throw new RuntimeException(String.format("error deleteBackup with undefined index %d", i)); } List stores = ((SnapshotManager) db).getDbs(); for (Chainbase store : stores) { diff --git a/framework/src/main/java/org/tron/core/net/TronNetDelegate.java b/framework/src/main/java/org/tron/core/net/TronNetDelegate.java index 69c7c8af61c..7a0c361165e 100644 --- a/framework/src/main/java/org/tron/core/net/TronNetDelegate.java +++ b/framework/src/main/java/org/tron/core/net/TronNetDelegate.java @@ -220,7 +220,7 @@ public void processBlock(BlockCapsule block, boolean isSync) throws P2pException && dbManager.getLatestSolidityNumShutDown() == dbManager.getDynamicPropertiesStore() .getLatestBlockHeaderNumberFromDB()) { - logger.info("begin shutdown, currentBlockNum:{}, DbBlockNum:{} ,solidifiedBlockNum:{}.", + logger.info("Begin shutdown, currentBlockNum:{}, DbBlockNum:{}, solidifiedBlockNum:{}", dbManager.getDynamicPropertiesStore().getLatestBlockHeaderNumber(), dbManager.getDynamicPropertiesStore().getLatestBlockHeaderNumberFromDB(), dbManager.getDynamicPropertiesStore().getLatestSolidifiedBlockNum()); @@ -252,7 +252,7 @@ public void processBlock(BlockCapsule block, boolean isSync) throws P2pException dbManager.pushBlock(block); Metrics.histogramObserve(timer); freshBlockId.put(blockId, System.currentTimeMillis()); - logger.info("Success process block {}.", blockId.getString()); + logger.info("Success process block {}", blockId.getString()); if (!backupServerStartFlag && System.currentTimeMillis() - block.getTimeStamp() < BLOCK_PRODUCED_INTERVAL) { backupServerStartFlag = true; @@ -278,7 +278,7 @@ public void processBlock(BlockCapsule block, boolean isSync) throws P2pException | ZksnarkException | EventBloomException e) { metricsService.failProcessBlock(block.getNum(), e.getMessage()); - logger.error("Process block failed, {}, reason: {}.", blockId.getString(), e.getMessage()); + logger.error("Process block failed, {}, reason: {}", blockId.getString(), e.getMessage()); throw new P2pException(TypeEnum.BAD_BLOCK, e); } } diff --git a/framework/src/main/java/org/tron/core/net/TronNetService.java b/framework/src/main/java/org/tron/core/net/TronNetService.java index 69d1bc0cebe..82568d6a7c0 100644 --- a/framework/src/main/java/org/tron/core/net/TronNetService.java +++ b/framework/src/main/java/org/tron/core/net/TronNetService.java @@ -5,6 +5,8 @@ import org.springframework.stereotype.Component; import org.tron.common.overlay.message.Message; import org.tron.common.overlay.server.ChannelManager; +import org.tron.common.prometheus.MetricKeys; +import org.tron.common.prometheus.Metrics; import org.tron.core.exception.P2pException; import org.tron.core.exception.P2pException.TypeEnum; import org.tron.core.net.message.BlockMessage; @@ -65,6 +67,9 @@ public class TronNetService { @Autowired private FetchBlockService fetchBlockService; + private static final String TAG = "~"; + private static final int DURATION_STEP = 50; + public void start() { channelManager.init(); advService.init(); @@ -72,18 +77,18 @@ public void start() { peerStatusCheck.init(); transactionsMsgHandler.init(); fetchBlockService.init(); - logger.info("TronNetService start successfully."); + logger.info("TronNetService start successfully"); } public void stop() { - logger.info("TronNetService closed start."); + logger.info("TronNetService closed start"); channelManager.close(); advService.close(); syncService.close(); peerStatusCheck.close(); transactionsMsgHandler.close(); fetchBlockService.close(); - logger.info("TronNetService closed successfully."); + logger.info("TronNetService closed successfully"); } public int fastBroadcastTransaction(TransactionMessage msg) { @@ -95,6 +100,7 @@ public void broadcast(Message msg) { } protected void onMessage(PeerConnection peer, TronMessage msg) { + long startTime = System.currentTimeMillis(); try { switch (msg.getType()) { case SYNC_BLOCK_CHAIN: @@ -123,6 +129,14 @@ protected void onMessage(PeerConnection peer, TronMessage msg) { } } catch (Exception e) { processException(peer, msg, e); + } finally { + long costs = System.currentTimeMillis() - startTime; + if (costs > DURATION_STEP) { + logger.info("Message processing costs {} ms, peer: {}, type: {}, time tag: {}", + costs, peer.getInetAddress(), msg.getType(), getTimeTag(costs)); + Metrics.histogramObserve(MetricKeys.Histogram.MESSAGE_PROCESS_LATENCY, + costs / Metrics.MILLISECONDS_PER_SECOND, msg.getType().name()); + } } } @@ -149,11 +163,14 @@ private void processException(PeerConnection peer, TronMessage msg, Exception ex case UNLINK_BLOCK: code = ReasonCode.UNLINKABLE; break; + case DB_ITEM_NOT_FOUND: + code = ReasonCode.FETCH_FAIL; + break; default: code = ReasonCode.UNKNOWN; break; } - logger.warn("Message from {} process failed, {} \n type: {}, detail: {}.", + logger.warn("Message from {} process failed, {} \n type: {}, detail: {}", peer.getInetAddress(), msg, type, ex.getMessage()); } else { code = ReasonCode.UNKNOWN; @@ -163,4 +180,13 @@ private void processException(PeerConnection peer, TronMessage msg, Exception ex peer.disconnect(code); } + + private String getTimeTag(long duration) { + StringBuilder tag = new StringBuilder(TAG); + long tagCount = duration / DURATION_STEP; + for (; tagCount > 0; tagCount--) { + tag.append(TAG); + } + return tag.toString(); + } } diff --git a/framework/src/main/java/org/tron/core/net/message/ChainInventoryMessage.java b/framework/src/main/java/org/tron/core/net/message/ChainInventoryMessage.java index 5b0a0398778..b38961d2e51 100644 --- a/framework/src/main/java/org/tron/core/net/message/ChainInventoryMessage.java +++ b/framework/src/main/java/org/tron/core/net/message/ChainInventoryMessage.java @@ -50,7 +50,7 @@ public List getBlockIds() { .map(blockId -> new BlockId(blockId.getHash(), blockId.getNumber())) .collect(Collectors.toCollection(ArrayList::new)); } catch (Exception e) { - logger.info("breakPoint"); + logger.info("Failed to get blockIds of chain inventory message"); } return null; } diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/BlockMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/BlockMsgHandler.java index 8e7eac91670..5d6d4eb7f75 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/BlockMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/BlockMsgHandler.java @@ -7,6 +7,8 @@ import org.bouncycastle.util.encoders.Hex; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.tron.common.prometheus.MetricKeys; +import org.tron.common.prometheus.Metrics; import org.tron.core.Constant; import org.tron.core.capsule.BlockCapsule; import org.tron.core.capsule.BlockCapsule.BlockId; @@ -76,7 +78,11 @@ public void processMessage(PeerConnection peer, TronMessage msg) throws P2pExcep if (null != time) { MetricsUtil.histogramUpdateUnCheck(MetricsKey.NET_LATENCY_FETCH_BLOCK + peer.getNode().getHost(), now - time); + Metrics.histogramObserve(MetricKeys.Histogram.BLOCK_FETCH_LATENCY, + (now - time) / Metrics.MILLISECONDS_PER_SECOND); } + Metrics.histogramObserve(MetricKeys.Histogram.BLOCK_RECEIVE_DELAY, + (now - blockMessage.getBlockCapsule().getTimeStamp()) / Metrics.MILLISECONDS_PER_SECOND); fetchBlockService.blockFetchSuccess(blockId); long interval = blockId.getNum() - tronNetDelegate.getHeadBlockId().getNum(); processBlock(peer, blockMessage.getBlockCapsule()); @@ -113,7 +119,7 @@ private void check(PeerConnection peer, BlockMessage msg) throws P2pException { private void processBlock(PeerConnection peer, BlockCapsule block) throws P2pException { BlockId blockId = block.getBlockId(); if (!tronNetDelegate.containBlock(block.getParentBlockId())) { - logger.warn("Get unlink block {} from {}, head is {}.", blockId.getString(), + logger.warn("Get unlink block {} from {}, head is {}", blockId.getString(), peer.getInetAddress(), tronNetDelegate.getHeadBlockId().getString()); syncService.startSync(peer); return; diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/ChainInventoryMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/ChainInventoryMsgHandler.java index 500d33904be..81f0d2290a1 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/ChainInventoryMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/ChainInventoryMsgHandler.java @@ -66,12 +66,15 @@ public void processMessage(PeerConnection peer, TronMessage msg) throws P2pExcep synchronized (tronNetDelegate.getBlockLock()) { try { + BlockId blockId = null; while (!peer.getSyncBlockToFetch().isEmpty() && tronNetDelegate .containBlock(peer.getSyncBlockToFetch().peek())) { - BlockId blockId = peer.getSyncBlockToFetch().pop(); + blockId = peer.getSyncBlockToFetch().pop(); peer.setBlockBothHave(blockId); + } + if (blockId != null) { logger.info("Block {} from {} is processed", - blockId.getString(), peer.getNode().getHost()); + blockId.getString(), peer.getNode().getHost()); } } catch (NoSuchElementException e) { logger.warn("Process ChainInventoryMessage failed, peer {}, isDisconnect:{}", diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/FetchInvDataMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/FetchInvDataMsgHandler.java index 8e9daa8ddc9..c719674966d 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/FetchInvDataMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/FetchInvDataMsgHandler.java @@ -34,7 +34,6 @@ import org.tron.core.net.service.SyncService; import org.tron.protos.Protocol.Inventory.InventoryType; import org.tron.protos.Protocol.PBFTMessage.Raw; -import org.tron.protos.Protocol.ReasonCode; import org.tron.protos.Protocol.Transaction; @Slf4j(topic = "net") @@ -73,9 +72,8 @@ public void processMessage(PeerConnection peer, TronMessage msg) throws P2pExcep try { message = tronNetDelegate.getData(hash, type); } catch (Exception e) { - logger.error("Fetch item {} failed. reason: {}", item, hash, e.getMessage()); - peer.disconnect(ReasonCode.FETCH_FAIL); - return; + throw new P2pException(TypeEnum.DB_ITEM_NOT_FOUND, + "Fetch item " + item + " failed. reason: " + e.getMessage()); } } @@ -145,7 +143,9 @@ private void check(PeerConnection peer, FetchInvDataMessage fetchInvDataMsg) thr .getCount(10); int maxCount = advService.getTrxCount().getCount(60); if (fetchCount > maxCount) { - logger.error("maxCount: " + maxCount + ", fetchCount: " + fetchCount); + logger.warn("Peer fetch too more transactions in 10 seconds, " + + "maxCount: {}, fetchCount: {}, peer: {}", + maxCount, fetchCount, peer.getInetAddress()); } } else { boolean isAdv = true; @@ -161,8 +161,9 @@ private void check(PeerConnection peer, FetchInvDataMessage fetchInvDataMsg) thr int outBlockCountIn1min = tronOutAdvBlock.getCount(60); int producedBlockIn2min = 120_000 / BLOCK_PRODUCED_INTERVAL; if (outBlockCountIn1min > producedBlockIn2min) { - logger.error("producedBlockIn2min: " + producedBlockIn2min + ", outBlockCountIn1min: " - + outBlockCountIn1min); + logger.warn("Peer fetch too more blocks in a minute, producedBlockIn2min: {}," + + " outBlockCountIn1min: {}, peer: {}", + producedBlockIn2min, outBlockCountIn1min, peer.getInetAddress()); } } else { if (!peer.isNeedSyncFromUs()) { diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/InventoryMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/InventoryMsgHandler.java index 6b8a8d85c92..96f022543f2 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/InventoryMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/InventoryMsgHandler.java @@ -49,7 +49,7 @@ private boolean check(PeerConnection peer, InventoryMessage inventoryMessage) { int size = inventoryMessage.getHashList().size(); if (peer.isNeedSyncFromPeer() || peer.isNeedSyncFromUs()) { - logger.warn("Drop inv: {} size: {} from Peer {}, syncFromUs: {}, syncFromPeer: {}.", + logger.warn("Drop inv: {} size: {} from Peer {}, syncFromUs: {}, syncFromPeer: {}", type, size, peer.getInetAddress(), peer.isNeedSyncFromUs(), peer.isNeedSyncFromPeer()); return false; } @@ -57,7 +57,7 @@ private boolean check(PeerConnection peer, InventoryMessage inventoryMessage) { if (type.equals(InventoryType.TRX)) { int count = peer.getNodeStatistics().messageStatistics.tronInTrxInventoryElement.getCount(10); if (count > maxCountIn10s) { - logger.warn("Drop inv: {} size: {} from Peer {}, Inv count: {} is overload.", + logger.warn("Drop inv: {} size: {} from Peer {}, Inv count: {} is overload", type, size, peer.getInetAddress(), count); if (Args.getInstance().isOpenPrintLog()) { logger.warn("[overload]Drop tx list is: {}", inventoryMessage.getHashList()); @@ -66,7 +66,7 @@ private boolean check(PeerConnection peer, InventoryMessage inventoryMessage) { } if (transactionsMsgHandler.isBusy()) { - logger.warn("Drop inv: {} size: {} from Peer {}, transactionsMsgHandler is busy.", + logger.warn("Drop inv: {} size: {} from Peer {}, transactionsMsgHandler is busy", type, size, peer.getInetAddress()); if (Args.getInstance().isOpenPrintLog()) { logger.warn("[isBusy]Drop tx list is: {}", inventoryMessage.getHashList()); diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/PbftDataSyncHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/PbftDataSyncHandler.java index 3525e53fe9b..5f107c9958d 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/PbftDataSyncHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/PbftDataSyncHandler.java @@ -92,11 +92,11 @@ private void processPBFTCommitMessage(PbftCommitMessage pbftCommitMessage) { if (raw.getDataType() == DataType.BLOCK && pbftSignDataStore.getBlockSignData(raw.getViewN()) == null) { pbftSignDataStore.putBlockSignData(raw.getViewN(), pbftCommitMessage.getPbftSignCapsule()); - logger.info("save the block {} pbft commit data", raw.getViewN()); + logger.info("Save the block {} pbft commit data", raw.getViewN()); } else if (raw.getDataType() == DataType.SRL && pbftSignDataStore.getSrSignData(raw.getEpoch()) == null) { pbftSignDataStore.putSrSignData(raw.getEpoch(), pbftCommitMessage.getPbftSignCapsule()); - logger.info("save the srl {} pbft commit data", raw.getEpoch()); + logger.info("Save the srl {} pbft commit data", raw.getEpoch()); } } catch (InvalidProtocolBufferException e) { logger.error("", e); diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java index 9f392b11e40..d9fc767ac6c 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/SyncBlockChainMsgHandler.java @@ -38,7 +38,7 @@ public void processMessage(PeerConnection peer, TronMessage msg) throws P2pExcep LinkedList blockIds = getLostBlockIds(summaryChainIds); if (blockIds.size() == 0) { - logger.error("Can't get lost block Ids."); + logger.warn("Can't get lost block Ids"); peer.disconnect(Protocol.ReasonCode.INCOMPATIBLE_CHAIN); return; } else if (blockIds.size() == 1) { diff --git a/framework/src/main/java/org/tron/core/net/messagehandler/TransactionsMsgHandler.java b/framework/src/main/java/org/tron/core/net/messagehandler/TransactionsMsgHandler.java index 8fc23196cd9..28cc68c2e44 100644 --- a/framework/src/main/java/org/tron/core/net/messagehandler/TransactionsMsgHandler.java +++ b/framework/src/main/java/org/tron/core/net/messagehandler/TransactionsMsgHandler.java @@ -64,18 +64,27 @@ public boolean isBusy() { public void processMessage(PeerConnection peer, TronMessage msg) throws P2pException { TransactionsMessage transactionsMessage = (TransactionsMessage) msg; check(peer, transactionsMessage); + int smartContractQueueSize = 0; + int trxHandlePoolQueueSize = 0; + int dropSmartContractCount = 0; for (Transaction trx : transactionsMessage.getTransactions().getTransactionsList()) { int type = trx.getRawData().getContract(0).getType().getNumber(); if (type == ContractType.TriggerSmartContract_VALUE || type == ContractType.CreateSmartContract_VALUE) { if (!smartContractQueue.offer(new TrxEvent(peer, new TransactionMessage(trx)))) { - logger.warn("Add smart contract failed, queueSize {}:{}", smartContractQueue.size(), - queue.size()); + smartContractQueueSize = smartContractQueue.size(); + trxHandlePoolQueueSize = queue.size(); + dropSmartContractCount++; } } else { trxHandlePool.submit(() -> handleTransaction(peer, new TransactionMessage(trx))); } } + + if (dropSmartContractCount > 0) { + logger.warn("Add smart contract failed, drop count: {}, queueSize {}:{}", + dropSmartContractCount, smartContractQueueSize, trxHandlePoolQueueSize); + } } private void check(PeerConnection peer, TransactionsMessage msg) throws P2pException { @@ -97,17 +106,17 @@ private void handleSmartContract() { trxHandlePool.submit(() -> handleTransaction(event.getPeer(), event.getMsg())); } } catch (InterruptedException e) { - logger.warn("Handle smart server interrupted."); + logger.warn("Handle smart server interrupted"); Thread.currentThread().interrupt(); } catch (Exception e) { - logger.error("Handle smart contract exception.", e); + logger.error("Handle smart contract exception", e); } }, 1000, 20, TimeUnit.MILLISECONDS); } private void handleTransaction(PeerConnection peer, TransactionMessage trx) { if (peer.isDisconnect()) { - logger.warn("Drop trx {} from {}, peer is disconnect.", trx.getMessageId(), + logger.warn("Drop trx {} from {}, peer is disconnect", trx.getMessageId(), peer.getInetAddress()); return; } @@ -126,7 +135,7 @@ private void handleTransaction(PeerConnection peer, TransactionMessage trx) { peer.disconnect(ReasonCode.BAD_TX); } } catch (Exception e) { - logger.error("Trx {} from peer {} process failed.", trx.getMessageId(), peer.getInetAddress(), + logger.error("Trx {} from peer {} process failed", trx.getMessageId(), peer.getInetAddress(), e); } } diff --git a/framework/src/main/java/org/tron/core/net/peer/PeerStatusCheck.java b/framework/src/main/java/org/tron/core/net/peer/PeerStatusCheck.java index 6600238fcc5..993d9e94781 100644 --- a/framework/src/main/java/org/tron/core/net/peer/PeerStatusCheck.java +++ b/framework/src/main/java/org/tron/core/net/peer/PeerStatusCheck.java @@ -27,7 +27,7 @@ public void init() { try { statusCheck(); } catch (Exception e) { - logger.error("", e); + logger.error("Check peers status processing failed", e); } }, 5, 2, TimeUnit.SECONDS); } @@ -46,7 +46,7 @@ public void statusCheck() { if (peer.isNeedSyncFromPeer() && peer.getBlockBothHaveUpdateTime() < now - blockUpdateTimeout) { - logger.warn("Peer {} not sync for a long time.", peer.getInetAddress()); + logger.warn("Peer {} not sync for a long time", peer.getInetAddress()); isDisconnected = true; } diff --git a/framework/src/main/java/org/tron/core/net/service/AdvService.java b/framework/src/main/java/org/tron/core/net/service/AdvService.java index e0dc4238ab7..b10e0542919 100644 --- a/framework/src/main/java/org/tron/core/net/service/AdvService.java +++ b/framework/src/main/java/org/tron/core/net/service/AdvService.java @@ -86,7 +86,7 @@ public void init() { try { consumerInvToSpread(); } catch (Exception exception) { - logger.error("Spread thread error. {}", exception.getMessage(), exception); + logger.error("Spread thread error", exception); } }, 100, 30, TimeUnit.MILLISECONDS); @@ -94,7 +94,7 @@ public void init() { try { consumerInvToFetch(); } catch (Exception exception) { - logger.error("Fetch thread error. {}", exception.getMessage(), exception); + logger.error("Fetch thread error", exception); } }, 100, 30, TimeUnit.MILLISECONDS); } @@ -151,7 +151,7 @@ public int fastBroadcastTransaction(TransactionMessage msg) { .collect(Collectors.toList()); if (peers.size() == 0) { - logger.warn("Broadcast transaction {} failed, no connection.", msg.getMessageId()); + logger.warn("Broadcast transaction {} failed, no connection", msg.getMessageId()); return 0; } @@ -173,7 +173,7 @@ public int fastBroadcastTransaction(TransactionMessage msg) { } } if (peersCount == 0) { - logger.warn("Broadcast transaction {} failed, no peers.", msg.getMessageId()); + logger.warn("Broadcast transaction {} failed, no peers", msg.getMessageId()); } return peersCount; } @@ -185,7 +185,7 @@ public void broadcast(Message msg) { } if (invToSpread.size() > MAX_SPREAD_SIZE) { - logger.warn("Drop message, type: {}, ID: {}.", msg.getType(), msg.getMessageId()); + logger.warn("Drop message, type: {}, ID: {}", msg.getType(), msg.getMessageId()); return; } @@ -269,7 +269,7 @@ private void consumerInvToFetch() { } invToFetch.forEach((item, time) -> { if (time < now - MSG_CACHE_DURATION_IN_BLOCKS * BLOCK_PRODUCED_INTERVAL) { - logger.info("This obj is too late to fetch, type: {} hash: {}.", item.getType(), + logger.info("This obj is too late to fetch, type: {} hash: {}", item.getType(), item.getHash()); invToFetch.remove(item); invToFetchCache.invalidate(item); diff --git a/framework/src/main/java/org/tron/core/net/service/FetchBlockService.java b/framework/src/main/java/org/tron/core/net/service/FetchBlockService.java index cfad7242471..ef541eb3559 100644 --- a/framework/src/main/java/org/tron/core/net/service/FetchBlockService.java +++ b/framework/src/main/java/org/tron/core/net/service/FetchBlockService.java @@ -58,8 +58,8 @@ public void init() { fetchBlockWorkerExecutor.scheduleWithFixedDelay(() -> { try { fetchBlockProcess(fetchBlockInfo); - } catch (Exception exception) { - logger.error("FetchBlockWorkerSchedule thread error. {}", exception.getMessage()); + } catch (Exception e) { + logger.error("FetchBlockWorkerSchedule thread error", e); } }, 0L, 50L, TimeUnit.MILLISECONDS); } @@ -69,6 +69,11 @@ public void close() { } public void fetchBlock(List sha256HashList, PeerConnection peer) { + if (sha256HashList.size() > 0) { + logger.info("Begin fetch block {} from {}", + new BlockCapsule.BlockId(sha256HashList.get(0)).getString(), + peer.getInetAddress()); + } if (null != fetchBlockInfo) { return; } @@ -78,12 +83,15 @@ public void fetchBlock(List sha256HashList, PeerConnection peer) { if (System.currentTimeMillis() - chainBaseManager.getHeadBlockTimeStamp() < BLOCK_FETCH_TIME_OUT_LIMIT) { fetchBlockInfo = new FetchBlockInfo(sha256Hash, peer, System.currentTimeMillis()); + logger.info("Set fetchBlockInfo, block: {}, peer: {}, time: {}", sha256Hash, + fetchBlockInfo.getPeer().getInetAddress(), fetchBlockInfo.getTime()); } }); } public void blockFetchSuccess(Sha256Hash sha256Hash) { + logger.info("Fetch block success, {}", new BlockCapsule.BlockId(sha256Hash).getString()); FetchBlockInfo fetchBlockInfoTemp = this.fetchBlockInfo; if (null == fetchBlockInfoTemp || !fetchBlockInfoTemp.getHash().equals(sha256Hash)) { return; @@ -98,6 +106,8 @@ private void fetchBlockProcess(FetchBlockInfo fetchBlock) { if (System.currentTimeMillis() - chainBaseManager.getHeadBlockTimeStamp() >= BLOCK_FETCH_TIME_OUT_LIMIT) { this.fetchBlockInfo = null; + logger.info("Clear fetchBlockInfo due to {} ms past head block time", + BLOCK_FETCH_TIME_OUT_LIMIT); return; } Item item = new Item(fetchBlock.getHash(), InventoryType.BLOCK); @@ -120,6 +130,8 @@ private void fetchBlockProcess(FetchBlockInfo fetchBlock) { }); } else { if (System.currentTimeMillis() - fetchBlock.getTime() >= fetchTimeOut) { + logger.info("Clear fetchBlockInfo due to fetch block {} timeout {}ms", + fetchBlock.getHash(), fetchTimeOut); this.fetchBlockInfo = null; } } diff --git a/framework/src/main/java/org/tron/core/net/service/SyncService.java b/framework/src/main/java/org/tron/core/net/service/SyncService.java index 8e4d7a49b3d..79697d7459f 100644 --- a/framework/src/main/java/org/tron/core/net/service/SyncService.java +++ b/framework/src/main/java/org/tron/core/net/service/SyncService.java @@ -72,7 +72,7 @@ public void init() { startFetchSyncBlock(); } } catch (Exception e) { - logger.error("Fetch sync block error.", e); + logger.error("Fetch sync block error", e); } }, 10, 1, TimeUnit.SECONDS); @@ -83,7 +83,7 @@ public void init() { handleSyncBlock(); } } catch (Exception e) { - logger.error("Handle sync block error.", e); + logger.error("Handle sync block error", e); } }, 10, 1, TimeUnit.SECONDS); } @@ -95,7 +95,7 @@ public void close() { public void startSync(PeerConnection peer) { if (peer.getTronState().equals(TronState.SYNCING)) { - logger.warn("Start sync failed, peer {} is in sync.", peer.getNode().getHost()); + logger.warn("Start sync failed, peer {} is in sync", peer.getNode().getHost()); return; } peer.setTronState(TronState.SYNCING); @@ -109,14 +109,14 @@ public void startSync(PeerConnection peer) { public void syncNext(PeerConnection peer) { try { if (peer.getSyncChainRequested() != null) { - logger.warn("Peer {} is in sync.", peer.getNode().getHost()); + logger.warn("Peer {} is in sync", peer.getNode().getHost()); return; } LinkedList chainSummary = getBlockChainSummary(peer); peer.setSyncChainRequested(new Pair<>(chainSummary, System.currentTimeMillis())); peer.sendMessage(new SyncBlockChainMessage(chainSummary)); } catch (Exception e) { - logger.error("Peer {} sync failed, reason: {}", peer.getInetAddress(), e.getMessage()); + logger.warn("Peer {} sync failed, reason: {}", peer.getInetAddress(), e.getMessage()); peer.disconnect(ReasonCode.SYNC_FAIL); } } @@ -273,7 +273,7 @@ private void processSyncBlock(BlockCapsule block) { tronNetDelegate.processBlock(block, true); pbftDataSyncHandler.processPBFTCommitData(block); } catch (Exception e) { - logger.error("Process sync block {} failed.", blockId.getString(), e); + logger.error("Process sync block {} failed", blockId.getString(), e); flag = false; } for (PeerConnection peer : tronNetDelegate.getActivePeer()) { diff --git a/framework/src/main/java/org/tron/core/services/http/GetBrokerageServlet.java b/framework/src/main/java/org/tron/core/services/http/GetBrokerageServlet.java index f36764d6489..1fbd94fe690 100644 --- a/framework/src/main/java/org/tron/core/services/http/GetBrokerageServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/GetBrokerageServlet.java @@ -1,8 +1,10 @@ package org.tron.core.services.http; +import java.io.IOException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import lombok.extern.slf4j.Slf4j; +import org.bouncycastle.util.encoders.DecoderException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.tron.core.db.Manager; @@ -24,6 +26,13 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { value = manager.getDelegationStore().getBrokerage(cycle, address); } response.getWriter().println("{\"brokerage\": " + value + "}"); + } catch (DecoderException | IllegalArgumentException e) { + try { + response.getWriter() + .println("{\"Error\": " + "\"INVALID address, " + e.getMessage() + "\"}"); + } catch (IOException ioe) { + logger.debug("IOException: {}", ioe.getMessage()); + } } catch (Exception e) { Util.processError(e, response); } diff --git a/framework/src/main/java/org/tron/core/services/http/GetRewardServlet.java b/framework/src/main/java/org/tron/core/services/http/GetRewardServlet.java index 14d6f11770d..c4d97f46c57 100644 --- a/framework/src/main/java/org/tron/core/services/http/GetRewardServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/GetRewardServlet.java @@ -25,7 +25,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { value = manager.getMortgageService().queryReward(address); } response.getWriter().println("{\"reward\": " + value + "}"); - } catch (DecoderException e) { + } catch (DecoderException | IllegalArgumentException e) { try { response.getWriter() .println("{\"Error\": " + "\"INVALID address, " + e.getMessage() + "\"}"); diff --git a/framework/src/main/java/org/tron/core/services/http/Util.java b/framework/src/main/java/org/tron/core/services/http/Util.java index 1a4f738a334..2fef8ff1425 100644 --- a/framework/src/main/java/org/tron/core/services/http/Util.java +++ b/framework/src/main/java/org/tron/core/services/http/Util.java @@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONException; import com.alibaba.fastjson.JSONObject; import com.google.protobuf.Any; import com.google.protobuf.ByteString; @@ -292,6 +293,8 @@ public static Transaction packTransaction(String strTransaction, boolean selfTyp logger.debug("ParseException: {}", e.getMessage()); } catch (ClassCastException e) { logger.debug("ClassCastException: {}", e.getMessage()); + } catch (JSONException e) { + logger.debug("JSONException: {}", e.getMessage()); } catch (Exception e) { logger.error("", e); } @@ -493,8 +496,10 @@ public static byte[] getAddress(HttpServletRequest request) throws Exception { String input = request.getReader().lines() .collect(Collectors.joining(System.lineSeparator())); Util.checkBodySize(input); - JSONObject jsonObject = JSONObject.parseObject(input); - addressStr = jsonObject.getString(addressParam); + JSONObject jsonObject = JSON.parseObject(input); + if (jsonObject != null) { + addressStr = jsonObject.getString(addressParam); + } } if (StringUtils.isNotBlank(addressStr)) { if (StringUtils.startsWith(addressStr, Constant.ADD_PRE_FIX_STRING_MAINNET)) { diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpc.java b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpc.java index caf0292a35c..52a3a2380d1 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpc.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpc.java @@ -152,7 +152,7 @@ TransactionResult getTransactionByBlockNumberAndIndex(String blockNumOrTag, Stri @JsonRpcError(exception = JsonRpcInvalidParamsException.class, code = -32602, data = "{}"), @JsonRpcError(exception = JsonRpcInternalException.class, code = -32000, data = "{}"), }) - String getCall(CallArguments transactionCall, String blockNumOrTag) + String getCall(CallArguments transactionCall, Object blockNumOrTag) throws JsonRpcInvalidParamsException, JsonRpcInvalidRequestException, JsonRpcInternalException; diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java index 82568f09acd..55bb167e771 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java @@ -14,6 +14,7 @@ import com.google.protobuf.ByteString; import com.google.protobuf.GeneratedMessageV3; import java.util.Arrays; +import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -134,6 +135,8 @@ public enum RequestSource { private static final String TAG_NOT_SUPPORT_ERROR = "TAG [earliest | pending] not supported"; private static final String QUANTITY_NOT_SUPPORT_ERROR = "QUANTITY not supported, just support TAG as latest"; + private static final String NO_BLOCK_HEADER = "header not found"; + private static final String NO_BLOCK_HEADER_BY_HASH = "header for hash not found"; private static final String ERROR_SELECTOR = "08c379a0"; // Function selector for Error(string) /** @@ -716,9 +719,54 @@ public TransactionReceipt getTransactionReceipt(String txId) } @Override - public String getCall(CallArguments transactionCall, String blockNumOrTag) + public String getCall(CallArguments transactionCall, Object blockParamObj) throws JsonRpcInvalidParamsException, JsonRpcInvalidRequestException, JsonRpcInternalException { + + String blockNumOrTag; + if (blockParamObj instanceof HashMap) { + HashMap paramMap; + paramMap = (HashMap) blockParamObj; + + if (paramMap.containsKey("blockNumber")) { + try { + blockNumOrTag = paramMap.get("blockNumber"); + } catch (ClassCastException e) { + throw new JsonRpcInvalidRequestException(JSON_ERROR); + } + + long blockNumber; + try { + blockNumber = ByteArray.hexToBigInteger(blockNumOrTag).longValue(); + } catch (Exception e) { + throw new JsonRpcInvalidParamsException(BLOCK_NUM_ERROR); + } + + if (wallet.getBlockByNum(blockNumber) == null) { + throw new JsonRpcInternalException(NO_BLOCK_HEADER); + } + + } else if (paramMap.containsKey("blockHash")) { + try { + blockNumOrTag = paramMap.get("blockHash"); + } catch (ClassCastException e) { + throw new JsonRpcInvalidRequestException(JSON_ERROR); + } + + if (getBlockByJsonHash(blockNumOrTag) == null) { + throw new JsonRpcInternalException(NO_BLOCK_HEADER_BY_HASH); + } + } else { + throw new JsonRpcInvalidRequestException(JSON_ERROR); + } + + blockNumOrTag = LATEST_STR; + } else if (blockParamObj instanceof String) { + blockNumOrTag = (String) blockParamObj; + } else { + throw new JsonRpcInvalidRequestException(JSON_ERROR); + } + if (EARLIEST_STR.equalsIgnoreCase(blockNumOrTag) || PENDING_STR.equalsIgnoreCase(blockNumOrTag)) { throw new JsonRpcInvalidParamsException(TAG_NOT_SUPPORT_ERROR); diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/types/BlockResult.java b/framework/src/main/java/org/tron/core/services/jsonrpc/types/BlockResult.java index 4d4d506502d..65d0dd56ca9 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/types/BlockResult.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/types/BlockResult.java @@ -79,10 +79,10 @@ public class BlockResult { @Getter @Setter - private String baseFeePerGas = null; + private String baseFeePerGas = "0x0"; @Getter @Setter - private String mixHash = null; + private String mixHash = ByteArray.toJsonHex(new byte[32]); public BlockResult(Block block, boolean fullTx, Wallet wallet) { BlockCapsule blockCapsule = new BlockCapsule(block); @@ -91,20 +91,26 @@ public BlockResult(Block block, boolean fullTx, Wallet wallet) { hash = ByteArray.toJsonHex(blockCapsule.getBlockId().getBytes()); parentHash = ByteArray.toJsonHex(block.getBlockHeader().getRawData().getParentHash().toByteArray()); - nonce = null; // no value - sha3Uncles = null; // no value + nonce = ByteArray.toJsonHex(new byte[8]); // no value + sha3Uncles = ByteArray.toJsonHex(new byte[32]); // no value logsBloom = ByteArray.toJsonHex(new byte[256]); // no value transactionsRoot = ByteArray .toJsonHex(block.getBlockHeader().getRawData().getTxTrieRoot().toByteArray()); stateRoot = ByteArray .toJsonHex(block.getBlockHeader().getRawData().getAccountStateRoot().toByteArray()); - receiptsRoot = null; // no value - miner = ByteArray.toJsonHexAddress(blockCapsule.getWitnessAddress().toByteArray()); - difficulty = null; // no value - totalDifficulty = null; // no value - extraData = null; // no value + receiptsRoot = ByteArray.toJsonHex(new byte[32]); // no value + + if (blockCapsule.getNum() == 0) { + miner = ByteArray.toJsonHex(new byte[20]); + } else { + miner = ByteArray.toJsonHexAddress(blockCapsule.getWitnessAddress().toByteArray()); + } + + difficulty = "0x0"; // no value + totalDifficulty = "0x0"; // no value + extraData = "0x"; // no value size = ByteArray.toJsonHex(block.getSerializedSize()); - timestamp = ByteArray.toJsonHex(blockCapsule.getTimeStamp()); + timestamp = ByteArray.toJsonHex(blockCapsule.getTimeStamp() / 1000); long gasUsedInBlock = 0; long gasLimitInBlock = 0; diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/types/TransactionResult.java b/framework/src/main/java/org/tron/core/services/jsonrpc/types/TransactionResult.java index 091b8091113..5dd46d71235 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/types/TransactionResult.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/types/TransactionResult.java @@ -15,6 +15,8 @@ import org.tron.protos.Protocol; import org.tron.protos.Protocol.Transaction; import org.tron.protos.Protocol.Transaction.Contract; +import org.tron.protos.Protocol.Transaction.Contract.ContractType; +import org.tron.protos.contract.SmartContractOuterClass.TriggerSmartContract; @JsonPropertyOrder(alphabetic = true) @ToString @@ -56,9 +58,9 @@ public class TransactionResult { private void parseSignature(Transaction tx) { if (tx.getSignatureCount() == 0) { - v = null; - r = null; - s = null; + v = ByteArray.toJsonHex(new byte[1]); + r = ByteArray.toJsonHex(new byte[32]); + s = ByteArray.toJsonHex(new byte[32]); return; } @@ -75,59 +77,89 @@ private void parseSignature(Transaction tx) { s = ByteArray.toJsonHex(sByte); } + private String parseInput(Transaction tx) { + String data; + if (tx.getRawData().getContractCount() == 0) { + data = "0x"; + } else { + Contract contract = tx.getRawData().getContract(0); + if (contract.getType() == ContractType.TriggerSmartContract) { + try { + TriggerSmartContract triggerSmartContract = contract.getParameter() + .unpack(TriggerSmartContract.class); + data = ByteArray.toJsonHex(triggerSmartContract.getData().toByteArray()); + } catch (Exception e) { + data = "0x"; + } + } else { + data = "0x"; + } + } + + return data; + } + public TransactionResult(BlockCapsule blockCapsule, int index, Protocol.Transaction tx, long energyUsageTotal, long energyFee, Wallet wallet) { - byte[] txId = new TransactionCapsule(tx).getTransactionId().getBytes(); + TransactionCapsule capsule = new TransactionCapsule(tx); + byte[] txId = capsule.getTransactionId().getBytes(); hash = ByteArray.toJsonHex(txId); - nonce = null; // no value + nonce = ByteArray.toJsonHex(new byte[8]); // no value blockHash = ByteArray.toJsonHex(blockCapsule.getBlockId().getBytes()); blockNumber = ByteArray.toJsonHex(blockCapsule.getNum()); transactionIndex = ByteArray.toJsonHex(index); if (!tx.getRawData().getContractList().isEmpty()) { Contract contract = tx.getRawData().getContract(0); - byte[] fromByte = TransactionCapsule.getOwner(contract); + byte[] fromByte = capsule.getOwnerAddress(); byte[] toByte = getToAddress(tx); - from = ByteArray.toJsonHexAddress(fromByte); + + if (blockCapsule.getNum() == 0) { + from = ByteArray.toJsonHex(new byte[20]); + } else { + from = ByteArray.toJsonHexAddress(fromByte); + } + to = ByteArray.toJsonHexAddress(toByte); value = ByteArray.toJsonHex(getTransactionAmount(contract, hash, wallet)); } else { - from = null; - to = null; - value = null; + from = ByteArray.toJsonHex(new byte[20]); + to = ByteArray.toJsonHex(new byte[20]); + value = "0x0"; } gas = ByteArray.toJsonHex(energyUsageTotal); gasPrice = ByteArray.toJsonHex(energyFee); - input = ByteArray.toJsonHex(tx.getRawData().getData().toByteArray()); + input = parseInput(tx); parseSignature(tx); } public TransactionResult(Transaction tx, Wallet wallet) { - byte[] txid = new TransactionCapsule(tx).getTransactionId().getBytes(); + TransactionCapsule capsule = new TransactionCapsule(tx); + byte[] txid = capsule.getTransactionId().getBytes(); hash = ByteArray.toJsonHex(txid); - nonce = null; // no value + nonce = ByteArray.toJsonHex(new byte[8]); // no value blockHash = "0x"; blockNumber = "0x"; transactionIndex = "0x"; if (!tx.getRawData().getContractList().isEmpty()) { Contract contract = tx.getRawData().getContract(0); - byte[] fromByte = TransactionCapsule.getOwner(contract); + byte[] fromByte = capsule.getOwnerAddress(); byte[] toByte = getToAddress(tx); from = ByteArray.toJsonHexAddress(fromByte); to = ByteArray.toJsonHexAddress(toByte); value = ByteArray.toJsonHex(getTransactionAmount(contract, hash, wallet)); } else { - from = null; - to = null; - value = null; + from = ByteArray.toJsonHex(new byte[20]); + to = ByteArray.toJsonHex(new byte[20]); + value = "0x0"; } gas = "0x0"; gasPrice = "0x"; - input = ByteArray.toJsonHex(tx.getRawData().getData().toByteArray()); + input = parseInput(tx); parseSignature(tx); } diff --git a/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java b/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java index 037760e201a..ad32214840e 100644 --- a/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java +++ b/framework/src/main/java/org/tron/tool/litefullnode/LiteFullNodeTool.java @@ -4,6 +4,7 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; import com.beust.jcommander.internal.Lists; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import com.google.common.collect.Maps; import com.google.common.primitives.Bytes; @@ -16,9 +17,10 @@ import java.nio.file.Paths; import java.util.Arrays; import java.util.List; +import java.util.NoSuchElementException; import java.util.Objects; import java.util.Optional; -import java.util.stream.LongStream; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.rocksdb.RocksDBException; @@ -42,55 +44,30 @@ public class LiteFullNodeTool { private static final long START_TIME = System.currentTimeMillis() / 1000; + private static long RECENT_BLKS = 65536; + private static final String SNAPSHOT_DIR_NAME = "snapshot"; private static final String HISTORY_DIR_NAME = "history"; private static final String INFO_FILE_NAME = "info.properties"; private static final String BACKUP_DIR_PREFIX = ".bak_"; private static final String CHECKPOINT_DB = "tmp"; - private static final long VM_NEED_RECENT_BLKS = 256; - + private static final String CHECKPOINT_DB_V2 = "checkpoint"; private static final String BLOCK_DB_NAME = "block"; private static final String BLOCK_INDEX_DB_NAME = "block-index"; - private static final String TRANS_CACHE_DB_NAME = "trans-cache"; + private static final String TRANS_DB_NAME = "trans"; private static final String COMMON_DB_NAME = "common"; + private static final String TRANSACTION_RET_DB_NAME = "transactionRetStore"; + private static final String TRANSACTION_HISTORY_DB_NAME = "transactionHistoryStore"; + private static final String PROPERTIES_DB_NAME = "properties"; private static final String DIR_FORMAT_STRING = "%s%s%s"; private static List archiveDbs = Arrays.asList( BLOCK_DB_NAME, BLOCK_INDEX_DB_NAME, - "trans", - "transactionRetStore", - "transactionHistoryStore"); - private static List minimumDbsForLiteNode = Arrays.asList( - "DelegatedResource", - "DelegatedResourceAccountIndex", - "IncrementalMerkleTree", - "account", - "account-index", - "accountTrie", - "accountid-index", - "asset-issue", - "asset-issue-v2", - //"block_KDB", - "code", - //"common", - "contract", - "delegation", - "exchange", - "exchange-v2", - //"nullifier", - "properties", - "proposal", - "recent-block", - //"recent-transaction", - "storage-row", - //TRANS_CACHE_DB_NAME, - //"tree-block-index", - "votes", - "witness", - "witness_schedule" - ); + TRANS_DB_NAME, + TRANSACTION_RET_DB_NAME, + TRANSACTION_HISTORY_DB_NAME); /** * Create the snapshot dataset. @@ -100,24 +77,23 @@ public class LiteFullNodeTool { * @param snapshotDir the path that stores the snapshot dataset */ public void generateSnapshot(String sourceDir, String snapshotDir) { - logger.info("start create snapshot."); + logger.info("Start create snapshot."); long start = System.currentTimeMillis(); snapshotDir = Paths.get(snapshotDir, SNAPSHOT_DIR_NAME).toString(); try { + hasEnoughBlock(sourceDir); List snapshotDbs = getSnapshotDbs(sourceDir); split(sourceDir, snapshotDir, snapshotDbs); mergeCheckpoint2Snapshot(sourceDir, snapshotDir); - // write genesisBlock and latestBlock - fillSnapshotBlockDb(sourceDir, snapshotDir); - // create tran-cache if not exist, for compatible - checkTranCacheStore(sourceDir, snapshotDir); + // write genesisBlock , latest recent blocks and trans + fillSnapshotBlockAndTransDb(sourceDir, snapshotDir); generateInfoProperties(Paths.get(snapshotDir, INFO_FILE_NAME).toString(), sourceDir); } catch (IOException | RocksDBException e) { - logger.error("create snapshot failed, " + e.getMessage()); + logger.error("Create snapshot failed, {}.", e.getMessage()); return; } long end = System.currentTimeMillis(); - logger.info("create snapshot finished, take {}s.", (end - start) / 1000); + logger.info("Create snapshot finished, take {} s.\n", (end - start) / 1000); } /** @@ -128,19 +104,24 @@ public void generateSnapshot(String sourceDir, String snapshotDir) { * @param historyDir the path that stores the history dataset */ public void generateHistory(String sourceDir, String historyDir) { - logger.info("start create history."); + logger.info("Start create history."); long start = System.currentTimeMillis(); historyDir = Paths.get(historyDir, HISTORY_DIR_NAME).toString(); try { + if (isLite(sourceDir)) { + throw new IllegalStateException( + String.format("Unavailable sourceDir: %s is not fullNode data.", sourceDir)); + } + hasEnoughBlock(sourceDir); split(sourceDir, historyDir, archiveDbs); mergeCheckpoint2History(sourceDir, historyDir); generateInfoProperties(Paths.get(historyDir, INFO_FILE_NAME).toString(), sourceDir); } catch (IOException | RocksDBException e) { - logger.error("create history failed, " + e.getMessage()); + logger.error("Create history failed, {}.", e.getMessage()); return; } long end = System.currentTimeMillis(); - logger.info("create history finished, take {}s.\n", (end - start) / 1000); + logger.info("Create history finished, take {} s.\n", (end - start) / 1000); } /** @@ -151,10 +132,16 @@ public void generateHistory(String sourceDir, String historyDir) { * @param databaseDir lite fullnode database path */ public void completeHistoryData(String historyDir, String databaseDir) { - logger.info("start merge history to lite fullnode."); + logger.info("Start merge history to lite node."); long start = System.currentTimeMillis(); BlockNumInfo blockNumInfo = null; try { + // check historyDir is from lite data + if (isLite(historyDir)) { + throw new IllegalStateException( + String.format("Unavailable history: %s is not generated by fullNode data.", + historyDir)); + } // 1. check block number and genesis block are compatible, // and return the block numbers of snapshot and history blockNumInfo = checkAndGetBlockNumInfo(historyDir, databaseDir); @@ -169,11 +156,11 @@ public void completeHistoryData(String historyDir, String databaseDir) { // 6. delete snapshot flag deleteSnapshotFlag(databaseDir); } catch (IOException | RocksDBException | BadItemException e) { - logger.error("merge history data to database failed, " + e.getMessage()); + logger.error("Merge history data to database failed, {}.", e.getMessage()); return; } long end = System.currentTimeMillis(); - logger.info("merge history finished, take {}s \n", (end - start) / 1000); + logger.info("Merge history finished, take {} s. \n", (end - start) / 1000); } private List getSnapshotDbs(String sourceDir) { @@ -183,11 +170,6 @@ private List getSnapshotDbs(String sourceDir) { .filter(File::isDirectory) .filter(dir -> !archiveDbs.contains(dir.getName())) .forEach(dir -> snapshotDbs.add(dir.getName())); - for (String dir : minimumDbsForLiteNode) { - if (!snapshotDbs.contains(dir)) { - throw new RuntimeException("databaseDir does not contain all the necessary databases"); - } - } return snapshotDbs; } @@ -201,85 +183,116 @@ private void mergeCheckpoint2History(String sourceDir, String destDir) { } private void split(String sourceDir, String destDir, List dbs) throws IOException { - logger.info("-- begin to split the dbs."); + logger.info("Begin to split the dbs."); if (!new File(sourceDir).isDirectory()) { - throw new RuntimeException("sourceDir must be a directory, sourceDir: " + sourceDir); + throw new RuntimeException(String.format("sourceDir: %s must be a directory ", sourceDir)); } File destPath = new File(destDir); if (new File(destDir).exists()) { - throw new RuntimeException("destDir is already exist, please remove it first"); + throw new RuntimeException(String.format( + "destDir: %s is already exist, please remove it first", destDir)); } if (!destPath.mkdirs()) { - throw new RuntimeException("destDir create failed, please check"); + throw new RuntimeException(String.format("destDir: %s create failed, please check", destDir)); } Util.copyDatabases(Paths.get(sourceDir), Paths.get(destDir), dbs); } private void mergeCheckpoint(String sourceDir, String destDir, List destDbs) { - logger.info("-- begin to merge checkpoint to dataset"); + logger.info("Begin to merge checkpoint to dataset."); try { - DBInterface tmpDb = DbTool.getDB(sourceDir, CHECKPOINT_DB); - try (DBIterator iterator = tmpDb.iterator()) { - for (iterator.seekToFirst(); iterator.hasNext(); iterator.next()) { - byte[] key = iterator.getKey(); - byte[] value = iterator.getValue(); - String dbName = SnapshotManager.simpleDecode(key); - byte[] realKey = Arrays.copyOfRange(key, dbName.getBytes().length + 4, key.length); - byte[] realValue = value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); - if (destDbs != null && destDbs.contains(dbName)) { - DBInterface destDb = DbTool.getDB(destDir, dbName); - if (realValue != null) { - destDb.put(realKey, realValue); - } else { - destDb.delete(realKey); - } - } + List cpList = getCheckpointV2List(sourceDir); + if (cpList.size() > 0) { + for (String cp: cpList) { + DBInterface checkpointDb = DbTool.getDB(sourceDir + "/" + CHECKPOINT_DB_V2, cp); + recover(checkpointDb, destDir, destDbs); } + } else { + DBInterface tmpDb = DbTool.getDB(sourceDir, CHECKPOINT_DB); + recover(tmpDb, destDir, destDbs); } } catch (IOException | RocksDBException e) { throw new RuntimeException(e); } } + private void recover(DBInterface db, String destDir, List destDbs) + throws IOException, RocksDBException { + try (DBIterator iterator = db.iterator()) { + for (iterator.seekToFirst(); iterator.hasNext(); iterator.next()) { + byte[] key = iterator.getKey(); + byte[] value = iterator.getValue(); + String dbName = SnapshotManager.simpleDecode(key); + byte[] realKey = Arrays.copyOfRange(key, dbName.getBytes().length + 4, key.length); + byte[] realValue = + value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); + if (destDbs != null && destDbs.contains(dbName)) { + DBInterface destDb = DbTool.getDB(destDir, dbName); + if (realValue != null) { + destDb.put(realKey, realValue); + } else { + destDb.delete(realKey); + } + } + } + } + } + private void generateInfoProperties(String propertyfile, String databaseDir) throws IOException, RocksDBException { - logger.info("-- create {} for dataset", INFO_FILE_NAME); + logger.info("Create {} for dataset.", INFO_FILE_NAME); if (!FileUtil.createFileIfNotExists(propertyfile)) { - throw new RuntimeException("create properties file failed..."); + throw new RuntimeException("Create properties file failed."); } if (!PropUtil.writeProperty(propertyfile, Constant.SPLIT_BLOCK_NUM, Long.toString(getLatestBlockHeaderNum(databaseDir)))) { - throw new RuntimeException("write properties file failed..."); + throw new RuntimeException("Write properties file failed."); } } private long getLatestBlockHeaderNum(String databaseDir) throws IOException, RocksDBException { // query latest_block_header_number from checkpoint first final String latestBlockHeaderNumber = "latest_block_header_number"; - byte[] value = DbTool.getDB(databaseDir, CHECKPOINT_DB).get( - Bytes.concat(simpleEncode(CHECKPOINT_DB), latestBlockHeaderNumber.getBytes())); - if (value != null && value.length > 1) { - return ByteArray.toLong(Arrays.copyOfRange(value, 1, value.length)); + List cpList = getCheckpointV2List(databaseDir); + DBInterface checkpointDb = null; + if (cpList.size() > 0) { + String lastestCp = cpList.get(cpList.size() - 1); + checkpointDb = DbTool.getDB(databaseDir + "/" + CHECKPOINT_DB_V2, lastestCp); + } else { + checkpointDb = DbTool.getDB(databaseDir, CHECKPOINT_DB); + } + Long blockNumber = getLatestBlockHeaderNumFromCP(checkpointDb, + latestBlockHeaderNumber.getBytes()); + if (blockNumber != null) { + return blockNumber; } // query from propertiesDb if checkpoint not contains latest_block_header_number - DBInterface propertiesDb = DbTool.getDB(databaseDir, "properties"); + DBInterface propertiesDb = DbTool.getDB(databaseDir, PROPERTIES_DB_NAME); return Optional.ofNullable(propertiesDb.get(ByteArray.fromString(latestBlockHeaderNumber))) .map(ByteArray::toLong) .orElseThrow( () -> new IllegalArgumentException("not found latest block header number")); } + private Long getLatestBlockHeaderNumFromCP(DBInterface db, byte[] key) { + byte[] value = db.get(Bytes.concat(simpleEncode(PROPERTIES_DB_NAME), key)); + if (value != null && value.length > 1) { + return ByteArray.toLong(Arrays.copyOfRange(value, 1, value.length)); + } + return null; + } + /** - * Syncing block from peer that needs latest block and genesis block, - * also VM need recent blocks. + * recent blocks, trans and genesis block. */ - private void fillSnapshotBlockDb(String sourceDir, String snapshotDir) + private void fillSnapshotBlockAndTransDb(String sourceDir, String snapshotDir) throws IOException, RocksDBException { - logger.info("-- begin to fill latest block and genesis block to snapshot"); + logger.info("Begin to fill {} block, genesis block and trans to snapshot.", RECENT_BLKS); DBInterface sourceBlockIndexDb = DbTool.getDB(sourceDir, BLOCK_INDEX_DB_NAME); DBInterface sourceBlockDb = DbTool.getDB(sourceDir, BLOCK_DB_NAME); DBInterface destBlockDb = DbTool.getDB(snapshotDir, BLOCK_DB_NAME); DBInterface destBlockIndexDb = DbTool.getDB(snapshotDir, BLOCK_INDEX_DB_NAME); + DBInterface destTransDb = DbTool.getDB(snapshotDir, TRANS_DB_NAME); // put genesis block and block-index into snapshot long genesisBlockNum = 0L; byte[] genesisBlockID = sourceBlockIndexDb.get(ByteArray.fromLong(genesisBlockNum)); @@ -287,71 +300,33 @@ private void fillSnapshotBlockDb(String sourceDir, String snapshotDir) destBlockDb.put(genesisBlockID, sourceBlockDb.get(genesisBlockID)); long latestBlockNum = getLatestBlockHeaderNum(sourceDir); - long startIndex = latestBlockNum > VM_NEED_RECENT_BLKS - ? latestBlockNum - VM_NEED_RECENT_BLKS : 0; - // put the recent blocks in snapshot, VM needs recent 256 blocks. - LongStream.rangeClosed(startIndex, latestBlockNum).forEach( - blockNum -> { - byte[] blockId = null; - byte[] block = null; - try { - blockId = getDataFromSourceDB(sourceDir, BLOCK_INDEX_DB_NAME, - Longs.toByteArray(blockNum)); - block = getDataFromSourceDB(sourceDir, BLOCK_DB_NAME, blockId); - } catch (IOException | RocksDBException e) { - throw new RuntimeException(e.getMessage()); - } - // put recent blocks index into snapshot - destBlockIndexDb.put(ByteArray.fromLong(blockNum), blockId); - // put latest blocks into snapshot - destBlockDb.put(blockId, block); - }); + long startIndex = latestBlockNum - RECENT_BLKS + 1; + // put the recent blocks and trans in snapshot + for (long blockNum = startIndex; blockNum <= latestBlockNum; blockNum++) { + try { + byte[] blockId = getDataFromSourceDB(sourceDir, BLOCK_INDEX_DB_NAME, + Longs.toByteArray(blockNum)); + byte[] block = getDataFromSourceDB(sourceDir, BLOCK_DB_NAME, blockId); + // put block + destBlockDb.put(blockId, block); + // put block index + destBlockIndexDb.put(ByteArray.fromLong(blockNum), blockId); + // put trans + long finalBlockNum = blockNum; + new BlockCapsule(block).getTransactions().stream().map( + tc -> tc.getTransactionId().getBytes()) + .map(bytes -> Maps.immutableEntry(bytes, Longs.toByteArray(finalBlockNum))) + .forEach(e -> destTransDb.put(e.getKey(), e.getValue())); + } catch (IOException | RocksDBException | BadItemException e) { + throw new RuntimeException(e.getMessage()); + } + } DBInterface destCommonDb = DbTool.getDB(snapshotDir, COMMON_DB_NAME); destCommonDb.put(DB_KEY_NODE_TYPE, ByteArray.fromInt(Constant.NODE_TYPE_LIGHT_NODE)); destCommonDb.put(DB_KEY_LOWEST_BLOCK_NUM, ByteArray.fromLong(startIndex)); } - private void checkTranCacheStore(String sourceDir, String snapshotDir) - throws IOException, RocksDBException { - logger.info("-- create trans-cache db if not exists."); - if (FileUtil.isExists(String.format(DIR_FORMAT_STRING, snapshotDir, - File.separator, TRANS_CACHE_DB_NAME))) { - return; - } - // fullnode is old version, create trans-cache database - DBInterface recentBlockDb = DbTool.getDB(snapshotDir, "recent-block"); - DBInterface transCacheDb = DbTool.getDB(snapshotDir, TRANS_CACHE_DB_NAME); - long headNum = getLatestBlockHeaderNum(sourceDir); - long recentBlockCount = recentBlockDb.size(); - - LongStream.rangeClosed(headNum - recentBlockCount + 1, headNum).forEach( - blockNum -> { - byte[] blockId = null; - byte[] block = null; - try { - blockId = getDataFromSourceDB(sourceDir, BLOCK_INDEX_DB_NAME, - Longs.toByteArray(blockNum)); - block = getDataFromSourceDB(sourceDir, BLOCK_DB_NAME, blockId); - } catch (IOException | RocksDBException e) { - throw new RuntimeException(e.getMessage()); - } - BlockCapsule blockCapsule = null; - try { - blockCapsule = new BlockCapsule(block); - } catch (BadItemException e) { - throw new RuntimeException("construct block failed, num: " + blockNum); - } - if (blockCapsule.getTransactions().isEmpty()) { - return; - } - blockCapsule.getTransactions().stream() - .map(tc -> tc.getTransactionId().getBytes()) - .map(bytes -> Maps.immutableEntry(bytes, Longs.toByteArray(blockNum))) - .forEach(e -> transCacheDb.put(e.getKey(), e.getValue())); - }); - } - private byte[] getGenesisBlockHash(String parentDir) throws IOException, RocksDBException { long genesisBlockNum = 0L; DBInterface blockIndexDb = DbTool.getDB(parentDir, BLOCK_INDEX_DB_NAME); @@ -373,14 +348,14 @@ private static byte[] simpleEncode(String s) { private BlockNumInfo checkAndGetBlockNumInfo(String historyDir, String databaseDir) throws IOException, RocksDBException { - logger.info("-- check the compatibility of this history"); + logger.info("Check the compatibility of this history."); String snapshotInfo = String.format( DIR_FORMAT_STRING, databaseDir, File.separator, INFO_FILE_NAME); String historyInfo = String.format( DIR_FORMAT_STRING, historyDir, File.separator, INFO_FILE_NAME); if (!FileUtil.isExists(snapshotInfo)) { throw new FileNotFoundException( - "snapshot property file is not found. maybe this is a complete fullnode?"); + "Snapshot property file is not found. maybe this is a complete fullnode?"); } if (!FileUtil.isExists(historyInfo)) { throw new FileNotFoundException("history property file is not found."); @@ -390,15 +365,17 @@ private BlockNumInfo checkAndGetBlockNumInfo(String historyDir, String databaseD long historyBlkNum = Long.parseLong(PropUtil.readProperty(historyInfo, Constant .SPLIT_BLOCK_NUM)); if (historyBlkNum < snapshotBlkNum) { - logger.error("history latest block number is lower than snapshot, history: {}, snapshot: {}", - historyBlkNum, snapshotBlkNum); - throw new RuntimeException("history latest block number is lower than snapshot."); + throw new RuntimeException( + String.format( + "History latest block number is lower than snapshot, history: %d, snapshot: %d", + historyBlkNum, snapshotBlkNum)); } // check genesis block is equal if (!Arrays.equals(getGenesisBlockHash(databaseDir), getGenesisBlockHash(historyDir))) { - logger.error("genesis block hash is not equal, history: {}, database: {}", - getGenesisBlockHash(historyDir), getGenesisBlockHash(databaseDir)); - throw new RuntimeException("genesis block is not equal."); + throw new RuntimeException(String.format( + "Genesis block hash is not equal, history: %s, database: %s", + Arrays.toString(getGenesisBlockHash(historyDir)), + Arrays.toString(getGenesisBlockHash(databaseDir)))); } return new BlockNumInfo(snapshotBlkNum, historyBlkNum); } @@ -406,26 +383,26 @@ private BlockNumInfo checkAndGetBlockNumInfo(String historyDir, String databaseD private void backupArchiveDbs(String databaseDir) throws IOException { String bakDir = String.format("%s%s%s%d", databaseDir, File.separator, BACKUP_DIR_PREFIX, START_TIME); - logger.info("-- backup the archive dbs to {}", bakDir); + logger.info("Backup the archive dbs to {}.", bakDir); if (!FileUtil.createDirIfNotExists(bakDir)) { - throw new RuntimeException("create bak dir failed"); + throw new RuntimeException(String.format("create bak dir %s failed", bakDir)); } Util.copyDatabases(Paths.get(databaseDir), Paths.get(bakDir), archiveDbs); archiveDbs.forEach(db -> FileUtil.deleteDir(new File(databaseDir, db))); } private void copyHistory2Database(String historyDir, String databaseDir) throws IOException { - logger.info("-- begin to copy history to database"); + logger.info("Begin to copy history to database."); Util.copyDatabases(Paths.get(historyDir), Paths.get(databaseDir), archiveDbs); } private void trimHistory(String databaseDir, BlockNumInfo blockNumInfo) throws BadItemException, IOException, RocksDBException { - logger.info("-- begin to trim the history data."); + logger.info("Begin to trim the history data."); DBInterface blockIndexDb = DbTool.getDB(databaseDir, BLOCK_INDEX_DB_NAME); DBInterface blockDb = DbTool.getDB(databaseDir, BLOCK_DB_NAME); - DBInterface transDb = DbTool.getDB(databaseDir, "trans"); - DBInterface tranRetDb = DbTool.getDB(databaseDir, "transactionRetStore"); + DBInterface transDb = DbTool.getDB(databaseDir, TRANS_DB_NAME); + DBInterface tranRetDb = DbTool.getDB(databaseDir, TRANSACTION_RET_DB_NAME); for (long n = blockNumInfo.getHistoryBlkNum(); n > blockNumInfo.getSnapshotBlkNum(); n--) { byte[] blockIdHash = blockIndexDb.get(ByteArray.fromLong(n)); BlockCapsule block = new BlockCapsule(blockDb.get(blockIdHash)); @@ -445,7 +422,7 @@ private void trimHistory(String databaseDir, BlockNumInfo blockNumInfo) private void mergeBak2Database(String databaseDir) throws IOException, RocksDBException { String bakDir = String.format("%s%s%s%d", databaseDir, File.separator, BACKUP_DIR_PREFIX, START_TIME); - logger.info("-- begin to merge {} to database", bakDir); + logger.info("Begin to merge {} to database.", bakDir); for (String dbName : archiveDbs) { DBInterface bakDb = DbTool.getDB(bakDir, dbName); DBInterface destDb = DbTool.getDB(databaseDir, dbName); @@ -460,12 +437,15 @@ private void mergeBak2Database(String databaseDir) throws IOException, RocksDBEx private byte[] getDataFromSourceDB(String sourceDir, String dbName, byte[] key) throws IOException, RocksDBException { DBInterface sourceDb = DbTool.getDB(sourceDir, dbName); - DBInterface checkpointDb = DbTool.getDB(sourceDir, "tmp"); - byte[] value = sourceDb.get(key); - if (isEmptyBytes(value)) { - byte[] valueFromTmp = checkpointDb.get(Bytes.concat(simpleEncode(dbName), key)); + DBInterface checkpointDb = DbTool.getDB(sourceDir, CHECKPOINT_DB); + // get data from tmp first. + byte[] valueFromTmp = checkpointDb.get(Bytes.concat(simpleEncode(dbName), key)); + byte[] value; + if (isEmptyBytes(valueFromTmp)) { + value = sourceDb.get(key); + } else { value = valueFromTmp.length == 1 - ? null : Arrays.copyOfRange(valueFromTmp, 1, valueFromTmp.length); + ? null : Arrays.copyOfRange(valueFromTmp, 1, valueFromTmp.length); } if (isEmptyBytes(value)) { throw new RuntimeException(String.format("data not found in store, dbName: %s, key: %s", @@ -487,19 +467,63 @@ private static boolean isEmptyBytes(byte[] b) { } private void deleteSnapshotFlag(String databaseDir) throws IOException, RocksDBException { - logger.info("-- delete the info file."); + logger.info("Delete the info file from {}.", databaseDir); Files.delete(Paths.get(databaseDir, INFO_FILE_NAME)); - DBInterface destBlockIndexDb = DbTool.getDB(databaseDir, BLOCK_INDEX_DB_NAME); - if (destBlockIndexDb.get(ByteArray.fromLong(1)) != null) { + if (!isLite(databaseDir)) { DBInterface destCommonDb = DbTool.getDB(databaseDir, COMMON_DB_NAME); destCommonDb.delete(DB_KEY_NODE_TYPE); destCommonDb.delete(DB_KEY_LOWEST_BLOCK_NUM); - logger.info("-- deleted node_type and lowest_block_num from " - + "common to identify this node is a real fullnode."); + logger.info("Deleted {} and {} from {} to identify this node is a real fullnode.", + "node_type", "lowest_block_num", COMMON_DB_NAME); } } + private void hasEnoughBlock(String sourceDir) throws RocksDBException, IOException { + // check latest + long latest = getLatestBlockHeaderNum(sourceDir); + // check second ,skip 0; + long second = getSecondBlock(sourceDir); + if (latest - second + 1 < RECENT_BLKS) { + throw new NoSuchElementException( + String.format("At least %d blocks in block store, actual latestBlock:%d, firstBlock:%d.", + RECENT_BLKS, latest, second)); + } + } + + private boolean isLite(String databaseDir) throws RocksDBException, IOException { + return getSecondBlock(databaseDir) > 1; + } + + private long getSecondBlock(String databaseDir) throws RocksDBException, IOException { + long num = 0; + DBInterface sourceBlockIndexDb = DbTool.getDB(databaseDir, BLOCK_INDEX_DB_NAME); + DBIterator iterator = sourceBlockIndexDb.iterator(); + iterator.seek(ByteArray.fromLong(1)); + if (iterator.hasNext()) { + num = Longs.fromByteArray(iterator.getKey()); + } + return num; + } + + @VisibleForTesting + public static void setRecentBlks(long recentBlks) { + RECENT_BLKS = recentBlks; + } + + @VisibleForTesting + public static void reSetRecentBlks() { + RECENT_BLKS = 65536; + } + + private List getCheckpointV2List(String sourceDir) { + File file = new File(Paths.get(sourceDir, CHECKPOINT_DB_V2).toString()); + if (file.exists() && file.isDirectory() && file.list() != null) { + return Arrays.stream(file.list()).sorted().collect(Collectors.toList()); + } + return Lists.newArrayList(); + } + private void run(Args argv) { if (StringUtils.isBlank(argv.fnDataPath) || StringUtils.isBlank(argv.datasetPath)) { throw new ParameterException("fnDataPath or datasetPath can't be null"); @@ -544,8 +568,8 @@ public static void main(String[] args) { } else { tool.run(argv); } - } catch (ParameterException parameterException) { - logger.error(parameterException.toString()); + } catch (Exception e) { + logger.error(e.getMessage()); jct.usage(); } } diff --git a/framework/src/main/resources/config-localtest.conf b/framework/src/main/resources/config-localtest.conf index 239b55d7b4f..15488c53743 100644 --- a/framework/src/main/resources/config-localtest.conf +++ b/framework/src/main/resources/config-localtest.conf @@ -51,7 +51,8 @@ storage { // maxOpenFiles = 100 // }, ] - + checkpoint.version = 2 + checkpoint.sync = true } node.discovery = { diff --git a/framework/src/main/resources/config.conf b/framework/src/main/resources/config.conf index aa1f51507bf..6a06f4ed0fe 100644 --- a/framework/src/main/resources/config.conf +++ b/framework/src/main/resources/config.conf @@ -90,10 +90,12 @@ storage { balance.history.lookup = false + # checkpoint.version = 2 + # checkpoint.sync = true + # the estimated number of block transactions (default 1000, min 100, max 10000). # so the total number of cached transactions is 65536 * txCache.estimatedTransactions # txCache.estimatedTransactions = 1000 - } node.discovery = { diff --git a/framework/src/test/java/org/tron/common/overlay/discover/table/NodeEntryTest.java b/framework/src/test/java/org/tron/common/overlay/discover/table/NodeEntryTest.java index 76d2baae9bc..f30b02d3953 100644 --- a/framework/src/test/java/org/tron/common/overlay/discover/table/NodeEntryTest.java +++ b/framework/src/test/java/org/tron/common/overlay/discover/table/NodeEntryTest.java @@ -3,28 +3,67 @@ import org.junit.Assert; import org.junit.Test; import org.tron.common.overlay.discover.node.Node; +import org.tron.common.utils.ByteArray; public class NodeEntryTest { @Test public void test() throws InterruptedException { Node node1 = Node.instanceOf("127.0.0.1:10001"); - NodeEntry nodeEntry = new NodeEntry(node1); - int distance = nodeEntry.getDistance(); - Assert.assertEquals(-256, distance); + NodeEntry nodeEntry = new NodeEntry(Node.getNodeId(), node1); long lastModified = nodeEntry.getModified(); - //System.out.println(lastModified); Thread.sleep(1); nodeEntry.touch(); long nowModified = nodeEntry.getModified(); - //System.out.println(nowModified); Assert.assertNotEquals(lastModified, nowModified); Node node2 = Node.instanceOf("127.0.0.1:10002"); - NodeEntry nodeEntry2 = new NodeEntry(node2); + NodeEntry nodeEntry2 = new NodeEntry(Node.getNodeId(), node2); boolean isDif = nodeEntry.equals(nodeEntry2); Assert.assertTrue(isDif); } + @Test + public void testDistance() { + byte[] randomId = Node.getNodeId(); + String hexRandomIdStr = ByteArray.toHexString(randomId); + Assert.assertEquals(128, hexRandomIdStr.length()); + + byte[] nodeId1 = ByteArray.fromHexString( + "0000000000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000000"); + byte[] nodeId2 = ByteArray.fromHexString( + "a000000000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000000"); + Assert.assertEquals(256, NodeEntry.distance(nodeId1, nodeId2)); + + byte[] nodeId3 = ByteArray.fromHexString( + "0000000000000000000000000000000000000000000000000000000000000001" + + "0000000000000000000000000000000000000000000000000000000000000000"); + Assert.assertEquals(1, NodeEntry.distance(nodeId1, nodeId3)); + + byte[] nodeId4 = ByteArray.fromHexString( + "0000000000000000000000000000000000000000000000000000000000000000" + + "8000000000000000000000000000000000000000000000000000000000000000"); + Assert.assertEquals(0, NodeEntry.distance(nodeId1, nodeId4)); // => 0 + + byte[] nodeId5 = ByteArray.fromHexString( + "0000000000000000000000000000000000000000000000000000000000000000" + + "4000000000000000000000000000000000000000000000000000000000000000"); + Assert.assertEquals(-1, NodeEntry.distance(nodeId1, nodeId5)); // => 0 + + byte[] nodeId6 = ByteArray.fromHexString( + "0000000000000000000000000000000000000000000000000000000000000000" + + "2000000000000000000000000000000000000000000000000000000000000000"); + Assert.assertEquals(-2, NodeEntry.distance(nodeId1, nodeId6)); // => 0 + + byte[] nodeId7 = ByteArray.fromHexString( + "0000000000000000000000000000000000000000000000000000000000000000" + + "0000000000000000000000000000000000000000000000000000000000000001"); + Assert.assertEquals(-255, NodeEntry.distance(nodeId1, nodeId7)); // => 0 + + Assert.assertEquals(-256, NodeEntry.distance(nodeId1, nodeId1)); // => 0 + } + } diff --git a/framework/src/test/java/org/tron/common/overlay/discover/table/NodeTableTest.java b/framework/src/test/java/org/tron/common/overlay/discover/table/NodeTableTest.java index ab0062f5d82..07d52849092 100644 --- a/framework/src/test/java/org/tron/common/overlay/discover/table/NodeTableTest.java +++ b/framework/src/test/java/org/tron/common/overlay/discover/table/NodeTableTest.java @@ -133,6 +133,9 @@ public void dropNodeTest() { Assert.assertTrue(nodeTable.contains(node)); nodeTable.dropNode(node); Assert.assertTrue(!nodeTable.contains(node)); + nodeTable.addNode(node); + nodeTable.dropNode(new Node(ids.get(1), ips[0], 10000, 10000)); + Assert.assertTrue(!nodeTable.contains(node)); } @Test diff --git a/framework/src/test/java/org/tron/common/overlay/discover/table/TimeComparatorTest.java b/framework/src/test/java/org/tron/common/overlay/discover/table/TimeComparatorTest.java index 616ec29b772..b1c3a82f50e 100644 --- a/framework/src/test/java/org/tron/common/overlay/discover/table/TimeComparatorTest.java +++ b/framework/src/test/java/org/tron/common/overlay/discover/table/TimeComparatorTest.java @@ -9,10 +9,10 @@ public class TimeComparatorTest { @Test public void test() throws InterruptedException { Node node1 = Node.instanceOf("127.0.0.1:10001"); - NodeEntry ne1 = new NodeEntry(node1); + NodeEntry ne1 = new NodeEntry(Node.getNodeId(), node1); Thread.sleep(1); Node node2 = Node.instanceOf("127.0.0.1:10002"); - NodeEntry ne2 = new NodeEntry(node2); + NodeEntry ne2 = new NodeEntry(Node.getNodeId(), node2); TimeComparator tc = new TimeComparator(); int result = tc.compare(ne1, ne2); Assert.assertEquals(1, result); diff --git a/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java b/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java index a62c6bbefb9..1b261fd8b98 100644 --- a/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java +++ b/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java @@ -271,7 +271,9 @@ public void testCheckOrInitEngine() { try { dataSource.initDB(); } catch (Exception e) { - Assert.assertTrue(e.getMessage().contains("Failed to")); + Assert.assertEquals(String.format("failed to check database: %s, engine do not match", + "test_engine"), + e.getMessage()); } Assert.assertNull(dataSource.getDatabase()); PropUtil.writeProperty(enginePath, "ENGINE", "ROCKSDB"); diff --git a/framework/src/test/java/org/tron/common/utils/PublicMethod.java b/framework/src/test/java/org/tron/common/utils/PublicMethod.java new file mode 100644 index 00000000000..b61ea6c8c4d --- /dev/null +++ b/framework/src/test/java/org/tron/common/utils/PublicMethod.java @@ -0,0 +1,105 @@ +package org.tron.common.utils; + +import com.google.protobuf.ByteString; +import java.math.BigInteger; +import org.tron.api.GrpcAPI; +import org.tron.api.WalletGrpc; +import org.tron.common.crypto.ECKey; +import org.tron.core.Wallet; +import org.tron.protos.Protocol; +import org.tron.protos.contract.BalanceContract; +import stest.tron.wallet.common.client.utils.TransactionUtils; + +public class PublicMethod { + + /** + * Convert to pub. + * @param priKey private key + * @return public addr + */ + public static byte[] getFinalAddress(String priKey) { + Wallet.setAddressPreFixByte((byte) 0x41); + ECKey key = ECKey.fromPrivate(new BigInteger(priKey, 16)); + return key.getAddress(); + } + + /** + * Transfer TRX. + * @param to addr receives the asset + * @param amount asset amount + * @param owner sender + * @param priKey private key of the sender + * @param blockingStubFull Grpc interface + * @return true or false + */ + public static Boolean sendcoin(byte[] to, long amount, byte[] owner, String priKey, + WalletGrpc.WalletBlockingStub blockingStubFull) { + Wallet.setAddressPreFixByte((byte) 0x41); + ECKey temKey = null; + try { + BigInteger priK = new BigInteger(priKey, 16); + temKey = ECKey.fromPrivate(priK); + } catch (Exception ex) { + ex.printStackTrace(); + } + final ECKey ecKey = temKey; + + int times = 0; + while (times++ <= 2) { + + BalanceContract.TransferContract.Builder builder = + BalanceContract.TransferContract.newBuilder(); + com.google.protobuf.ByteString bsTo = com.google.protobuf.ByteString.copyFrom(to); + com.google.protobuf.ByteString bsOwner = ByteString.copyFrom(owner); + builder.setToAddress(bsTo); + builder.setOwnerAddress(bsOwner); + builder.setAmount(amount); + + BalanceContract.TransferContract contract = builder.build(); + Protocol.Transaction transaction = blockingStubFull.createTransaction(contract); + if (transaction == null || transaction.getRawData().getContractCount() == 0) { + continue; + } + transaction = signTransaction(ecKey, transaction); + GrpcAPI.Return response = broadcastTransaction(transaction, blockingStubFull); + return response.getResult(); + } + return false; + } + + /** + * Sign TX. + * @param ecKey ecKey of the private key + * @param transaction transaction object + */ + public static Protocol.Transaction signTransaction(ECKey ecKey, + Protocol.Transaction transaction) { + if (ecKey == null || ecKey.getPrivKey() == null) { + return null; + } + transaction = TransactionUtils.setTimestamp(transaction); + return TransactionUtils.sign(transaction, ecKey); + } + + /** + * Broadcast TX. + * @param transaction transaction object + * @param blockingStubFull Grpc interface + */ + public static GrpcAPI.Return broadcastTransaction( + Protocol.Transaction transaction, WalletGrpc.WalletBlockingStub blockingStubFull) { + int i = 10; + GrpcAPI.Return response = blockingStubFull.broadcastTransaction(transaction); + while (!response.getResult() && response.getCode() == GrpcAPI.Return.response_code.SERVER_BUSY + && i > 0) { + try { + Thread.sleep(300); + } catch (InterruptedException e) { + e.printStackTrace(); + } + i--; + response = blockingStubFull.broadcastTransaction(transaction); + } + return response; + } +} diff --git a/framework/src/test/java/org/tron/core/db/ManagerTest.java b/framework/src/test/java/org/tron/core/db/ManagerTest.java index 4407f4572aa..3812c2335c5 100755 --- a/framework/src/test/java/org/tron/core/db/ManagerTest.java +++ b/framework/src/test/java/org/tron/core/db/ManagerTest.java @@ -298,7 +298,8 @@ public void adjustBalanceTest() { Assert.assertTrue(false); } catch (BalanceInsufficientException e) { Assert.assertEquals( - StringUtil.createReadableString(account.createDbKey()) + " insufficient balance", + StringUtil.createReadableString(account.createDbKey()) + " insufficient balance" + + ", balance: " + account.getBalance() + ", amount: " + 40, e.getMessage()); } @@ -354,7 +355,8 @@ public void adjustAssetBalanceV2Test() { } catch (BalanceInsufficientException e) { Assert.assertTrue(e instanceof BalanceInsufficientException); Assert.assertEquals( - "reduceAssetAmount failed !", e.getMessage()); + "reduceAssetAmount failed! account: " + StringUtil.encode58Check(account.createDbKey()), + e.getMessage()); } account.setBalance(30); @@ -389,7 +391,8 @@ public void pushBlockInvalidMerkelRoot() { Assert.assertTrue(false); } catch (BadBlockException e) { Assert.assertTrue(e instanceof BadBlockException); - Assert.assertEquals("The merkle hash is not validated", e.getMessage()); + Assert.assertEquals("The merkle hash is not validated for " + + blockCapsule2.getNum(), e.getMessage()); } catch (Exception e) { Assert.assertFalse(e instanceof Exception); } @@ -403,7 +406,7 @@ public void adjustTotalShieldPoolValueTest() { Assert.assertTrue(false); } catch (BalanceInsufficientException e) { Assert.assertTrue(e instanceof BalanceInsufficientException); - Assert.assertEquals("Total shielded pool value can not below 0", e.getMessage()); + Assert.assertEquals("total shielded pool value can not below 0, actual: -1", e.getMessage()); } long beforeTotalShieldValue = chainManager.getDynamicPropertiesStore() @@ -444,7 +447,8 @@ public void pushBlockTooMuchShieldedTransactions() { Assert.assertTrue(false); } catch (BadBlockException e) { Assert.assertTrue(e instanceof BadBlockException); - Assert.assertEquals("shielded transaction count > " + SHIELDED_TRANS_IN_BLOCK_COUNTS, + Assert.assertEquals("num: " + blockCapsule2.getNum() + + ", shielded transaction count > " + SHIELDED_TRANS_IN_BLOCK_COUNTS, e.getMessage()); } catch (Exception e) { Assert.assertFalse(e instanceof Exception); diff --git a/framework/src/test/java/org/tron/core/db2/CheckpointV2Test.java b/framework/src/test/java/org/tron/core/db2/CheckpointV2Test.java new file mode 100644 index 00000000000..dff2d376fd5 --- /dev/null +++ b/framework/src/test/java/org/tron/core/db2/CheckpointV2Test.java @@ -0,0 +1,106 @@ +package org.tron.core.db2; + +import com.google.common.collect.Maps; +import com.google.common.primitives.Bytes; +import com.google.common.primitives.Longs; +import com.google.protobuf.ByteString; +import java.io.File; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.tron.common.application.Application; +import org.tron.common.application.ApplicationFactory; +import org.tron.common.application.TronApplicationContext; +import org.tron.common.utils.FileUtil; +import org.tron.common.utils.Sha256Hash; +import org.tron.core.Constant; +import org.tron.core.capsule.BlockCapsule; +import org.tron.core.config.DefaultConfig; +import org.tron.core.config.args.Args; +import org.tron.core.db2.RevokingDbWithCacheNewValueTest.TestRevokingTronStore; +import org.tron.core.db2.core.Chainbase; +import org.tron.core.db2.core.SnapshotManager; + +@Slf4j +public class CheckpointV2Test { + + private SnapshotManager revokingDatabase; + private TronApplicationContext context; + private Application appT; + private TestRevokingTronStore tronDatabase; + + @Before + public void init() { + Args.setParam(new String[]{"-d", "output_SnapshotManager_test"}, + Constant.TEST_CONF); + Args.getInstance().getStorage().setCheckpointVersion(2); + Args.getInstance().getStorage().setCheckpointSync(true); + context = new TronApplicationContext(DefaultConfig.class); + appT = ApplicationFactory.create(context); + revokingDatabase = context.getBean(SnapshotManager.class); + revokingDatabase.enable(); + tronDatabase = new TestRevokingTronStore("testSnapshotManager-test"); + revokingDatabase.add(tronDatabase.getRevokingDB()); + } + + @After + public void removeDb() { + Args.clearParam(); + context.destroy(); + tronDatabase.close(); + FileUtil.deleteDir(new File("output_SnapshotManager_test")); + revokingDatabase.getCheckTmpStore().close(); + tronDatabase.close(); + } + + @Test + public void testCheckpointV2() { + while (revokingDatabase.size() != 0) { + revokingDatabase.pop(); + } + + revokingDatabase.setMaxFlushCount(0); + revokingDatabase.setUnChecked(false); + revokingDatabase.setMaxSize(0); + List dbList = revokingDatabase.getDbs(); + Map dbMap = dbList.stream() + .map(db -> Maps.immutableEntry(db.getDbName(), db)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + for (int i = 1; i <= 5; i++) { + BlockCapsule blockCapsule = new BlockCapsule(i, Sha256Hash.ZERO_HASH, + System.currentTimeMillis(), ByteString.EMPTY); + try (ISession tmpSession = revokingDatabase.buildSession()) { + dbMap.get("block").put(Longs.toByteArray(i), blockCapsule.getData()); + tmpSession.commit(); + } + } + revokingDatabase.buildSession(); + + Iterator> iterator = dbMap.get("block").iterator(); + Sha256Hash preDbHash = Sha256Hash.ZERO_HASH; + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + byte[] hashBytes = Bytes.concat(entry.getKey(), entry.getValue()); + preDbHash = Sha256Hash.of(true, Bytes.concat(preDbHash.getBytes(), hashBytes)); + } + + revokingDatabase.check(); + revokingDatabase.buildSession(); + + Iterator> iterator2 = dbMap.get("block").iterator(); + Sha256Hash afterDbHash = Sha256Hash.ZERO_HASH; + while (iterator2.hasNext()) { + Map.Entry entry = iterator2.next(); + byte[] hashBytes = Bytes.concat(entry.getKey(), entry.getValue()); + afterDbHash = Sha256Hash.of(true, Bytes.concat(afterDbHash.getBytes(), hashBytes)); + } + + Assert.assertEquals(0, preDbHash.compareTo(afterDbHash)); + } +} diff --git a/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java b/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java index 81681346799..966e947d828 100644 --- a/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java +++ b/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java @@ -1,6 +1,14 @@ package org.tron.core.db2; +import com.google.common.collect.Maps; +import com.google.common.primitives.Bytes; +import com.google.common.primitives.Longs; +import com.google.protobuf.ByteString; import java.io.File; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Assert; @@ -10,11 +18,14 @@ import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; import org.tron.common.utils.FileUtil; +import org.tron.common.utils.Sha256Hash; import org.tron.core.Constant; +import org.tron.core.capsule.BlockCapsule; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; import org.tron.core.db2.RevokingDbWithCacheNewValueTest.TestRevokingTronStore; import org.tron.core.db2.SnapshotRootTest.ProtoCapsuleTest; +import org.tron.core.db2.core.Chainbase; import org.tron.core.db2.core.SnapshotManager; import org.tron.core.exception.BadItemException; import org.tron.core.exception.ItemNotFoundException; @@ -59,11 +70,18 @@ public synchronized void testRefresh() revokingDatabase.setMaxFlushCount(0); revokingDatabase.setUnChecked(false); revokingDatabase.setMaxSize(5); + List dbList = revokingDatabase.getDbs(); + Map dbMap = dbList.stream() + .map(db -> Maps.immutableEntry(db.getDbName(), db)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); ProtoCapsuleTest protoCapsule = new ProtoCapsuleTest("refresh".getBytes()); for (int i = 1; i < 11; i++) { ProtoCapsuleTest testProtoCapsule = new ProtoCapsuleTest(("refresh" + i).getBytes()); try (ISession tmpSession = revokingDatabase.buildSession()) { tronDatabase.put(protoCapsule.getData(), testProtoCapsule); + BlockCapsule blockCapsule = new BlockCapsule(i, Sha256Hash.ZERO_HASH, + System.currentTimeMillis(), ByteString.EMPTY); + dbMap.get("block").put(Longs.toByteArray(i), blockCapsule.getData()); tmpSession.commit(); } } diff --git a/framework/src/test/java/org/tron/core/jsonrpc/JsonrpcServiceTest.java b/framework/src/test/java/org/tron/core/jsonrpc/JsonrpcServiceTest.java index 8ac70c4661e..143f257d85a 100644 --- a/framework/src/test/java/org/tron/core/jsonrpc/JsonrpcServiceTest.java +++ b/framework/src/test/java/org/tron/core/jsonrpc/JsonrpcServiceTest.java @@ -1,6 +1,5 @@ package org.tron.core.jsonrpc; -import com.google.protobuf.Any; import com.google.protobuf.ByteString; import java.io.File; import lombok.extern.slf4j.Slf4j; @@ -230,8 +229,7 @@ public void testGetBlockByNumber() { Assert.assertEquals(ByteArray.toJsonHex(blockCapsule.getNum()), blockResult.getNumber()); Assert .assertEquals(blockCapsule.getTransactions().size(), blockResult.getTransactions().length); - Assert.assertNull(blockResult.getNonce()); - + Assert.assertEquals("0x0000000000000000", blockResult.getNonce()); } diff --git a/framework/src/test/java/org/tron/program/LiteFullNodeToolTest.java b/framework/src/test/java/org/tron/program/LiteFullNodeToolTest.java index 174c708a7c7..4549b2cde4d 100644 --- a/framework/src/test/java/org/tron/program/LiteFullNodeToolTest.java +++ b/framework/src/test/java/org/tron/program/LiteFullNodeToolTest.java @@ -6,21 +6,22 @@ import java.io.File; import java.math.BigInteger; import java.nio.file.Paths; +import org.junit.After; +import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.tron.api.DatabaseGrpc; import org.tron.api.GrpcAPI; import org.tron.api.WalletGrpc; -import org.tron.api.WalletSolidityGrpc; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; import org.tron.common.config.DbBackupConfig; import org.tron.common.crypto.ECKey; import org.tron.common.utils.FileUtil; +import org.tron.common.utils.PublicMethod; import org.tron.common.utils.Utils; import org.tron.core.Wallet; import org.tron.core.config.DefaultConfig; @@ -37,12 +38,7 @@ public class LiteFullNodeToolTest { private static final Logger logger = LoggerFactory.getLogger("Test"); private TronApplicationContext context; - private ManagedChannel channelFull = null; private WalletGrpc.WalletBlockingStub blockingStubFull = null; - private WalletSolidityGrpc.WalletSolidityBlockingStub blockingStubSolidity = null; - private DatabaseGrpc.DatabaseBlockingStub databaseBlockingStub = null; - private RpcApiService rpcApiService; - private RpcApiServiceOnSolidity rpcApiServiceOnSolidity; private Application appTest; private String databaseDir; @@ -50,38 +46,40 @@ public class LiteFullNodeToolTest { @Rule public ExpectedException thrown = ExpectedException.none(); + + private static final String DB_PATH = "output_lite_fn"; + /** * init logic. */ public void startApp() { context = new TronApplicationContext(DefaultConfig.class); appTest = ApplicationFactory.create(context); - rpcApiService = context.getBean(RpcApiService.class); - rpcApiServiceOnSolidity = context.getBean(RpcApiServiceOnSolidity.class); - appTest.addService(rpcApiService); - appTest.addService(rpcApiServiceOnSolidity); + appTest.addService(context.getBean(RpcApiService.class)); + appTest.addService(context.getBean(RpcApiServiceOnSolidity.class)); appTest.initServices(Args.getInstance()); appTest.startServices(); appTest.startup(); String fullnode = String.format("%s:%d", "127.0.0.1", Args.getInstance().getRpcPort()); - channelFull = ManagedChannelBuilder.forTarget(fullnode) + ManagedChannel channelFull = ManagedChannelBuilder.forTarget(fullnode) .usePlaintext(true) .build(); blockingStubFull = WalletGrpc.newBlockingStub(channelFull); - blockingStubSolidity = WalletSolidityGrpc.newBlockingStub(channelFull); - databaseBlockingStub = DatabaseGrpc.newBlockingStub(channelFull); } /** * Delete the database when exit. */ - public static void destory(String dbPath) { - if (FileUtil.deleteDir(new File(dbPath))) { - logger.info("Release resources successful."); - } else { - logger.info("Release resources failure."); + public static void destroy(String dbPath) { + File f = new File(dbPath); + if (f.exists()) { + if (FileUtil.deleteDir(f)) { + logger.info("Release resources successful."); + } else { + logger.info("Release resources failure."); + } } } @@ -94,84 +92,100 @@ public void shutdown() { context.destroy(); } - @Test - public void testToolsWithLevelDB() { - String dbPath = "output_lite_fn_leveldb_test"; - Args.setParam(new String[]{"-d", dbPath, "-w"}, "config-localtest.conf"); + @Before + public void init() { + destroy(DB_PATH); // delete if prev failed + Args.setParam(new String[]{"-d", DB_PATH, "-w"}, "config-localtest.conf"); // allow account root Args.getInstance().setAllowAccountStateRoot(1); databaseDir = Args.getInstance().getStorage().getDbDirectory(); - testTools("LEVELDB", dbPath); - destory(dbPath); + // init dbBackupConfig to avoid NPE + Args.getInstance().dbBackupConfig = DbBackupConfig.getInstance(); + } + + @After + public void clear() { + destroy(DB_PATH); + Args.clearParam(); + } + + @Test + public void testToolsWithLevelDB() { + logger.info("testToolsWithLevelDB start"); + testTools("LEVELDB", 1); + } + + @Test + public void testToolsWithLevelDBV2() { + logger.info("testToolsWithLevelDB start"); + testTools("LEVELDB", 2); } @Test public void testToolsWithRocksDB() { - String dbPath = "output_lite_fn_rocksdb_test"; - Args.setParam(new String[]{"-d", dbPath, "-w"}, "config-localtest.conf"); - // allow account root - Args.getInstance().setAllowAccountStateRoot(1); - databaseDir = Args.getInstance().getStorage().getDbDirectory(); - // init dbBackupConfig to avoid NPE - Args.getInstance().dbBackupConfig = DbBackupConfig.getInstance(); - testTools("ROCKSDB", dbPath); - destory(dbPath); + logger.info("testToolsWithRocksDB start"); + testTools("ROCKSDB", 1); } - private void testTools(String dbType, String dbPath) { + private void testTools(String dbType, int checkpointVersion) { final String[] argsForSnapshot = new String[]{"-o", "split", "-t", "snapshot", "--fn-data-path", - dbPath + File.separator + databaseDir, "--dataset-path", dbPath}; + DB_PATH + File.separator + databaseDir, "--dataset-path", + DB_PATH}; final String[] argsForHistory = new String[]{"-o", "split", "-t", "history", "--fn-data-path", - dbPath + File.separator + databaseDir, "--dataset-path", dbPath}; + DB_PATH + File.separator + databaseDir, "--dataset-path", + DB_PATH}; final String[] argsForMerge = - new String[]{"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir, - "--dataset-path", dbPath + File.separator + "history"}; + new String[]{"-o", "merge", "--fn-data-path", DB_PATH + File.separator + databaseDir, + "--dataset-path", DB_PATH + File.separator + "history"}; Args.getInstance().getStorage().setDbEngine(dbType); + Args.getInstance().getStorage().setCheckpointVersion(checkpointVersion); + LiteFullNodeTool.setRecentBlks(3); // start fullnode startApp(); - // produce transactions for 10 seconds - generateSomeTransactions(10); + // produce transactions for 18 seconds + generateSomeTransactions(18); // stop the node shutdown(); // delete tran-cache - FileUtil.deleteDir(Paths.get(dbPath, databaseDir, "trans-cache").toFile()); + FileUtil.deleteDir(Paths.get(DB_PATH, databaseDir, "trans-cache").toFile()); // generate snapshot LiteFullNodeTool.main(argsForSnapshot); // start fullnode startApp(); - // produce transactions for 10 seconds - generateSomeTransactions(4); + // produce transactions for 6 seconds + generateSomeTransactions(6); // stop the node shutdown(); // generate history LiteFullNodeTool.main(argsForHistory); // backup original database to database_bak - File database = new File(Paths.get(dbPath, databaseDir).toString()); - if (!database.renameTo(new File(Paths.get(dbPath, databaseDir + "_bak").toString()))) { + File database = new File(Paths.get(DB_PATH, databaseDir).toString()); + if (!database.renameTo(new File(Paths.get(DB_PATH, databaseDir + "_bak").toString()))) { throw new RuntimeException( String.format("rename %s to %s failed", database.getPath(), - Paths.get(dbPath, databaseDir).toString())); + Paths.get(DB_PATH, databaseDir).toString())); } // change snapshot to the new database - File snapshot = new File(Paths.get(dbPath, "snapshot").toString()); - if (!snapshot.renameTo(new File(Paths.get(dbPath, databaseDir).toString()))) { + File snapshot = new File(Paths.get(DB_PATH, "snapshot").toString()); + if (!snapshot.renameTo(new File(Paths.get(DB_PATH, databaseDir).toString()))) { throw new RuntimeException( String.format("rename snapshot to %s failed", - Paths.get(dbPath, databaseDir).toString())); + Paths.get(DB_PATH, databaseDir).toString())); } // start and validate the snapshot startApp(); - generateSomeTransactions(4); + generateSomeTransactions(6); // stop the node shutdown(); // merge history LiteFullNodeTool.main(argsForMerge); // start and validate startApp(); - generateSomeTransactions(4); + generateSomeTransactions(6); shutdown(); + LiteFullNodeTool.reSetRecentBlks(); } private void generateSomeTransactions(int during) { @@ -183,8 +197,8 @@ private void generateSomeTransactions(int during) { byte[] address = ecKey2.getAddress(); String sunPri = "cba92a516ea09f620a16ff7ee95ce0df1d56550a8babe9964981a7144c8a784a"; - byte[] sunAddress = getFinalAddress(sunPri); - sendcoin(address, 1L, + byte[] sunAddress = PublicMethod.getFinalAddress(sunPri); + PublicMethod.sendcoin(address, 1L, sunAddress, sunPri, blockingStubFull); try { Thread.sleep(sleepOnce); @@ -196,97 +210,4 @@ private void generateSomeTransactions(int during) { } } } - - /** - * Set public for future use. - * @param priKey private key - * @return public addr - */ - public static byte[] getFinalAddress(String priKey) { - Wallet.setAddressPreFixByte((byte) 0x41); - ECKey key = ECKey.fromPrivate(new BigInteger(priKey, 16)); - return key.getAddress(); - } - - /** - * Set public for future use. - * @param to addr receives the asset - * @param amount asset amount - * @param owner sender - * @param priKey private key of the sender - * @param blockingStubFull Grpc interface - * @return true or false - */ - public static Boolean sendcoin(byte[] to, long amount, byte[] owner, String priKey, - WalletGrpc.WalletBlockingStub blockingStubFull) { - Wallet.setAddressPreFixByte((byte) 0x41); - ECKey temKey = null; - try { - BigInteger priK = new BigInteger(priKey, 16); - temKey = ECKey.fromPrivate(priK); - } catch (Exception ex) { - ex.printStackTrace(); - } - final ECKey ecKey = temKey; - - int times = 0; - while (times++ <= 2) { - - BalanceContract.TransferContract.Builder builder = - BalanceContract.TransferContract.newBuilder(); - ByteString bsTo = ByteString.copyFrom(to); - ByteString bsOwner = ByteString.copyFrom(owner); - builder.setToAddress(bsTo); - builder.setOwnerAddress(bsOwner); - builder.setAmount(amount); - - BalanceContract.TransferContract contract = builder.build(); - Protocol.Transaction transaction = blockingStubFull.createTransaction(contract); - if (transaction == null || transaction.getRawData().getContractCount() == 0) { - continue; - } - transaction = signTransaction(ecKey, transaction); - GrpcAPI.Return response = broadcastTransaction(transaction, blockingStubFull); - return response.getResult(); - } - return false; - } - - /** - * Set public for future use. - * @param ecKey ecKey of the private key - * @param transaction transaction object - */ - public static Protocol.Transaction signTransaction(ECKey ecKey, - Protocol.Transaction transaction) { - if (ecKey == null || ecKey.getPrivKey() == null) { - logger.warn("Warning: Can't sign,there is no private key !!"); - return null; - } - transaction = TransactionUtils.setTimestamp(transaction); - return TransactionUtils.sign(transaction, ecKey); - } - - /** - * Set public for future use. - * @param transaction transaction object - * @param blockingStubFull Grpc interface - */ - public static GrpcAPI.Return broadcastTransaction( - Protocol.Transaction transaction, WalletGrpc.WalletBlockingStub blockingStubFull) { - int i = 10; - GrpcAPI.Return response = blockingStubFull.broadcastTransaction(transaction); - while (!response.getResult() && response.getCode() == GrpcAPI.Return.response_code.SERVER_BUSY - && i > 0) { - try { - Thread.sleep(300); - } catch (InterruptedException e) { - e.printStackTrace(); - } - i--; - response = blockingStubFull.broadcastTransaction(transaction); - } - return response; - } - } diff --git a/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java b/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java index 5771323138f..144d0c45579 100644 --- a/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java +++ b/plugins/src/main/java/org/tron/plugins/ArchiveManifest.java @@ -108,6 +108,11 @@ public static int run(String[] args) { return 404; } + if (!dbDirectory.isDirectory()) { + logger.info("{} is not directory.", parameters.databaseDirectory); + return 405; + } + List files = Arrays.stream(Objects.requireNonNull(dbDirectory.listFiles())) .filter(File::isDirectory).collect( Collectors.toList()); diff --git a/plugins/src/main/java/org/tron/plugins/Db.java b/plugins/src/main/java/org/tron/plugins/Db.java index e03a412c1a9..702f505b147 100644 --- a/plugins/src/main/java/org/tron/plugins/Db.java +++ b/plugins/src/main/java/org/tron/plugins/Db.java @@ -12,7 +12,7 @@ mixinStandardHelpOptions = true, version = "db command 1.0", description = "An rich command set that provides high-level operations for dbs.", - subcommands = {CommandLine.HelpCommand.class, DbMove.class}, + subcommands = {CommandLine.HelpCommand.class, DbMove.class, DbArchive.class}, commandListHeading = "%nCommands:%n%nThe most commonly used git commands are:%n" ) public class Db { diff --git a/plugins/src/main/java/org/tron/plugins/DbArchive.java b/plugins/src/main/java/org/tron/plugins/DbArchive.java new file mode 100644 index 00000000000..333098949fe --- /dev/null +++ b/plugins/src/main/java/org/tron/plugins/DbArchive.java @@ -0,0 +1,248 @@ +package org.tron.plugins; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.iq80.leveldb.impl.Iq80DBFactory.factory; + +import java.io.BufferedInputStream; +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Properties; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import me.tongfei.progressbar.ProgressBar; +import org.iq80.leveldb.CompressionType; +import org.iq80.leveldb.DB; +import org.iq80.leveldb.Options; +import org.iq80.leveldb.impl.Filename; +import picocli.CommandLine; +import picocli.CommandLine.Option; + +@Slf4j(topic = "archive") +@CommandLine.Command(name = "archive", description = "a helper to rewrite leveldb manifest.") +public class DbArchive implements Callable { + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + + @Option(names = {"-d", "--database-directory"}, + defaultValue = "output-directory/database", + description = "java-tron database directory. Default: ${DEFAULT-VALUE}") + private String databaseDirectory; + + @Option(names = {"-b", "--batch-size"}, + defaultValue = "80000", + description = "deal manifest batch size. Default: ${DEFAULT-VALUE}") + private int maxBatchSize; + + @Option(names = {"-m", "--manifest-size"}, + defaultValue = "0", + description = "manifest min size(M) to archive. Default: ${DEFAULT-VALUE}") + private int maxManifestSize; + + @Option(names = {"-h", "--help"}, help = true) + private boolean help; + + + @Override + public Integer call() throws Exception { + if (help) { + spec.commandLine().usage(System.out); + return 0; + } + + File dbDirectory = new File(databaseDirectory); + if (!dbDirectory.exists()) { + spec.commandLine().getErr().format("Directory %s does not exist.", + databaseDirectory).println(); + logger.info("Directory {} does not exist.", databaseDirectory); + return 404; + } + + if (!dbDirectory.isDirectory()) { + spec.commandLine().getErr().format(" %s is not directory.", + databaseDirectory).println(); + logger.info("{} is not directory.", databaseDirectory); + return 405; + } + + List files = Arrays.stream(Objects.requireNonNull(dbDirectory.listFiles())) + .filter(File::isDirectory).collect( + Collectors.toList()); + + if (files.isEmpty()) { + spec.commandLine().getErr().format("Directory %s does not contain any database.", + databaseDirectory).println(); + logger.info("Directory {} does not contain any database.", databaseDirectory); + return 0; + } + final long time = System.currentTimeMillis(); + List services = new ArrayList<>(); + files.forEach(f -> services.add(new ArchiveManifest(databaseDirectory, f.getName(), + maxManifestSize, maxBatchSize))); + ProgressBar.wrap(services.stream(), "archive task").parallel().forEach(Archive::doArchive); + spec.commandLine().getOut().println("archive db done."); + + logger.info("DatabaseDirectory:{}, maxManifestSize:{}, maxBatchSize:{}," + + "database reopen use {} seconds total.", + databaseDirectory, maxManifestSize, maxBatchSize, + (System.currentTimeMillis() - time) / 1000); + + return 0; + } + + + interface Archive { + + default void doArchive() { + + } + } + + static class ArchiveManifest implements Archive { + + private static final String KEY_ENGINE = "ENGINE"; + private static final String LEVELDB = "LEVELDB"; + + private final Path srcDbPath; + private final String name; + private final Options options; + private final long startTime; + + public ArchiveManifest(String src, String name, int maxManifestSize, int maxBatchSize) { + this.name = name; + this.srcDbPath = Paths.get(src, name); + this.startTime = System.currentTimeMillis(); + this.options = newDefaultLevelDbOptions(); + this.options.maxManifestSize(maxManifestSize); + this.options.maxBatchSize(maxBatchSize); + } + + public static Options newDefaultLevelDbOptions() { + Options dbOptions = new Options(); + dbOptions.createIfMissing(true); + dbOptions.paranoidChecks(true); + dbOptions.verifyChecksums(true); + dbOptions.compressionType(CompressionType.SNAPPY); + dbOptions.blockSize(4 * 1024); + dbOptions.writeBufferSize(10 * 1024 * 1024); + dbOptions.cacheSize(10 * 1024 * 1024L); + dbOptions.maxOpenFiles(1000); + dbOptions.maxBatchSize(64_000); + dbOptions.maxManifestSize(128); + return dbOptions; + } + + public void open() throws IOException { + DB database = factory.open(this.srcDbPath.toFile(), this.options); + database.close(); + } + + public boolean checkManifest(String dir) throws IOException { + // Read "CURRENT" file, which contains a pointer to the current manifest file + File currentFile = new File(dir, Filename.currentFileName()); + if (!currentFile.exists()) { + return false; + } + String currentName = com.google.common.io.Files.asCharSource(currentFile, UTF_8).read(); + if (currentName.isEmpty() || currentName.charAt(currentName.length() - 1) != '\n') { + return false; + } + currentName = currentName.substring(0, currentName.length() - 1); + File current = new File(dir, currentName); + if (!current.isFile()) { + return false; + } + long maxSize = options.maxManifestSize(); + if (maxSize < 0) { + return false; + } + logger.info("CurrentName {}/{},size {} kb.", dir, currentName, current.length() / 1024); + if ("market_pair_price_to_order".equalsIgnoreCase(this.name)) { + logger.info("Db {} ignored.", this.name); + return false; + } + return current.length() >= maxSize * 1024 * 1024; + } + + @Override + public void doArchive() { + File levelDbFile = srcDbPath.toFile(); + if (!levelDbFile.exists()) { + logger.info("File {},does not exist, ignored.", srcDbPath); + return; + } + if (!checkEngine()) { + logger.info("Db {},not leveldb, ignored.", this.name); + return; + } + try { + if (checkManifest(levelDbFile.toString())) { + open(); + logger.info("Db {} archive use {} ms.", this.name, + (System.currentTimeMillis() - startTime)); + } else { + logger.info("Db {},no need, ignored.", levelDbFile); + } + } catch (Exception e) { + throw new RuntimeException("Db " + this.name + " archive failed.", e); + } + } + + public boolean checkEngine() { + String dir = this.srcDbPath.toString(); + String enginePath = dir + File.separator + "engine.properties"; + if (!new File(enginePath).exists() && !writeProperty(enginePath, KEY_ENGINE, LEVELDB)) { + return false; + } + String engine = readProperty(enginePath, KEY_ENGINE); + return LEVELDB.equals(engine); + } + + public static String readProperty(String file, String key) { + try (FileInputStream fileInputStream = new FileInputStream(file); + InputStream inputStream = new BufferedInputStream(fileInputStream)) { + Properties prop = new Properties(); + prop.load(inputStream); + return new String(prop.getProperty(key, "").getBytes(StandardCharsets.ISO_8859_1), + UTF_8); + } catch (Exception e) { + logger.error("readProperty", e); + return ""; + } + } + + public static boolean writeProperty(String file, String key, String value) { + try (OutputStream out = new FileOutputStream(file); + FileInputStream fis = new FileInputStream(file); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, UTF_8))) { + BufferedReader bf = new BufferedReader(new InputStreamReader(fis, UTF_8)); + Properties properties = new Properties(); + properties.load(bf); + properties.setProperty(key, value); + properties.store(bw, "Generated by the application. PLEASE DO NOT EDIT! "); + } catch (Exception e) { + logger.warn("writeProperty", e); + return false; + } + return true; + } + + } + +} \ No newline at end of file diff --git a/plugins/src/test/java/org/tron/plugins/DbArchiveTest.java b/plugins/src/test/java/org/tron/plugins/DbArchiveTest.java new file mode 100644 index 00000000000..10bed418764 --- /dev/null +++ b/plugins/src/test/java/org/tron/plugins/DbArchiveTest.java @@ -0,0 +1,135 @@ +package org.tron.plugins; + +import static org.iq80.leveldb.impl.Iq80DBFactory.factory; + +import java.io.BufferedReader; +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.nio.file.Paths; +import java.util.Properties; +import java.util.UUID; +import lombok.extern.slf4j.Slf4j; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import picocli.CommandLine; + +@Slf4j +public class DbArchiveTest { + + private static final String OUTPUT_DIRECTORY = "output-directory/database/dbArchive"; + + private static final String ENGINE = "ENGINE"; + private static final String LEVELDB = "LEVELDB"; + private static final String ROCKSDB = "ROCKSDB"; + private static final String ACCOUNT = "account"; + private static final String ACCOUNT_ROCKSDB = "account-rocksdb"; + private static final String MARKET = "market_pair_price_to_order"; + private static final String ENGINE_FILE = "engine.properties"; + + @BeforeClass + public static void init() throws IOException { + File file = new File(OUTPUT_DIRECTORY,ACCOUNT); + factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); + writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + + file = new File(OUTPUT_DIRECTORY,MARKET); + factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); + writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + + file = new File(OUTPUT_DIRECTORY,ACCOUNT_ROCKSDB); + factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); + writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,ROCKSDB); + + } + + @AfterClass + public static void destroy() { + deleteDir(new File(OUTPUT_DIRECTORY)); + } + + @Test + public void testRun() { + String[] args = new String[] {"db", "archive", "-d", OUTPUT_DIRECTORY }; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testHelp() { + String[] args = new String[] {"db", "archive", "-h"}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testMaxManifest() { + String[] args = new String[] {"db", "archive", "-d", OUTPUT_DIRECTORY, "-m", "128"}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testNotExist() { + String[] args = new String[] {"db", "archive", "-d", + OUTPUT_DIRECTORY + File.separator + UUID.randomUUID()}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(404, cli.execute(args)); + } + + @Test + public void testEmpty() { + File file = new File(OUTPUT_DIRECTORY + File.separator + UUID.randomUUID()); + file.mkdirs(); + file.deleteOnExit(); + String[] args = new String[] {"db", "archive", "-d", file.toString()}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(0, cli.execute(args)); + } + + private static void writeProperty(String filename, String key, String value) throws IOException { + File file = new File(filename); + if (!file.exists()) { + file.createNewFile(); + } + + try (FileInputStream fis = new FileInputStream(file); + OutputStream out = new FileOutputStream(file); + BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(out, + StandardCharsets.UTF_8))) { + BufferedReader bf = new BufferedReader(new InputStreamReader(fis, StandardCharsets.UTF_8)); + Properties properties = new Properties(); + properties.load(bf); + properties.setProperty(key, value); + properties.store(bw, "Generated by the application. PLEASE DO NOT EDIT! "); + } catch (Exception e) { + logger.warn("{}", e); + } + } + + /** + * delete directory. + */ + private static boolean deleteDir(File dir) { + if (dir.isDirectory()) { + String[] children = dir.list(); + assert children != null; + for (String child : children) { + boolean success = deleteDir(new File(dir, child)); + if (!success) { + logger.warn("can't delete dir:" + dir); + return false; + } + } + } + return dir.delete(); + } +}