Compare commits

...
14 Commits
Author SHA1 Message Date
Brannon King eba414966c Add hard fork to include more claim data
Signed-off-by: Anthony Fieroni <bvbfan@abv.bg>
2020-06-29 19:56:25 +03:00
Brannon King 33edd67d66 Merge pull request #391 from lbryio/merkle_improve_v19 2020-06-29 07:29:10 -06:00
Brannon King 701cab7df3 scaled back the dbcache change 2020-06-25 09:52:49 +03:00
Brannon King 909bb331d0 use a better index on support table 2020-06-25 09:52:07 +03:00
Anthony Fieroni 2248bfacec Increase default db cache, optimize db before sync
Signed-off-by: Anthony Fieroni <bvbfan@abv.bg>
2020-06-17 15:02:47 +03:00
Anthony Fieroni b7e26d3e75 Add claims hash to node table
Signed-off-by: Anthony Fieroni <bvbfan@abv.bg>
2020-06-17 15:02:29 +03:00
Brannon King 68ecea3fdb made the functional test workaround not break testnet 2020-04-22 08:14:18 -06:00
Brannon King 8028a44de5 Merge branch 'block_filter_in_sqlite' into v19_master 2020-04-22 06:17:32 -06:00
Brannon King 50bf41e9c0 keep the block filter entirely in sqlite
and make it work right, fixing issue 383
2020-04-21 07:02:15 -06:00
Brannon King f1d8061534 repair unintended hard fork at 745383
adjustments to hf sync


fixed compilation
2020-04-20 07:33:36 -06:00
Brannon King 165ab09a7d reset testnet 2020-04-20 07:03:51 -06:00
Brannon King 012fb0718f fix max block filesize param int parsing 2020-04-19 10:42:42 -06:00
Brannon King 905f744baf fixed missing flags in getblocktemplate 2020-04-19 10:34:43 -06:00
Brannon King 52d1eddadc fix max block filesize 2020-04-18 11:00:57 -06:00
38 changed files with 882 additions and 530 deletions

No files matched your search

+1 -1
View File
@@ -3,7 +3,7 @@ AC_PREREQ([2.60])
define(_CLIENT_VERSION_MAJOR, 0)
define(_CLIENT_VERSION_MINOR, 19)
define(_CLIENT_VERSION_REVISION, 1)
define(_CLIENT_VERSION_BUILD, 1)
define(_CLIENT_VERSION_BUILD, 2)
define(_CLIENT_VERSION_RC, 0)
define(_CLIENT_VERSION_IS_RELEASE, true)
define(_COPYRIGHT_YEAR, 2020)
@@ -35,6 +35,7 @@ function set_config() {
override_config_option RPC_ALLOW_IP rpcallowip $MERGED_CONFIG
override_config_option RPC_PORT rpcport $MERGED_CONFIG
override_config_option RPC_BIND rpcbind $MERGED_CONFIG
override_config_option TX_INDEX txindex $MERGED_CONFIG
override_config_option MAX_TX_FEE maxtxfee $MERGED_CONFIG
override_config_option DUST_RELAY_FEE dustrelayfee $MERGED_CONFIG
# Make the new merged config file the new CONFIG_PATH
+1
View File
@@ -121,6 +121,7 @@ BITCOIN_TESTS =\
test/claimtrieexpirationfork_tests.cpp \
test/claimtriefixture.cpp \
test/claimtriehashfork_tests.cpp \
test/claimtrieclaiminfo_tests.cpp \
test/claimtrienormalization_tests.cpp \
test/claimtrierpc_tests.cpp \
test/nameclaim_tests.cpp \
+23 -20
View File
@@ -136,8 +136,9 @@ public:
consensus.nAllowMinDiffMaxHeight = -1;
consensus.nNormalizedNameForkHeight = 539940; // targeting 21 March 2019
consensus.nMinRemovalWorkaroundHeight = 297706;
consensus.nMaxRemovalWorkaroundHeight = 100000000;
consensus.nMaxRemovalWorkaroundHeight = 658300;
consensus.nAllClaimsInMerkleForkHeight = 658310; // targeting 30 Oct 2019
consensus.nClaimInfoInMerkleForkHeight = 900000; // FIXME pick height
consensus.fPowAllowMinDifficultyBlocks = false;
consensus.fPowNoRetargeting = false;
consensus.nRuleChangeActivationThreshold = 1916; // 95% of a half week
@@ -147,10 +148,10 @@ public:
consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nTimeout = 1230767999; // December 31, 2008
// The best chain should have at least this much work.
consensus.nMinimumChainWork = uint256S("0000000000000000000000000000000000000000000002bfdb5232f364d6774e"); //700k
consensus.nMinimumChainWork = uint256S("0000000000000000000000000000000000000000000003253077412df5b49766"); //749k
// By default assume that the signatures in ancestors of this block are valid.
consensus.defaultAssumeValid = uint256S("beaf6432c9a7be3ea8c333bd7a90d4b3e07b0f20c86aa2e5dfebc9eba340201c"); //700k
consensus.defaultAssumeValid = uint256S("b9676f45be594438a2011407c93bb530d817fa365846e7a6ecdf2790e4a0ad6b"); //749k
/**
* The message start string is designed to be unlikely to occur in normal data.
@@ -227,26 +228,27 @@ public:
consensus.nMajorityRejectBlockOutdated = 75;
consensus.nMajorityWindow = 100;
consensus.BIP16Exception = uint256S("0x00000000dd30457c001f4095d208cc1296b0eed002427aa599874af7a432b105");
consensus.BIP34Height = 21111;
consensus.BIP34Hash = uint256S("0x0000000023b3a96d3484e5abb3755c413e7d41500f8e2a5c3f0dd01299cd8ef8");
consensus.BIP34Height = 14;
consensus.BIP34Hash = uint256S("0x079557d16edcd640c4057c9fddb81257263014fe384c4aa348c5b9d190650a46");
// FIXME: adjust heights
consensus.BIP65Height = 1200000;
consensus.BIP66Height = 1200000;
consensus.BIP65Height = 120;
consensus.BIP66Height = 120;
consensus.powLimit = uint256S("0000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff");
consensus.nPowTargetTimespan = 150;
consensus.CSVHeight = 1200000;
consensus.SegwitHeight = 1198600;
consensus.MinBIP9WarningHeight = 1200616; // segwit activation height + miner confirmation window
consensus.CSVHeight = 0;
consensus.SegwitHeight = 120;
consensus.MinBIP9WarningHeight = 2136; // segwit activation height + miner confirmation window
consensus.nPowTargetSpacing = 150;
consensus.nOriginalClaimExpirationTime = 262974;
consensus.nExtendedClaimExpirationTime = 2102400;
consensus.nExtendedClaimExpirationForkHeight = 278160;
consensus.nAllowMinDiffMinHeight = 277299;
consensus.nAllowMinDiffMaxHeight = 1100000;
consensus.nNormalizedNameForkHeight = 993380; // targeting, 21 Feb 2019
consensus.nExtendedClaimExpirationForkHeight = 1;
consensus.nAllowMinDiffMinHeight = 1;
consensus.nAllowMinDiffMaxHeight = 2;
consensus.nNormalizedNameForkHeight = 1;
consensus.nMinRemovalWorkaroundHeight = 99;
consensus.nMaxRemovalWorkaroundHeight = 100000000;
consensus.nAllClaimsInMerkleForkHeight = 1198560; // targeting 30 Sep 2019
consensus.nMaxRemovalWorkaroundHeight = 100;
consensus.nAllClaimsInMerkleForkHeight = 110;
consensus.nClaimInfoInMerkleForkHeight = 1500000; // FIXME pick height
consensus.fPowAllowMinDifficultyBlocks = true;
consensus.fPowNoRetargeting = false;
consensus.nRuleChangeActivationThreshold = 1512; // 75% for testchains
@@ -256,10 +258,10 @@ public:
consensus.vDeployments[Consensus::DEPLOYMENT_TESTDUMMY].nTimeout = 1230767999; // December 31, 2008
// The best chain should have at least this much work.
consensus.nMinimumChainWork = uint256S("0x000000000000000000000000000000000000000000000000000a0c3931735170");
consensus.nMinimumChainWork = uint256S("00000000000000000000000000000000000000000000000000000054b7d280af"); // 8400
// By default assume that the signatures in ancestors of this block are valid.
consensus.defaultAssumeValid = uint256S("9812b0bcb7e889e58d999c897e9eaddb2dab98122ff1cfb238ebeef5351bd48c"); // 1
consensus.defaultAssumeValid = uint256S("50b68b892f4e0f2ef649df37ef10b702e826c8913cc785c5e8ec16dd6be83f8b"); // 8400
pchMessageStart[0] = 0xfa;
pchMessageStart[1] = 0xe4;
@@ -282,8 +284,8 @@ public:
vFixedSeeds.clear();
vSeeds.clear();
vSeeds.emplace_back("testdnsseed1.lbry.io");
vSeeds.emplace_back("testdnsseed2.lbry.io");
vSeeds.emplace_back("testdnsseed1.lbry.com");
vSeeds.emplace_back("testdnsseed2.lbry.com");
base58Prefixes[PUBKEY_ADDRESS] = std::vector<unsigned char>(1,111);
base58Prefixes[SCRIPT_ADDRESS] = std::vector<unsigned char>(1,196);
@@ -345,6 +347,7 @@ public:
consensus.nMinRemovalWorkaroundHeight = -1;
consensus.nMaxRemovalWorkaroundHeight = -1;
consensus.nAllClaimsInMerkleForkHeight = 350;
consensus.nClaimInfoInMerkleForkHeight = 1350;
consensus.fPowAllowMinDifficultyBlocks = false;
consensus.fPowNoRetargeting = false;
consensus.nRuleChangeActivationThreshold = 108; // 75% for testchains
+45 -1
View File
@@ -4,8 +4,13 @@
#include <coins.h>
#include <claimscriptop.h>
#include <consensus/validation.h>
#include <logging.h>
#include <nameclaim.h>
#include <validation.h>
#include <util/translation.h>
#include <boost/locale.hpp>
CClaimScriptAddOp::CClaimScriptAddOp(const COutPoint& point, CAmount nValue, int nHeight)
: point(point), nValue(nValue), nHeight(nHeight)
@@ -176,10 +181,49 @@ bool ProcessClaim(CClaimScriptOp& claimOp, CClaimTrieCache& trieCache, const CSc
case OP_UPDATE_CLAIM:
return claimOp.updateClaim(trieCache, vchToString(vvchParams[0]), uint160(vvchParams[1]));
default:
throw std::runtime_error("Unimplemented OP handler.");
throw std::runtime_error(_("Unimplemented OP handler.").translated);
}
}
static bool isUtf8(const std::string& name)
{
using namespace boost::locale::conv;
try {
return to_utf<char>(name, "UTF-8", stop) == name;
} catch (const conversion_error&) {
return false;
}
}
bool ValidateClaimName(const CScript& scriptPubKey, std::string& reason)
{
int op;
std::vector<std::vector<unsigned char> > vvchParams;
if (!DecodeClaimScript(scriptPubKey, op, vvchParams))
return true;
switch (op) {
case OP_CLAIM_NAME:
case OP_UPDATE_CLAIM:
case OP_SUPPORT_CLAIM:
break;
default:
reason = _("Unsupported operation").translated;
return false;
}
const auto name = vchToString(vvchParams[0]);
if (!isUtf8(name)) {
reason = _("Claim name is not valid UTF8 string").translated;
return false;
}
static const std::string disallowedSymbols("=&#:$@%?;/\\\"<>*\n\t\r\b\0", 20);
if (name.find_first_of(disallowedSymbols) != std::string::npos) {
reason = _("Claim name contains invalid symbol").translated;
return false;
}
return true;
}
void UpdateCache(const CTransaction& tx, CClaimTrieCache& trieCache, const CCoinsViewCache& view, int nHeight, const CUpdateCacheCallbacks& callbacks)
{
class CSpendClaimHistory : public CClaimScriptSpendOp
+7
View File
@@ -242,4 +242,11 @@ struct CUpdateCacheCallbacks
*/
void UpdateCache(const CTransaction& tx, CClaimTrieCache& trieCache, const CCoinsViewCache& view, int nHeight, const CUpdateCacheCallbacks& callbacks = {});
/**
* Function to validate that we process only UTF8 claim names
* @param[in] scriptPubKey claim script to be decoded
* @param[in] reason rejection reason if any
* */
bool ValidateClaimName(const CScript& scriptPubKey, std::string& reason);
#endif // BITCOIN_CLAIMSCRIPTOP_H
+48
View File
@@ -2,6 +2,7 @@
#include <data.h>
#include <algorithm>
#include <cassert>
#include <sstream>
CClaimValue::CClaimValue(COutPoint outPoint, uint160 claimId, int64_t nAmount, int nHeight, int nValidAtHeight)
@@ -121,3 +122,50 @@ CClaimTrieProofNode::CClaimTrieProofNode(std::vector<std::pair<unsigned char, ui
: children(std::move(children)), hasValue(hasValue), valHash(std::move(valHash))
{
}
// Sequence ordering is pretty similar to bid one (ASC)
// what we have in CClaimValue::operator< just ignoring effective amount
bool CClaimInfo::operator<(const CClaimInfo& other) const
{
if (originalHeight < other.originalHeight)
return true;
if (originalHeight != other.originalHeight)
return false;
if (updateHeight > other.updateHeight)
return true;
if (updateHeight != other.updateHeight)
return false;
return outPoint != other.outPoint && !(outPoint < other.outPoint);
}
bool CClaimInfo::operator==(const CClaimInfo& other) const
{
return claimId == other.claimId;
}
static CClaimInfo claimInfo(const CClaimNsupports& claimNsupports)
{
CClaimInfo info;
info.originalHeight = claimNsupports.originalHeight;
info.updateHeight = claimNsupports.claim.nHeight;
info.outPoint = claimNsupports.claim.outPoint;
return info;
}
std::vector<CClaimNsupports> seqSort(const std::vector<CClaimNsupports>& source)
{
auto claimsNsupports = source;
std::sort(claimsNsupports.begin(), claimsNsupports.end(), [](const CClaimNsupports& lhs, const CClaimNsupports& rhs) {
return claimInfo(lhs) < claimInfo(rhs);
});
return claimsNsupports;
}
std::size_t indexOf(const std::vector<CClaimNsupports>& source, const uint160& claimId)
{
auto it = std::find_if(source.begin(), source.end(), [&claimId](const CClaimNsupports& claimNsupports) {
return claimNsupports.claim.claimId == claimId;
});
assert(it != source.end());
return std::distance(source.begin(), it);
}
+15
View File
@@ -132,4 +132,19 @@ struct CClaimTrieProof
COutPoint outPoint;
};
struct CClaimInfo
{
CClaimInfo() = default;
bool operator<(const CClaimInfo& other) const;
bool operator==(const CClaimInfo& other) const;
uint160 claimId;
COutPoint outPoint;
int updateHeight = 0;
int originalHeight = 0;
};
std::vector<CClaimNsupports> seqSort(const std::vector<CClaimNsupports>& source);
std::size_t indexOf(const std::vector<CClaimNsupports>& source, const uint160& claimId);
#endif // CLAIMTRIE_DATA_H
+168 -55
View File
@@ -4,6 +4,10 @@
#include <log.h>
#include <trie.h>
#include <iomanip>
#include <sstream>
#include <string>
#include <boost/locale.hpp>
#include <boost/locale/conversion.hpp>
#include <boost/locale/localization_backend.hpp>
@@ -139,10 +143,10 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary()
// make the new nodes
db << "INSERT INTO node(name) SELECT NORMALIZED(name) AS nn FROM claim WHERE nn != nodeName "
"AND activationHeight <= ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL" << nNextHeight;
"AND activationHeight <= ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL" << nNextHeight;
// there's a subtlety here: names in supports don't make new nodes
db << "UPDATE node SET hash = NULL WHERE name IN "
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN "
"(SELECT NORMALIZED(name) AS nn FROM support WHERE nn != nodeName "
"AND activationHeight <= ?1 AND expirationHeight > ?1)" << nNextHeight;
@@ -151,7 +155,7 @@ bool CClaimTrieCacheNormalizationFork::normalizeAllNamesInTrieIfNecessary()
db << "UPDATE support SET nodeName = NORMALIZED(name) WHERE activationHeight <= ?1 AND expirationHeight > ?1" << nNextHeight;
// remove the old nodes
db << "UPDATE node SET hash = NULL WHERE name NOT IN "
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name NOT IN "
"(SELECT nodeName FROM claim WHERE activationHeight <= ?1 AND expirationHeight > ?1 "
"UNION SELECT nodeName FROM support WHERE activationHeight <= ?1 AND expirationHeight > ?1)" << nNextHeight;
@@ -167,9 +171,9 @@ bool CClaimTrieCacheNormalizationFork::unnormalizeAllNamesInTrieIfNecessary()
ensureTransacting();
db << "INSERT INTO node(name) SELECT name FROM claim WHERE name != nodeName "
"AND activationHeight < ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL" << nNextHeight;
"AND activationHeight < ?1 AND expirationHeight > ?1 ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL" << nNextHeight;
db << "UPDATE node SET hash = NULL WHERE name IN "
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN "
"(SELECT nodeName FROM support WHERE name != nodeName "
"UNION SELECT nodeName FROM claim WHERE name != nodeName)";
@@ -177,7 +181,7 @@ bool CClaimTrieCacheNormalizationFork::unnormalizeAllNamesInTrieIfNecessary()
db << "UPDATE support SET nodeName = name";
// we need to let the tree structure method do the actual node delete
db << "UPDATE node SET hash = NULL WHERE name NOT IN "
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name NOT IN "
"(SELECT DISTINCT name FROM claim)";
return true;
@@ -227,7 +231,7 @@ CClaimTrieCacheHashFork::CClaimTrieCacheHashFork(CClaimTrie* base) : CClaimTrieC
{
}
extern uint256 verifyEmptyTrie(const std::string&);
static const auto emptyTrieHash = uint256S("0000000000000000000000000000000000000000000000000000000000000001");
static const auto leafHash = uint256S("0000000000000000000000000000000000000000000000000000000000000002");
static const auto emptyHash = uint256S("0000000000000000000000000000000000000000000000000000000000000003");
@@ -243,34 +247,80 @@ uint256 ComputeMerkleRoot(std::vector<uint256> hashes)
return hashes.empty() ? uint256{} : hashes[0];
}
uint256 CClaimTrieCacheHashFork::computeNodeHash(const std::string& name, int takeoverHeight)
std::vector<uint256> CClaimTrieCacheHashFork::childrenHashes(const std::string& name, const std::function<void(const std::string&)>& callback)
{
using row_type = sqlite::row_iterator::reference;
std::function<void(row_type)> visitor;
if (callback)
visitor = [&callback](row_type row) {
std::string childName;
row >> childName;
callback(childName);
};
else
visitor = [](row_type row) {
/* name */ row.index()++;
};
std::vector<uint256> childHashes;
for (auto&& row: childHashQuery << name) {
visitor(row);
row >> *childHashes.emplace(childHashes.end());
}
childHashQuery++;
return childHashes;
}
std::vector<uint256> CClaimTrieCacheHashFork::claimsHashes(const std::string& name, int takeoverHeight, const std::function<void(const CClaimInfo&)>& callback)
{
using row_type = sqlite::row_iterator::reference;
std::function<COutPoint(row_type)> visitor;
if (callback)
visitor = [&callback](row_type row) -> COutPoint {
CClaimInfo info;
row >> info.outPoint.hash >> info.outPoint.n
>> info.claimId >> info.updateHeight;
// activationHeight, amount
row.index() += 2;
row >> info.originalHeight;
callback(info);
return info.outPoint;
};
else
visitor = [](row_type row) -> COutPoint {
COutPoint outPoint;
row >> outPoint.hash >> outPoint.n;
return outPoint;
};
std::vector<uint256> claimHashes;
for (auto&& row: claimHashQuery << nNextHeight << name)
claimHashes.push_back(getValueHash(visitor(row), takeoverHeight));
claimHashQuery++;
return claimHashes;
}
uint256 CClaimTrieCacheHashFork::computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight)
{
if (nNextHeight < base->nAllClaimsInMerkleForkHeight)
return CClaimTrieCacheNormalizationFork::computeNodeHash(name, takeoverHeight);
return CClaimTrieCacheNormalizationFork::computeNodeHash(name, claimsHash, takeoverHeight);
std::vector<uint256> childHashes;
childHashQuery << name >> [&childHashes](std::string, uint256 hash) {
childHashes.push_back(std::move(hash));
};
childHashQuery++;
auto childHashes = childrenHashes(name);
std::vector<uint256> claimHashes;
if (takeoverHeight > 0) {
COutPoint p;
for (auto &&row: claimHashQuery << nNextHeight << name) {
row >> p.hash >> p.n;
claimHashes.push_back(getValueHash(p, takeoverHeight));
if (claimsHash.IsNull()) {
auto hashes = claimsHashes(name, takeoverHeight);
claimsHash = hashes.empty() ? emptyHash : ComputeMerkleRoot(std::move(hashes));
}
claimHashQuery++;
} else {
claimsHash = emptyHash;
}
if (childHashes.empty() && claimHashes.empty())
return verifyEmptyTrie(name);
if (name.empty() && childHashes.empty() && claimsHash == emptyHash
&& base->nMaxRemovalWorkaroundHeight < 0) // detecting regtest, but maybe all on next hard-fork?
return emptyTrieHash; // here for compatibility with the functional tests
auto left = childHashes.empty() ? leafHash : ComputeMerkleRoot(std::move(childHashes));
auto right = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(std::move(claimHashes));
const auto& childrenHash = childHashes.empty() ? leafHash : ComputeMerkleRoot(std::move(childHashes));
return Hash(left.begin(), left.end(), right.begin(), right.end());
return Hash2(childrenHash, claimsHash);
}
std::vector<uint256> ComputeMerklePath(const std::vector<uint256>& hashes, uint32_t idx)
@@ -330,10 +380,10 @@ std::vector<uint256> ComputeMerklePath(const std::vector<uint256>& hashes, uint3
extern const std::string proofClaimQuery_s;
bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, const uint160& claim, CClaimTrieProof& proof)
bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, const uint160& claimId, CClaimTrieProof& proof)
{
if (nNextHeight < base->nAllClaimsInMerkleForkHeight)
return CClaimTrieCacheNormalizationFork::getProofForName(name, claim, proof);
return CClaimTrieCacheNormalizationFork::getProofForName(name, claimId, proof);
auto fillPairs = [&proof](const std::vector<uint256>& hashes, uint32_t idx) {
auto partials = ComputeMerklePath(hashes, idx);
@@ -348,32 +398,26 @@ bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, const uin
std::string key;
int takeoverHeight;
row >> key >> takeoverHeight;
uint32_t nextCurrentIdx = 0;
std::vector<uint256> childHashes;
for (auto&& child : childHashQuery << key) {
std::string childKey;
uint256 childHash;
child >> childKey >> childHash;
if (name.find(childKey) == 0)
nextCurrentIdx = uint32_t(childHashes.size());
childHashes.push_back(childHash);
}
childHashQuery++;
std::vector<uint256> claimHashes;
uint32_t claimIdx = 0;
for (auto&& child: claimHashQuery << nNextHeight << key) {
COutPoint childOutPoint;
uint160 childClaimID;
child >> childOutPoint.hash >> childOutPoint.n >> childClaimID;
if (childClaimID == claim && key == name) {
claimIdx = uint32_t(claimHashes.size());
proof.outPoint = childOutPoint;
proof.hasValue = true;
}
claimHashes.push_back(getValueHash(childOutPoint, takeoverHeight));
}
claimHashQuery++;
uint32_t childIdx = 0, idx = 0;
auto childHashes = childrenHashes(key, [&](const std::string& childKey) {
if (name.find(childKey) == 0)
childIdx = idx;
++idx;
});
uint32_t claimIdx = 0; idx = 0;
std::function<void(const CClaimInfo&)> visitor;
if (key == name)
visitor = [&](const CClaimInfo& info) {
if (info.claimId == claimId) {
claimIdx = idx;
proof.hasValue = true;
proof.outPoint = info.outPoint;
}
++idx;
};
auto claimHashes = claimsHashes(key, takeoverHeight, visitor);
// I am on a node; I need a hash(children, claims)
// if I am the last node on the list, it will be hash(children, x)
@@ -388,7 +432,7 @@ bool CClaimTrieCacheHashFork::getProofForName(const std::string& name, const uin
auto hash = claimHashes.empty() ? emptyHash : ComputeMerkleRoot(std::move(claimHashes));
proof.pairs.emplace_back(false, hash);
if (!childHashes.empty())
fillPairs(childHashes, nextCurrentIdx);
fillPairs(childHashes, childIdx);
}
}
std::reverse(proof.pairs.begin(), proof.pairs.end());
@@ -401,7 +445,7 @@ void CClaimTrieCacheHashFork::initializeIncrement()
// we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests)
if (nNextHeight == base->nAllClaimsInMerkleForkHeight - 1) {
ensureTransacting();
db << "UPDATE node SET hash = NULL";
db << "UPDATE node SET hash = NULL, claimsHash = NULL";
}
}
@@ -410,7 +454,7 @@ bool CClaimTrieCacheHashFork::finalizeDecrement()
auto ret = CClaimTrieCacheNormalizationFork::finalizeDecrement();
if (ret && nNextHeight == base->nAllClaimsInMerkleForkHeight - 1) {
ensureTransacting();
db << "UPDATE node SET hash = NULL";
db << "UPDATE node SET hash = NULL, claimsHash = NULL";
}
return ret;
}
@@ -419,3 +463,72 @@ bool CClaimTrieCacheHashFork::allowSupportMetadata() const
{
return nNextHeight >= base->nAllClaimsInMerkleForkHeight;
}
CClaimTrieCacheClaimInfoHashFork::CClaimTrieCacheClaimInfoHashFork(CClaimTrie* base) : CClaimTrieCacheHashFork(base)
{
}
extern std::vector<unsigned char> heightToVch(int n);
// NOTE: the name is supposed to be the final one
// normalized or not is the caller responsibility
uint256 claimInfoHash(const std::string& name, const COutPoint& outPoint, int bid, int seq, int nHeightOfLastTakeover)
{
auto hash = Hash2(Hash(name), Hash(outPoint.hash));
hash = Hash2(hash, std::to_string(outPoint.n));
hash = Hash2(hash, std::to_string(bid));
hash = Hash2(hash, std::to_string(seq));
return Hash2(hash, heightToVch(nHeightOfLastTakeover));
}
inline std::size_t indexOf(const std::vector<CClaimInfo>& infos, const CClaimInfo& info)
{
return std::distance(infos.begin(), std::find(infos.begin(), infos.end(), info));
}
std::vector<uint256> CClaimTrieCacheClaimInfoHashFork::claimsHashes(const std::string& name, int takeoverHeight, const std::function<void(const CClaimInfo&)>& callback)
{
if (nNextHeight < base->nClaimInfoInMerkleForkHeight)
return CClaimTrieCacheHashFork::claimsHashes(name, takeoverHeight, callback);
std::vector<CClaimInfo> claimsByBid;
for (auto&& row: claimHashQuery << nNextHeight << name) {
auto& cb = *claimsByBid.emplace(claimsByBid.end());
row >> cb.outPoint.hash >> cb.outPoint.n
>> cb.claimId >> cb.updateHeight;
// activationHeight, amount
row.index() += 2;
row >> cb.originalHeight;
if (callback) callback(cb);
}
claimHashQuery++;
auto claimsBySeq = claimsByBid;
std::sort(claimsBySeq.begin(), claimsBySeq.end());
std::vector<uint256> claimsHashes;
for (auto i = 0u; i < claimsByBid.size(); ++i) {
auto& cb = claimsByBid[i];
claimsHashes.push_back(claimInfoHash(name, cb.outPoint, i, indexOf(claimsBySeq, cb), takeoverHeight));
}
return claimsHashes;
}
void CClaimTrieCacheClaimInfoHashFork::initializeIncrement()
{
CClaimTrieCacheHashFork::initializeIncrement();
// we could do this in the constructor, but that would not allow for multiple increments in a row (as done in unit tests)
if (nNextHeight == base->nClaimInfoInMerkleForkHeight - 1) {
ensureTransacting();
db << "UPDATE node SET hash = NULL";
}
}
bool CClaimTrieCacheClaimInfoHashFork::finalizeDecrement()
{
auto ret = CClaimTrieCacheHashFork::finalizeDecrement();
if (ret && nNextHeight == base->nClaimInfoInMerkleForkHeight - 1) {
ensureTransacting();
db << "UPDATE node SET hash = NULL";
}
return ret;
}
+20 -2
View File
@@ -66,9 +66,27 @@ public:
bool allowSupportMetadata() const;
protected:
uint256 computeNodeHash(const std::string& name, int takeoverHeight) override;
virtual std::vector<uint256> childrenHashes(const std::string& name,
const std::function<void(const std::string&)>& callback = {});
virtual std::vector<uint256> claimsHashes(const std::string& name, int takeoverHeight,
const std::function<void(const CClaimInfo&)>& callback = {});
uint256 computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight) override;
};
typedef CClaimTrieCacheHashFork CClaimTrieCache;
class CClaimTrieCacheClaimInfoHashFork : public CClaimTrieCacheHashFork
{
public:
explicit CClaimTrieCacheClaimInfoHashFork(CClaimTrie* base);
CClaimTrieCacheClaimInfoHashFork(CClaimTrieCacheClaimInfoHashFork&&) = default;
void initializeIncrement() override;
bool finalizeDecrement() override;
protected:
std::vector<uint256> claimsHashes(const std::string& name, int takeoverHeight,
const std::function<void(const CClaimInfo&)>& callback = {}) override;
};
typedef CClaimTrieCacheClaimInfoHashFork CClaimTrieCache;
#endif // CLAIMTRIE_FORKS_H
+35
View File
@@ -28,6 +28,41 @@ uint256 Hash(TIterator begin, TIterator end, Args... args)
return CalcHash(&sha, begin, end, args...);
}
template <typename T>
struct IsHashable {
static const bool value = false;
};
template <typename T>
struct IsHashable<std::vector<T>> {
static const bool value = true;
};
template <>
struct IsHashable<std::string> {
static const bool value = true;
};
template <>
struct IsHashable<uint256> {
static const bool value = true;
};
template <typename T>
uint256 Hash(const T& c)
{
static_assert(IsHashable<T>::value, "T is not hashable");
return Hash(c.begin(), c.end());
}
template <typename T1, typename T2>
uint256 Hash2(const T1& c1, const T2& c2)
{
static_assert(IsHashable<T1>::value, "T1 is not hashable");
static_assert(IsHashable<T2>::value, "T2 is not hashable");
return Hash(c1.begin(), c1.end(), c2.begin(), c2.end());
}
extern std::function<void(std::vector<uint256>&)> sha256n_way;
#endif // CLAIMTRIE_HASHES_H
+5 -1
View File
@@ -20,6 +20,10 @@
%ignore CBaseBlob(CBaseBlob &&);
%ignore CClaimNsupports(CClaimNsupports &&);
%ignore CClaimTrieCacheHashFork(CClaimTrieCacheHashFork &&);
%ignore CClaimTrieCacheExpirationFork(CClaimTrieCacheExpirationFork &&);
%ignore CClaimTrieCacheNormalizationFork(CClaimTrieCacheNormalizationFork &&);
%ignore CClaimTrieCacheClaimInfoHashFork(CClaimTrieCacheClaimInfoHashFork &&);
%ignore CClaimTrieProof(CClaimTrieProof &&);
%ignore CClaimTrieProofNode(CClaimTrieProofNode &&);
%ignore CClaimValue(CClaimValue &&);
@@ -39,7 +43,7 @@
%include "txoutpoint.h"
%include "data.h"
%rename(CClaimTrieCache) CClaimTrieCacheHashFork;
%rename(CClaimTrieCache) CClaimTrieCacheClaimInfoHashFork;
%include "trie.h"
%include "forks.h"
+12
View File
@@ -452,4 +452,16 @@ const boost::container::flat_map<std::pair<int, std::string>, int> takeoverWorka
{{658098, "le-temps-nous-glisse-entre-les-doigts"}, 600099},
};
inline bool isTakeoverWorkaroundActive(int nHeight, const std::string& name)
{
// This is a super ugly hack to work around bug in old code.
// The bug: un/support a name then update it. This will cause its takeover height to be reset to current.
// This is because the old code with add to the cache without setting block originals when dealing in supports.
if (nHeight < 658300) {
auto wit = takeoverWorkarounds.find(std::make_pair(nHeight, name));
return wit != takeoverWorkarounds.end();
}
return false;
}
#endif // TAKEOVERWORKAROUNDS_H
+160 -89
View File
@@ -24,11 +24,9 @@ std::vector<unsigned char> heightToVch(int n)
uint256 getValueHash(const COutPoint& outPoint, int nHeightOfLastTakeover)
{
auto hash1 = Hash(outPoint.hash.begin(), outPoint.hash.end());
auto snOut = std::to_string(outPoint.n);
auto hash2 = Hash(snOut.begin(), snOut.end());
auto vchHash = heightToVch(nHeightOfLastTakeover);
auto hash3 = Hash(vchHash.begin(), vchHash.end());
auto hash1 = Hash(outPoint.hash);
auto hash2 = Hash(std::to_string(outPoint.n));
auto hash3 = Hash(heightToVch(nHeightOfLastTakeover));
return Hash(hash1.begin(), hash1.end(), hash2.begin(), hash2.end(), hash3.begin(), hash3.end());
}
@@ -56,6 +54,7 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
int64_t nExtendedClaimExpirationTime,
int64_t nExtendedClaimExpirationForkHeight,
int64_t nAllClaimsInMerkleForkHeight,
int64_t nClaimInfoInMerkleForkHeight,
int proportionalDelayFactor) :
nNextHeight(height),
dbCacheBytes(cacheBytes),
@@ -67,13 +66,14 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
nOriginalClaimExpirationTime(nOriginalClaimExpirationTime),
nExtendedClaimExpirationTime(nExtendedClaimExpirationTime),
nExtendedClaimExpirationForkHeight(nExtendedClaimExpirationForkHeight),
nAllClaimsInMerkleForkHeight(nAllClaimsInMerkleForkHeight)
nAllClaimsInMerkleForkHeight(nAllClaimsInMerkleForkHeight),
nClaimInfoInMerkleForkHeight(nClaimInfoInMerkleForkHeight)
{
applyPragmas(db, cacheBytes >> 10U); // in KB
db << "CREATE TABLE IF NOT EXISTS node (name BLOB NOT NULL PRIMARY KEY, "
"parent BLOB REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
"hash BLOB)";
"hash BLOB, claimsHash BLOB)";
db << "CREATE TABLE IF NOT EXISTS claim (claimID BLOB NOT NULL PRIMARY KEY, name BLOB NOT NULL, "
"nodeName BLOB NOT NULL REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
@@ -96,6 +96,8 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
db << "DELETE FROM takeover";
}
doNodeTableMigration();
db << "CREATE INDEX IF NOT EXISTS node_hash_len_name ON node (hash, LENGTH(name) DESC)";
// db << "CREATE UNIQUE INDEX IF NOT EXISTS node_parent_name ON node (parent, name)"; // no apparent gain
db << "CREATE INDEX IF NOT EXISTS node_parent ON node (parent)";
@@ -114,10 +116,33 @@ CClaimTrie::CClaimTrie(std::size_t cacheBytes, bool fWipe, int height,
db << "INSERT OR IGNORE INTO node(name, hash) VALUES(x'', ?)" << emptyTrieHash; // ensure that we always have our root node
}
void CClaimTrie::doNodeTableMigration()
{
try {
isNodeMigrationStart = false;
for (auto&& row : db << "SELECT claimsHash FROM node WHERE name = x''")
break;
} catch (const sqlite::sqlite_exception&) {
logPrint << "Running one-time upgrade of node table to cache the hash of claims..." << Clog::endl;
isNodeMigrationStart = true;
// new node schema
db << "CREATE TABLE node_new (name BLOB NOT NULL PRIMARY KEY, "
"parent BLOB REFERENCES node(name) DEFERRABLE INITIALLY DEFERRED, "
"hash BLOB, claimsHash BLOB)";
db << "INSERT OR REPLACE INTO node_new(name, parent, hash) "
"SELECT name, parent, hash FROM node";
db << "DROP TABLE node";
db << "ALTER TABLE node_new RENAME TO node";
}
}
CClaimTrieCacheBase::~CClaimTrieCacheBase()
{
if (transacting) {
db << "rollback";
db << "ROLLBACK";
transacting = false;
}
claimHashQuery.used(true);
@@ -132,6 +157,7 @@ std::size_t CClaimTrie::cache()
bool CClaimTrie::SyncToDisk()
{
db << "PRAGMA optimize";
// alternatively, switch to full sync after we are caught up on the chain
return sqlite::sync(db) == SQLITE_OK;
}
@@ -200,6 +226,27 @@ bool CClaimTrieCacheBase::haveSupportInQueue(const std::string& name, const COut
return false;
}
bool emptyNodeShouldExistAt(const sqlite::database& db, const std::string& name, int nNextHeight, int requiredChildren) {
auto end = name + std::string(256, std::numeric_limits<char>::max()); // 256 == MAX_CLAIM_NAME_SIZE + 1
auto query = db << "SELECT DISTINCT nodeName FROM claim "
"WHERE nodeName BETWEEN ?1 AND ?2 "
"AND activationHeight < ?3 AND expirationHeight >= ?3 "
"ORDER BY nodeName"
<< name << end << nNextHeight;
std::unordered_set<char> ss;
for (auto&& row: query) {
std::string nn;
row >> nn;
if (nn == name)
return false;
assert(nn.size() > name.size());
ss.insert(nn[name.size()]);
if (ss.size() >= requiredChildren)
return true;
}
return false;
}
bool CClaimTrieCacheBase::deleteNodeIfPossible(const std::string& name, std::string& parent, int64_t& claims)
{
if (name.empty()) return false;
@@ -258,7 +305,7 @@ void CClaimTrieCacheBase::ensureTreeStructureIsUpToDate()
"name IN (WITH RECURSIVE prefix(p) AS (VALUES(?) UNION ALL "
"SELECT POPS(p) FROM prefix WHERE p != x'') SELECT p FROM prefix)";
auto insertQuery = db << "INSERT INTO node(name, parent, hash) VALUES(?, ?, NULL) "
auto insertQuery = db << "INSERT INTO node(name, parent, hash, claimsHash) VALUES(?, ?, NULL, NULL) "
"ON CONFLICT(name) DO UPDATE SET parent = excluded.parent, hash = NULL";
auto nodeQuery = db << "SELECT name FROM node WHERE parent = ?";
@@ -346,7 +393,7 @@ int64_t CClaimTrieCacheBase::getTotalValueOfClaimsInTrie(bool fControllingOnly)
int64_t ret = 0;
const std::string query = fControllingOnly ?
"SELECT SUM(amount) FROM (SELECT c.amount as amount, "
"(SELECT(SELECT IFNULL(SUM(s.amount),0)+c.amount FROM support s "
"(SELECT(SELECT IFNULL(SUM(s.amount),0)+c.amount FROM support s INDEXED BY support_supportedClaimID "
"WHERE s.supportedClaimID = c.claimID AND c.nodeName = s.nodeName "
"AND s.activationHeight < ?1 AND s.expirationHeight >= ?1) as effective "
"ORDER BY effective DESC LIMIT 1) as winner FROM claim c "
@@ -417,15 +464,7 @@ void completeHash(uint256& partialHash, const std::string& key, int to)
partialHash = Hash(it, it + 1, partialHash.begin(), partialHash.end());
}
uint256 verifyEmptyTrie(const std::string& name)
{
if (!name.empty())
logPrint << "Corrupt trie near: " << name << Clog::endl;
assert(name.empty());
return emptyTrieHash;
}
uint256 CClaimTrieCacheBase::computeNodeHash(const std::string& name, int takeoverHeight)
uint256 CClaimTrieCacheBase::computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight)
{
const auto pos = name.size();
std::vector<uint8_t> vchToHash;
@@ -438,14 +477,16 @@ uint256 CClaimTrieCacheBase::computeNodeHash(const std::string& name, int takeov
childHashQuery++;
if (takeoverHeight > 0) {
CClaimValue claim;
if (getInfoForName(name, claim)) {
auto valueHash = getValueHash(claim.outPoint, takeoverHeight);
vchToHash.insert(vchToHash.end(), valueHash.begin(), valueHash.end());
if (claimsHash.IsNull()) {
CClaimValue claim;
if (getInfoForName(name, claim))
claimsHash = getValueHash(claim.outPoint, takeoverHeight);
}
if (!claimsHash.IsNull())
vchToHash.insert(vchToHash.end(), claimsHash.begin(), claimsHash.end());
}
return vchToHash.empty() ? verifyEmptyTrie(name) : Hash(vchToHash.begin(), vchToHash.end());
return vchToHash.empty() ? emptyTrieHash : Hash(vchToHash.begin(), vchToHash.end());
}
bool CClaimTrieCacheBase::checkConsistency()
@@ -460,22 +501,47 @@ bool CClaimTrieCacheBase::checkConsistency()
}
}
if (base->isNodeMigrationStart)
ensureTransacting();
auto updateQuery = db << "UPDATE node SET claimsHash = ? WHERE name = ?";
// not checking everything as it takes too long
auto query = db << "SELECT n.name, n.hash, "
auto query = db << (base->isNodeMigrationStart ?
"SELECT n.name, n.hash, n.claimsHash, "
"IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END "
"FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) FROM node n "
"WHERE n.name IN (SELECT r.name FROM node r ORDER BY RANDOM() LIMIT 100000) OR n.parent = x''";
"FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) "
"FROM node n ORDER BY LENGTH(n.name) DESC"
:
"SELECT n.name, n.hash, n.claimsHash, "
"IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END "
"FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) "
"FROM node n WHERE n.name IN "
"(SELECT r.name FROM node r ORDER BY RANDOM() LIMIT 100000) OR n.parent = x''");
for (auto&& row: query) {
std::string name;
uint256 hash;
int takeoverHeight;
row >> name >> hash >> takeoverHeight;
auto computedHash = computeNodeHash(name, takeoverHeight);
uint256 hash, claimsHash, computedClaimsHash;
row >> name >> hash >> claimsHash >> takeoverHeight;
auto computedHash = computeNodeHash(name, computedClaimsHash, takeoverHeight);
if (computedHash != hash) {
logPrint << "Invalid hash at " << name << Clog::endl;
return false;
}
if (base->isNodeMigrationStart) {
assert(!computedClaimsHash.IsNull());
updateQuery << computedClaimsHash << name;
updateQuery++;
} else if (computedClaimsHash != claimsHash) {
logPrint << "Invalid claimsHash at " << name << Clog::endl;
return false;
}
}
updateQuery.used(true);
if (base->isNodeMigrationStart)
return flush();
return true;
}
@@ -516,8 +582,8 @@ bool CClaimTrieCacheBase::flush()
const std::string childHashQuery_s = "SELECT name, hash FROM node WHERE parent = ? ORDER BY name";
const std::string claimHashQuery_s =
"SELECT c.txID, c.txN, c.claimID, c.updateHeight, c.activationHeight, c.amount, "
"(SELECT IFNULL(SUM(s.amount),0)+c.amount FROM support s "
"SELECT c.txID, c.txN, c.claimID, c.updateHeight, c.activationHeight, c.amount, c.originalHeight, "
"(SELECT IFNULL(SUM(s.amount),0)+c.amount FROM support s INDEXED BY support_supportedClaimID "
"WHERE s.supportedClaimID = c.claimID AND s.nodeName = c.nodeName "
"AND s.activationHeight < ?1 AND s.expirationHeight >= ?1) as effectiveAmount "
"FROM claim c WHERE c.nodeName = ?2 AND c.activationHeight < ?1 AND c.expirationHeight >= ?1 "
@@ -579,20 +645,20 @@ uint256 CClaimTrieCacheBase::getMerkleHash()
{
ensureTreeStructureIsUpToDate();
uint256 hash;
db << "SELECT hash FROM node WHERE name = x''"
>> [&hash](std::unique_ptr<uint256> rootHash) {
if (rootHash)
hash = std::move(*rootHash);
};
if (!hash.IsNull())
return hash;
for (auto&& row : db << "SELECT hash FROM node WHERE name = x''") {
row >> hash;
if (!hash.IsNull())
return hash;
}
assert(transacting); // no data changed but we didn't have the root hash there already?
auto updateQuery = db << "UPDATE node SET hash = ? WHERE name = ?";
db << "SELECT n.name, IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END FROM takeover t WHERE t.name = n.name "
"ORDER BY t.height DESC LIMIT 1), 0) FROM node n WHERE n.hash IS NULL ORDER BY LENGTH(n.name) DESC" // assumes n.name is blob
>> [this, &hash, &updateQuery](const std::string& name, int takeoverHeight) {
hash = computeNodeHash(name, takeoverHeight);
updateQuery << hash << name;
auto updateQuery = db << "UPDATE node SET hash = ?, claimsHash = ? WHERE name = ?";
db << "SELECT n.name, n.claimsHash, "
"IFNULL((SELECT CASE WHEN t.claimID IS NULL THEN 0 ELSE t.height END "
"FROM takeover t WHERE t.name = n.name ORDER BY t.height DESC LIMIT 1), 0) "
"FROM node n WHERE n.hash IS NULL ORDER BY LENGTH(n.name) DESC" // assumes n.name is blob
>> [&](const std::string& name, uint256 claimsHash, int takeoverHeight) {
hash = computeNodeHash(name, claimsHash, takeoverHeight);
updateQuery << hash << claimsHash << name;
updateQuery++;
};
updateQuery.used(true);
@@ -646,7 +712,7 @@ bool CClaimTrieCacheBase::addClaim(const std::string& name, const COutPoint& out
<< originalHeight << nHeight << nValidHeight << nValidHeight << expires;
if (nValidHeight < nNextHeight)
db << "INSERT INTO node(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL" << nodeName;
db << "INSERT INTO node(name) VALUES(?) ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL" << nodeName;
return true;
}
@@ -667,7 +733,7 @@ bool CClaimTrieCacheBase::addSupport(const std::string& name, const COutPoint& o
<< supportedClaimId << name << nodeName << outPoint.hash << outPoint.n << nAmount << nHeight << nValidHeight << nValidHeight << expires;
if (nValidHeight < nNextHeight)
db << "UPDATE node SET hash = NULL WHERE name = ?" << nodeName;
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name = ?" << nodeName;
return true;
}
@@ -675,44 +741,37 @@ bool CClaimTrieCacheBase::addSupport(const std::string& name, const COutPoint& o
bool CClaimTrieCacheBase::removeClaim(const uint160& claimId, const COutPoint& outPoint, std::string& nodeName,
int& validHeight, int& originalHeight)
{
ensureTransacting();
// this gets tricky in that we may be removing an update
// when going forward we spend a claim (aka, call removeClaim) before updating it (aka, call addClaim)
// when going backwards we first remove the update by calling removeClaim
// we then undo the spend of the previous one by calling addClaim with the original data
// in order to maintain the proper takeover height the updater will need to use our height returned here
auto query = db << "SELECT nodeName, activationHeight, originalHeight FROM claim WHERE claimID = ? AND txID = ? AND txN = ? AND expirationHeight >= ?"
auto query = db << "SELECT nodeName, activationHeight, originalHeight FROM claim "
"WHERE claimID = ? AND txID = ? AND txN = ? AND expirationHeight >= ?"
<< claimId << outPoint.hash << outPoint.n << nNextHeight;
auto it = query.begin();
if (it == query.end())
return false;
*it >> nodeName >> validHeight >> originalHeight;
db << "DELETE FROM claim WHERE claimID = ? AND txID = ? and txN = ?" << claimId << outPoint.hash << outPoint.n;
ensureTransacting();
db << "DELETE FROM claim WHERE claimID = ? AND txID = ? AND txN = ?"
<< claimId << outPoint.hash << outPoint.n;
if (!db.rows_modified())
return false;
db << "UPDATE node SET hash = NULL WHERE name = ?" << nodeName;
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name = ?" << nodeName;
// when node should be deleted from cache but instead it's kept
// because it's a parent one and should not be effectively erased
// we had a bug in the old code where that situation would force a zero delay on re-add
if (nNextHeight >= base->nMinRemovalWorkaroundHeight
&& nNextHeight < base->nMaxRemovalWorkaroundHeight) { // TODO: hard fork this out (which we already tried once but failed)
// neither LIKE nor SUBSTR will use an index on a blob, but BETWEEN is a good, fast alternative
auto end = nodeName + std::string( 256, std::numeric_limits<char>::max()); // 256 == MAX_CLAIM_NAME_SIZE + 1
auto innerQuery = db << "SELECT nodeName FROM claim WHERE nodeName BETWEEN ?1 AND ?2 "
"AND activationHeight < ?3 AND expirationHeight >= ?3 ORDER BY nodeName LIMIT 1"
<< nodeName << end << nNextHeight;
for (auto&& row: innerQuery) {
std::string shortestMatch;
row >> shortestMatch;
if (shortestMatch != nodeName)
// set this when there are no more claims on that name and that node still has children
removalWorkaround.insert(nodeName);
}
&& nNextHeight < base->nMaxRemovalWorkaroundHeight
) {
if (emptyNodeShouldExistAt(db, nodeName, nNextHeight, 1))
removalWorkaround.insert(nodeName);
}
return true;
}
@@ -734,13 +793,11 @@ bool CClaimTrieCacheBase::removeSupport(const COutPoint& outPoint, std::string&
db << "DELETE FROM support WHERE txID = ? AND txN = ?" << outPoint.hash << outPoint.n;
if (!db.rows_modified())
return false;
db << "UPDATE node SET hash = NULL WHERE name = ?" << nodeName;
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name = ?" << nodeName;
return true;
}
// hardcoded claims that should trigger a takeover
#include <takeoverworkarounds.h>
bool CClaimTrieCacheBase::incrementBlock()
{
// the plan:
@@ -751,11 +808,11 @@ bool CClaimTrieCacheBase::incrementBlock()
db << "INSERT INTO node(name) SELECT nodeName FROM claim INDEXED BY claim_activationHeight "
"WHERE activationHeight = ?1 AND expirationHeight > ?1 "
"ON CONFLICT(name) DO UPDATE SET hash = NULL"
"ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL"
<< nNextHeight;
// don't make new nodes for items in supports or items that expire this block that don't exist in claims
db << "UPDATE node SET hash = NULL WHERE name IN "
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN "
"(SELECT nodeName FROM claim WHERE expirationHeight = ?1 "
"UNION SELECT nodeName FROM support WHERE expirationHeight = ?1 OR activationHeight = ?1)"
<< nNextHeight;
@@ -766,7 +823,11 @@ bool CClaimTrieCacheBase::incrementBlock()
return true;
}
void CClaimTrieCacheBase::insertTakeovers(bool allowReplace) {
// hardcoded claims that should trigger a takeover
#include <takeoverworkarounds.h>
void CClaimTrieCacheBase::insertTakeovers(bool allowReplace)
{
auto insertTakeoverQuery = allowReplace ?
db << "INSERT OR REPLACE INTO takeover(name, height, claimID) VALUES(?, ?, ?)" :
db << "INSERT INTO takeover(name, height, claimID) VALUES(?, ?, ?)";
@@ -787,13 +848,7 @@ void CClaimTrieCacheBase::insertTakeovers(bool allowReplace) {
if (takeoverHappening && activateAllFor(nameWithTakeover))
hasCandidate = getInfoForName(nameWithTakeover, candidateValue, 1);
// This is a super ugly hack to work around bug in old code.
// The bug: un/support a name then update it. This will cause its takeover height to be reset to current.
// This is because the old code with add to the cache without setting block originals when dealing in supports.
if (nNextHeight < 658300) {
auto wit = takeoverWorkarounds.find(std::make_pair(nNextHeight, nameWithTakeover));
takeoverHappening |= wit != takeoverWorkarounds.end();
}
takeoverHappening |= isTakeoverWorkaroundActive(nNextHeight, nameWithTakeover);
logPrint << "Takeover on " << nameWithTakeover << " at " << nNextHeight << ", happening: " << takeoverHappening << ", set before: " << hasCurrentWinner << Clog::endl;
@@ -830,10 +885,10 @@ bool CClaimTrieCacheBase::decrementBlock()
nNextHeight--;
db << "INSERT INTO node(name) SELECT nodeName FROM claim "
"WHERE expirationHeight = ? ON CONFLICT(name) DO UPDATE SET hash = NULL"
"WHERE expirationHeight = ? ON CONFLICT(name) DO UPDATE SET hash = NULL, claimsHash = NULL"
<< nNextHeight;
db << "UPDATE node SET hash = NULL WHERE name IN("
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN("
"SELECT nodeName FROM support WHERE expirationHeight = ?1 OR activationHeight = ?1 "
"UNION SELECT nodeName FROM claim WHERE activationHeight = ?1)"
<< nNextHeight;
@@ -849,7 +904,7 @@ bool CClaimTrieCacheBase::decrementBlock()
bool CClaimTrieCacheBase::finalizeDecrement()
{
db << "UPDATE node SET hash = NULL WHERE name IN "
db << "UPDATE node SET hash = NULL, claimsHash = NULL WHERE name IN "
"(SELECT nodeName FROM claim WHERE activationHeight = ?1 AND expirationHeight > ?1 "
"UNION SELECT nodeName FROM support WHERE activationHeight = ?1 AND expirationHeight > ?1 "
"UNION SELECT name FROM takeover WHERE height = ?1)" << nNextHeight;
@@ -869,14 +924,30 @@ int CClaimTrieCacheBase::getDelayForName(const std::string& name, const uint160&
return 0;
}
// NOTE: old code had a bug in it where nodes with no claims but with children would get left in the cache after removal.
// This would cause the getNumBlocksOfContinuousOwnership to return zero (causing incorrect takeover height calc).
auto hit = removalWorkaround.find(name);
if (hit != removalWorkaround.end()) {
removalWorkaround.erase(hit);
return 0;
if (nNextHeight > base->nMaxRemovalWorkaroundHeight) {
if (!hasCurrentWinner)
return 0;
// TODO: hard fork this out! It's wrong but kept for backwards compatibility
// Plan: if we have no claims for this node but we do have multiple children
// such that we have an implicit node here then return a 0
if (emptyNodeShouldExistAt(db, name, nNextHeight, 2))
return 0;
}
return hasCurrentWinner ? std::min((nNextHeight - winningTakeoverHeight) / base->nProportionalDelayFactor, 4032) : 0;
else {
// NOTE: old code had a bug in it where nodes with no claims but with children would get left in the cache after removal.
// This would cause the getNumBlocksOfContinuousOwnership to return zero (causing incorrect takeover height calc).
auto hit = removalWorkaround.find(name);
if (hit != removalWorkaround.end()) {
removalWorkaround.erase(hit);
return 0;
}
}
if (!hasCurrentWinner)
return 0;
return std::min((nNextHeight - winningTakeoverHeight) / base->nProportionalDelayFactor, 4032);
}
std::string CClaimTrieCacheBase::adjustNameForValidHeight(const std::string& name, int validHeight) const
+18 -6
View File
@@ -18,18 +18,23 @@ uint256 getValueHash(const COutPoint& outPoint, int nHeightOfLastTakeover);
class CClaimTrie
{
// for unit tests
friend class CClaimTrieCacheBase;
friend class ClaimTrieChainFixture;
friend class ValidationBlockTests;
// forks have to be friends of trie
friend class CClaimTrieCacheHashFork;
friend class CClaimTrieCacheExpirationFork;
friend class CClaimTrieCacheNormalizationFork;
friend class ValidationBlockTests;
friend class CClaimTrieCacheClaimInfoHashFork;
public:
CClaimTrie() = delete;
CClaimTrie(CClaimTrie&&) = delete;
CClaimTrie(const CClaimTrie&) = delete;
CClaimTrie(std::size_t cacheBytes, bool fWipe, int height = 0,
CClaimTrie(std::size_t cacheBytes,
bool fWipe, int height = 0,
const std::string& dataDir = ".",
int nNormalizedNameForkHeight = 1,
int nMinRemovalWorkaroundHeight = 1,
@@ -38,6 +43,7 @@ public:
int64_t nExtendedClaimExpirationTime = 1,
int64_t nExtendedClaimExpirationForkHeight = 1,
int64_t nAllClaimsInMerkleForkHeight = 1,
int64_t nClaimInfoInMerkleForkHeight = 1,
int proportionalDelayFactor = 32);
CClaimTrie& operator=(CClaimTrie&&) = delete;
@@ -52,6 +58,7 @@ protected:
const std::size_t dbCacheBytes;
const std::string dbFile;
sqlite::database db;
bool isNodeMigrationStart;
const int nProportionalDelayFactor;
const int nNormalizedNameForkHeight;
@@ -61,6 +68,10 @@ protected:
const int64_t nExtendedClaimExpirationTime;
const int64_t nExtendedClaimExpirationForkHeight;
const int64_t nAllClaimsInMerkleForkHeight;
const int64_t nClaimInfoInMerkleForkHeight;
private:
void doNodeTableMigration();
};
class CClaimTrieCacheBase
@@ -93,7 +104,7 @@ public:
int nHeight, int nValidHeight = -1);
bool removeClaim(const uint160& claimId, const COutPoint& outPoint, std::string& nodeName,
int& validHeight, int& originalHeight);
int& validHeight, int& originalHeight);
bool removeSupport(const COutPoint& outPoint, std::string& nodeName, int& validHeight);
virtual bool incrementBlock();
@@ -121,10 +132,9 @@ protected:
int nNextHeight; // Height of the block that is being worked on, which is
CClaimTrie* base;
sqlite::database db;
mutable std::unordered_set<std::string> removalWorkaround;
sqlite::database_binder childHashQuery, claimHashQuery, claimHashQueryLimit;
virtual uint256 computeNodeHash(const std::string& name, int takeoverHeight);
virtual uint256 computeNodeHash(const std::string& name, uint256& claimsHash, int takeoverHeight);
supportEntryType getSupportsForName(const std::string& name) const;
virtual int getDelayForName(const std::string& name, const uint160& claimId) const;
@@ -132,10 +142,12 @@ protected:
bool deleteNodeIfPossible(const std::string& name, std::string& parent, int64_t& claims);
void ensureTreeStructureIsUpToDate();
void ensureTransacting();
void insertTakeovers(bool allowReplace=false);
void insertTakeovers(bool allowReplace = false);
private:
bool transacting;
mutable std::unordered_set<std::string> removalWorkaround;
// for unit test
friend struct ClaimTrieChainFixture;
friend class CClaimTrieCacheTest;
-1
View File
@@ -6,7 +6,6 @@
COutPoint::COutPoint() noexcept : n(std::numeric_limits<uint32_t>::max())
{
}
COutPoint::COutPoint(uint256 hashIn, uint32_t nIn) : hash(std::move(hashIn)), n(nIn)
+1
View File
@@ -103,6 +103,7 @@ struct Params {
}
/** blocks before the hard fork that adds all claims into the merkle hash */
int64_t nAllClaimsInMerkleForkHeight;
int64_t nClaimInfoInMerkleForkHeight;
int64_t DifficultyAdjustmentInterval() const { return nPowTargetTimespan / nPowTargetSpacing; }
uint256 nMinimumChainWork;
uint256 defaultAssumeValid;
+56 -273
View File
@@ -6,7 +6,6 @@
#include <clientversion.h>
#include <index/blockfilterindex.h>
#include <streams.h>
#include <sqlite.h>
#include <util/system.h>
#include <validation.h>
@@ -18,29 +17,6 @@
* active chain can always be retrieved, alleviating timing concerns.
*/
constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000; // 16 MiB
/** The pre-allocation chunk size for fltr?????.dat files */
constexpr unsigned int FLTR_FILE_CHUNK_SIZE = 0x100000; // 1 MiB
namespace {
struct DBVal {
uint256 hash;
uint256 header;
FlatFilePos pos;
ADD_SERIALIZE_METHODS;
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(hash);
READWRITE(header);
READWRITE(pos);
}
};
} // namespace
static std::map<BlockFilterType, BlockFilterIndex> g_filter_indexes;
BlockFilterIndex::BlockFilterIndex(BlockFilterType filter_type,
@@ -50,313 +26,120 @@ BlockFilterIndex::BlockFilterIndex(BlockFilterType filter_type,
const std::string& filter_name = BlockFilterTypeName(filter_type);
if (filter_name.empty()) throw std::invalid_argument("unknown filter_type");
fs::path path = GetDataDir() / "filter" / filter_name;
fs::path path = GetDataDir() / "filter";
fs::create_directories(path);
path /= filter_name;
m_name = filter_name + " block filter index";
m_db = MakeUnique<BaseIndex::DB>(path, n_cache_size, f_memory, f_wipe);
m_filter_fileseq = MakeUnique<FlatFileSeq>(std::move(path), "fltr", FLTR_FILE_CHUNK_SIZE);
(*m_db) << "CREATE TABLE IF NOT EXISTS block (height INTEGER, hash BLOB NOT NULL COLLATE BINARY, "
(*m_db) << "CREATE TABLE IF NOT EXISTS block (height INTEGER NOT NULL, hash BLOB NOT NULL COLLATE BINARY, "
"filter_hash BLOB NOT NULL COLLATE BINARY, header BLOB NOT NULL COLLATE BINARY, "
"file INTEGER NOT NULL, pos INTEGER NOT NULL, "
"filter_data BLOB NOT NULL COLLATE BINARY, "
"PRIMARY KEY(height, hash));";
(*m_db) << "CREATE TABLE IF NOT EXISTS file_pos (file INTEGER NOT NULL, pos INTEGER NOT NULL);";
if (f_wipe) {
(*m_db) << "DELETE FROM file_pos";
(*m_db) << "DELETE FROM block";
}
}
bool BlockFilterIndex::Init()
{
if (!ReadFilePos(m_next_filter_pos)) {
m_next_filter_pos.nFile = 0;
m_next_filter_pos.nPos = 0;
}
return BaseIndex::Init();
}
bool BlockFilterIndex::ReadFilePos(FlatFilePos& file) const
{
file.SetNull();
bool success = false;
for (auto&& row : (*m_db) << "SELECT file, pos FROM file_pos") {
row >> file.nFile >> file.nPos;
success = true;
}
return success;
}
bool BlockFilterIndex::WriteFilePos(const FlatFilePos& file)
{
(*m_db) << "DELETE FROM file_pos";
(*m_db) << "INSERT INTO file_pos VALUES(?, ?)" << file.nFile << file.nPos;
return m_db->rows_modified() > 0;
}
bool BlockFilterIndex::CommitInternal()
{
const FlatFilePos& pos = m_next_filter_pos;
// Flush current filter file to disk.
CAutoFile file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
if (file.IsNull()) {
return error("%s: Failed to open filter file %d", __func__, pos.nFile);
}
if (!FileCommit(file.Get())) {
return error("%s: Failed to commit filter file %d", __func__, pos.nFile);
}
return BaseIndex::CommitInternal();
}
bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const
{
CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION);
if (filein.IsNull()) {
return false;
}
uint256 block_hash;
std::vector<unsigned char> encoded_filter;
try {
filein >> block_hash >> encoded_filter;
filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter));
}
catch (const std::exception& e) {
return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what());
}
return true;
}
size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter)
{
assert(filter.GetFilterType() == GetFilterType());
size_t data_size =
GetSerializeSize(filter.GetBlockHash(), CLIENT_VERSION) +
GetSerializeSize(filter.GetEncodedFilter(), CLIENT_VERSION);
// If writing the filter would overflow the file, flush and move to the next one.
if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) {
CAutoFile last_file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
if (last_file.IsNull()) {
LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile);
return 0;
}
if (!TruncateFile(last_file.Get(), pos.nPos)) {
LogPrintf("%s: Failed to truncate filter file %d\n", __func__, pos.nFile);
return 0;
}
if (!FileCommit(last_file.Get())) {
LogPrintf("%s: Failed to commit filter file %d\n", __func__, pos.nFile);
return 0;
}
pos.nFile++;
pos.nPos = 0;
}
// Pre-allocate sufficient space for filter data.
bool out_of_space;
m_filter_fileseq->Allocate(pos, data_size, out_of_space);
if (out_of_space) {
LogPrintf("%s: out of disk space\n", __func__);
return 0;
}
CAutoFile fileout(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
if (fileout.IsNull()) {
LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile);
return 0;
}
fileout << filter.GetBlockHash() << filter.GetEncodedFilter();
return data_size;
}
bool BlockFilterIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
{
CBlockUndo block_undo;
uint256 prev_header;
if (pindex->nHeight > 0) {
if (!UndoReadFromDisk(block_undo, pindex)) {
return false;
}
uint256 block_hash;
auto query = (*m_db) << "SELECT hash, header FROM block WHERE height = ?"
<< pindex->nHeight - 1;
auto it = query.begin();
if (it == query.end())
if (!UndoReadFromDisk(block_undo, pindex))
return false;
*it >> block_hash >> prev_header;
uint256 expected_block_hash = pindex->pprev->GetBlockHash();
if (block_hash != expected_block_hash) {
return error("%s: previous block header belongs to unexpected block %s; expected %s",
__func__, block_hash.ToString(), expected_block_hash.ToString());
}
if (!LookupFilterHeader(pindex->pprev, prev_header))
return false;
}
BlockFilter filter(m_filter_type, block, block_undo);
size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter);
if (bytes_written == 0)
return false;
const auto filterHash = filter.GetHash(); // trying to avoid temps
const auto filterHeader = filter.ComputeHeader(prev_header);
(*m_db) << "INSERT OR REPLACE INTO block VALUES(?, ?, ?, ?, ?, ?)"
(*m_db) << "INSERT OR REPLACE INTO block VALUES(?, ?, ?, ?, ?)"
<< pindex->nHeight
<< pindex->hash
<< filterHash
<< filterHeader
<< m_next_filter_pos.nFile
<< m_next_filter_pos.nPos;
<< filter.GetEncodedFilter();
if (m_db->rows_modified() <= 0) {
return false;
}
m_next_filter_pos.nPos += bytes_written;
return true;
}
static bool CopyHeightIndexToHashIndex(sqlite::database& db, int start_height, int stop_height)
{
if (start_height > stop_height)
return true;
db << "UPDATE block SET height = NULL WHERE height BETWEEN ? AND ?"
<< start_height << stop_height;
return db.rows_modified() > 0;
}
bool BlockFilterIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip)
{
assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip);
// During a reorg, we need to copy all filters for blocks that are getting disconnected from the
// height index to the hash index so we can still find them when the height index entries are
// overwritten.
if (!CopyHeightIndexToHashIndex(*m_db, new_tip->nHeight + 1, current_tip->nHeight)) {
return false;
}
// The latest filter position gets written in Commit by the call to the BaseIndex::Rewind.
// But since this creates new references to the filter, the position should get updated here
// atomically as well in case Commit fails.
if (!WriteFilePos(m_next_filter_pos)) return false;
return BaseIndex::Rewind(current_tip, new_tip);
}
static bool LookupOne(sqlite::database& db, const CBlockIndex* block_index, DBVal& result)
{
// First check if the result is stored under the height index and the value there matches the
// block hash. This should be the case if the block is on the active chain.
auto query = db << "SELECT filter_hash, header, file, pos FROM block WHERE (height = ? "
"OR height IS NULL) AND hash = ? LIMIT 1"
<< block_index->nHeight << block_index->GetBlockHash();
for (auto&& row : query) {
row >> result.hash >> result.header >> result.pos.nFile >> result.pos.nPos;
return true;
}
return false;
}
static bool LookupRange(sqlite::database& db, const std::string& index_name, int start_height,
const CBlockIndex* stop_index, std::vector<DBVal>& results)
{
if (start_height < 0) {
return error("%s: start height (%d) is negative", __func__, start_height);
}
if (start_height > stop_index->nHeight) {
return error("%s: start height (%d) is greater than stop height (%d)",
__func__, start_height, stop_index->nHeight);
}
size_t results_size = static_cast<size_t>(stop_index->nHeight - start_height + 1);
results.resize(results_size);
// Iterate backwards through block indexes collecting results in order to access the block hash
// of each entry in case we need to look it up in the hash index.
for (const CBlockIndex* block_index = stop_index;
block_index && block_index->nHeight >= start_height;
block_index = block_index->pprev) {
size_t i = static_cast<size_t>(block_index->nHeight - start_height);
if (!LookupOne(db, block_index, results[i])) {
return error("%s: unable to read value in %s at key %s", __func__,
index_name, block_index->GetBlockHash().ToString());
}
}
return true;
return m_db->rows_modified() > 0;
}
bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const
{
DBVal entry;
if (!LookupOne(*m_db, block_index, entry)) {
return false;
auto query = *m_db << "SELECT filter_data FROM block WHERE height = ? and hash = ?"
<< block_index->nHeight << block_index->hash;
for (auto&& row: query) {
std::vector<uint8_t> data;
row >> data;
filter_out = BlockFilter(m_filter_type, block_index->hash, data);
return true;
}
return ReadFilterFromDisk(entry.pos, filter_out);
return false;
}
bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const
{
DBVal entry;
if (!LookupOne(*m_db, block_index, entry)) {
return false;
auto query = *m_db << "SELECT header FROM block WHERE height = ? and hash = ?"
<< block_index->nHeight << block_index->hash;
for (auto&& row: query) {
row >> header_out;
return true;
}
header_out = entry.header;
return true;
return false;
}
bool BlockFilterIndex::LookupFilterRange(int start_height, const CBlockIndex* stop_index,
std::vector<BlockFilter>& filters_out) const
{
std::vector<DBVal> entries;
if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
return false;
}
filters_out.resize(entries.size());
auto filter_pos_it = filters_out.begin();
for (const auto& entry : entries) {
if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) {
assert(start_height >= 0);
assert(stop_index->nHeight >= start_height);
filters_out.reserve(stop_index->nHeight - start_height + 1);
while(stop_index && stop_index->nHeight >= start_height) {
filters_out.emplace_back();
if (!LookupFilter(stop_index, filters_out.back()))
return false;
}
++filter_pos_it;
stop_index = stop_index->pprev;
}
std::reverse(filters_out.begin(), filters_out.end());
return true;
}
bool LookupFilterHash(sqlite::database& db, const CBlockIndex* block_index, uint256& hash_out)
{
auto query = db << "SELECT filter_hash FROM block WHERE height = ? and hash = ?"
<< block_index->nHeight << block_index->hash;
for (auto&& row: query) {
row >> hash_out;
return true;
}
return true;
return false;
}
bool BlockFilterIndex::LookupFilterHashRange(int start_height, const CBlockIndex* stop_index,
std::vector<uint256>& hashes_out) const
{
std::vector<DBVal> entries;
if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
return false;
}
hashes_out.clear();
hashes_out.reserve(entries.size());
for (const auto& entry : entries) {
hashes_out.push_back(entry.hash);
assert(start_height >= 0);
assert(stop_index->nHeight >= start_height);
hashes_out.reserve(stop_index->nHeight - start_height + 1);
while(stop_index && stop_index->nHeight >= start_height) {
hashes_out.emplace_back();
if (!LookupFilterHash(*m_db, stop_index, hashes_out.back()))
return false;
stop_index = stop_index->pprev;
}
std::reverse(hashes_out.begin(), hashes_out.end());
return true;
}
-18
View File
@@ -24,29 +24,11 @@ private:
std::string m_name;
std::unique_ptr<BaseIndex::DB> m_db;
FlatFilePos m_next_filter_pos;
std::unique_ptr<FlatFileSeq> m_filter_fileseq;
bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const;
size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter);
protected:
bool Init() override;
bool CommitInternal() override;
bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override;
bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override;
BaseIndex::DB& GetDB() const override { return *m_db; }
const char* GetName() const override { return m_name.c_str(); }
bool ReadFilePos(FlatFilePos& file) const;
bool WriteFilePos(const FlatFilePos& file);
public:
/** Constructs the index, which becomes available to be queried. */
explicit BlockFilterIndex(BlockFilterType filter_type,
+1 -1
View File
@@ -366,7 +366,6 @@ void SetupServerArgs()
gArgs.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
#if HAVE_SYSTEM
gArgs.AddArg("-alertnotify=<cmd>", "Execute command when a relevant alert is received or we see a really long fork (%s in cmd is replaced by message)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
gArgs.AddArg("-maxblockfilesize=<n>", strprintf("Keep the block files below n megabytes each. (default: %d, minimum: 8)", DEFAULT_MAX_BLOCKFILE_SIZE >> 20U), false, OptionsCategory::OPTIONS);
#endif
gArgs.AddArg("-assumevalid=<hex>", strprintf("If this block is in the chain assume that it and its ancestors are valid and potentially skip their script verification (0 to verify all, default: %s, testnet: %s)", defaultChainParams->GetConsensus().defaultAssumeValid.GetHex(), testnetChainParams->GetConsensus().defaultAssumeValid.GetHex()), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
gArgs.AddArg("-blocksdir=<dir>", "Specify directory to hold blocks subdirectory for *.dat files (default: <datadir>)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
@@ -383,6 +382,7 @@ void SetupServerArgs()
gArgs.AddArg("-feefilter", strprintf("Tell other nodes to filter invs to us by our mempool min fee (default: %u)", DEFAULT_FEEFILTER), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS);
gArgs.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
gArgs.AddArg("-loadblock=<file>", "Imports blocks from external blk000??.dat file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
gArgs.AddArg("-maxblockfilesize=<n>", strprintf("Keep the block files below n megabytes each. (default: %d, minimum: 8)", DEFAULT_MAX_BLOCKFILE_SIZE >> 20U), ArgsManager::ALLOW_INT, OptionsCategory::OPTIONS);
gArgs.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
gArgs.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
gArgs.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+4 -1
View File
@@ -9,7 +9,7 @@
#include <consensus/validation.h>
#include <coins.h>
#include <claimscriptop.h>
#include <nameclaim.h>
@@ -114,6 +114,9 @@ bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, const CFeeR
unsigned int nDataOut = 0;
txnouttype whichType;
for (const CTxOut& txout : tx.vout) {
if (!ValidateClaimName(txout.scriptPubKey, reason))
return false;
if (!::IsStandard(StripClaimScriptPrefix(txout.scriptPubKey), whichType)) {
reason = "scriptpubkey";
return false;
+10 -29
View File
@@ -79,7 +79,8 @@ void RollBackTo(const CBlockIndex* targetIndex, CCoinsViewCache& coinsCache, CCl
trieCache.getMerkleHash(); // update the hash tree
}
std::string escapeNonUtf8(const std::string& name)
// it ensures name is utf8 like string
std::string convertToUtf8(const std::string& name)
{
using namespace boost::locale::conv;
try {
@@ -120,26 +121,6 @@ static bool extractValue(const CScript& scriptPubKey, std::string& name, std::st
return true;
}
std::vector<CClaimNsupports> seqSort(const std::vector<CClaimNsupports>& source)
{
auto claimsNsupports = source;
std::sort(claimsNsupports.begin(), claimsNsupports.end(), [](const CClaimNsupports& lhs, const CClaimNsupports& rhs) {
return lhs.originalHeight < rhs.originalHeight || (lhs.originalHeight == rhs.originalHeight && lhs.claim < rhs.claim);
});
return claimsNsupports;
}
std::size_t indexOf(const std::vector<CClaimNsupports>& source, const uint160& claimId)
{
auto it = std::find_if(source.begin(), source.end(), [&claimId](const CClaimNsupports& claimNsupports) {
return claimNsupports.claim.claimId == claimId;
});
assert(it != source.end());
return std::distance(source.begin(), it);
}
UniValue claimToJSON(const CCoinsViewCache& coinsCache, const CClaimValue& claim)
{
UniValue result(UniValue::VOBJ);
@@ -148,7 +129,7 @@ UniValue claimToJSON(const CCoinsViewCache& coinsCache, const CClaimValue& claim
if (!coin.IsSpent()) {
std::string name, value;
if (extractValue(coin.out.scriptPubKey, name, value)) {
result.pushKV(T_NAME, escapeNonUtf8(name));
result.pushKV(T_NAME, convertToUtf8(name));
result.pushKV(T_VALUE, value);
}
@@ -245,7 +226,7 @@ static UniValue getnamesintrie(const JSONRPCRequest& request)
UniValue ret(UniValue::VARR);
trieCache.getNamesInTrie([&ret](const std::string& name) {
ret.push_back(escapeNonUtf8(name));
ret.push_back(convertToUtf8(name));
if (ShutdownRequested())
throw JSONRPCError(RPC_INTERNAL_ERROR, "Shutdown requested");
@@ -294,7 +275,7 @@ static UniValue getvalueforname(const JSONRPCRequest& request)
bid = indexOf(csToName.claimsNsupports, claimIdIn);
}
ret.pushKV(T_NORMALIZEDNAME, escapeNonUtf8(csToName.name));
ret.pushKV(T_NORMALIZEDNAME, convertToUtf8(csToName.name));
ret.pushKVs(claimAndSupportsToJSON(coinsCache, claimNsupports));
ret.pushKV(T_LASTTAKEOVERHEIGHT, csToName.nLastTakeoverHeight);
ret.pushKV(T_BID, (int)bid);
@@ -320,7 +301,7 @@ UniValue getclaimsforname(const JSONRPCRequest& request)
auto csToName = trieCache.getClaimsForName(name);
UniValue result(UniValue::VOBJ);
result.pushKV(T_NORMALIZEDNAME, escapeNonUtf8(csToName.name));
result.pushKV(T_NORMALIZEDNAME, convertToUtf8(csToName.name));
auto seqOrder = seqSort(csToName.claimsNsupports);
@@ -378,7 +359,7 @@ UniValue getclaimbybid(const JSONRPCRequest& request)
seq = indexOf(seqOrder, claimNsupports.claim.claimId);
}
result.pushKV(T_NORMALIZEDNAME, escapeNonUtf8(csToName.name));
result.pushKV(T_NORMALIZEDNAME, convertToUtf8(csToName.name));
result.pushKVs(claimAndSupportsToJSON(coinsCache, claimNsupports));
result.pushKV(T_LASTTAKEOVERHEIGHT, csToName.nLastTakeoverHeight);
result.pushKV(T_BID, bid);
@@ -423,7 +404,7 @@ UniValue getclaimbyseq(const JSONRPCRequest& request)
return claimNsupports;
}();
result.pushKV(T_NORMALIZEDNAME, escapeNonUtf8(csToName.name));
result.pushKV(T_NORMALIZEDNAME, convertToUtf8(csToName.name));
result.pushKVs(claimAndSupportsToJSON(coinsCache, claimNsupports));
result.pushKV(T_LASTTAKEOVERHEIGHT, csToName.nLastTakeoverHeight);
result.pushKV(T_BID, (int)bid);
@@ -458,7 +439,7 @@ UniValue getclaimbyid(const JSONRPCRequest& request)
seq = indexOf(seqOrder, foundClaim.claimId);
bid = indexOf(csToName.claimsNsupports, foundClaim.claimId);
}
ret.pushKV(T_NORMALIZEDNAME, escapeNonUtf8(csToName.name));
ret.pushKV(T_NORMALIZEDNAME, convertToUtf8(csToName.name));
ret.pushKVs(claimAndSupportsToJSON(coinsCache, claimNsupports));
ret.pushKV(T_LASTTAKEOVERHEIGHT, csToName.nLastTakeoverHeight);
ret.pushKV(T_BID, (int)bid);
@@ -529,7 +510,7 @@ UniValue getclaimsfortx(const JSONRPCRequest& request)
UniValue o(UniValue::VOBJ);
o.pushKV(T_N, static_cast<int64_t>(i));
std::string sName(vvchParams[0].begin(), vvchParams[0].end());
o.pushKV(T_NAME, escapeNonUtf8(sName));
o.pushKV(T_NAME, convertToUtf8(sName));
if (op == OP_CLAIM_NAME) {
uint160 claimId = ClaimIdHash(hash, i);
o.pushKV(T_CLAIMID, claimId.GetHex());
+12 -1
View File
@@ -369,6 +369,7 @@ static UniValue getblocktemplate(const JSONRPCRequest& request)
UniValue lpval = NullUniValue;
std::set<std::string> setClientRules;
int64_t nMaxVersionPreVB = -1;
UniValue aMutable(UniValue::VARR);
if (!request.params[0].isNull())
{
const UniValue& oparam = request.params[0].get_obj();
@@ -383,6 +384,13 @@ static UniValue getblocktemplate(const JSONRPCRequest& request)
throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid mode");
lpval = find_value(oparam, "longpollid");
const UniValue& capval = find_value(oparam, "capabilities");
if (capval.isArray()) {
for (std::size_t i = 0; i < capval.size(); ++i)
if (capval[i].get_str().rfind("coinbase/", 0) == 0) // we don't care what they do to the coinbase
aMutable.push_back(capval[i]);
}
if (strMode == "proposal")
{
const UniValue& dataval = find_value(oparam, "data");
@@ -573,7 +581,6 @@ static UniValue getblocktemplate(const JSONRPCRequest& request)
arith_uint256 hashTarget = arith_uint256().SetCompact(pblock->nBits);
UniValue aMutable(UniValue::VARR);
aMutable.push_back("time");
aMutable.push_back("transactions");
aMutable.push_back("prevblock");
@@ -623,6 +630,10 @@ static UniValue getblocktemplate(const JSONRPCRequest& request)
}
}
}
if (pindexPrev->nHeight + 1 >= consensusParams.CSVHeight)
aRules.push_back("csv");
if (pindexPrev->nHeight + 1 >= consensusParams.SegwitHeight)
aRules.push_back("segwit");
result.pushKV("version", pblock->nVersion);
result.pushKV("rules", aRules);
result.pushKV("vbavailable", vbavailable);
+26
View File
@@ -24,6 +24,32 @@ BOOST_AUTO_TEST_CASE(claim_replace_test) {
BOOST_CHECK(fixture.is_best_claim("bassfisher", tx2));
}
BOOST_AUTO_TEST_CASE(triple_update_test)
{
ClaimTrieChainFixture fixture;
CMutableTransaction tx1 = fixture.MakeClaim(fixture.GetCoinbase(), "test", "one", 10);
fixture.IncrementBlocks(1);
CMutableTransaction tx1a = fixture.MakeClaim(fixture.GetCoinbase(), "tester", "one", 3);
CMutableTransaction tx2 = fixture.MakeClaim(fixture.GetCoinbase(), "test", "one", 5);
auto height = ::ChainActive().Tip()->nHeight;
fixture.IncrementBlocks(9);
CMutableTransaction tx3 = fixture.MakeUpdate(tx1, "test", "two", ClaimIdHash(tx1.GetHash(), 0), 10);
CMutableTransaction tx4 = fixture.MakeUpdate(tx2, "test", "two", ClaimIdHash(tx2.GetHash(), 0), 15);
CMutableTransaction tx5 = fixture.MakeUpdate(tx4, "test", "two", ClaimIdHash(tx2.GetHash(), 0), 5);
fixture.IncrementBlocks(1);
BOOST_CHECK(fixture.is_best_claim("test", tx3));
uint160 id; int takeover;
BOOST_REQUIRE(fixture.getLastTakeoverForName("test", id, takeover));
BOOST_CHECK_EQUAL(takeover, height);
id = ClaimIdHash(tx2.GetHash(), 0);
std::string name;
CClaimValue value;
BOOST_REQUIRE(fixture.getClaimById(id, name, value));
BOOST_CHECK_EQUAL(value.nValidAtHeight, height+20);
}
BOOST_AUTO_TEST_CASE(takeover_stability_test) {
// no competing bids
ClaimTrieChainFixture fixture;
+124
View File
@@ -0,0 +1,124 @@
// Copyright (c) 2015-2019 The LBRY Foundation
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://opensource.org/licenses/mit-license.php
#include <miner.h>
#include <test/claimtriefixture.h>
#include <validation.h>
using namespace std;
extern std::vector<CClaimNsupports> seqSort(const std::vector<CClaimNsupports>& source);
extern std::size_t indexOf(const std::vector<CClaimNsupports>& source, const uint160& claimId);
extern boost::test_tools::predicate_result ValidatePairs(CClaimTrieCache& cache, const std::vector<std::pair<bool, uint256>>& pairs, uint256 claimHash);
extern uint256 claimInfoHash(const std::string& name, const COutPoint& outPoint, int bid, int seq, int nHeightOfLastTakeover);
BOOST_FIXTURE_TEST_SUITE(claimtrieclaiminfo_tests, RegTestingSetup)
BOOST_AUTO_TEST_CASE(hash_includes_all_claiminfo_rollback_test)
{
ClaimTrieChainFixture fixture;
fixture.setClaimInfoForkHeight(5);
CMutableTransaction tx1 = fixture.MakeClaim(fixture.GetCoinbase(), "test", "one", 1);
fixture.IncrementBlocks(1);
auto currentRoot = fixture.getMerkleHash();
fixture.IncrementBlocks(1);
BOOST_CHECK_EQUAL(currentRoot, fixture.getMerkleHash());
fixture.IncrementBlocks(3);
BOOST_CHECK_NE(currentRoot, fixture.getMerkleHash());
fixture.DecrementBlocks(3);
BOOST_CHECK_EQUAL(currentRoot, fixture.getMerkleHash());
}
BOOST_AUTO_TEST_CASE(hash_includes_all_claiminfo_single_test)
{
ClaimTrieChainFixture fixture;
fixture.setClaimInfoForkHeight(2);
fixture.IncrementBlocks(4);
CMutableTransaction tx1 = fixture.MakeClaim(fixture.GetCoinbase(), "test", "one", 1);
fixture.IncrementBlocks(1);
COutPoint outPoint(tx1.GetHash(), 0);
uint160 claimId = ClaimIdHash(tx1.GetHash(), 0);
CClaimTrieProof proof;
BOOST_CHECK(fixture.getProofForName("test", claimId, proof));
BOOST_CHECK(proof.hasValue);
BOOST_CHECK_EQUAL(proof.outPoint, outPoint);
auto claimHash = claimInfoHash("test", outPoint, 0, 0, proof.nHeightOfLastTakeover);
BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
}
BOOST_AUTO_TEST_CASE(hash_includes_all_claiminfo_triple_test)
{
ClaimTrieChainFixture fixture;
fixture.setClaimInfoForkHeight(2);
fixture.IncrementBlocks(4);
std::string names[] = {"test", "tester", "tester2"};
CMutableTransaction tx1 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "one", 1);
CMutableTransaction tx2 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "two", 2);
CMutableTransaction tx3 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "thr", 3);
CMutableTransaction tx7 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "for", 4);
CMutableTransaction tx8 = fixture.MakeClaim(fixture.GetCoinbase(), names[0], "fiv", 5);
CMutableTransaction tx4 = fixture.MakeClaim(fixture.GetCoinbase(), names[1], "two", 2);
CMutableTransaction tx5 = fixture.MakeClaim(fixture.GetCoinbase(), names[1], "thr", 3);
CMutableTransaction tx6 = fixture.MakeClaim(fixture.GetCoinbase(), names[2], "one", 1);
fixture.IncrementBlocks(1);
for (const auto& name : names) {
int bid = 0;
auto cfn = fixture.getClaimsForName(name);
auto seqOrder = seqSort(cfn.claimsNsupports);
for (auto& claimSupports : cfn.claimsNsupports) {
CClaimTrieProof proof;
BOOST_CHECK_EQUAL(name, cfn.name); // normalization depends
auto& claim = claimSupports.claim;
BOOST_CHECK(fixture.getProofForName(name, claim.claimId, proof));
BOOST_CHECK(proof.hasValue);
BOOST_CHECK_EQUAL(proof.outPoint, claim.outPoint);
auto seq = indexOf(seqOrder, claim.claimId);
auto claimHash = claimInfoHash(name, claim.outPoint, bid++, seq, proof.nHeightOfLastTakeover);
BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
}
}
}
BOOST_AUTO_TEST_CASE(hash_includes_all_claiminfo_branched_test)
{
ClaimTrieChainFixture fixture;
fixture.setClaimInfoForkHeight(2);
fixture.IncrementBlocks(4);
std::string names[] = {"test", "toast", "tot", "top", "toa", "toad"};
for (const auto& name : names)
fixture.MakeClaim(fixture.GetCoinbase(), name, "one", 1);
fixture.MakeClaim(fixture.GetCoinbase(), "toa", "two", 2);
fixture.MakeClaim(fixture.GetCoinbase(), "toa", "tre", 3);
fixture.MakeClaim(fixture.GetCoinbase(), "toa", "qua", 4);
fixture.MakeClaim(fixture.GetCoinbase(), "toa", "cin", 5);
fixture.IncrementBlocks(1);
for (const auto& name : names) {
int bid = 0;
auto cfn = fixture.getClaimsForName(name);
auto seqOrder = seqSort(cfn.claimsNsupports);
for (auto& claimSupports : cfn.claimsNsupports) {
CClaimTrieProof proof;
BOOST_CHECK_EQUAL(name, cfn.name); // normalization depends
auto& claim = claimSupports.claim;
BOOST_CHECK(fixture.getProofForName(name, claim.claimId, proof));
BOOST_CHECK(proof.hasValue);
BOOST_CHECK_EQUAL(proof.outPoint, claim.outPoint);
auto seq = indexOf(seqOrder, claim.claimId);
auto claimHash = claimInfoHash(name, claim.outPoint, bid++, seq, proof.nHeightOfLastTakeover);
BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
}
}
}
BOOST_AUTO_TEST_SUITE_END()
+6 -5
View File
@@ -824,11 +824,12 @@ BOOST_AUTO_TEST_CASE(removal_workaround_functions)
fixture.IncrementBlocks(1);
BOOST_CHECK(fixture.is_best_claim("a", tx3a));
fixture.IncrementBlocks(7);
CMutableTransaction tx4a = fixture.MakeUpdate(tx2a, "b", "b", ClaimIdHash(tx2a.GetHash(), 0), 4);
CMutableTransaction tx4b = fixture.MakeUpdate(tx2b, "b", "b", ClaimIdHash(tx2b.GetHash(), 0), 5); // trigger a takeover
fixture.IncrementBlocks(1);
BOOST_CHECK(fixture.is_best_claim("b", tx4b));
// TODO: bring this back after we hard fork out the removal workaround part 2!
// fixture.IncrementBlocks(7);
// CMutableTransaction tx4a = fixture.MakeUpdate(tx2a, "b", "b", ClaimIdHash(tx2a.GetHash(), 0), 4);
// CMutableTransaction tx4b = fixture.MakeUpdate(tx2b, "b", "b", ClaimIdHash(tx2b.GetHash(), 0), 5); // trigger a takeover
// fixture.IncrementBlocks(1);
// BOOST_CHECK(fixture.is_best_claim("b", tx4b));
}
BOOST_AUTO_TEST_SUITE_END()
+22 -3
View File
@@ -76,9 +76,10 @@ BlockAssembler AssemblerForTest()
ClaimTrieChainFixture::ClaimTrieChainFixture() : CClaimTrieCache(&::Claimtrie()),
unique_block_counter(0), normalization_original(-1), expirationForkHeight(-1), forkhash_original(-1),
minRemovalWorkaroundHeight(-1), maxRemovalWorkaroundHeight(-1)
claiminfo_original(-1), minRemovalWorkaroundHeight(-1), maxRemovalWorkaroundHeight(-1)
{
fRequireStandard = false;
lRequireStandard = fRequireStandard;
fRequireStandard = false; // we don't require standard tx
BOOST_CHECK_EQUAL(nNextHeight, ::ChainActive().Height() + 1);
setNormalizationForkHeight(1000000);
@@ -98,6 +99,7 @@ ClaimTrieChainFixture::ClaimTrieChainFixture() : CClaimTrieCache(&::Claimtrie())
ClaimTrieChainFixture::~ClaimTrieChainFixture()
{
added_unchecked = false;
fRequireStandard = lRequireStandard;
DecrementBlocks(::ChainActive().Height());
auto& consensus = const_cast<Consensus::Params&>(Params().GetConsensus());
if (normalization_original >= 0) {
@@ -116,6 +118,10 @@ ClaimTrieChainFixture::~ClaimTrieChainFixture()
consensus.nAllClaimsInMerkleForkHeight = forkhash_original;
const_cast<int64_t&>(base->nAllClaimsInMerkleForkHeight) = forkhash_original;
}
if (claiminfo_original >= 0) {
consensus.nClaimInfoInMerkleForkHeight = claiminfo_original;
const_cast<int64_t&>(base->nClaimInfoInMerkleForkHeight) = claiminfo_original;
}
if (minRemovalWorkaroundHeight >= 0) {
consensus.nMinRemovalWorkaroundHeight = minRemovalWorkaroundHeight;
consensus.nMaxRemovalWorkaroundHeight = maxRemovalWorkaroundHeight;
@@ -124,7 +130,8 @@ ClaimTrieChainFixture::~ClaimTrieChainFixture()
}
}
void ClaimTrieChainFixture::setRemovalWorkaroundHeight(int targetMinusCurrent, int blocks = 1000) {
void ClaimTrieChainFixture::setRemovalWorkaroundHeight(int targetMinusCurrent, int blocks = 1000)
{
int target = ::ChainActive().Height() + targetMinusCurrent;
auto& consensus = const_cast<Consensus::Params&>(Params().GetConsensus());
if (minRemovalWorkaroundHeight < 0) {
@@ -174,6 +181,18 @@ void ClaimTrieChainFixture::setHashForkHeight(int targetMinusCurrent)
const_cast<int64_t&>(base->nAllClaimsInMerkleForkHeight) = target;
}
void ClaimTrieChainFixture::setClaimInfoForkHeight(int targetMinusCurrent)
{
if (targetMinusCurrent)
setHashForkHeight(targetMinusCurrent - 1);
int target = ::ChainActive().Height() + targetMinusCurrent;
auto& consensus = const_cast<Consensus::Params&>(Params().GetConsensus());
if (claiminfo_original < 0)
claiminfo_original = consensus.nClaimInfoInMerkleForkHeight;
consensus.nClaimInfoInMerkleForkHeight = target;
const_cast<int64_t&>(base->nClaimInfoInMerkleForkHeight) = target;
}
bool ClaimTrieChainFixture::CreateBlock(const std::unique_ptr<CBlockTemplate>& pblocktemplate)
{
CBlock* pblock = &pblocktemplate->block;
+4
View File
@@ -42,11 +42,13 @@ struct ClaimTrieChainFixture: public CClaimTrieCache
int normalization_original;
unsigned int num_txs_for_next_block;
bool added_unchecked;
bool lRequireStandard;
int64_t expirationForkHeight;
int64_t originalExpiration;
int64_t extendedExpiration;
int64_t forkhash_original;
int64_t claiminfo_original;
int minRemovalWorkaroundHeight, maxRemovalWorkaroundHeight;
using CClaimTrieCache::getSupportsForName;
@@ -61,6 +63,8 @@ struct ClaimTrieChainFixture: public CClaimTrieCache
void setHashForkHeight(int targetMinusCurrent);
void setClaimInfoForkHeight(int targetMinusCurrent);
void setRemovalWorkaroundHeight(int targetMinusCurrent, int blocks);
bool CreateBlock(const std::unique_ptr<CBlockTemplate>& pblocktemplate);
+11 -6
View File
@@ -8,7 +8,7 @@
using namespace std;
void ValidatePairs(CClaimTrieCache& cache, const std::vector<std::pair<bool, uint256>>& pairs, uint256 claimHash)
boost::test_tools::predicate_result ValidatePairs(CClaimTrieCache& cache, const std::vector<std::pair<bool, uint256>>& pairs, uint256 claimHash)
{
for (auto& pair : pairs)
if (pair.first) // we're on the right because we were an odd index number
@@ -16,7 +16,12 @@ void ValidatePairs(CClaimTrieCache& cache, const std::vector<std::pair<bool, uin
else
claimHash = Hash(claimHash.begin(), claimHash.end(), pair.second.begin(), pair.second.end());
BOOST_CHECK_EQUAL(cache.getMerkleHash(), claimHash);
if (cache.getMerkleHash() == claimHash)
return true;
boost::test_tools::predicate_result res(false);
res.message() << cache.getMerkleHash().ToString()
<< " != " << claimHash.ToString();
return res;
}
BOOST_FIXTURE_TEST_SUITE(claimtriehashfork_tests, RegTestingSetup)
@@ -55,7 +60,7 @@ BOOST_AUTO_TEST_CASE(hash_includes_all_claims_single_test)
BOOST_CHECK(proof.hasValue);
BOOST_CHECK_EQUAL(proof.outPoint, outPoint);
auto claimHash = getValueHash(outPoint, proof.nHeightOfLastTakeover);
ValidatePairs(fixture, proof.pairs, claimHash);
BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
}
BOOST_AUTO_TEST_CASE(hash_includes_all_claims_triple_test)
@@ -84,7 +89,7 @@ BOOST_AUTO_TEST_CASE(hash_includes_all_claims_triple_test)
BOOST_CHECK(proof.hasValue);
BOOST_CHECK_EQUAL(proof.outPoint, claim.outPoint);
auto claimHash = getValueHash(claim.outPoint, proof.nHeightOfLastTakeover);
ValidatePairs(fixture, proof.pairs, claimHash);
BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
}
}
}
@@ -113,7 +118,7 @@ BOOST_AUTO_TEST_CASE(hash_includes_all_claims_branched_test)
BOOST_CHECK(proof.hasValue);
BOOST_CHECK_EQUAL(proof.outPoint, claim.outPoint);
auto claimHash = getValueHash(claim.outPoint, proof.nHeightOfLastTakeover);
ValidatePairs(fixture, proof.pairs, claimHash);
BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
}
}
}
@@ -174,7 +179,7 @@ BOOST_AUTO_TEST_CASE(hash_claims_children_fuzzer_test)
BOOST_CHECK(proof.hasValue);
BOOST_CHECK_EQUAL(proof.outPoint, claim.outPoint);
auto claimHash = getValueHash(claim.outPoint, proof.nHeightOfLastTakeover);
ValidatePairs(fixture, proof.pairs, claimHash);
BOOST_CHECK(ValidatePairs(fixture, proof.pairs, claimHash));
}
}
}
+5 -5
View File
@@ -8,7 +8,7 @@
using namespace std;
extern void ValidatePairs(CClaimTrieCache& cache, const std::vector<std::pair<bool, uint256>>& pairs, uint256 claimHash);
extern boost::test_tools::predicate_result ValidatePairs(CClaimTrieCache& cache, const std::vector<std::pair<bool, uint256>>& pairs, uint256 claimHash);
BOOST_FIXTURE_TEST_SUITE(claimtrierpc_tests, RegTestingSetup)
@@ -325,7 +325,7 @@ BOOST_AUTO_TEST_CASE(hash_bid_seq_claim_changes_test)
result = getnameproof(req);
auto claimHash = getValueHash(COutPoint(tx2.GetHash(), 1), result[T_LASTTAKEOVERHEIGHT].get_int());
ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash);
BOOST_CHECK(ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash));
// check by partial id (can be even 2 chars)
req.params = UniValue(UniValue::VARR);
@@ -335,7 +335,7 @@ BOOST_AUTO_TEST_CASE(hash_bid_seq_claim_changes_test)
result = getnameproof(req);
claimHash = getValueHash(COutPoint(tx2.GetHash(), 0), result[T_LASTTAKEOVERHEIGHT].get_int());
ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash);
BOOST_CHECK(ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash));
auto getclaimproofbybid = tableRPC["getclaimproofbybid"];
req.params = UniValue(UniValue::VARR);
@@ -344,7 +344,7 @@ BOOST_AUTO_TEST_CASE(hash_bid_seq_claim_changes_test)
result = getclaimproofbybid(req);
claimHash = getValueHash(COutPoint(tx1.GetHash(), 0), result[T_LASTTAKEOVERHEIGHT].get_int());
ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash);
BOOST_CHECK(ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash));
auto getclaimproofbyseq = tableRPC["getclaimproofbyseq"];
req.params = UniValue(UniValue::VARR);
@@ -353,7 +353,7 @@ BOOST_AUTO_TEST_CASE(hash_bid_seq_claim_changes_test)
result = getclaimproofbyseq(req);
claimHash = getValueHash(COutPoint(tx3.GetHash(), 0), result[T_LASTTAKEOVERHEIGHT].get_int());
ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash);
BOOST_CHECK(ValidatePairs(fixture, jsonToPairs(result[T_PAIRS]), claimHash));
auto getchangesinblock = tableRPC["getchangesinblock"];
req.params = UniValue(UniValue::VARR);
+1 -1
View File
@@ -59,7 +59,7 @@ class uint256;
//! No need to periodic flush if at least this much space still available.
static constexpr int MAX_BLOCK_COINSDB_USAGE = 10;
//! -dbcache default (MiB)
static const int64_t nDefaultDbCache = 480;
static const int64_t nDefaultDbCache = 800;
//! -dbbatchsize default (bytes)
static const int64_t nDefaultDbBatchSize = 16 << 20;
//! max. -dbcache (MiB)
+6 -5
View File
@@ -141,6 +141,7 @@ CClaimTrie& Claimtrie() {
consensus.nExtendedClaimExpirationTime,
consensus.nExtendedClaimExpirationForkHeight,
consensus.nAllClaimsInMerkleForkHeight,
consensus.nClaimInfoInMerkleForkHeight,
Params().NetworkIDString() == CBaseChainParams::MAIN ? 32 : 1);
};
return *g_claimtrie;
@@ -2347,6 +2348,9 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl
// TODO: if the "just check" flag is set, we should reduce the work done here. Incrementing blocks twice per mine is not efficient.
assert(trieCache.incrementBlock());
int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2;
LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal);
if (trieCache.getMerkleHash() != block.hashClaimTrie) {
return state.Invalid(ValidationInvalidReason::CLAIMTRIE_HASH,
error("ConnectBlock() : the merkle root of the claim trie does not match "
@@ -2354,9 +2358,6 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl
block.hashClaimTrie.GetHex(), pindex->nHeight), REJECT_INVALID, "bad-claim-merkle-hash");
}
int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2;
LogPrint(BCLog::BENCH, " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs (%.2fms/blk)]\n", (unsigned)block.vtx.size(), MILLI * (nTime3 - nTime2), MILLI * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : MILLI * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * MICRO, nTimeConnect * MILLI / nBlocksTotal);
CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, chainparams.GetConsensus());
if (block.vtx[0]->GetValueOut() > blockReward)
return state.Invalid(ValidationInvalidReason::CONSENSUS,
@@ -2366,8 +2367,8 @@ bool CChainState::ConnectBlock(const CBlock& block, CValidationState& state, CBl
if (!control.Wait())
return state.Invalid(ValidationInvalidReason::CONSENSUS, error("%s: CheckQueue failed", __func__), REJECT_INVALID, "block-validation-failed");
int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2;
LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs (%.2fms/blk)]\n", nInputs - 1, MILLI * (nTime4 - nTime2), nInputs <= 1 ? 0 : MILLI * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * MICRO, nTimeVerify * MILLI / nBlocksTotal);
int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime3;
LogPrint(BCLog::BENCH, " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs (%.2fms/blk)]\n", nInputs - 1, MILLI * (nTime4 - nTime3), nInputs <= 1 ? 0 : MILLI * (nTime4 - nTime3) / (nInputs-1), nTimeVerify * MICRO, nTimeVerify * MILLI / nBlocksTotal);
if (fJustCheck)
return true;
+2 -2
View File
@@ -571,7 +571,7 @@ UniValue abandonclaim(const JSONRPCRequest& request)
static void MaybePushAddress(UniValue& entry, const CTxDestination &dest);
extern std::string escapeNonUtf8(const std::string&);
extern std::string convertToUtf8(const std::string&);
void ListNameClaims(interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx, CWallet* const pwallet, int nMinDepth,
UniValue& ret, const bool include_supports, bool list_spent)
@@ -599,7 +599,7 @@ void ListNameClaims(interfaces::Chain::Lock& locked_chain, const CWalletTx& wtx,
continue;
std::string sName (vvchParams[0].begin(), vvchParams[0].end());
entry.pushKV("name", escapeNonUtf8(sName));
entry.pushKV("name", convertToUtf8(sName));
if (op == OP_CLAIM_NAME)
{
uint160 claimId = ClaimIdHash(wtx.GetHash(), s.vout);
+22
View File
@@ -526,5 +526,27 @@ BOOST_AUTO_TEST_CASE(remove_pruned_funds)
PruneAbandonFunds(txid);
}
class HasMessage {
public:
explicit HasMessage(const std::string& reason) : m_reason(reason) {}
bool operator() (const UniValue& e) const {
return e["message"].get_str().find(m_reason) != std::string::npos;
};
private:
const std::string m_reason;
};
BOOST_AUTO_TEST_CASE(invalid_utf8_claim_name)
{
// nClaimInfoInMerkleForkHeight = 1350
generateBlock(1350);
// disallow non UTF8 strings
BOOST_CHECK_EXCEPTION(ClaimAName("\xFF\xFF", "deadbeef", "1.0"), UniValue, HasMessage("Claim name is not valid UTF8 string"));
// disallow \0 in string
BOOST_CHECK_EXCEPTION(ClaimAName(std::string("test\0ab", 7), "deadbeef", "1.0"), UniValue, HasMessage("Claim name contains invalid symbol"));
// allow ^ in string
ClaimAName("test^ab", "deadbeef", "1.0");
}
BOOST_AUTO_TEST_SUITE_END()
+4 -1
View File
@@ -340,8 +340,9 @@ BOOST_AUTO_TEST_CASE(LoadReceiveRequests)
class ListCoinsTestingSetup : public TestChain100Setup
{
public:
ListCoinsTestingSetup()
ListCoinsTestingSetup() : lRequireStandard(fRequireStandard)
{
fRequireStandard = false; // ensure all txs goes to wallet
CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey()));
wallet = MakeUnique<CWallet>(m_chain.get(), WalletLocation(), WalletDatabase::CreateMock());
bool firstRun;
@@ -359,6 +360,7 @@ public:
~ListCoinsTestingSetup()
{
wallet.reset();
fRequireStandard = lRequireStandard;
}
CWalletTx& AddTx(CRecipient recipient)
@@ -389,6 +391,7 @@ public:
return it->second;
}
bool lRequireStandard;
std::unique_ptr<interfaces::Chain> m_chain = interfaces::MakeChain();
std::unique_ptr<CWallet> wallet;
};
+6 -3
View File
@@ -15,7 +15,7 @@
#include <key_io.h>
#include <nameclaim.h>
#include <policy/fees.h>
#include <policy/policy.h>
#include <policy/settings.h>
#include <primitives/block.h>
#include <primitives/transaction.h>
#include <script/descriptor.h>
@@ -3378,8 +3378,12 @@ bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve
{
{
auto locked_chain = chain().lock();
LOCK(cs_wallet);
std::string reason;
// reject non standard tx before adding to mempool
if (fRequireStandard && !IsStandardTx(*tx, reason))
return state.Invalid(ValidationInvalidReason::TX_NOT_STANDARD, false, REJECT_NONSTANDARD, reason);
LOCK(cs_wallet);
CWalletTx wtxNew(this, std::move(tx));
wtxNew.mapValue = std::move(mapValue);
wtxNew.vOrderForm = std::move(orderForm);
@@ -3411,7 +3415,6 @@ bool CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::ve
std::string err_string;
if (!wtx.SubmitMemoryPoolAndRelay(err_string, true, *locked_chain)) {
WalletLogPrintf("CommitTransaction(): Transaction cannot be broadcast immediately, %s\n", err_string);
// TODO: if we expect the failure to be long term or permanent, instead delete wtx from the wallet and return failure.
}
}
}
View File
File mode changed.
View File
File mode changed.