Compare commits

...
Author SHA1 Message Date
Alex Grintsvayg 4e9977ac23 update staked_amount to include supports for claims in the channel's stake. fixes #3072
ci / lint (push) Waiting to run
ci / tests / unit (macos-latest) (push) Waiting to run
ci / tests / unit (ubuntu-latest) (push) Waiting to run
ci / tests / unit (windows-latest) (push) Waiting to run
ci / tests / integration (postgres, blockchain) (push) Waiting to run
ci / tests / integration (sqlite, blockchain) (push) Waiting to run
2020-12-18 14:57:22 -05:00
Victor Shyba 784343878b fix windows complaints on permissions for wallet file test 2020-12-15 14:40:56 -03:00
Victor Shyba 350eec35ee account for the number of variables on each iteration 2020-12-15 14:18:10 -03:00
Victor Shyba 76d0e31544 fixes for old keys 2020-12-11 19:31:39 -03:00
Victor Shyba cd3f189cf6 ClaimMeta attr typo 2020-12-08 20:50:37 -03:00
Victor Shyba 240f0a9a05 fix initial sync for sync without claims 2020-12-08 20:50:11 -03:00
Victor Shyba 94d76cb13d fix bug on index application 2020-12-08 18:34:40 -03:00
11 changed files with 113 additions and 54 deletions

No files matched your search

+43 -16
View File
@@ -38,26 +38,46 @@ def channel_content_count_calc(signable):
support = TXO.alias('support')
def staked_support_aggregation(aggregate):
def staked_support_subquery(claim_hash_column, aggregate):
"""Return a query that selects unspent supports"""
content = Claim.alias("content")
return (
select(aggregate).where(
(support.c.txo_type == TXO_TYPES['support']) &
select(
aggregate
).select_from(
support
.join(content, support.c.claim_hash == content.c.claim_hash)
).where(
((content.c.claim_hash == claim_hash_column) | (content.c.channel_hash == claim_hash_column)) &
(support.c.txo_type == TXO_TYPES["support"]) &
(support.c.spent_height == 0)
).scalar_subquery()
)
.scalar_subquery()
)
def staked_support_amount_calc(other):
return (
staked_support_aggregation(func.coalesce(func.sum(support.c.amount), 0))
.where(support.c.claim_hash == other.c.claim_hash)
)
def staked_support_amount_calc(claim_hash):
"""Return a query that sums unspent supports for a claim"""
return staked_support_subquery(claim_hash, func.coalesce(func.sum(support.c.amount), 0))
def staked_support_count_calc(other):
def staked_support_count_calc(claim_hash):
"""Return a query that counts unspent supports for a claim"""
return staked_support_subquery(claim_hash, func.coalesce(func.count('*'), 0))
def claims_in_channel_amount_calc(claim_hash):
"""Return a query that sums the amount of all the claims in a channel"""
content = Claim.alias("content")
return (
staked_support_aggregation(func.coalesce(func.count('*'), 0))
.where(support.c.claim_hash == other.c.claim_hash)
select(
func.coalesce(func.sum(content.c.amount), 0)
).select_from(
content
).where(
content.c.channel_hash == claim_hash
)
.scalar_subquery()
)
@@ -87,8 +107,9 @@ def select_claims_for_saving(
channel_txo = TXO.alias('channel_txo')
return select(
*minimum_txo_columns, TXO.c.claim_hash,
staked_support_amount_calc(TXO).label('staked_support_amount'),
staked_support_count_calc(TXO).label('staked_support_count'),
claims_in_channel_amount_calc(TXO.c.claim_hash).label('claims_in_channel_amount'),
staked_support_amount_calc(TXO.c.claim_hash).label('staked_support_amount'),
staked_support_count_calc(TXO.c.claim_hash).label('staked_support_count'),
reposted_claim_count_calc(TXO).label('reposted_count'),
TXO.c.signature, TXO.c.signature_digest,
case([(
@@ -110,6 +131,7 @@ def select_claims_for_saving(
def row_to_claim_for_saving(row) -> Tuple[Output, dict]:
return row_to_txo(row), {
'claims_in_channel_amount': int(row.claims_in_channel_amount),
'staked_support_amount': int(row.staked_support_amount),
'staked_support_count': int(row.staked_support_count),
'reposted_count': int(row.reposted_count),
@@ -247,8 +269,13 @@ def update_stakes(blocks: Tuple[int, int], claims: int, p: ProgressContext):
Claim.update()
.where(where_claims_with_changed_supports(blocks))
.values(
staked_support_amount=staked_support_amount_calc(Claim),
staked_support_count=staked_support_count_calc(Claim),
staked_amount=(
Claim.c.amount +
claims_in_channel_amount_calc(Claim.c.claim_hash) +
staked_support_amount_calc(Claim.c.claim_hash)
),
staked_support_amount=staked_support_amount_calc(Claim.c.claim_hash),
staked_support_count=staked_support_count_calc(Claim.c.claim_hash),
)
)
result = p.ctx.execute(sql)
+1 -1
View File
@@ -259,7 +259,7 @@ class BlockchainSync(Sync):
async def sync_claims(self, blocks) -> bool:
delete_claims = takeovers = claims_with_changed_supports = claims_with_changed_reposts = 0
initial_sync = not await self.db.has_claims()
initial_sync = not await self.db.has_filters()
with Progress(self.db.message_queue, CLAIMS_INIT_EVENT) as p:
if initial_sync:
total, batches = await self.distribute_unspent_txos(CLAIM_TYPE_CODES)
+3 -2
View File
@@ -60,7 +60,8 @@ def get_all_addresses(self):
def add_keys(pubkeys):
c = context()
for start in range(0, len(pubkeys), c.variable_limit - 1):
batch = pubkeys[start:(start + c.variable_limit - 1)]
current_limit = c.variable_limit // len(pubkeys[0]) # (overall limit) // (maximum on a query)
for start in range(0, len(pubkeys), current_limit - 1):
batch = pubkeys[start:(start + current_limit - 1)]
c.execute(c.insert_or_ignore(PubkeyAddress).values([{'address': k['address']} for k in batch]))
c.execute(c.insert_or_ignore(AccountAddress).values(batch))
+2 -1
View File
@@ -107,7 +107,8 @@ def check_version_and_create_tables():
for table in metadata.sorted_tables:
disable_trigger_and_constraints(table.name)
if ctx.is_postgres:
ctx.execute(text(pg_add_account_address_constraints_and_indexes))
for statement in pg_add_account_address_constraints_and_indexes:
ctx.execute(text(statement))
def disable_trigger_and_constraints(table_name):
+9 -4
View File
@@ -34,7 +34,12 @@ BASE_SELECT_SUPPORT_COLUMNS = BASE_SELECT_TXO_COLUMNS + [
def compat_layer(**constraints):
# for old sdk, to be removed later
replacements = {"effective_amount": "staked_amount"}
replacements = {
"effective_amount": "staked_amount",
"trending_mixed": "trend_mixed",
"trending_group": "trend_group",
"trending_local": "trend_local"
}
for old_key, new_key in replacements.items():
if old_key in constraints:
constraints[new_key] = constraints.pop(old_key)
@@ -115,7 +120,7 @@ BASE_SELECT_CLAIM_COLUMNS = BASE_SELECT_TXO_COLUMNS + [
Claim.c.short_url,
Claim.c.signed_claim_count,
Claim.c.signed_support_count,
(Claim.c.amount + Claim.c.staked_support_amount).label('staked_amount'),
Claim.c.staked_amount,
Claim.c.staked_support_amount,
Claim.c.staked_support_count,
Claim.c.is_signature_valid,
@@ -150,8 +155,8 @@ def select_claims(cols: List = None, for_count=False, **constraints) -> Select:
column = 'claim_name'
table = "trend" if column.startswith('trend') else "claim"
column = f"{table}.{column}"
if column in ('trend_group', 'trend_mixed', 'release_time'):
column = f"COALESCE({column}, {1<<32})"
if column in ('trend.trend_group', 'trend.trend_mixed', 'claim.release_time'):
column = f"COALESCE({column}, {-1 * (1<<32)})"
sql_order_by.append(
f"{column} {'ASC' if is_asc else 'DESC'}"
)
+19 -12
View File
@@ -143,28 +143,35 @@ def distribute_unspent_txos(
return total, buckets
def where_changed_support_txos(blocks: Optional[Tuple[int, int]]):
def claims_with_changed_supports(blocks: Optional[Tuple[int, int]]) -> Select:
has_changed_supports = (
select(Claim.c.claim_hash.label("claim_hash"), Claim.c.channel_hash.label("channel_hash"))
.join(Claim, Claim.c.claim_hash == TXO.c.claim_hash)
.where(
(TXO.c.txo_type == TXO_TYPES['support']) &
(between(TXO.c.height, blocks[0], blocks[-1]) | between(TXO.c.spent_height, blocks[0], blocks[-1]))
)
.cte("has_changed_supports")
)
return (
(TXO.c.txo_type == TXO_TYPES['support']) & (
between(TXO.c.height, blocks[0], blocks[-1]) |
between(TXO.c.spent_height, blocks[0], blocks[-1])
select(has_changed_supports.c.claim_hash.label("claim_hash"))
.union_all( # UNION ALL is faster than UNION because it does not remove duplicates
select(has_changed_supports.c.channel_hash)
.where(has_changed_supports.c.channel_hash.isnot(None))
)
)
def where_claims_with_changed_supports(blocks: Optional[Tuple[int, int]]):
def where_claims_with_changed_supports(blocks: Optional[Tuple[int, int]]) -> Select:
return Claim.c.claim_hash.in_(
select(TXO.c.claim_hash).where(
where_changed_support_txos(blocks)
)
claims_with_changed_supports(blocks)
)
def count_claims_with_changed_supports(blocks: Optional[Tuple[int, int]]) -> int:
sql = (
select(func.count(distinct(TXO.c.claim_hash)).label('total'))
.where(where_changed_support_txos(blocks))
)
sub_query = claims_with_changed_supports(blocks).subquery()
sql = select(func.count(distinct(sub_query.c.claim_hash)).label('total')).select_from(sub_query)
return context().fetchone(sql)['total']
+3 -3
View File
@@ -532,8 +532,8 @@ class BulkLoader:
return row
def claim_to_rows(
self, txo: Output, staked_support_amount: int, staked_support_count: int, reposted_count: int,
signature: bytes = None, signature_digest: bytes = None, channel_public_key: bytes = None,
self, txo: Output, claims_in_channel_amount: int, staked_support_amount: int, staked_support_count: int,
reposted_count: int, signature: bytes = None, signature_digest: bytes = None, channel_public_key: bytes = None,
) -> Tuple[dict, List]:
tx = txo.tx_ref
@@ -545,7 +545,7 @@ class BulkLoader:
'height': tx.height,
'timestamp': tx.timestamp,
# support
'staked_amount': txo.amount + staked_support_amount,
'staked_amount': txo.amount + claims_in_channel_amount + staked_support_amount,
'staked_support_amount': staked_support_amount,
'staked_support_count': staked_support_count,
# basic metadata
+4 -4
View File
@@ -113,10 +113,10 @@ class Outputs:
'expiration_height': claim.expiration_height,
'effective_amount': claim.effective_amount,
'support_amount': claim.support_amount,
'trend_group': claim.trend_group,
'trend_mixed': claim.trend_mixed,
'trend_local': claim.trend_local,
'trend_global': claim.trend_global,
'trend_group': claim.trending_group,
'trend_mixed': claim.trending_mixed,
'trend_local': claim.trending_local,
'trend_global': claim.trending_global,
}
if claim.HasField('channel'):
txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout]
+1 -1
View File
@@ -1603,7 +1603,7 @@ class API:
'include_total': pagination['include_total'],
'order_by': order_by
})
_, kwargs = pop_kwargs("no_totals", kwargs) # deprecated, ignoring
kwargs.pop("no_totals", None) # deprecated, ignoring
claim_filter_dict.update(kwargs)
if protobuf:
return await self.service.protobuf_search_claims(**remove_nulls(claim_filter_dict))
@@ -1032,7 +1032,7 @@ class TestGeneralBlockchainSync(SyncingBlockchainTestCase):
self.assertEqual(total, 1600000000)
# create a claim in channel A and have channel B support that claim
claim_a = await self.get_claim(await self.create_claim(name="bob", amount='2.0', sign=channel_a))
claim_a = await self.get_claim(await self.create_claim(name="alfred", amount='2.0', sign=channel_a))
await self.support_claim(claim_a, '1.0', sign=channel_b)
await self.generate(1)
@@ -1097,6 +1097,22 @@ class TestGeneralBlockchainSync(SyncingBlockchainTestCase):
])
self.assertEqual(total, 1100000000)
# add a few supports and make sure `staked_amount` is right
claim_b = await self.get_claim(await self.create_claim(name="bob", amount='0.7', sign=channel_a))
await self.generate(1)
await self.support_claim(claim_b, '1.3', sign=channel_b)
await self.support_claim(claim_b, '1.3', sign=channel_c)
await self.support_claim(channel_b, '1.7', sign=channel_c) # this should not be counted for channel a
await self.generate(1)
ch_c, ch_b, ch_a = await self.db.search_claims(order_by=['name'], claim_type="channel", limit=3)
_, total = await self.db.sum_supports(channel_a.claim_hash, include_channel_content=True, exclude_own_supports=False)
self.assertEqual('@A', ch_a.claim_name)
self.assertEqual(1000000, ch_a.amount)
self.assertEqual(2460000000, total) # sum just the supports, including unlocked (abandoned) tips
self.assertEqual(2031000000, ch_a.meta['staked_amount']) # sum claims and supports
async def test_meta_fields_are_translated_to_protobuf(self):
chan_ab = await self.get_claim(
await self.create_claim(claim_id_startswith='ab', is_channel=True))
@@ -1149,7 +1165,7 @@ class TestGeneralBlockchainSync(SyncingBlockchainTestCase):
claim1 = await self.get_claim(await self.create_claim(name="one"))
claim2 = await self.get_claim(await self.create_claim(name="two"))
await self.generate(1)
results = await self.db.search_claims(order_by=["trend_group", "trend_mixed"])
results = await self.db.search_claims(order_by=["trending_group", "trending_mixed"]) # test compat layer
self.assertEqual(0, results.rows[0].meta['trend_mixed'])
self.assertEqual(0, results.rows[1].meta['trend_mixed'])
self.assertEqual(0, results.rows[0].meta['trend_group'])
+10 -8
View File
@@ -189,16 +189,18 @@ class TestFileBasedWalletManager(AsyncioTestCase):
manager = WalletManager(self.db)
await manager.storage.prepare()
with tempfile.NamedTemporaryFile(suffix='.json') as wallet_file:
wallet_file_name = None
with tempfile.NamedTemporaryFile(suffix='.json', delete=False) as wallet_file:
wallet_file.write(b'{"version": 1}')
wallet_file.seek(0)
wallet_file_name = wallet_file.name
# create and write wallet to a file
wallet = await manager.load(wallet_file.name)
account = await wallet.accounts.generate()
await manager.storage.save(wallet)
# create and write wallet to a file
wallet = await manager.load(wallet_file_name)
account = await wallet.accounts.generate()
await manager.storage.save(wallet)
# read wallet from file
wallet = await manager.load(wallet_file.name)
# read wallet from file
wallet = await manager.load(wallet_file_name)
self.assertEqual(account.public_key.address, wallet.accounts.default.public_key.address)
self.assertEqual(account.public_key.address, wallet.accounts.default.public_key.address)