Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4e9977ac23 | ||
|
|
784343878b | ||
|
|
350eec35ee | ||
|
|
76d0e31544 | ||
|
|
cd3f189cf6 | ||
|
|
240f0a9a05 | ||
|
|
94d76cb13d |
No files matched your search
@@ -38,26 +38,46 @@ def channel_content_count_calc(signable):
|
||||
support = TXO.alias('support')
|
||||
|
||||
|
||||
def staked_support_aggregation(aggregate):
|
||||
def staked_support_subquery(claim_hash_column, aggregate):
|
||||
"""Return a query that selects unspent supports"""
|
||||
content = Claim.alias("content")
|
||||
return (
|
||||
select(aggregate).where(
|
||||
(support.c.txo_type == TXO_TYPES['support']) &
|
||||
select(
|
||||
aggregate
|
||||
).select_from(
|
||||
support
|
||||
.join(content, support.c.claim_hash == content.c.claim_hash)
|
||||
).where(
|
||||
((content.c.claim_hash == claim_hash_column) | (content.c.channel_hash == claim_hash_column)) &
|
||||
(support.c.txo_type == TXO_TYPES["support"]) &
|
||||
(support.c.spent_height == 0)
|
||||
).scalar_subquery()
|
||||
)
|
||||
.scalar_subquery()
|
||||
)
|
||||
|
||||
|
||||
def staked_support_amount_calc(other):
|
||||
return (
|
||||
staked_support_aggregation(func.coalesce(func.sum(support.c.amount), 0))
|
||||
.where(support.c.claim_hash == other.c.claim_hash)
|
||||
)
|
||||
def staked_support_amount_calc(claim_hash):
|
||||
"""Return a query that sums unspent supports for a claim"""
|
||||
return staked_support_subquery(claim_hash, func.coalesce(func.sum(support.c.amount), 0))
|
||||
|
||||
|
||||
def staked_support_count_calc(other):
|
||||
def staked_support_count_calc(claim_hash):
|
||||
"""Return a query that counts unspent supports for a claim"""
|
||||
return staked_support_subquery(claim_hash, func.coalesce(func.count('*'), 0))
|
||||
|
||||
|
||||
def claims_in_channel_amount_calc(claim_hash):
|
||||
"""Return a query that sums the amount of all the claims in a channel"""
|
||||
content = Claim.alias("content")
|
||||
return (
|
||||
staked_support_aggregation(func.coalesce(func.count('*'), 0))
|
||||
.where(support.c.claim_hash == other.c.claim_hash)
|
||||
select(
|
||||
func.coalesce(func.sum(content.c.amount), 0)
|
||||
).select_from(
|
||||
content
|
||||
).where(
|
||||
content.c.channel_hash == claim_hash
|
||||
)
|
||||
.scalar_subquery()
|
||||
)
|
||||
|
||||
|
||||
@@ -87,8 +107,9 @@ def select_claims_for_saving(
|
||||
channel_txo = TXO.alias('channel_txo')
|
||||
return select(
|
||||
*minimum_txo_columns, TXO.c.claim_hash,
|
||||
staked_support_amount_calc(TXO).label('staked_support_amount'),
|
||||
staked_support_count_calc(TXO).label('staked_support_count'),
|
||||
claims_in_channel_amount_calc(TXO.c.claim_hash).label('claims_in_channel_amount'),
|
||||
staked_support_amount_calc(TXO.c.claim_hash).label('staked_support_amount'),
|
||||
staked_support_count_calc(TXO.c.claim_hash).label('staked_support_count'),
|
||||
reposted_claim_count_calc(TXO).label('reposted_count'),
|
||||
TXO.c.signature, TXO.c.signature_digest,
|
||||
case([(
|
||||
@@ -110,6 +131,7 @@ def select_claims_for_saving(
|
||||
|
||||
def row_to_claim_for_saving(row) -> Tuple[Output, dict]:
|
||||
return row_to_txo(row), {
|
||||
'claims_in_channel_amount': int(row.claims_in_channel_amount),
|
||||
'staked_support_amount': int(row.staked_support_amount),
|
||||
'staked_support_count': int(row.staked_support_count),
|
||||
'reposted_count': int(row.reposted_count),
|
||||
@@ -247,8 +269,13 @@ def update_stakes(blocks: Tuple[int, int], claims: int, p: ProgressContext):
|
||||
Claim.update()
|
||||
.where(where_claims_with_changed_supports(blocks))
|
||||
.values(
|
||||
staked_support_amount=staked_support_amount_calc(Claim),
|
||||
staked_support_count=staked_support_count_calc(Claim),
|
||||
staked_amount=(
|
||||
Claim.c.amount +
|
||||
claims_in_channel_amount_calc(Claim.c.claim_hash) +
|
||||
staked_support_amount_calc(Claim.c.claim_hash)
|
||||
),
|
||||
staked_support_amount=staked_support_amount_calc(Claim.c.claim_hash),
|
||||
staked_support_count=staked_support_count_calc(Claim.c.claim_hash),
|
||||
)
|
||||
)
|
||||
result = p.ctx.execute(sql)
|
||||
|
||||
@@ -259,7 +259,7 @@ class BlockchainSync(Sync):
|
||||
|
||||
async def sync_claims(self, blocks) -> bool:
|
||||
delete_claims = takeovers = claims_with_changed_supports = claims_with_changed_reposts = 0
|
||||
initial_sync = not await self.db.has_claims()
|
||||
initial_sync = not await self.db.has_filters()
|
||||
with Progress(self.db.message_queue, CLAIMS_INIT_EVENT) as p:
|
||||
if initial_sync:
|
||||
total, batches = await self.distribute_unspent_txos(CLAIM_TYPE_CODES)
|
||||
|
||||
@@ -60,7 +60,8 @@ def get_all_addresses(self):
|
||||
|
||||
def add_keys(pubkeys):
|
||||
c = context()
|
||||
for start in range(0, len(pubkeys), c.variable_limit - 1):
|
||||
batch = pubkeys[start:(start + c.variable_limit - 1)]
|
||||
current_limit = c.variable_limit // len(pubkeys[0]) # (overall limit) // (maximum on a query)
|
||||
for start in range(0, len(pubkeys), current_limit - 1):
|
||||
batch = pubkeys[start:(start + current_limit - 1)]
|
||||
c.execute(c.insert_or_ignore(PubkeyAddress).values([{'address': k['address']} for k in batch]))
|
||||
c.execute(c.insert_or_ignore(AccountAddress).values(batch))
|
||||
@@ -107,7 +107,8 @@ def check_version_and_create_tables():
|
||||
for table in metadata.sorted_tables:
|
||||
disable_trigger_and_constraints(table.name)
|
||||
if ctx.is_postgres:
|
||||
ctx.execute(text(pg_add_account_address_constraints_and_indexes))
|
||||
for statement in pg_add_account_address_constraints_and_indexes:
|
||||
ctx.execute(text(statement))
|
||||
|
||||
|
||||
def disable_trigger_and_constraints(table_name):
|
||||
|
||||
@@ -34,7 +34,12 @@ BASE_SELECT_SUPPORT_COLUMNS = BASE_SELECT_TXO_COLUMNS + [
|
||||
|
||||
def compat_layer(**constraints):
|
||||
# for old sdk, to be removed later
|
||||
replacements = {"effective_amount": "staked_amount"}
|
||||
replacements = {
|
||||
"effective_amount": "staked_amount",
|
||||
"trending_mixed": "trend_mixed",
|
||||
"trending_group": "trend_group",
|
||||
"trending_local": "trend_local"
|
||||
}
|
||||
for old_key, new_key in replacements.items():
|
||||
if old_key in constraints:
|
||||
constraints[new_key] = constraints.pop(old_key)
|
||||
@@ -115,7 +120,7 @@ BASE_SELECT_CLAIM_COLUMNS = BASE_SELECT_TXO_COLUMNS + [
|
||||
Claim.c.short_url,
|
||||
Claim.c.signed_claim_count,
|
||||
Claim.c.signed_support_count,
|
||||
(Claim.c.amount + Claim.c.staked_support_amount).label('staked_amount'),
|
||||
Claim.c.staked_amount,
|
||||
Claim.c.staked_support_amount,
|
||||
Claim.c.staked_support_count,
|
||||
Claim.c.is_signature_valid,
|
||||
@@ -150,8 +155,8 @@ def select_claims(cols: List = None, for_count=False, **constraints) -> Select:
|
||||
column = 'claim_name'
|
||||
table = "trend" if column.startswith('trend') else "claim"
|
||||
column = f"{table}.{column}"
|
||||
if column in ('trend_group', 'trend_mixed', 'release_time'):
|
||||
column = f"COALESCE({column}, {1<<32})"
|
||||
if column in ('trend.trend_group', 'trend.trend_mixed', 'claim.release_time'):
|
||||
column = f"COALESCE({column}, {-1 * (1<<32)})"
|
||||
sql_order_by.append(
|
||||
f"{column} {'ASC' if is_asc else 'DESC'}"
|
||||
)
|
||||
|
||||
+19
-12
@@ -143,28 +143,35 @@ def distribute_unspent_txos(
|
||||
return total, buckets
|
||||
|
||||
|
||||
def where_changed_support_txos(blocks: Optional[Tuple[int, int]]):
|
||||
def claims_with_changed_supports(blocks: Optional[Tuple[int, int]]) -> Select:
|
||||
has_changed_supports = (
|
||||
select(Claim.c.claim_hash.label("claim_hash"), Claim.c.channel_hash.label("channel_hash"))
|
||||
.join(Claim, Claim.c.claim_hash == TXO.c.claim_hash)
|
||||
.where(
|
||||
(TXO.c.txo_type == TXO_TYPES['support']) &
|
||||
(between(TXO.c.height, blocks[0], blocks[-1]) | between(TXO.c.spent_height, blocks[0], blocks[-1]))
|
||||
)
|
||||
.cte("has_changed_supports")
|
||||
)
|
||||
|
||||
return (
|
||||
(TXO.c.txo_type == TXO_TYPES['support']) & (
|
||||
between(TXO.c.height, blocks[0], blocks[-1]) |
|
||||
between(TXO.c.spent_height, blocks[0], blocks[-1])
|
||||
select(has_changed_supports.c.claim_hash.label("claim_hash"))
|
||||
.union_all( # UNION ALL is faster than UNION because it does not remove duplicates
|
||||
select(has_changed_supports.c.channel_hash)
|
||||
.where(has_changed_supports.c.channel_hash.isnot(None))
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def where_claims_with_changed_supports(blocks: Optional[Tuple[int, int]]):
|
||||
def where_claims_with_changed_supports(blocks: Optional[Tuple[int, int]]) -> Select:
|
||||
return Claim.c.claim_hash.in_(
|
||||
select(TXO.c.claim_hash).where(
|
||||
where_changed_support_txos(blocks)
|
||||
)
|
||||
claims_with_changed_supports(blocks)
|
||||
)
|
||||
|
||||
|
||||
def count_claims_with_changed_supports(blocks: Optional[Tuple[int, int]]) -> int:
|
||||
sql = (
|
||||
select(func.count(distinct(TXO.c.claim_hash)).label('total'))
|
||||
.where(where_changed_support_txos(blocks))
|
||||
)
|
||||
sub_query = claims_with_changed_supports(blocks).subquery()
|
||||
sql = select(func.count(distinct(sub_query.c.claim_hash)).label('total')).select_from(sub_query)
|
||||
return context().fetchone(sql)['total']
|
||||
|
||||
|
||||
|
||||
@@ -532,8 +532,8 @@ class BulkLoader:
|
||||
return row
|
||||
|
||||
def claim_to_rows(
|
||||
self, txo: Output, staked_support_amount: int, staked_support_count: int, reposted_count: int,
|
||||
signature: bytes = None, signature_digest: bytes = None, channel_public_key: bytes = None,
|
||||
self, txo: Output, claims_in_channel_amount: int, staked_support_amount: int, staked_support_count: int,
|
||||
reposted_count: int, signature: bytes = None, signature_digest: bytes = None, channel_public_key: bytes = None,
|
||||
) -> Tuple[dict, List]:
|
||||
|
||||
tx = txo.tx_ref
|
||||
@@ -545,7 +545,7 @@ class BulkLoader:
|
||||
'height': tx.height,
|
||||
'timestamp': tx.timestamp,
|
||||
# support
|
||||
'staked_amount': txo.amount + staked_support_amount,
|
||||
'staked_amount': txo.amount + claims_in_channel_amount + staked_support_amount,
|
||||
'staked_support_amount': staked_support_amount,
|
||||
'staked_support_count': staked_support_count,
|
||||
# basic metadata
|
||||
|
||||
@@ -113,10 +113,10 @@ class Outputs:
|
||||
'expiration_height': claim.expiration_height,
|
||||
'effective_amount': claim.effective_amount,
|
||||
'support_amount': claim.support_amount,
|
||||
'trend_group': claim.trend_group,
|
||||
'trend_mixed': claim.trend_mixed,
|
||||
'trend_local': claim.trend_local,
|
||||
'trend_global': claim.trend_global,
|
||||
'trend_group': claim.trending_group,
|
||||
'trend_mixed': claim.trending_mixed,
|
||||
'trend_local': claim.trending_local,
|
||||
'trend_global': claim.trending_global,
|
||||
}
|
||||
if claim.HasField('channel'):
|
||||
txo.channel = tx_map[claim.channel.tx_hash].outputs[claim.channel.nout]
|
||||
|
||||
+1
-1
@@ -1603,7 +1603,7 @@ class API:
|
||||
'include_total': pagination['include_total'],
|
||||
'order_by': order_by
|
||||
})
|
||||
_, kwargs = pop_kwargs("no_totals", kwargs) # deprecated, ignoring
|
||||
kwargs.pop("no_totals", None) # deprecated, ignoring
|
||||
claim_filter_dict.update(kwargs)
|
||||
if protobuf:
|
||||
return await self.service.protobuf_search_claims(**remove_nulls(claim_filter_dict))
|
||||
|
||||
@@ -1032,7 +1032,7 @@ class TestGeneralBlockchainSync(SyncingBlockchainTestCase):
|
||||
self.assertEqual(total, 1600000000)
|
||||
|
||||
# create a claim in channel A and have channel B support that claim
|
||||
claim_a = await self.get_claim(await self.create_claim(name="bob", amount='2.0', sign=channel_a))
|
||||
claim_a = await self.get_claim(await self.create_claim(name="alfred", amount='2.0', sign=channel_a))
|
||||
await self.support_claim(claim_a, '1.0', sign=channel_b)
|
||||
await self.generate(1)
|
||||
|
||||
@@ -1097,6 +1097,22 @@ class TestGeneralBlockchainSync(SyncingBlockchainTestCase):
|
||||
])
|
||||
self.assertEqual(total, 1100000000)
|
||||
|
||||
# add a few supports and make sure `staked_amount` is right
|
||||
claim_b = await self.get_claim(await self.create_claim(name="bob", amount='0.7', sign=channel_a))
|
||||
await self.generate(1)
|
||||
await self.support_claim(claim_b, '1.3', sign=channel_b)
|
||||
await self.support_claim(claim_b, '1.3', sign=channel_c)
|
||||
await self.support_claim(channel_b, '1.7', sign=channel_c) # this should not be counted for channel a
|
||||
await self.generate(1)
|
||||
|
||||
ch_c, ch_b, ch_a = await self.db.search_claims(order_by=['name'], claim_type="channel", limit=3)
|
||||
_, total = await self.db.sum_supports(channel_a.claim_hash, include_channel_content=True, exclude_own_supports=False)
|
||||
self.assertEqual('@A', ch_a.claim_name)
|
||||
self.assertEqual(1000000, ch_a.amount)
|
||||
self.assertEqual(2460000000, total) # sum just the supports, including unlocked (abandoned) tips
|
||||
self.assertEqual(2031000000, ch_a.meta['staked_amount']) # sum claims and supports
|
||||
|
||||
|
||||
async def test_meta_fields_are_translated_to_protobuf(self):
|
||||
chan_ab = await self.get_claim(
|
||||
await self.create_claim(claim_id_startswith='ab', is_channel=True))
|
||||
@@ -1149,7 +1165,7 @@ class TestGeneralBlockchainSync(SyncingBlockchainTestCase):
|
||||
claim1 = await self.get_claim(await self.create_claim(name="one"))
|
||||
claim2 = await self.get_claim(await self.create_claim(name="two"))
|
||||
await self.generate(1)
|
||||
results = await self.db.search_claims(order_by=["trend_group", "trend_mixed"])
|
||||
results = await self.db.search_claims(order_by=["trending_group", "trending_mixed"]) # test compat layer
|
||||
self.assertEqual(0, results.rows[0].meta['trend_mixed'])
|
||||
self.assertEqual(0, results.rows[1].meta['trend_mixed'])
|
||||
self.assertEqual(0, results.rows[0].meta['trend_group'])
|
||||
|
||||
@@ -189,16 +189,18 @@ class TestFileBasedWalletManager(AsyncioTestCase):
|
||||
manager = WalletManager(self.db)
|
||||
await manager.storage.prepare()
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix='.json') as wallet_file:
|
||||
wallet_file_name = None
|
||||
with tempfile.NamedTemporaryFile(suffix='.json', delete=False) as wallet_file:
|
||||
wallet_file.write(b'{"version": 1}')
|
||||
wallet_file.seek(0)
|
||||
wallet_file_name = wallet_file.name
|
||||
|
||||
# create and write wallet to a file
|
||||
wallet = await manager.load(wallet_file.name)
|
||||
account = await wallet.accounts.generate()
|
||||
await manager.storage.save(wallet)
|
||||
# create and write wallet to a file
|
||||
wallet = await manager.load(wallet_file_name)
|
||||
account = await wallet.accounts.generate()
|
||||
await manager.storage.save(wallet)
|
||||
|
||||
# read wallet from file
|
||||
wallet = await manager.load(wallet_file.name)
|
||||
# read wallet from file
|
||||
wallet = await manager.load(wallet_file_name)
|
||||
|
||||
self.assertEqual(account.public_key.address, wallet.accounts.default.public_key.address)
|
||||
self.assertEqual(account.public_key.address, wallet.accounts.default.public_key.address)
|
||||
Reference in new issue
Block a user